def step_impl(context, table, id_format, message_type):
folder = streaming_data_helper.generate_fixture_data_folder(message_type)
context.latest_metadata_store_ids = []
wrap_id_value = id_format == "wrapped"
table_name = streaming_data_helper.get_metadata_store_table_name(
table, context.metadata_store_tables)
for topic in context.topics_for_test:
topic_name = template_helper.get_topic_name(topic["topic"])
temp_folder_for_topic = os.path.join(context.temp_folder, topic_name)
full_folder_path = file_helper.generate_edited_files_folder(
temp_folder_for_topic, folder)
latest_file_path = file_helper.get_file_from_folder_with_latest_timestamp(
full_folder_path)
(
record_id,
record_timestamp,
results,
) = metadata_store_helper.get_metadata_for_id_and_timestamp_from_file(
table_name, latest_file_path, topic_name, wrap_id_value)
console_printer.print_info(
f"Received {len(results)} responses in topic '{topic_name}'")
console_printer.print_info(
f"Actual metadata store results are '{results}' in topic '{topic_name}'"
)
console_printer.print_info(
f"Asserting exactly one response received for id of '{record_id}' and timestamp of '{str(record_timestamp)}' in topic '{topic_name}'"
)
assert (
len(results) == 1
), "Metadata table result not returned, try restarting the k2hb consumers"
results_iterator = iter(results.items())
result_row_key_value_pair = next(results_iterator)
result_row_key = result_row_key_value_pair[0]
result_row_value = result_row_key_value_pair[1]
console_printer.print_info(
f"Asserting the key value for the result in topic '{topic_name}'")
assert record_id in result_row_key
console_printer.print_info(
f"Asserted key value of '{result_row_key}' contains expected id of '{record_id}' in topic '{topic_name}'"
)
console_printer.print_info(
f"Asserting the field values for the result in topic '{topic_name}'"
)
assert record_id in result_row_value["hbase_id"]
assert record_timestamp == result_row_value["hbase_timestamp"]
assert topic_name == result_row_value["topic_name"]
context.latest_metadata_store_ids.append(
[topic_name, record_id, record_timestamp])
def step_impl(context, message_type, date, key):
folder = streaming_data_helper.generate_fixture_data_folder(message_type)
topic_prefix = streaming_data_helper.generate_topic_prefix(message_type)
qualified_key = None if key == "None" else key
date_qualified = (None if date == "None" else datetime.strptime(
date, "%Y-%m-%dT%H:%M:%S.%f"))
for topic in context.topics_for_test:
topic_name = template_helper.get_topic_name(topic["topic"])
generated_files = kafka_data_generator.generate_kafka_files(
test_run_name=context.test_run_name,
s3_input_bucket=context.s3_ingest_bucket,
input_template_name=input_template,
output_template_name=output_template,
new_uuid=qualified_key,
local_files_temp_folder=os.path.join(context.temp_folder,
topic_name),
fixture_files_root=context.fixture_path_local,
s3_output_prefix=context.s3_temp_output_path,
record_count=1,
topic_name=topic["topic"],
snapshots_output_folder=context.
snapshot_files_hbase_records_temp_folder,
seconds_timeout=context.timeout,
fixture_data_folder=folder,
custom_base_timestamp=date_qualified,
)
files_to_send_to_kafka_broker = [
generated_file[0] for generated_file in generated_files
]
aws_helper.send_files_to_kafka_producer_sns(
dynamodb_table_name=context.dynamo_db_table_name,
s3_input_bucket=context.s3_ingest_bucket,
aws_acc_id=context.aws_acc,
sns_topic_name=context.aws_sns_topic_name,
fixture_files=files_to_send_to_kafka_broker,
message_key=key,
topic_name=topic["topic"],
topic_prefix=topic_prefix,
region=context.aws_region_main,
)
def step_impl(context, id_format, message_type):
folder = streaming_data_helper.generate_fixture_data_folder(message_type)
for topic in context.topics_for_test:
topic_name = template_helper.get_topic_name(topic["topic"])
temp_folder_for_topic = os.path.join(context.temp_folder, topic_name)
full_folder_path = file_helper.generate_edited_files_folder(
temp_folder_for_topic, folder)
latest_file_path = file_helper.get_file_from_folder_with_latest_timestamp(
full_folder_path)
wrap_id_value = id_format == "wrapped"
file_comparer.assert_specific_file_stored_in_hbase(
topic_name,
latest_file_path,
60,
record_expected_in_hbase=False,
wrap_id=wrap_id_value,
)
def step_impl(
context,
record_count,
message_type,
input_file_name,
output_file_name,
dlq_file_name,
snapshot_record_file_name,
encrypt_in_sender,
wait_for_sending,
key_method,
):
context.uploaded_id = uuid.uuid4()
folder = streaming_data_helper.generate_fixture_data_folder(message_type)
topic_prefix = streaming_data_helper.generate_topic_prefix(message_type)
skip_encryption = "true" if encrypt_in_sender == "false" else "false"
output_template = None if output_file_name == "None" else output_file_name
dlq_template = None if dlq_file_name == "None" else dlq_file_name
snapshot_record_file_name = (None if snapshot_record_file_name == "None"
else snapshot_record_file_name)
wait_for_sending_bool = wait_for_sending.lower() == "true"
message_volume = (context.kafka_message_volume
if context.kafka_message_volume else "1")
random_keys = context.kafka_random_key if context.kafka_random_key else "false"
context.kafka_generated_dlq_output_files = []
for topic in context.topics_for_test:
key = None
if key_method.lower() == "static":
key = context.uploaded_id
elif key_method.lower() == "topic":
key = uuid.uuid4()
topic_name = template_helper.get_topic_name(topic["topic"])
generated_files = kafka_data_generator.generate_kafka_files(
test_run_name=context.test_run_name,
s3_input_bucket=context.s3_ingest_bucket,
input_template_name=input_file_name,
output_template_name=output_template,
new_uuid=key,
local_files_temp_folder=os.path.join(context.temp_folder,
topic_name),
fixture_files_root=context.fixture_path_local,
s3_output_prefix=context.s3_temp_output_path,
record_count=record_count,
topic_name=topic["topic"],
snapshots_output_folder=context.
snapshot_files_hbase_records_temp_folder,
seconds_timeout=context.timeout,
fixture_data_folder=folder,
dlq_template_name=dlq_template,
snapshot_record_template_name=snapshot_record_file_name,
)
files_to_send_to_kafka_broker = [
generated_file[0] for generated_file in generated_files
]
aws_helper.send_files_to_kafka_producer_sns(
dynamodb_table_name=context.dynamo_db_table_name,
s3_input_bucket=context.s3_ingest_bucket,
aws_acc_id=context.aws_acc,
sns_topic_name=context.aws_sns_topic_name,
fixture_files=files_to_send_to_kafka_broker,
message_key=context.uploaded_id,
topic_name=topic["topic"],
topic_prefix=topic_prefix,
region=context.aws_region_main,
skip_encryption=skip_encryption,
kafka_message_volume=message_volume,
kafka_random_key=random_keys,
wait_for_job_completion=wait_for_sending_bool,
)
dlq_files_for_topic = []
for generated_file in generated_files:
if len(generated_file) > 3:
dlq_files_for_topic.append(generated_file[3])
context.kafka_generated_dlq_output_files.append(
(topic["topic"], dlq_files_for_topic))
def step_impl(context, streaming_type, id_format, message_type):
folder = streaming_data_helper.generate_fixture_data_folder(message_type)
manifest_bucket = context.k2hb_manifest_write_s3_bucket
valid_prefixes = {
"main": context.k2hb_main_manifest_write_s3_prefix,
"equalities": context.k2hb_equality_manifest_write_s3_prefix,
"audit": context.k2hb_audit_manifest_write_s3_prefix,
}
manifest_base_prefix = valid_prefixes.get(streaming_type, "NOT_SET")
if manifest_base_prefix == "NOT_SET":
raise AssertionError(
f"Could not find manifest prefix for streaming of '{streaming_type}'"
)
for topic in context.topics_for_test:
topic_name = template_helper.get_topic_name(topic["topic"])
temp_folder_for_topic = os.path.join(context.temp_folder, topic_name)
full_folder_path = file_helper.generate_edited_files_folder(
temp_folder_for_topic, folder)
latest_file_path = file_helper.get_file_from_folder_with_latest_timestamp(
full_folder_path)
wrap_id_value = id_format == "wrapped"
file_pattern = f"^.*_.*_\d+-.*_.*_\d+.txt$"
console_printer.print_info(
f"Looking for manifest files in '{manifest_bucket}' bucket with prefix of '{manifest_base_prefix}' and pattern of '{file_pattern}'"
)
manifest_files = aws_helper.retrieve_files_from_s3(
manifest_bucket,
manifest_base_prefix,
file_pattern,
)
console_printer.print_info(
f"Found '{len(manifest_files)}' manifest files")
manifest_lines = []
for manifest_file in manifest_files:
manifest_lines_in_file = manifest_file.splitlines()
manifest_lines.extend([
manifest_line_in_file.replace('""', '"')
for manifest_line_in_file in manifest_lines_in_file
])
record_id = file_helper.get_id_object_from_json_file(latest_file_path)
record_timestamp = file_helper.get_timestamp_as_long_from_json_file(
latest_file_path)
expected = streaming_manifests_helper.generate_correct_manifest_line(
record_id, record_timestamp, topic_name, wrap_id=wrap_id_value)
console_printer.print_info(
f"Expecting manifest line with data of '{expected}'")
console_printer.print_info(
f"Actual manifest lines were '{manifest_lines}'")
assert expected in manifest_lines