def step_impl(context, table, id_format, message_type): folder = streaming_data_helper.generate_fixture_data_folder(message_type) context.latest_metadata_store_ids = [] wrap_id_value = id_format == "wrapped" table_name = streaming_data_helper.get_metadata_store_table_name( table, context.metadata_store_tables) for topic in context.topics_for_test: topic_name = template_helper.get_topic_name(topic["topic"]) temp_folder_for_topic = os.path.join(context.temp_folder, topic_name) full_folder_path = file_helper.generate_edited_files_folder( temp_folder_for_topic, folder) latest_file_path = file_helper.get_file_from_folder_with_latest_timestamp( full_folder_path) ( record_id, record_timestamp, results, ) = metadata_store_helper.get_metadata_for_id_and_timestamp_from_file( table_name, latest_file_path, topic_name, wrap_id_value) console_printer.print_info( f"Received {len(results)} responses in topic '{topic_name}'") console_printer.print_info( f"Actual metadata store results are '{results}' in topic '{topic_name}'" ) console_printer.print_info( f"Asserting exactly one response received for id of '{record_id}' and timestamp of '{str(record_timestamp)}' in topic '{topic_name}'" ) assert ( len(results) == 1 ), "Metadata table result not returned, try restarting the k2hb consumers" results_iterator = iter(results.items()) result_row_key_value_pair = next(results_iterator) result_row_key = result_row_key_value_pair[0] result_row_value = result_row_key_value_pair[1] console_printer.print_info( f"Asserting the key value for the result in topic '{topic_name}'") assert record_id in result_row_key console_printer.print_info( f"Asserted key value of '{result_row_key}' contains expected id of '{record_id}' in topic '{topic_name}'" ) console_printer.print_info( f"Asserting the field values for the result in topic '{topic_name}'" ) assert record_id in result_row_value["hbase_id"] assert record_timestamp == result_row_value["hbase_timestamp"] assert topic_name == result_row_value["topic_name"] context.latest_metadata_store_ids.append( [topic_name, record_id, record_timestamp])
def step_impl(context, message_type, date, key): folder = streaming_data_helper.generate_fixture_data_folder(message_type) topic_prefix = streaming_data_helper.generate_topic_prefix(message_type) qualified_key = None if key == "None" else key date_qualified = (None if date == "None" else datetime.strptime( date, "%Y-%m-%dT%H:%M:%S.%f")) for topic in context.topics_for_test: topic_name = template_helper.get_topic_name(topic["topic"]) generated_files = kafka_data_generator.generate_kafka_files( test_run_name=context.test_run_name, s3_input_bucket=context.s3_ingest_bucket, input_template_name=input_template, output_template_name=output_template, new_uuid=qualified_key, local_files_temp_folder=os.path.join(context.temp_folder, topic_name), fixture_files_root=context.fixture_path_local, s3_output_prefix=context.s3_temp_output_path, record_count=1, topic_name=topic["topic"], snapshots_output_folder=context. snapshot_files_hbase_records_temp_folder, seconds_timeout=context.timeout, fixture_data_folder=folder, custom_base_timestamp=date_qualified, ) files_to_send_to_kafka_broker = [ generated_file[0] for generated_file in generated_files ] aws_helper.send_files_to_kafka_producer_sns( dynamodb_table_name=context.dynamo_db_table_name, s3_input_bucket=context.s3_ingest_bucket, aws_acc_id=context.aws_acc, sns_topic_name=context.aws_sns_topic_name, fixture_files=files_to_send_to_kafka_broker, message_key=key, topic_name=topic["topic"], topic_prefix=topic_prefix, region=context.aws_region_main, )
def step_impl(context, id_format, message_type): folder = streaming_data_helper.generate_fixture_data_folder(message_type) for topic in context.topics_for_test: topic_name = template_helper.get_topic_name(topic["topic"]) temp_folder_for_topic = os.path.join(context.temp_folder, topic_name) full_folder_path = file_helper.generate_edited_files_folder( temp_folder_for_topic, folder) latest_file_path = file_helper.get_file_from_folder_with_latest_timestamp( full_folder_path) wrap_id_value = id_format == "wrapped" file_comparer.assert_specific_file_stored_in_hbase( topic_name, latest_file_path, 60, record_expected_in_hbase=False, wrap_id=wrap_id_value, )
def step_impl( context, record_count, message_type, input_file_name, output_file_name, dlq_file_name, snapshot_record_file_name, encrypt_in_sender, wait_for_sending, key_method, ): context.uploaded_id = uuid.uuid4() folder = streaming_data_helper.generate_fixture_data_folder(message_type) topic_prefix = streaming_data_helper.generate_topic_prefix(message_type) skip_encryption = "true" if encrypt_in_sender == "false" else "false" output_template = None if output_file_name == "None" else output_file_name dlq_template = None if dlq_file_name == "None" else dlq_file_name snapshot_record_file_name = (None if snapshot_record_file_name == "None" else snapshot_record_file_name) wait_for_sending_bool = wait_for_sending.lower() == "true" message_volume = (context.kafka_message_volume if context.kafka_message_volume else "1") random_keys = context.kafka_random_key if context.kafka_random_key else "false" context.kafka_generated_dlq_output_files = [] for topic in context.topics_for_test: key = None if key_method.lower() == "static": key = context.uploaded_id elif key_method.lower() == "topic": key = uuid.uuid4() topic_name = template_helper.get_topic_name(topic["topic"]) generated_files = kafka_data_generator.generate_kafka_files( test_run_name=context.test_run_name, s3_input_bucket=context.s3_ingest_bucket, input_template_name=input_file_name, output_template_name=output_template, new_uuid=key, local_files_temp_folder=os.path.join(context.temp_folder, topic_name), fixture_files_root=context.fixture_path_local, s3_output_prefix=context.s3_temp_output_path, record_count=record_count, topic_name=topic["topic"], snapshots_output_folder=context. snapshot_files_hbase_records_temp_folder, seconds_timeout=context.timeout, fixture_data_folder=folder, dlq_template_name=dlq_template, snapshot_record_template_name=snapshot_record_file_name, ) files_to_send_to_kafka_broker = [ generated_file[0] for generated_file in generated_files ] aws_helper.send_files_to_kafka_producer_sns( dynamodb_table_name=context.dynamo_db_table_name, s3_input_bucket=context.s3_ingest_bucket, aws_acc_id=context.aws_acc, sns_topic_name=context.aws_sns_topic_name, fixture_files=files_to_send_to_kafka_broker, message_key=context.uploaded_id, topic_name=topic["topic"], topic_prefix=topic_prefix, region=context.aws_region_main, skip_encryption=skip_encryption, kafka_message_volume=message_volume, kafka_random_key=random_keys, wait_for_job_completion=wait_for_sending_bool, ) dlq_files_for_topic = [] for generated_file in generated_files: if len(generated_file) > 3: dlq_files_for_topic.append(generated_file[3]) context.kafka_generated_dlq_output_files.append( (topic["topic"], dlq_files_for_topic))
def step_impl(context, streaming_type, id_format, message_type): folder = streaming_data_helper.generate_fixture_data_folder(message_type) manifest_bucket = context.k2hb_manifest_write_s3_bucket valid_prefixes = { "main": context.k2hb_main_manifest_write_s3_prefix, "equalities": context.k2hb_equality_manifest_write_s3_prefix, "audit": context.k2hb_audit_manifest_write_s3_prefix, } manifest_base_prefix = valid_prefixes.get(streaming_type, "NOT_SET") if manifest_base_prefix == "NOT_SET": raise AssertionError( f"Could not find manifest prefix for streaming of '{streaming_type}'" ) for topic in context.topics_for_test: topic_name = template_helper.get_topic_name(topic["topic"]) temp_folder_for_topic = os.path.join(context.temp_folder, topic_name) full_folder_path = file_helper.generate_edited_files_folder( temp_folder_for_topic, folder) latest_file_path = file_helper.get_file_from_folder_with_latest_timestamp( full_folder_path) wrap_id_value = id_format == "wrapped" file_pattern = f"^.*_.*_\d+-.*_.*_\d+.txt$" console_printer.print_info( f"Looking for manifest files in '{manifest_bucket}' bucket with prefix of '{manifest_base_prefix}' and pattern of '{file_pattern}'" ) manifest_files = aws_helper.retrieve_files_from_s3( manifest_bucket, manifest_base_prefix, file_pattern, ) console_printer.print_info( f"Found '{len(manifest_files)}' manifest files") manifest_lines = [] for manifest_file in manifest_files: manifest_lines_in_file = manifest_file.splitlines() manifest_lines.extend([ manifest_line_in_file.replace('""', '"') for manifest_line_in_file in manifest_lines_in_file ]) record_id = file_helper.get_id_object_from_json_file(latest_file_path) record_timestamp = file_helper.get_timestamp_as_long_from_json_file( latest_file_path) expected = streaming_manifests_helper.generate_correct_manifest_line( record_id, record_timestamp, topic_name, wrap_id=wrap_id_value) console_printer.print_info( f"Expecting manifest line with data of '{expected}'") console_printer.print_info( f"Actual manifest lines were '{manifest_lines}'") assert expected in manifest_lines