def step_impl(context, module_name):
schema_config = context.kickstart_schema_config[module_name]
if schema_config["record_layout"].lower() == "csv":
for collection in schema_config["schema"].keys():
s3_result_key = os.path.join(context.kickstart_hive_result_path,
f"e2e_{collection}.csv")
console_printer.print_info(f"S3 Request Location: {s3_result_key}")
file_content = aws_helper.get_s3_object(
None, context.published_bucket, s3_result_key).decode("utf-8")
actual_content = (file_content.replace("\t", ",").replace(
"NULL", "None").strip().splitlines())
expected_file_name = [
file for file in context.kickstart_current_run_input_files
if collection in file
][0]
console_printer.print_info(
f"Expected File Name: {expected_file_name}")
expected_content = file_helper.get_contents_of_file(
expected_file_name, False).splitlines()[1:]
for input_line, output_line in zip(actual_content,
expected_content):
assert (
input_line.lower() == output_line.lower()
), f"Expected result of '{input_line}', does not match '{output_line}' for collection {collection}"
elif schema_config["record_layout"].lower() == "json":
for collection in schema_config["schema"].keys():
s3_result_key = os.path.join(context.kickstart_hive_result_path,
f"e2e_{collection}.csv")
console_printer.print_info(f"S3 Request Location: {s3_result_key}")
file_content = aws_helper.get_s3_object(
None, context.published_bucket, s3_result_key).decode("utf-8")
actual_content = file_content.replace("NULL",
"None").strip().splitlines()
console_printer.print_info(
f"This the local file name in the list: {context.kickstart_current_run_input_files}"
)
expected_file_name = [
file for file in context.kickstart_current_run_input_files
if f"{module_name}-{collection}" in file
][0]
console_printer.print_info(
f"Expected File Name: {expected_file_name}")
expected_json = json.loads(
file_helper.get_contents_of_file(expected_file_name,
False))["data"]
expected_content = "\n".join([
"\t".join([str(record[field]) for field in record])
for record in expected_json
]).splitlines()
for input_line, output_line in zip(actual_content,
expected_content):
assert (
input_line.lower() == output_line.lower()
), f"Expected result of '{input_line}', does not match '{output_line}' for collection {collection}"
def step_(context, input_json):
expected_file_name = os.path.join(
context.fixture_path_local, "pdm_data", "input_data", input_json
)
input_file = file_helper.get_contents_of_file(expected_file_name, False)
inputs_s3_key = os.path.join(context.pdm_test_input_s3_prefix, input_json)
aws_helper.put_object_in_s3(input_file, context.published_bucket, inputs_s3_key)
def step_impl(context, dlq_file_template):
for topic in context.topics_for_test:
dlq_file = None
for dlq_files_and_topic_tuple in context.kafka_generated_dlq_output_files:
if topic["topic"] == dlq_files_and_topic_tuple[0]:
for dlq_file_for_topic in dlq_files_and_topic_tuple[1]:
if dlq_file_template in dlq_file_for_topic:
dlq_file = dlq_file_for_topic
if dlq_file is None:
raise AssertionError(
f"No generated dlq file could be found for dlq template of {dlq_file_template}"
)
expected_file_content = file_helper.get_contents_of_file(
dlq_file, True)
id_object = file_helper.get_id_object_from_json_file(dlq_file)
dlq_full_file_path_s3 = os.path.join(
context.s3_dlq_path_and_date_prefix, id_object)
try:
console_printer.print_info(
f"Waiting for dlq file in s3 with prefix of '{dlq_full_file_path_s3}' in bucket '{context.s3_ingest_bucket}'"
)
aws_helper.wait_for_file_to_be_in_s3(context.s3_ingest_bucket,
dlq_full_file_path_s3,
context.timeout)
except Exception as ex:
raise AssertionError(ex)
file_data = aws_helper.retrieve_files_from_s3(context.s3_ingest_bucket,
dlq_full_file_path_s3)
number_files = len(file_data)
if number_files != 1:
raise AssertionError(
f"There should a single dlq file found for {dlq_full_file_path_s3} but was {number_files}"
)
input_json = json.loads(expected_file_content)
output_json = json.loads(file_data[0])
console_printer.print_info(
f"Asserting unformatted dlq message actual '{input_json}' is like expected '{output_json}'"
)
assert input_json["key"] in output_json["body"]
assert input_json["reason"] in output_json["reason"]
input_json_formatted = file_helper.get_json_with_replaced_values(
expected_file_content)
output_json_formatted = file_helper.get_json_with_replaced_values(
file_data[0])
console_printer.print_info(
f"Asserting formatted dlq message actual '{output_json_formatted}' is like expected '{input_json_formatted}'"
)
assert input_json_formatted == output_json_formatted
def step_(context, expected_result_file_name):
console_printer.print_info(f"S3 Request Location: {context.pdm_results_s3_file}")
actual = aws_helper.get_s3_object(
None, context.published_bucket, context.pdm_results_s3_file
).decode("ascii")
actual_comma_deliminated = actual.replace("\t", ",").strip()
expected_file_name = os.path.join(
context.fixture_path_local, "pdm_data", "expected", expected_result_file_name
)
expected = file_helper.get_contents_of_file(expected_file_name, False)
expected_comma_deliminated = expected.replace("\t", ",").strip()
assert (
expected_comma_deliminated == actual_comma_deliminated
), f"Expected result of '{expected_comma_deliminated}', does not match '{actual_comma_deliminated}'"
def get_locally_generated_snapshot_file_records(snapshot_file):
"""Returns the sorted contents of the local snapshot file as an array.
Keyword arguments:
snapshot_file -- full path to the file
"""
expected_snapshot_file_s3_contents = file_helper.get_contents_of_file(
snapshot_file, False)
snapshot_records = []
for line in expected_snapshot_file_s3_contents.splitlines():
unsorted_json_record = json.loads(line)
snapshot_records.append(
json.dumps(unsorted_json_record, sort_keys=True))
return snapshot_records
def step_(context, expected_result_file_name):
console_printer.print_info(
f"S3 Request Location: {context.adg_results_s3_file}")
actual = (aws_helper.get_s3_object(
None, context.published_bucket,
context.adg_results_s3_file).decode("ascii").replace("\t", "").replace(
" ", "").strip())
expected_file_name = os.path.join(
context.fixture_path_local,
"snapshot_data",
"expected",
expected_result_file_name,
)
expected = (file_helper.get_contents_of_file(
expected_file_name, False).replace("\t", "").replace(" ", "").strip())
assert (expected == actual
), f"Expected result of '{expected}', does not match '{actual}'"
def step_impl(context, dlq_file_template):
for topic in context.topics_for_test:
dlq_file = None
for dlq_files_and_topic_tuple in context.kafka_generated_dlq_output_files:
if topic["topic"] == dlq_files_and_topic_tuple[0]:
for dlq_file_for_topic in dlq_files_and_topic_tuple[1]:
if dlq_file_template in dlq_file_for_topic:
dlq_file = dlq_file_for_topic
if dlq_file is None:
raise AssertionError(
f"No generated dlq file could be found for dlq template of {dlq_file_template}"
)
expected_file_content = file_helper.get_contents_of_file(
dlq_file, True)
id_object = file_helper.get_id_object_from_json_file(dlq_file)
test_run_topic_name = template_helper.get_topic_name(topic["topic"])
file_comparer.assert_specific_id_missing_in_hbase(
test_run_topic_name, id_object, 5, True)
def files_upload_to_s3(context, local_file_list, folder_name, upload_method):
for file in local_file_list:
if upload_method.lower() == "unencrypted":
console_printer.print_info(
f"Data will be uploaded in {upload_method} format to s3 bucket"
)
console_printer.print_info(f"The file name is {file}")
file_name = os.path.basename(file)
input_file = file_helper.get_contents_of_file(file, False)
inputs_s3_key = os.path.join(folder_name, file_name)
console_printer.print_info(
f"Uploading the local file {file} with basename as {file_name} into s3 bucket {context.published_bucket} using key name as {inputs_s3_key}"
)
aws_helper.put_object_in_s3(
input_file, context.published_bucket, inputs_s3_key
)
elif upload_method.lower() == "encrypted":
console_printer.print_info(
f"Data will be uploaded in {upload_method} format to s3 bucket"
)
console_printer.print_info(f"The input file name is {file}")
file_name = os.path.basename(file)
encrypted_key = context.encryption_encrypted_key
master_key = context.encryption_master_key_id
plaintext_key = context.encryption_plaintext_key
[
file_iv_int,
file_iv_whole,
] = historic_data_load_generator.generate_initialisation_vector()
console_printer.print_info(f"Extracting the raw data from local directory")
data = file_helper.get_contents_of_file(file, False).encode("utf-8")
console_printer.print_info(f"Applying encryption to the raw data")
input_data = historic_data_load_generator.encrypt(
file_iv_whole, plaintext_key, data
)
inputs_s3_key = os.path.join(folder_name, file_name + ".enc")
all_metadata = json.loads(
historic_data_load_generator.generate_encryption_metadata_for_metadata_file(
encrypted_key, master_key, plaintext_key, file_iv_int
)
)
console_printer.print_info("Metadata of for encrypted file is \n")
console_printer.print_info(f"{json.dumps(all_metadata)}")
metadata = {
"iv": all_metadata["initialisationVector"],
"ciphertext": all_metadata["encryptedEncryptionKey"],
"datakeyencryptionkeyid": all_metadata["keyEncryptionKeyId"],
}
console_printer.print_info(
f"Uploading the local file {file} with basename as {file_name} into s3 bucket {context.published_bucket} using key name as {inputs_s3_key} and along with metadata"
)
aws_helper.put_object_in_s3_with_metadata(
input_data, context.published_bucket, inputs_s3_key, metadata=metadata
)
def _generate_input_data_files_threaded(
s3_bucket,
s3_prefix,
file_count,
record_count,
short_topic,
input_template,
encrypted_key,
plaintext_key,
master_key,
input_folder,
max_worker_count,
):
"""Generates required historic data files from the files in the given folder using multiple threads.
Keyword arguments:
s3_bucket -- the s3 bucket to send input files to
s3_prefix -- the s3 prefix to send input files to
file_count -- the number of files to create for the collection
record_count -- the number of records per file
short_topic -- the short topic
input_template -- the name and location for the input template json file
encrypted_key -- the encrypted version of the plaintext key
plaintext_key -- the plaintext data key for encrypting the data file
master_key -- the master key used to encrypt the data key
input_folder -- the folder to store the generated input files in
max_worker_count -- max thread number
"""
input_base_content = file_helper.get_contents_of_file(
input_template, False)
[file_iv_int, file_iv_whole] = generate_initialisation_vector()
encryption_json_text = generate_encryption_metadata_for_metadata_file(
encrypted_key, master_key, plaintext_key, file_iv_int)
with ThreadPoolExecutor(max_workers=max_worker_count) as executor_input:
future_results_input = []
for file_number in range(1, int(file_count) + 1):
future_results_input.append(
executor_input.submit(
_generate_input_files_with_different_timestamps,
s3_bucket,
s3_prefix,
file_number,
record_count,
file_count,
short_topic,
file_iv_whole,
input_base_content,
encrypted_key,
plaintext_key,
master_key,
input_folder,
encryption_json_text,
))
wait(future_results_input)
for future in future_results_input:
try:
yield future.result()
except Exception as ex:
console_printer.print_error_text(
f"Individual file generation failed with error: '{ex}'")
def assert_specific_file_stored_in_hbase(topic_name,
file_full_path,
timeout,
record_expected_in_hbase=True,
wrap_id=False):
"""Checks the specific file in stored in HBase and raises assertion error if not.
Keyword arguments:
topic_name -- full topic name
file_full_path -- the full path and name of the file to check
timeout -- the timeout in seconds
record_expected_in_hbase -- true if the record should be in HBase and false if it should not (default True)
wrap_id -- True is the id format should be wrapped with an "id" object (default False)
"""
file_contents = file_helper.get_contents_of_file(file_full_path, True)
id_object = file_helper.get_id_object_from_json_file(file_full_path)
input_formatted = file_helper.get_json_with_replaced_values(file_contents)
console_printer.print_info(
f"Retrieving specific file from HBase using topic name '{topic_name}', "
+
f"id of '{id_object}', expected setting of '{record_expected_in_hbase}' "
+ f"and wrap id setting of '{wrap_id}'")
file_found = False
file_matches = False
count = 1
while (not file_found or (record_expected_in_hbase
and not file_matches)) and count <= timeout:
hbase_data = aws_helper.retrieve_data_from_hbase(
id_object, topic_name, wrap_id)
if hbase_data:
file_found = True
hbase_data_json = json.loads(hbase_data)
hbase_contents = json.dumps(hbase_data_json, sort_keys=True)
hbase_formatted = file_helper.get_json_with_replaced_values(
hbase_contents)
if input_formatted == hbase_formatted:
file_matches = True
time.sleep(1)
count += 1
if record_expected_in_hbase:
if not file_found:
raise AssertionError(
f"File {file_full_path}: content '{input_formatted}' not found in HBase"
)
elif not file_matches:
raise AssertionError(
f"Input mismatch for file '{file_full_path}': content '{input_formatted}' not matched by HBase content '{hbase_formatted}'"
)
elif file_found:
raise AssertionError(
f"File '{file_full_path}': content '{input_formatted}' found in HBase"
)
elif file_matches:
raise AssertionError(
f"Input mismatch for file '{file_full_path}': content '{input_formatted}' matched by HBase content '{hbase_formatted}'"
)
return topic_name