def _generate_kafka_input_file(
s3_input_bucket,
local_folder,
template_file_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
s3_output_prefix,
seconds_timeout,
timestamp=None,
):
"""Generates a Kafka input file and sends it to S3 and returns remote location.
Keyword arguments:
s3_input_bucket - the bucket for the remote fixture files
local_folder -- the local parent folder for the template file
template_file_name -- the template file name
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
fixture_files_root -- the local path to the feature file to send
s3_output_prefix -- the output path for the edited file in s3
seconds_timeout -- the timeout in seconds for the test
timestamp -- set to a value to override timestamp in file
"""
timestamp_string = "NOT_SET" if timestamp is None else str(timestamp)
console_printer.print_debug(
f"Generating Kafka input file for template of {template_file_name} and id of {new_uuid} and timestamp of {timestamp_string}"
)
file_name = os.path.join(fixture_files_root, local_folder,
template_file_name)
with open(file_name) as open_file:
record = open_file.read()
record = record.replace("||newid||", f"{new_uuid}")
if timestamp is not None:
record = record.replace("||timestamp||", f"{timestamp}")
output_files_folder = os.path.join(local_folder, "input-only-files")
output_file_local = file_helper.generate_local_output_file(
output_files_folder, template_file_name, local_files_temp_folder)
console_printer.print_debug(
f"Writing Kafka input file to '{output_file_local}'")
with open(f"{output_file_local}", "w") as output_file_data:
output_file_data.write(record)
output_file_full_path_s3 = os.path.join(s3_output_prefix, str(new_uuid))
full_file_path_s3 = os.path.join(output_file_full_path_s3,
template_file_name)
aws_helper.upload_file_to_s3_and_wait_for_consistency(
output_file_local, s3_input_bucket, seconds_timeout, full_file_path_s3)
return (full_file_path_s3, output_file_local)
def _upload_data_output_file_to_s3(
topic_name,
record,
s3_input_bucket,
new_uuid,
local_files_temp_folder,
s3_output_prefix,
seconds_timeout,
timestamp,
):
"""Uploads a gzipped version of the file to S3 and returns full S3 path.
Keyword arguments:
topic_name -- the topic name
record -- the data for the output file
s3_input_bucket -- the bucket for the remote fixture files
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
s3_output_prefix -- the output path for the edited file in s3
seconds_timeout -- the timeout in seconds for the test
timestamp -- the timestamp for this file
"""
output_file_local = file_helper.generate_local_output_file(
"tmp_corporate_data", f"{str(new_uuid)}.gzip", local_files_temp_folder)
console_printer.print_debug(
f"Writing corporate data tmp gzip file to '{output_file_local}'")
dumped_record = json.dumps(json.loads(record))
with gzip.open(output_file_local, "wb") as output_file_data:
output_file_data.write(dumped_record.encode("utf_8"))
(
database,
collection,
) = template_helper.get_database_and_collection_from_topic_name(topic_name)
file_name = f"{topic_name}_0_1-{str(new_uuid)}.jsonl.gz"
output_file_full_path_s3 = data_load_helper.generate_corporate_data_s3_prefix(
s3_output_prefix,
database,
collection,
timestamp,
)
full_file_path_s3 = os.path.join(output_file_full_path_s3, file_name)
aws_helper.upload_file_to_s3_and_wait_for_consistency(
output_file_local, s3_input_bucket, seconds_timeout, full_file_path_s3)
os.remove(output_file_local)
return full_file_path_s3
def _generate_kafka_output_file(
s3_input_bucket,
local_folder,
template_file_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
timestamp=None,
):
"""Generates a kafka output value to check against HBase and returns lcoal file name for edited file.
Keyword arguments:
s3_input_bucket - the bucket for the remote fixture files
local_folder -- the local parent folder for the template file
template_file_name -- the template file name
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
fixture_files_root -- the local path to the feature file to send
timestamp -- set to a value to override timestamp in file
"""
timestamp_string = "NOT_SET" if timestamp is None else str(timestamp)
console_printer.print_debug(
f"Generating Kafka output file for template of {template_file_name} and id of {new_uuid} and timestamp of {timestamp_string}"
)
file_name = os.path.join(fixture_files_root, local_folder,
template_file_name)
with open(file_name) as open_file:
record = open_file.read()
record = record.replace("||newid||", f"{new_uuid}")
if timestamp is not None:
record = record.replace("||timestamp||", f"{timestamp}")
output_file_local = file_helper.generate_local_output_file(
local_folder, template_file_name, local_files_temp_folder)
console_printer.print_debug(
f"Writing Kafka output file to '{output_file_local}'")
with open(f"{output_file_local}", "w") as output_file_data:
output_file_data.write(record)
return output_file_local
def _generate_corporate_data_input_file(
local_folder,
template_file_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
timestamp_string,
):
"""Generates a corporate datainput file and sends it to S3 and returns local file location.
Keyword arguments:
local_folder -- the local parent folder for the template file
template_file_name -- the template file name
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
fixture_files_root -- the local path to the feature file to send
timestamp_string -- the formatted string of the timestamp in the file
"""
console_printer.print_debug(
f"Generating corporate data input file for template of {template_file_name} and id of {new_uuid} and timestamp of {timestamp_string}"
)
file_name = os.path.join(fixture_files_root, local_folder,
template_file_name)
with open(file_name) as open_file:
record = open_file.read()
record = record.replace("||newid||", f"{new_uuid}")
record = record.replace("||timestamp||", f"{timestamp_string}")
output_files_folder = os.path.join(local_folder, "input-only-files")
output_file_local = file_helper.generate_local_output_file(
output_files_folder, template_file_name, local_files_temp_folder)
console_printer.print_debug(
f"Writing corporate data input file to '{output_file_local}'")
with open(f"{output_file_local}", "w") as output_file_data:
output_file_data.write(record)
return output_file_local
def _generate_kafka_file(
test_run_name,
s3_input_bucket,
input_template_name,
output_template_name,
dlq_template_name,
snapshot_record_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
s3_output_prefix,
topic_name,
snapshots_output_folder,
seconds_timeout,
fixture_data_folder,
timestamp=None,
):
"""Generates Kafka input and check files, returns tuple of (input s3 location, output local file).
Keyword arguments:
test_run_name -- unique name for this test run
s3_input_bucket - the bucket for the remote fixture files
input_template_name -- the input template file
output_template_name -- the output template file (None if no output needed)
dlq_template_name -- the output template file (None if no dlq file needed)
snapshot_record_template_name -- the snapshot record template file (None if no snapshot file needed)
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
fixture_files_root -- the local path to the feature file to send
s3_output_prefix -- the output path for the edited file in s3
topic_name -- the topic name which records are generated for
snapshots_output_folder -- the snapshots output folder (None if no snapshot file needed)
seconds_timeout -- the timeout in seconds for the test
fixture_data_folder -- the folder from the root of the fixture data
timestamp -- set to a value to override timestamp in file
"""
timestamp_string = "NOT_SET" if timestamp is None else str(timestamp)
console_printer.print_debug(
f"Generating Kafka file for input template of {input_template_name}, output template of {output_template_name}, id of {new_uuid} and timestamp of {timestamp_string}"
)
input_s3_prefix, input_file_local = _generate_kafka_input_file(
s3_input_bucket,
fixture_data_folder,
input_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
s3_output_prefix,
seconds_timeout,
timestamp,
)
output_local_file = None
if output_template_name is not None:
output_local_file = _generate_kafka_output_file(
s3_input_bucket,
fixture_data_folder,
output_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
timestamp,
)
dlq_local_file = None
if dlq_template_name is not None:
dlq_local_file = _generate_kafka_dlq_file(
s3_input_bucket,
fixture_data_folder,
dlq_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
timestamp,
)
snapshot_local_file = None
if snapshot_record_template_name is not None:
snapshot_local_file = (
snapshot_data_generator.generate_hbase_record_for_snapshot_file(
snapshot_record_template_name,
timestamp,
new_uuid,
"E2E_KAFKA",
test_run_name,
topic_name,
fixture_files_root,
snapshots_output_folder,
))
return (
input_s3_prefix,
input_file_local,
output_local_file,
dlq_local_file,
snapshot_local_file,
)
def wait_for_statuses_in_export_status_table(timeout, export_status_table_name,
topics, correlation_id,
desired_statuses):
"""Returns true or false for if the items for the given correlation id after waiting for the timeout and topic list match given status.
Keyword arguments:
timeout -- the timeout in seconds
export_status_table_name -- the export table name
topics -- the array of topics to check
correlation_id -- the correlation id
desired_statuses -- an array of allowed statuses
"""
count = 1
matched_topics = []
console_printer.print_info(
f"Checking all export statuses for all topics match one of the desired statuses of '{desired_statuses}' for correlation_id of '{correlation_id}'"
)
while len(matched_topics) != len(topics) and count <= timeout:
for topic in topics:
if topic not in matched_topics:
topic_name = template_helper.get_topic_name(topic)
key_dict = {
"CorrelationId": {
"S": f"{correlation_id}"
},
"CollectionName": {
"S": f"{topic_name}"
},
}
item_details = aws_helper.get_item_from_dynamodb(
export_status_table_name, key_dict)
if "Item" not in item_details:
console_printer.print_debug(
f"No export status found for key dict of '{key_dict}'")
continue
collection_status = item_details["Item"]["CollectionStatus"][
"S"]
if collection_status not in desired_statuses:
console_printer.print_debug(
f"Status was '{collection_status}' which did not match any of '{desired_statuses}'"
)
continue
console_printer.print_info(
f"All export statuses match one of the desired statuses of '{desired_statuses}' for correlation_id of '{correlation_id}' and topic '{topic_name}'"
)
matched_topics.append(topic)
time.sleep(1)
count += 1
if len(matched_topics) != len(topics):
console_printer.print_info(
f"All export statuses for one or more topics did match one of the desired statuses of '{desired_statuses}' for correlation_id of '{correlation_id}' after '{timeout}' seconds"
)
return False
console_printer.print_info(
f"All export statuses match one of the desired statuses of '{desired_statuses}' for correlation_id of '{correlation_id}'"
)
return True
def _generate_corporate_data_file(
test_run_name,
s3_input_bucket,
input_template_name,
output_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
s3_output_prefix,
topic_name,
seconds_timeout,
timestamp,
timestamp_string,
):
"""Generates corporate data input and check files, returns tuple of (input s3 location, input local file, output local file).
Keyword arguments:
test_run_name -- unique name for this test run
s3_input_bucket - the bucket for the remote fixture files
input_template_name -- the input template file
output_template_name -- the output template file
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
fixture_files_root -- the local path to the feature file to send
s3_output_prefix -- the output path for the edited file in s3
topic_name -- the topic name which records are generated for
seconds_timeout -- the timeout in seconds for the test
timestamp -- the timestamp used in the file
timestamp_string -- the formatted string of the timestamp
"""
console_printer.print_debug(
f"Generating corporate datafile for input template of {input_template_name}, output template of {output_template_name}, id of {new_uuid} and timestamp of {timestamp}"
)
input_file_local = _generate_corporate_data_input_file(
"corporate_data",
input_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
timestamp_string,
)
(output_s3_prefix,
output_local_file) = _generate_corporate_data_output_file(
topic_name,
s3_input_bucket,
"corporate_data",
output_template_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
s3_output_prefix,
seconds_timeout,
timestamp,
timestamp_string,
)
return (
input_file_local,
output_local_file,
output_s3_prefix,
)
def _generate_corporate_data_output_file(
topic_name,
s3_input_bucket,
local_folder,
template_file_name,
new_uuid,
local_files_temp_folder,
fixture_files_root,
s3_output_prefix,
seconds_timeout,
timestamp,
timestamp_string,
):
"""Generates a corporate data output value to check against HBase and returns tuple of local and s3 file name for edited file.
Keyword arguments:
topic_name -- the topic name
s3_input_bucket -- the bucket for the remote fixture files
local_folder -- the local parent folder for the template file
template_file_name -- the template file name
new_uuid -- the uuid to use for the id
local_files_temp_folder -- the root folder for the temporary files to sit in
fixture_files_root -- the local path to the feature file to send
s3_output_prefix -- the output path for the edited file in s3
seconds_timeout -- the timeout in seconds for the test
timestamp -- the timestamp for the file
timestamp_string -- the formatted string of the timestamp in the file
"""
console_printer.print_debug(
f"Generating corporate data output file for template of {template_file_name} and id of {new_uuid} and timestamp of {timestamp_string}"
)
file_name = os.path.join(fixture_files_root, local_folder,
template_file_name)
with open(file_name) as open_file:
record = open_file.read()
record = record.replace("||newid||", f"{new_uuid}")
record = record.replace("||timestamp||", f"{timestamp_string}")
output_file_local = file_helper.generate_local_output_file(
local_folder, template_file_name, local_files_temp_folder)
console_printer.print_debug(
f"Writing corporate data output file to '{output_file_local}'")
with open(f"{output_file_local}", "w") as output_file_data:
output_file_data.write(record)
full_file_path_s3 = _upload_data_output_file_to_s3(
topic_name,
record,
s3_input_bucket,
new_uuid,
local_files_temp_folder,
s3_output_prefix,
seconds_timeout,
timestamp,
)
return (full_file_path_s3, output_file_local)