def step_(context, snapshot_type, step_name):
context.adg_s3_prefix = os.path.join(context.mongo_snapshot_path,
context.test_run_name)
context.adg_export_date = datetime.now().strftime("%Y-%m-%d")
payload = {
CORRELATION_ID: context.test_run_name,
S3_PREFIX: context.adg_s3_prefix,
SNAPSHOT_TYPE: snapshot_type,
EXPORT_DATE: context.adg_export_date,
}
payload_json = json.dumps(payload)
cluster_response = invoke_lambda.invoke_adg_emr_launcher_lambda(
payload_json)
cluster_arn = cluster_response[CLUSTER_ARN]
cluster_arn_arr = cluster_arn.split(":")
cluster_identifier = cluster_arn_arr[len(cluster_arn_arr) - 1]
cluster_identifier_arr = cluster_identifier.split("/")
cluster_id = cluster_identifier_arr[len(cluster_identifier_arr) - 1]
console_printer.print_info(f"Started emr cluster : '{cluster_id}'")
step = aws_helper.get_emr_cluster_step(step_name, cluster_id)
context.adg_cluster_id = cluster_id
step_id = step["Id"]
console_printer.print_info(f"Step id for '{step_name}' : '{step_id}'")
if step is not None:
execution_state = aws_helper.poll_emr_cluster_step_status(
step_id, cluster_id, 2500)
if execution_state != COMPLETED_STATUS:
raise AssertionError(
f"'{step_name}' step failed with final status of '{execution_state}'"
)
def step_impl(context, step_type):
execution_state = aws_helper.poll_emr_cluster_step_status(
context.ingest_hbase_emr_job_step_id,
context.ingest_hbase_emr_cluster_id)
if execution_state != "COMPLETED":
raise AssertionError(
f"'{step_type}' step failed with final status of '{execution_state}'"
)
def step_impl(context):
execution_state = aws_helper.poll_emr_cluster_step_status(
context.create_hbase_tables_step_id,
context.ingest_hbase_emr_cluster_id, 1200)
if execution_state != "COMPLETED":
raise AssertionError(
f"Create tables did not complete successfully with final status of '{execution_state}'"
)
def step_impl(context, timeout_mins):
timeout_secs = int(timeout_mins) * 60
execution_state = aws_helper.poll_emr_cluster_step_status(
context.adg_cluster_step_id, context.adg_cluster_id, timeout_secs)
if execution_state != "COMPLETED":
raise AssertionError(
f"'{context.adg_cluster_step_name}' step failed with final status of '{execution_state}'"
)
def step_impl(context):
for step in context.kickstart_step_ids:
console_printer.print_info(
f"check if the step with {step} is complete or not")
execution_state = aws_helper.poll_emr_cluster_step_status(
step, context.kickstart_adg_cluster_id, 1200)
if execution_state != COMPLETED_STATUS:
raise AssertionError(
f"The step Id {step} failed with final status of '{execution_state}'"
)
def step_impl(context, timeout_setting):
if timeout_setting == "true":
timeout = context.timeout
elif timeout_setting == "long":
timeout = context.timeout * 5
else:
timeout = None
execution_state = aws_helper.poll_emr_cluster_step_status(
context.data_load_step_id, context.ingest_hbase_emr_cluster_id,
timeout)
if execution_state != "COMPLETED":
raise AssertionError(
f"Data load did not complete successfully with final status of '{execution_state}'"
)