def step_(context, snapshot_type, step_name): context.adg_s3_prefix = os.path.join(context.mongo_snapshot_path, context.test_run_name) context.adg_export_date = datetime.now().strftime("%Y-%m-%d") payload = { CORRELATION_ID: context.test_run_name, S3_PREFIX: context.adg_s3_prefix, SNAPSHOT_TYPE: snapshot_type, EXPORT_DATE: context.adg_export_date, } payload_json = json.dumps(payload) cluster_response = invoke_lambda.invoke_adg_emr_launcher_lambda( payload_json) cluster_arn = cluster_response[CLUSTER_ARN] cluster_arn_arr = cluster_arn.split(":") cluster_identifier = cluster_arn_arr[len(cluster_arn_arr) - 1] cluster_identifier_arr = cluster_identifier.split("/") cluster_id = cluster_identifier_arr[len(cluster_identifier_arr) - 1] console_printer.print_info(f"Started emr cluster : '{cluster_id}'") step = aws_helper.get_emr_cluster_step(step_name, cluster_id) context.adg_cluster_id = cluster_id step_id = step["Id"] console_printer.print_info(f"Step id for '{step_name}' : '{step_id}'") if step is not None: execution_state = aws_helper.poll_emr_cluster_step_status( step_id, cluster_id, 2500) if execution_state != COMPLETED_STATUS: raise AssertionError( f"'{step_name}' step failed with final status of '{execution_state}'" )
def step_impl(context, step_type): execution_state = aws_helper.poll_emr_cluster_step_status( context.ingest_hbase_emr_job_step_id, context.ingest_hbase_emr_cluster_id) if execution_state != "COMPLETED": raise AssertionError( f"'{step_type}' step failed with final status of '{execution_state}'" )
def step_impl(context): execution_state = aws_helper.poll_emr_cluster_step_status( context.create_hbase_tables_step_id, context.ingest_hbase_emr_cluster_id, 1200) if execution_state != "COMPLETED": raise AssertionError( f"Create tables did not complete successfully with final status of '{execution_state}'" )
def step_impl(context, timeout_mins): timeout_secs = int(timeout_mins) * 60 execution_state = aws_helper.poll_emr_cluster_step_status( context.adg_cluster_step_id, context.adg_cluster_id, timeout_secs) if execution_state != "COMPLETED": raise AssertionError( f"'{context.adg_cluster_step_name}' step failed with final status of '{execution_state}'" )
def step_impl(context): for step in context.kickstart_step_ids: console_printer.print_info( f"check if the step with {step} is complete or not") execution_state = aws_helper.poll_emr_cluster_step_status( step, context.kickstart_adg_cluster_id, 1200) if execution_state != COMPLETED_STATUS: raise AssertionError( f"The step Id {step} failed with final status of '{execution_state}'" )
def step_impl(context, timeout_setting): if timeout_setting == "true": timeout = context.timeout elif timeout_setting == "long": timeout = context.timeout * 5 else: timeout = None execution_state = aws_helper.poll_emr_cluster_step_status( context.data_load_step_id, context.ingest_hbase_emr_cluster_id, timeout) if execution_state != "COMPLETED": raise AssertionError( f"Data load did not complete successfully with final status of '{execution_state}'" )