def publish_sns_message(self, action, error_message, action_result_state): """ In order for the boto3 client to work we need the aws keys, session and a region. In addition the batch instance we will run in will need sts assume role permission to publish to sns_arn. :param action: :param error_message: :param action_result_state: We expect ActionResultState.SUCCESS or ActionResultState.FAILURE :return: nothing. """ if os.getenv('input_env'): # we should not be notifying if no input_env was provided. sns_client = None aws_access_key_id = None try: os.environ["AWS_DEFAULT_REGION"] = "us-east-1" # boto3 needs a defined region. aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action) sns_client = boto3.client('sns', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=security_token) except Exception as err: _logger.error("Could not create boto3 sns client with aws_access_key_id={0}, err={1}".format(aws_access_key_id, err)) self.publish_message(action_id=action.id, error_message=error_message, action_result_state=action_result_state, sns_arn=self.sns_arn, sns_client_publish=sns_client.publish, is_continue_on_failure=self.is_continue_on_failure)
def unload_to_s3(action, conn): """ :type action: dart.model.action.Action """ args = action.data.args aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action) now = datetime.utcnow() sql = """ UNLOAD ('{statement}') TO '{s3_path}' CREDENTIALS 'aws_access_key_id={aws_access_key_id};aws_secret_access_key={aws_secret_access_key}{token}' ALLOWOVERWRITE NULL AS 'NULL' ESCAPE DELIMITER '{delimiter}' PARALLEL {parallel} GZIP; """.format( statement=sanitized_query(args['source_sql_statement'].replace("'", "''")), s3_path=args['destination_s3_path'].replace( '{YEAR}', now.strftime('%Y')).replace( '{MONTH}', now.strftime('%m')).replace( '{DAY}', now.strftime('%d')).replace( '{HOUR}', now.strftime('%H')).replace( '{MINUTE}', now.strftime('%M')).replace( '{SECOND}', now.strftime('%S')), delimiter=args['delimiter'] if args.get('delimiter') else '\t', parallel='ON' if args['parallel'] else 'OFF', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, token=';token=%s' % security_token if security_token else '', ) conn.execute(sql)
def _load_stage_table(action, conn, dart, dataset, datastore, manifests, stage_schema_name, stage_table_name, step_num, steps_total): for step_num, s3_manifest_path in enumerate(manifests, start=2): aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action) sql = _get_copy_from_s3_sql(datastore, action, dataset, stage_schema_name, stage_table_name, s3_manifest_path, aws_access_key_id, aws_secret_access_key, security_token) conn.execute(sql) action = dart.patch_action(action, progress=_get_progress(step_num, steps_total)) return action, step_num+1
def _load_stage_table(action, conn, dart, dataset, datastore, manifests, stage_schema_name, stage_table_name, step_num, steps_total): for step_num, s3_manifest_path in enumerate(manifests, start=2): aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials( action) sql = _get_copy_from_s3_sql(datastore, action, dataset, stage_schema_name, stage_table_name, s3_manifest_path, aws_access_key_id, aws_secret_access_key, security_token) try: conn.execute(sql) except Exception as e: if "Check 'stl_load_errors' system table for details" in e.message: now = datetime.utcnow() filename = substitute_date_tokens(dataset.data.location, now) stl_load_error_sql = "SELECT * FROM pg_catalog.stl_load_errors WHERE filename LIKE '{filename}%%' " \ "ORDER BY starttime DESC LIMIT 1".format(filename=filename) stl_load_error = conn.execute(stl_load_error_sql).fetchone() exception_message = "Load into {table_name} failed.\n" \ "stl_load_error:\n" \ "Start Time: {starttime}\n" \ "Filename: {filename}\n" \ "Line No.: {line_number}\n" \ "Column Name: {colname}\n" \ "Type: {type}\n" \ "Column Length: {col_length}\n" \ "Position: {position}\n" \ "Raw Line: {raw_line}\n" \ "Raw Field Value: {raw_field_value}\n" \ "Error Code: {err_code}\n" \ "Error Reason: {err_reason}" exception_message = exception_message.format( table_name=stage_table_name, starttime=stl_load_error['starttime'], filename=stl_load_error['filename'], line_number=stl_load_error['line_number'], colname=stl_load_error['colname'], type=stl_load_error['type'], col_length=stl_load_error['col_length'], position=stl_load_error['position'], raw_line=stl_load_error['raw_line'], raw_field_value=stl_load_error['raw_field_value'], err_code=stl_load_error['err_code'], err_reason=stl_load_error['err_reason']) raise Exception(exception_message) else: raise e action = dart.patch_action(action, progress=_get_progress( step_num, steps_total)) return action, step_num + 1
def _load_stage_table(action, conn, dart, dataset, datastore, manifests, stage_schema_name, stage_table_name, step_num, steps_total): for step_num, s3_manifest_path in enumerate(manifests, start=2): aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action) sql = _get_copy_from_s3_sql(datastore, action, dataset, stage_schema_name, stage_table_name, s3_manifest_path, aws_access_key_id, aws_secret_access_key, security_token) try: conn.execute(sql) except Exception as e: if "Check 'stl_load_errors' system table for details" in e.message: now = datetime.utcnow() filename = substitute_date_tokens(dataset.data.location, now) stl_load_error_sql = "SELECT * FROM pg_catalog.stl_load_errors WHERE filename LIKE '{filename}%%' " \ "ORDER BY starttime DESC LIMIT 1".format(filename=filename) stl_load_error = conn.execute(stl_load_error_sql).fetchone() exception_message = "Load into {table_name} failed.\n" \ "stl_load_error:\n" \ "Start Time: {starttime}\n" \ "Filename: {filename}\n" \ "Line No.: {line_number}\n" \ "Column Name: {colname}\n" \ "Type: {type}\n" \ "Column Length: {col_length}\n" \ "Position: {position}\n" \ "Raw Line: {raw_line}\n" \ "Raw Field Value: {raw_field_value}\n" \ "Error Code: {err_code}\n" \ "Error Reason: {err_reason}" exception_message = exception_message.format( table_name=stage_table_name, starttime=stl_load_error['starttime'], filename=stl_load_error['filename'], line_number=stl_load_error['line_number'], colname=stl_load_error['colname'], type=stl_load_error['type'], col_length=stl_load_error['col_length'], position=stl_load_error['position'], raw_line=stl_load_error['raw_line'], raw_field_value=stl_load_error['raw_field_value'], err_code=stl_load_error['err_code'], err_reason=stl_load_error['err_reason'] ) raise Exception(exception_message) else: raise e action = dart.patch_action(action, progress=_get_progress(step_num, steps_total)) return action, step_num + 1
def publish_sns_message(self, action, error_message, action_result_state): """ In order for the boto3 client to work we need the aws keys, session and a region. In addition the batch instance we will run in will need sts assume role permission to publish to sns_arn. :param action: :param error_message: :param action_result_state: We expect ActionResultState.SUCCESS or ActionResultState.FAILURE :return: nothing. """ if os.getenv( 'input_env' ): # we should not be notifying if no input_env was provided. sns_client = None aws_access_key_id = None try: os.environ[ "AWS_DEFAULT_REGION"] = "us-east-1" # boto3 needs a defined region. aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials( action) sns_client = boto3.client( 'sns', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=security_token) except Exception as err: _logger.error( "Could not create boto3 sns client with aws_access_key_id={0}, err={1}" .format(aws_access_key_id, err)) self.publish_message( action_id=action.id, error_message=error_message, action_result_state=action_result_state, sns_arn=self.sns_arn, sns_client_publish=sns_client.publish, is_continue_on_failure=self.is_continue_on_failure)