def publish_sns_message(self, action, error_message, action_result_state):
"""
In order for the boto3 client to work we need the aws keys, session and a region.
In addition the batch instance we will run in will need sts assume role permission to publish to sns_arn.
:param action:
:param error_message:
:param action_result_state: We expect ActionResultState.SUCCESS or ActionResultState.FAILURE
:return: nothing.
"""
if os.getenv('input_env'): # we should not be notifying if no input_env was provided.
sns_client = None
aws_access_key_id = None
try:
os.environ["AWS_DEFAULT_REGION"] = "us-east-1" # boto3 needs a defined region.
aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action)
sns_client = boto3.client('sns',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=security_token)
except Exception as err:
_logger.error("Could not create boto3 sns client with aws_access_key_id={0}, err={1}".format(aws_access_key_id, err))
self.publish_message(action_id=action.id,
error_message=error_message,
action_result_state=action_result_state,
sns_arn=self.sns_arn,
sns_client_publish=sns_client.publish,
is_continue_on_failure=self.is_continue_on_failure)
def unload_to_s3(action, conn):
""" :type action: dart.model.action.Action """
args = action.data.args
aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action)
now = datetime.utcnow()
sql = """
UNLOAD ('{statement}') TO '{s3_path}'
CREDENTIALS 'aws_access_key_id={aws_access_key_id};aws_secret_access_key={aws_secret_access_key}{token}'
ALLOWOVERWRITE
NULL AS 'NULL'
ESCAPE
DELIMITER '{delimiter}'
PARALLEL {parallel}
GZIP;
""".format(
statement=sanitized_query(args['source_sql_statement'].replace("'", "''")),
s3_path=args['destination_s3_path'].replace(
'{YEAR}', now.strftime('%Y')).replace(
'{MONTH}', now.strftime('%m')).replace(
'{DAY}', now.strftime('%d')).replace(
'{HOUR}', now.strftime('%H')).replace(
'{MINUTE}', now.strftime('%M')).replace(
'{SECOND}', now.strftime('%S')),
delimiter=args['delimiter'] if args.get('delimiter') else '\t',
parallel='ON' if args['parallel'] else 'OFF',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
token=';token=%s' % security_token if security_token else '',
)
conn.execute(sql)
def _load_stage_table(action, conn, dart, dataset, datastore, manifests, stage_schema_name, stage_table_name,
step_num, steps_total):
for step_num, s3_manifest_path in enumerate(manifests, start=2):
aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action)
sql = _get_copy_from_s3_sql(datastore, action, dataset, stage_schema_name, stage_table_name,
s3_manifest_path, aws_access_key_id, aws_secret_access_key, security_token)
conn.execute(sql)
action = dart.patch_action(action, progress=_get_progress(step_num, steps_total))
return action, step_num+1
def _load_stage_table(action, conn, dart, dataset, datastore, manifests,
stage_schema_name, stage_table_name, step_num,
steps_total):
for step_num, s3_manifest_path in enumerate(manifests, start=2):
aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(
action)
sql = _get_copy_from_s3_sql(datastore, action, dataset,
stage_schema_name, stage_table_name,
s3_manifest_path, aws_access_key_id,
aws_secret_access_key, security_token)
try:
conn.execute(sql)
except Exception as e:
if "Check 'stl_load_errors' system table for details" in e.message:
now = datetime.utcnow()
filename = substitute_date_tokens(dataset.data.location, now)
stl_load_error_sql = "SELECT * FROM pg_catalog.stl_load_errors WHERE filename LIKE '{filename}%%' " \
"ORDER BY starttime DESC LIMIT 1".format(filename=filename)
stl_load_error = conn.execute(stl_load_error_sql).fetchone()
exception_message = "Load into {table_name} failed.\n" \
"stl_load_error:\n" \
"Start Time: {starttime}\n" \
"Filename: {filename}\n" \
"Line No.: {line_number}\n" \
"Column Name: {colname}\n" \
"Type: {type}\n" \
"Column Length: {col_length}\n" \
"Position: {position}\n" \
"Raw Line: {raw_line}\n" \
"Raw Field Value: {raw_field_value}\n" \
"Error Code: {err_code}\n" \
"Error Reason: {err_reason}"
exception_message = exception_message.format(
table_name=stage_table_name,
starttime=stl_load_error['starttime'],
filename=stl_load_error['filename'],
line_number=stl_load_error['line_number'],
colname=stl_load_error['colname'],
type=stl_load_error['type'],
col_length=stl_load_error['col_length'],
position=stl_load_error['position'],
raw_line=stl_load_error['raw_line'],
raw_field_value=stl_load_error['raw_field_value'],
err_code=stl_load_error['err_code'],
err_reason=stl_load_error['err_reason'])
raise Exception(exception_message)
else:
raise e
action = dart.patch_action(action,
progress=_get_progress(
step_num, steps_total))
return action, step_num + 1
def _load_stage_table(action, conn, dart, dataset, datastore, manifests, stage_schema_name, stage_table_name,
step_num, steps_total):
for step_num, s3_manifest_path in enumerate(manifests, start=2):
aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(action)
sql = _get_copy_from_s3_sql(datastore, action, dataset, stage_schema_name, stage_table_name,
s3_manifest_path, aws_access_key_id, aws_secret_access_key, security_token)
try:
conn.execute(sql)
except Exception as e:
if "Check 'stl_load_errors' system table for details" in e.message:
now = datetime.utcnow()
filename = substitute_date_tokens(dataset.data.location, now)
stl_load_error_sql = "SELECT * FROM pg_catalog.stl_load_errors WHERE filename LIKE '{filename}%%' " \
"ORDER BY starttime DESC LIMIT 1".format(filename=filename)
stl_load_error = conn.execute(stl_load_error_sql).fetchone()
exception_message = "Load into {table_name} failed.\n" \
"stl_load_error:\n" \
"Start Time: {starttime}\n" \
"Filename: {filename}\n" \
"Line No.: {line_number}\n" \
"Column Name: {colname}\n" \
"Type: {type}\n" \
"Column Length: {col_length}\n" \
"Position: {position}\n" \
"Raw Line: {raw_line}\n" \
"Raw Field Value: {raw_field_value}\n" \
"Error Code: {err_code}\n" \
"Error Reason: {err_reason}"
exception_message = exception_message.format(
table_name=stage_table_name,
starttime=stl_load_error['starttime'],
filename=stl_load_error['filename'],
line_number=stl_load_error['line_number'],
colname=stl_load_error['colname'],
type=stl_load_error['type'],
col_length=stl_load_error['col_length'],
position=stl_load_error['position'],
raw_line=stl_load_error['raw_line'],
raw_field_value=stl_load_error['raw_field_value'],
err_code=stl_load_error['err_code'],
err_reason=stl_load_error['err_reason']
)
raise Exception(exception_message)
else:
raise e
action = dart.patch_action(action, progress=_get_progress(step_num, steps_total))
return action, step_num + 1
def publish_sns_message(self, action, error_message, action_result_state):
"""
In order for the boto3 client to work we need the aws keys, session and a region.
In addition the batch instance we will run in will need sts assume role permission to publish to sns_arn.
:param action:
:param error_message:
:param action_result_state: We expect ActionResultState.SUCCESS or ActionResultState.FAILURE
:return: nothing.
"""
if os.getenv(
'input_env'
): # we should not be notifying if no input_env was provided.
sns_client = None
aws_access_key_id = None
try:
os.environ[
"AWS_DEFAULT_REGION"] = "us-east-1" # boto3 needs a defined region.
aws_access_key_id, aws_secret_access_key, security_token = lookup_credentials(
action)
sns_client = boto3.client(
'sns',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=security_token)
except Exception as err:
_logger.error(
"Could not create boto3 sns client with aws_access_key_id={0}, err={1}"
.format(aws_access_key_id, err))
self.publish_message(
action_id=action.id,
error_message=error_message,
action_result_state=action_result_state,
sns_arn=self.sns_arn,
sns_client_publish=sns_client.publish,
is_continue_on_failure=self.is_continue_on_failure)