def lambda_handler(event, context):
"""Calls custom transform developed by user
Arguments:
event {dict} -- Dictionary with details on previous processing step
context {dict} -- Dictionary with details on Lambda context
Returns:
{dict} -- Dictionary with Processed Bucket and Key(s)
"""
try:
logger.info('Fetching event data from previous step')
bucket = event['body']['bucket']
# keys_to_process = event['body']['keysToProcess']
team = event['body']['team']
pipeline = event['body']['pipeline']
stage = event['body']['pipeline_stage']
stage = stage.replace('A', 'B')
dataset = event['body']['dataset']
logger.info('Initializing Octagon client')
component = context.function_name.split('-')[-2].title()
octagon_client = (
octagon.OctagonClient()
.with_run_lambda(True)
.with_configuration_instance(event['body']['env'])
.build()
)
peh_id = octagon_client.start_pipeline_execution(
pipeline_name='{}-{}-stage-{}'.format(team,
pipeline, stage[-1].lower()),
dataset_name='{}-{}'.format(team, dataset),
comment=event
)
# Call custom transform created by user and process the file
logger.info('Calling user custom processing code')
transform_handler = TransformHandler().stage_transform(team, dataset, stage)
response = transform_handler().transform_object(
bucket, event['body'], team, dataset) # custom user code called
response['peh_id'] = peh_id
remove_content_tmp()
octagon_client.update_pipeline_execution(
status="{} {} Processing".format(stage, component), component=component)
except Exception as e:
logger.error("Fatal error", exc_info=True)
octagon_client.end_pipeline_execution_failed(component=component,
issue_comment="{} {} Error: {}".format(stage, component, repr(e)))
remove_content_tmp()
raise e
return response
def lambda_handler(event, context):
"""Calls custom job waiter developed by user
Arguments:
event {dict} -- Dictionary with details on previous processing step
context {dict} -- Dictionary with details on Lambda context
Returns:
{dict} -- Dictionary with Processed Bucket, Key(s) and Job Details
"""
try:
logger.info('Fetching event data from previous step')
bucket = event['body']['bucket']
keys_to_process = event['body']['key']
team = event['body']['team']
stage = event['body']['pipeline_stage']
dataset = event['body']['dataset']
job_details = event['body']['job']['jobDetails']
processed_keys_path = event['body']['job']['processedKeysPath']
logger.info('Initializing Octagon client')
component = context.function_name.split('-')[-2].title()
octagon_client = (octagon.OctagonClient().with_run_lambda(
True).with_configuration_instance(event['body']['env']).build())
logger.info('Checking Job Status with user custom code')
transform_handler = TransformHandler().stage_transform(
team, dataset, stage)
response = transform_handler().check_job_status(
bucket, keys_to_process, processed_keys_path,
job_details) # custom user code called
response['peh_id'] = event['body']['peh_id']
if event['body']['job']['jobDetails']['jobStatus'] == 'FAILED':
peh.PipelineExecutionHistoryAPI(
octagon_client).retrieve_pipeline_execution(response['peh_id'])
octagon_client.end_pipeline_execution_failed(
component=component,
issue_comment="{} {} Error: Check Job Logs".format(
stage, component))
except Exception as e:
logger.error("Fatal error", exc_info=True)
peh.PipelineExecutionHistoryAPI(
octagon_client).retrieve_pipeline_execution(
event['body']['peh_id'])
octagon_client.end_pipeline_execution_failed(
component=component,
issue_comment="{} {} Error: {}".format(stage, component, repr(e)))
raise e
return response
def lambda_handler(event, context):
"""Calls custom transform developed by user
Arguments:
event {dict} -- Dictionary with details on previous processing step
context {dict} -- Dictionary with details on Lambda context
Returns:
{dict} -- Dictionary with Processed Bucket and Key(s)
"""
try:
logger.info('Fetching event data from previous step')
bucket = event['body']['bucket']
key = event['body']['key']
team = event['body']['team']
stage = event['body']['pipeline_stage']
dataset = event['body']['dataset']
ddb_key = event['body']['manifest_ddb_key']
logger.info('Initializing Octagon client')
component = context.function_name.split('-')[-2].title()
octagon_client = (octagon.OctagonClient().with_run_lambda(
True).with_configuration_instance(event['body']['env']).build())
peh.PipelineExecutionHistoryAPI(
octagon_client).retrieve_pipeline_execution(
event['body']['peh_id'])
# Call custom transform created by user and process the file
logger.info('Calling user custom processing code')
transform_handler = TransformHandler().stage_transform(
team, dataset, stage)
response = transform_handler().transform_object(
bucket, key, team, dataset) # custom user code called
remove_content_tmp()
octagon_client.update_pipeline_execution(
status="{} {} Processing".format(stage, component),
component=component)
dynamo_interface.update_manifests_control_table_stagea(
ddb_key, "PROCESSING", response[0])
except Exception as e:
logger.error("Fatal error", exc_info=True)
octagon_client.end_pipeline_execution_failed(
component=component,
issue_comment="{} {} Error: {}".format(stage, component, repr(e)))
remove_content_tmp()
dynamo_interface.update_manifests_control_table_stagea(
ddb_key, "FAILED")
raise e
return response