def lambda_handler(event, context):
"""Lambda function that responds to changes in labeling job status, updating
the corresponding dynamo db tables and publishing to sns after a job is cancelled.
Parameters
----------
event: dict, required API gateway request with an input SQS arn, output SQS arn
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
log.log_request_and_context(event, context)
job_status = event["status"]
job_arns = event["job_arns"]
if len(job_arns) != 1:
raise ValueError("incorrect number of job arns in event: ", job_arns)
job_arn = job_arns[0]
# We received a new status for the job_arn.
process_new_status(job_arn, job_status, context.invoked_function_arn)
return "success"
def lambda_handler(event, context):
"""Lambda function that responds shows active batch information.
Parameters
----------
event: dict, required API gateway request with an input SQS arn, output SQS arn
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
Return doc:
https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
log_request_and_context(event, context)
try:
response = handle_request()
# Allow because we want to control status code and message if request has any failure.
# pylint: disable=broad-except
except Exception as err:
logger.error("Failed to handle request for workforce: %s", err)
return {
"statusCode": 500,
"body": "Error: failed to handle request.",
}
response = {
"statusCode": 200,
"body": json.dumps(response, default=str),
"isBase64Encoded": False,
}
return response
def lambda_handler(event, context):
"""Lambda function that copies any worker logs to s3 and publishes batch finish to SNS.
Parameters
----------
event: dict, required
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
"""
log_request_and_context(event, context)
try:
request_input = parse_input(event)
except KeyError as err:
logger.error("Input event missing required args: %s: %s", event, err)
raise Exception("Failed to parse input lambda handler") from err
batch_id = request_input["batch_id"]
# Mark the batch as completed.
try:
db.update_batch_status(batch_id, BatchStatus.COMPLETE)
except botocore.exceptions.ClientError as err:
raise Exception(f"failed to mark batch {batch_id} complete") from err
batch_metadata = db.get_batch_metadata(batch_id)
batch_info = input_batch_to_human_readable(batch_metadata)
message = {
"batchId": batch_id,
"message": "Batch processing has completed successfully.",
"batchInfo": batch_info,
"token": request_input["execution_id"],
"status": "SUCCESS",
}
output_sns_arn = os.getenv("DEFAULT_STATUS_SNS_ARN")
if request_input["output_sns_arn"]:
output_sns_arn = request_input["output_sns_arn"]
topic = sns.Topic(output_sns_arn)
try:
topic.publish(Message=json.dumps(message, indent=4, default=str), )
except botocore.exceptions.ClientError as err:
raise Exception(
f"Service error publishing SNS response for batch id: {batch_id}"
) from err
return {
"published_sns": message,
"output_sns_arn": output_sns_arn,
}
def lambda_handler(event, context):
"""Lambda function that ...
Reads the S3 Input manifest, and sends the batch of the data to the SMGT Job.
Parameters
----------
event: dict, required
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
"""
log.log_request_and_context(event, context)
parent_batch_id = event["parent_batch_id"]
job_level = event["job_level"]
parent_batch = db.get_batch_metadata(parent_batch_id)
if parent_batch is None:
raise Exception(f"Invalid parent batch id: {parent_batch_id}")
if job_level == 1:
meta_data_type = BatchMetadataType.FIRST_LEVEL
elif job_level == 2:
meta_data_type = BatchMetadataType.SECOND_LEVEL
elif job_level == 3:
meta_data_type = BatchMetadataType.THIRD_LEVEL
# Filter jobs by job level
labeling_jobs = parent_batch[BatchMetadataTableAttributes.LABELING_JOBS]
current_jobs = [job for job in labeling_jobs if job["jobLevel"] == job_level]
log.logging.info("Kicking off %d jobs for level %d", len(current_jobs), job_level)
batch_id = f"{parent_batch_id}-{meta_data_type.lower()}"
for job in current_jobs:
trigger_labeling_job(parent_batch_id, batch_id, job)
try:
db.insert_perform_labeling_job_metadata(
parent_batch_id=parent_batch_id,
batch_id=batch_id,
batch_status=BatchStatus.IN_PROGRESS,
batch_metadata_type=meta_data_type,
num_children_batches=len(current_jobs),
)
except botocore.exceptions.ClientError as err:
raise Exception(f"failed to put batch id {batch_id}") from err
return {
"batch_id": batch_id,
}
def lambda_handler(event, context):
"""Lambda function that responds shows active batch information.
Parameters
----------
event: dict, required API gateway request with an input SQS arn, output SQS arn
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
Return doc:
https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
log_request_and_context(event, context)
try:
request = parse_request(event)
except (KeyError, ValueError) as err:
logger.error("Failed to parse request: %s", err)
return {
"statusCode": 400,
"body": "Error: failed to parse request.",
}
try:
batch_info = handle_request(request)
except botocore.exceptions.ClientError as err:
logger.error(
"Boto call failed to execute during request handling: {err}")
return {
"statusCode": 500,
"body": "Error: internal error",
}
if batch_info is None:
logger.error("Batch id not found, request: %s", request)
return {
"statusCode": 400,
"body": f"batch id: {request['batchId']} not found",
"headers": {
"X-Amzn-ErrorType": "InvalidParameterException"
},
}
response = {
"statusCode": 200,
"body": json.dumps(batch_info, indent=4, default=str),
"isBase64Encoded": False,
}
return response
def lambda_handler(event, context):
"""Lambda function that executes batch creation API
Parameters
----------
event: dict, required API gateway request with an input SQS arn, output SQS arn
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
Return doc:
https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
log.log_request_and_context(event, context)
body = json.loads(event.get("body"))
return execute_multi_job(body)
def lambda_handler(event, context):
"""Lambda function that executes batch creation API
Parameters
----------
event: dict, required API gateway request with an input SQS arn, output SQS arn
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
Return doc:
https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
"""
sfn_client = get_boto_client("stepfunctions", context.invoked_function_arn)
log.log_request_and_context(event, context)
body = json.loads(event.get("body"))
batch_id = body.get("batchId")
down_sampling_rate = body.get("downSamplingRate")
input_metadata_batch_id = f"{batch_id}-{BatchMetadataType.HUMAN_INPUT_METADATA.lower()}"
input_metadata_batch = db.get_batch_metadata(input_metadata_batch_id)
if input_metadata_batch:
return construct_validation_error(
"The system indicates the you have already input the down sampling rate "
+ f'{input_metadata_batch.get("DownSamplingRate")}')
if batch_id is None:
return construct_validation_error("BatchId is required.")
if down_sampling_rate is None:
return construct_validation_error("DownSampling rate is required.")
batch_metadata = db.get_batch_metadata(batch_id)
if not batch_metadata:
return construct_validation_error(
f"BatchMetadata not found for the batchId: {batch_id}")
else:
if down_sampling_rate < 0 or down_sampling_rate > 100:
return construct_validation_error(
"Expected down sampling range in between 0 to 100.")
first_level_batch = db.get_child_batch_metadata(
batch_id, BatchMetadataType.FIRST_LEVEL)
job_output_location = first_level_batch[
BatchMetadataTableAttributes.JOB_OUTPUT_LOCATION]
state_token = batch_metadata.get(BatchMetadataTableAttributes.STATE_TOKEN)
if not state_token:
return construct_validation_error(
f"The system indicates the batch exeuction is not currently at the wait step {batch_metadata}"
)
sfn_client.send_task_success(
taskToken=batch_metadata[BatchMetadataTableAttributes.STATE_TOKEN],
output=json.dumps({
"batch_id":
batch_metadata[
BatchMetadataTableAttributes.FIRST_LEVEL_BATCH_METADATA_ID],
"s3_output_path":
job_output_location,
"down_sampling_rate":
down_sampling_rate,
"token_sent_source_arn":
context.invoked_function_arn,
}),
)
db.insert_batch_metadata_input(
batch_id=input_metadata_batch_id,
parent_batch_id=batch_id,
down_sampling_rate=down_sampling_rate,
input_manifest=job_output_location,
batch_status=BatchStatus.COMPLETE,
)
response = {
"statusCode": 200,
"body": "Successfully input metadata to resume batch execution : " +
f"batchId : {batch_id}, downSamplingRate: {down_sampling_rate}",
"isBase64Encoded": False,
}
return response
def lambda_handler(event, context):
"""Lambda function that copies any worker logs to s3 and publishes batch finish to SNS.
Parameters
----------
event: dict, required
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
"""
log_request_and_context(event, context)
execution_id = event["execution_id"]
request = event["input"]
validation_output = request.get("transformation_step_output")
if validation_output is None:
raise Exception(
"no batch id stored with validation output, can't write sns")
parent_batch_id = validation_output["batch_id"]
error_info = request["error-info"]
error_type = "Unknown"
error_message = ""
try:
# If cause is json parsable, get more specific error details (like from python exception).
# This avoids sending stack traces to the SNS receiver.
cause = json.loads(error_info["Cause"])
error_type = cause["errorType"]
error_message = cause["errorMessage"]
except (ValueError, KeyError):
# Error message isn't json parseable, default to just put the whole "Cause" string.
error_type = error_info["Error"]
error_message = error_info["Cause"]
try:
db.mark_batch_and_children_failed(parent_batch_id,
f"{error_type}: {error_message}")
except botocore.exceptions.ClientError as err:
# Soft failure, we want to still publish error sns even if we can't update db.
logger.error("failed to set batch status to error: %s", err)
message = {
"batchId": parent_batch_id,
"message": "Batch processing failed",
"errorType": error_type,
"errorString": error_message,
"token": execution_id,
"status": "FAILED",
}
output_sns_arn = os.getenv("DEFAULT_STATUS_SNS_ARN")
if "destinationSnsArn" in request:
output_sns_arn = request["destinationSnsArn"]
sns = boto3.resource("sns")
topic = sns.Topic(output_sns_arn)
try:
topic.publish(Message=json.dumps(message, indent=4), )
except botocore.exceptions.ClientError as err:
raise Exception(
f"Service error publishing SNS response for batch id: {parent_batch_id}"
) from err
return {
"published_sns": message,
"output_sns_arn": output_sns_arn,
}
def lambda_handler(event, context):
"""Lambda function that ...
Down sampling of the input manifest to send to the next step
Parameters
----------
event: dict, required
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
"""
log.log_request_and_context(event, context)
batch_id = event["batch_id"]
batch_metadata = db.get_batch_metadata(batch_id)
current_metadata_type = batch_metadata[BatchMetadataTableAttributes.BATCH_METADATA_TYPE]
if current_metadata_type == BatchMetadataType.FIRST_LEVEL:
processing_output_job_level = 1
elif current_metadata_type == BatchMetadataType.SECOND_LEVEL :
processing_output_job_level = 2
else :
processing_output_job_level = 3
parent_batch_id = batch_metadata[BatchMetadataTableAttributes.PARENT_BATCH_ID]
parent_batch_metadata = db.get_batch_metadata(parent_batch_id)
current_level_completed_labeling_jobs = []
future_level_labeling_jobs = []
labeling_jobs = parent_batch_metadata["LabelingJobs"]
for labeling_job in labeling_jobs:
if labeling_job["jobLevel"] == processing_output_job_level:
current_level_completed_labeling_jobs.append(labeling_job)
elif labeling_job["jobLevel"] > processing_output_job_level:
future_level_labeling_jobs.append(labeling_job)
for completed_labeling_job in current_level_completed_labeling_jobs:
completed_labeling_job_name = completed_labeling_job["jobName"]
for future_level_labeling_job in future_level_labeling_jobs:
if completed_labeling_job_name == future_level_labeling_job["inputConfig"]["chainFromJobName"] and \
future_level_labeling_job["inputConfig"].get("downSamplingRate"):
future_level_labeling_job_name = future_level_labeling_job["jobName"]
job_level_batch_metadata = db.get_batch_metadata_by_labeling_job_name(completed_labeling_job_name,
BatchMetadataType.JOB_LEVEL)[0]
completed_job_output_location = \
job_level_batch_metadata[BatchMetadataTableAttributes.JOB_OUTPUT_LOCATION]
s3_object = s3_accessor.fetch_s3(completed_job_output_location)
content = s3_object.decode('utf-8')
items = content.splitlines()
down_sample_rate = future_level_labeling_job["inputConfig"]["downSamplingRate"]
down_sampled_data = down_sample_to_proportion(items, down_sample_rate)
future_level_labeling_input_location = f"s3://{batch_processing_bucket_name}/batch_manifests/" \
f"{future_level_labeling_job_name}/processed/data.manifest"
s3_accessor.put_s3(future_level_labeling_input_location, "\n".join(down_sampled_data))
batch_id = f'{parent_batch_id}-{future_level_labeling_job_name}-' \
f'{BatchMetadataType.PROCESS_LEVEL.lower()}'
db.insert_processed_input_batch_metadata(parent_batch_id,
batch_id,
future_level_labeling_job_name,
future_level_labeling_input_location)
return None
def lambda_handler(event, context):
"""
Lambda function that transforms input data and stores inital DB entry
Parameters
----------
event: dict, required
context: object, required Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
Lambda Output Format: dict
"""
log.log_request_and_context(event, context)
labeling_jobs = event["labelingJobs"]
batch_id = event["batchId"]
error_message = ""
"""
Example database entry input for batch
{
"BatchCurrentStep": "INPUT",
"BatchId": "notebook-test-08f874a7",
"BatchMetadataType": "INPUT",
"BatchStatus": "INTERNAL_ERROR",
"LabelingJobs": [
{
"inputConfig": {
"inputManifestS3Uri": "s3://smgt-qa-batch-input-468814823616-us-east-1/two-frame-manifest.manifest"
},
"jobLevel": 1,
"jobModality": "PointCloudObjectDetectionAudit",
"jobName": "notebook-test-08f874a7-first-level",
"jobType": "BATCH",
"labelCategoryConfigS3Uri": "s3://smgt-qa-batch-input-468814823616-us-east-1/first-level-label-category-file.json",
"maxConcurrentTaskCount": 1,
"taskAvailabilityLifetimeInSeconds": 864000,
"taskTimeLimitInSeconds": 604800,
"workteamArn": "arn:aws:sagemaker:us-east-1:468814823616:workteam/private-crowd/first-level"
},
{
"inputConfig": {
"chainFromJobName": "notebook-test-08f874a7-first-level"
},
"jobLevel": 2,
"jobModality": "PointCloudObjectDetectionAudit",
"jobName": "notebook-test-08f874a7-second-level",
"jobType": "BATCH",
"maxConcurrentTaskCount": 1,
"taskAvailabilityLifetimeInSeconds": 864000,
"taskTimeLimitInSeconds": 604800,
"workteamArn": "arn:aws:sagemaker:us-east-1:468814823616:workteam/private-crowd/first-level"
}
]
}
"""
db.insert_transformed_input_batch_metadata(
batch_id=batch_id,
batch_current_step=BatchCurrentStep.INPUT,
batch_status=BatchStatus.IN_PROGRESS,
batch_metadata_type=BatchMetadataType.INPUT,
error_message=error_message,
labeling_jobs=labeling_jobs,
)
return {
"batch_id": batch_id,
}