def read_batch_status(batch_id):
with trace("Reading status for batch_id={}", batch_id):
object_key = "{}/status.json".format(batch_id)
s3_object = s3_resource.Object(WORK_BUCKET, object_key)
data = s3_object.get()['Body'].read()
json_doc = json.loads(data)
return json_doc
async def handle_event(event, lambda_context):
logger.info("Event: {}".format(json.dumps(event, indent=2)))
async with aioaws.resource("s3") as s3_resource:
records = [json.loads(record["body"]) for record in event["Records"]]
await asyncio.gather(
*[__gather(record, s3_resource) for record in records])
def read_task_result(batch_id, index):
object_key = "{}/done/{}.json".format(batch_id, index)
with trace("Reading task result {}/{} to s3", WORK_BUCKET, object_key):
s3_object = s3_resource.Object(WORK_BUCKET, object_key)
data = s3_object.get()['Body'].read()
json_doc = json.loads(data)
return json_doc
async def read_batch_output(batch_id, s3_resource) -> dict:
object_key = "{}.json".format(batch_id)
s3_object = await s3_resource.Object(OUTPUT_BUCKET, object_key)
response = await s3_object.get()
async with response["Body"] as stream:
data = await stream.read()
json_doc = json.loads(data)
return json_doc
async def read_pending_chunk(s3_bucket, s3_object_key, s3_resource):
async with trace("Reading pending chunk bucket={}/key={}", s3_bucket, s3_object_key):
s3_object = await s3_resource.Object(s3_bucket, s3_object_key)
response = await s3_object.get()
async with response["Body"] as stream:
data = await stream.read()
json_doc = json.loads(data)
return json_doc
async def read_batch_status(batch_id, s3_resource):
async with trace("Reading status for batch_id={}", batch_id):
object_key = "{}/status.json".format(batch_id)
s3_object = await s3_resource.Object(WORK_BUCKET, object_key)
response = await s3_object.get()
async with response["Body"] as stream:
data = await stream.read()
json_doc = json.loads(data)
return json_doc
async def read_chunk_result(batch_id, index, s3_resource):
async with trace("Reading chunk result batch_id={}/index={}", batch_id, index):
object_key = "{}/done/{}.done.json".format(batch_id, index)
s3_object = await s3_resource.Object(WORK_BUCKET, object_key)
response = await s3_object.get()
async with response["Body"] as stream:
data = await stream.read()
json_doc = json.loads(data)
return json_doc
def read_batch_input(bucket_name, object_key) -> dict:
with trace("Reading input {}/{} from s3", bucket_name, object_key):
if bucket_name != INPUT_BUCKET:
raise ValueError("Expected bucket {}, but was {}.".format(
INPUT_BUCKET, bucket_name))
s3_object = s3_resource.Object(bucket_name, object_key)
json_data = s3_object.get()['Body'].read()
json_doc = json.loads(json_data)
return json_doc
async def read_batch_input(bucket_name, object_key, s3_resource) -> dict:
async with trace("Reading input {}/{} from s3", bucket_name, object_key):
if bucket_name != INPUT_BUCKET:
raise ValueError("Expected bucket {}, but was {}.".format(INPUT_BUCKET, bucket_name))
s3_object = await s3_resource.Object(bucket_name, object_key)
response = await s3_object.get()
async with response["Body"] as stream:
data = await stream.read()
json_doc = json.loads(data)
return json_doc
async def handle_event(event, lambda_context):
logger.info("Event: {}".format(json.dumps(event, indent=2)))
async with aioaws.client("sqs") as sqs_client, \
aioaws.resource("s3") as s3_resource, \
aioaws.resource("dynamodb") as dynamodb_resource, \
await items_table.new_batch_writer(dynamodb_resource) as batch_writer:
chunks = [json.loads(record["body"]) for record in event["Records"]]
await asyncio.gather(*[__process(chunk, s3_resource, batch_writer) for chunk in chunks])
batch_ids = {chunk["batchId"] for chunk in chunks}
await asyncio.gather(*[__check_if_complete(batch_id, s3_resource, sqs_client) for batch_id in batch_ids])
def handle_event(event, lambda_context):
logger.info("Event: {}".format(json.dumps(event, indent=2)))
records = event["Records"]
with items_table.new_batch_writer() as batch_writer:
for record in records:
record = json.loads(record["body"])
with trace("Processing {}", json.dumps(record)):
index = record["index"]
batch_id = record["batchId"]
request = record["request"]
item_no = request["itemNo"]
items_table.put_item(
{
"itemNo": str(item_no),
"updateTimestamp": now_epoch_millis()
}, batch_writer)
work_bucket.write_task_result(batch_id, index, request, {
"success": True,
"message": "Ok"
})
work_bucket.delete_pending_task(batch_id, index)
if not work_bucket.exists_pending_task(batch_id):
gather_queue.send_batch_complete_message(batch_id)
def read_batch_output(batch_id) -> dict:
object_key = "{}.json".format(batch_id)
s3_object = s3_resource.Object(OUTPUT_BUCKET, object_key)
json_data = s3_object.get()['Body'].read()
json_doc = json.loads(json_data)
return json_doc
