def read_batch_status(batch_id): with trace("Reading status for batch_id={}", batch_id): object_key = "{}/status.json".format(batch_id) s3_object = s3_resource.Object(WORK_BUCKET, object_key) data = s3_object.get()['Body'].read() json_doc = json.loads(data) return json_doc
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) async with aioaws.resource("s3") as s3_resource: records = [json.loads(record["body"]) for record in event["Records"]] await asyncio.gather( *[__gather(record, s3_resource) for record in records])
def read_task_result(batch_id, index): object_key = "{}/done/{}.json".format(batch_id, index) with trace("Reading task result {}/{} to s3", WORK_BUCKET, object_key): s3_object = s3_resource.Object(WORK_BUCKET, object_key) data = s3_object.get()['Body'].read() json_doc = json.loads(data) return json_doc
async def read_batch_output(batch_id, s3_resource) -> dict: object_key = "{}.json".format(batch_id) s3_object = await s3_resource.Object(OUTPUT_BUCKET, object_key) response = await s3_object.get() async with response["Body"] as stream: data = await stream.read() json_doc = json.loads(data) return json_doc
async def read_pending_chunk(s3_bucket, s3_object_key, s3_resource): async with trace("Reading pending chunk bucket={}/key={}", s3_bucket, s3_object_key): s3_object = await s3_resource.Object(s3_bucket, s3_object_key) response = await s3_object.get() async with response["Body"] as stream: data = await stream.read() json_doc = json.loads(data) return json_doc
async def read_batch_status(batch_id, s3_resource): async with trace("Reading status for batch_id={}", batch_id): object_key = "{}/status.json".format(batch_id) s3_object = await s3_resource.Object(WORK_BUCKET, object_key) response = await s3_object.get() async with response["Body"] as stream: data = await stream.read() json_doc = json.loads(data) return json_doc
async def read_chunk_result(batch_id, index, s3_resource): async with trace("Reading chunk result batch_id={}/index={}", batch_id, index): object_key = "{}/done/{}.done.json".format(batch_id, index) s3_object = await s3_resource.Object(WORK_BUCKET, object_key) response = await s3_object.get() async with response["Body"] as stream: data = await stream.read() json_doc = json.loads(data) return json_doc
def read_batch_input(bucket_name, object_key) -> dict: with trace("Reading input {}/{} from s3", bucket_name, object_key): if bucket_name != INPUT_BUCKET: raise ValueError("Expected bucket {}, but was {}.".format( INPUT_BUCKET, bucket_name)) s3_object = s3_resource.Object(bucket_name, object_key) json_data = s3_object.get()['Body'].read() json_doc = json.loads(json_data) return json_doc
async def read_batch_input(bucket_name, object_key, s3_resource) -> dict: async with trace("Reading input {}/{} from s3", bucket_name, object_key): if bucket_name != INPUT_BUCKET: raise ValueError("Expected bucket {}, but was {}.".format(INPUT_BUCKET, bucket_name)) s3_object = await s3_resource.Object(bucket_name, object_key) response = await s3_object.get() async with response["Body"] as stream: data = await stream.read() json_doc = json.loads(data) return json_doc
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) async with aioaws.client("sqs") as sqs_client, \ aioaws.resource("s3") as s3_resource, \ aioaws.resource("dynamodb") as dynamodb_resource, \ await items_table.new_batch_writer(dynamodb_resource) as batch_writer: chunks = [json.loads(record["body"]) for record in event["Records"]] await asyncio.gather(*[__process(chunk, s3_resource, batch_writer) for chunk in chunks]) batch_ids = {chunk["batchId"] for chunk in chunks} await asyncio.gather(*[__check_if_complete(batch_id, s3_resource, sqs_client) for batch_id in batch_ids])
def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) records = event["Records"] with items_table.new_batch_writer() as batch_writer: for record in records: record = json.loads(record["body"]) with trace("Processing {}", json.dumps(record)): index = record["index"] batch_id = record["batchId"] request = record["request"] item_no = request["itemNo"] items_table.put_item( { "itemNo": str(item_no), "updateTimestamp": now_epoch_millis() }, batch_writer) work_bucket.write_task_result(batch_id, index, request, { "success": True, "message": "Ok" }) work_bucket.delete_pending_task(batch_id, index) if not work_bucket.exists_pending_task(batch_id): gather_queue.send_batch_complete_message(batch_id)
def read_batch_output(batch_id) -> dict: object_key = "{}.json".format(batch_id) s3_object = s3_resource.Object(OUTPUT_BUCKET, object_key) json_data = s3_object.get()['Body'].read() json_doc = json.loads(json_data) return json_doc