async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_objects = __get_s3_objects_from(event) async with aioaws.resource("s3") as s3_resource, \ aioaws.resource("dynamodb") as dynamodb_resource, \ await items_table.new_batch_writer(dynamodb_resource) as batch_writer: await asyncio.gather(*[ __process(s3_object, s3_resource, batch_writer) for s3_object in s3_objects ])
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) async with aioaws.client("sqs") as sqs_client, \ aioaws.resource("s3") as s3_resource, \ aioaws.resource("dynamodb") as dynamodb_resource, \ await items_table.new_batch_writer(dynamodb_resource) as batch_writer: chunks = [json.loads(record["body"]) for record in event["Records"]] await asyncio.gather(*[__process(chunk, s3_resource, batch_writer) for chunk in chunks]) batch_ids = {chunk["batchId"] for chunk in chunks} await asyncio.gather(*[__check_if_complete(batch_id, s3_resource, sqs_client) for batch_id in batch_ids])
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) async with aioaws.resource("s3") as s3_resource: records = [json.loads(record["body"]) for record in event["Records"]] await asyncio.gather( *[__gather(record, s3_resource) for record in records])
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_objects = __get_s3_objects_from(event) batch_ids = set([__extract_batch_id(key[1]) for key in s3_objects]) async with aioaws.client("sqs") as sqs_client, \ aioaws.resource("s3") as s3_resource: await asyncio.gather(*[ __check_if_complete(batch_id, s3_resource, sqs_client) for batch_id in batch_ids ])
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_object = __get_s3_object_from(event) if s3_object is None: return batch_id = __extract_batch_id(s3_object[1]) async with trace("Scattering {}", batch_id): async with aioaws.resource("s3") as s3_resource, \ aioaws.client("sqs") as sqs_client, \ aioaws.resource("dynamodb") as dynamodb_resource: batch_doc = await input_bucket.read_batch_input( s3_object[0], s3_object[1], s3_resource) validate_input(batch_doc) records = batch_doc.get("records", []) record_batch_started(batch_id) await batch_status_table.put_batch_status(batch_id, len(records), dynamodb_resource) await __write_chunks_and_send_messages(batch_id, records, dynamodb_resource, sqs_client) await input_bucket.delete_batch_input(s3_object[0], s3_object[1], s3_resource) record_scatter_finished(batch_id, len(records))
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_object = __get_s3_object_from(event) if s3_object is None: logger.info("Is s3 test event. Skipping.") return batch_id = __extract_batch_id(s3_object[1]) async with trace("Scattering {}", batch_id): async with aioaws.resource("s3") as s3_resource, aioaws.client("sqs") as sqs_client: batch_doc = await input_bucket.read_batch_input(s3_object[0], s3_object[1], s3_resource) validate_input(batch_doc) records = batch_doc.get("records", []) record_batch_started(batch_id) await work_bucket.write_batch_status(batch_id, len(records), CHUNK_SIZE, s3_resource) await __write_chunks(batch_id, records, s3_resource, sqs_client) await input_bucket.delete_batch_input(s3_object[0], s3_object[1], s3_resource) record_scatter_finished(batch_id, len(records))
async def run(batch_id, batch): async with aioaws.resource("s3") as s3_resource: await input_bucket.write_batch_input(batch_id, batch, s3_resource) await __wait_for_output_json_available_in_s3(batch_id, s3_resource)