async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) async with aioaws.resource("s3") as s3_resource: records = [json.loads(record["body"]) for record in event["Records"]] await asyncio.gather( *[__gather(record, s3_resource) for record in records])
def __exit__(self, exc_type, exc_value, tb): self.end = time.time_ns() self.duration_milis = int((self.end - self.start) / 1000 / 1000) if exc_type is None: logger.info("SUCCESS of \"%s\". Duration %d millis.", str(self.message), self.duration_milis) else: logger.info("FAILURE of \"%s\". Duration %d millis.", str(self.message), self.duration_milis, exc_info=True)
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_objects = __get_s3_objects_from(event) batch_ids = set([__extract_batch_id(key[1]) for key in s3_objects]) async with aioaws.client("sqs") as sqs_client, \ aioaws.resource("s3") as s3_resource: await asyncio.gather(*[ __check_if_complete(batch_id, s3_resource, sqs_client) for batch_id in batch_ids ])
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_objects = __get_s3_objects_from(event) async with aioaws.resource("s3") as s3_resource, \ aioaws.resource("dynamodb") as dynamodb_resource, \ await items_table.new_batch_writer(dynamodb_resource) as batch_writer: await asyncio.gather(*[ __process(s3_object, s3_resource, batch_writer) for s3_object in s3_objects ])
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) async with aioaws.client("sqs") as sqs_client, \ aioaws.resource("s3") as s3_resource, \ aioaws.resource("dynamodb") as dynamodb_resource, \ await items_table.new_batch_writer(dynamodb_resource) as batch_writer: chunks = [json.loads(record["body"]) for record in event["Records"]] await asyncio.gather(*[__process(chunk, s3_resource, batch_writer) for chunk in chunks]) batch_ids = {chunk["batchId"] for chunk in chunks} await asyncio.gather(*[__check_if_complete(batch_id, s3_resource, sqs_client) for batch_id in batch_ids])
def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_object = __get_s3_object_from(event) if s3_object is None: return batch_id = __extract_batch_id(s3_object[1]) record_batch_started(batch_id) with trace("Scattering {}", batch_id): batch_doc = input_bucket.read_batch_input(s3_object[0], s3_object[1]) validate_input(batch_doc) records = batch_doc.get("records", []) work_bucket.write_batch_status(batch_id, len(records)) __write_tasks_and_send_messages(batch_id, records) input_bucket.delete_batch_input(s3_object[0], s3_object[1]) record_scatter_finished(batch_id, len(records))
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_object = __get_s3_object_from(event) if s3_object is None: logger.info("Is s3 test event. Skipping.") return batch_id = __extract_batch_id(s3_object[1]) async with trace("Scattering {}", batch_id): async with aioaws.resource("s3") as s3_resource, aioaws.client("sqs") as sqs_client: batch_doc = await input_bucket.read_batch_input(s3_object[0], s3_object[1], s3_resource) validate_input(batch_doc) records = batch_doc.get("records", []) record_batch_started(batch_id) await work_bucket.write_batch_status(batch_id, len(records), CHUNK_SIZE, s3_resource) await __write_chunks(batch_id, records, s3_resource, sqs_client) await input_bucket.delete_batch_input(s3_object[0], s3_object[1], s3_resource) record_scatter_finished(batch_id, len(records))
async def _flush(self): if self._queue_url is None: response = await self._sqs_client.get_queue_url(QueueName=self._queue_name) self._queue_url = response["QueueUrl"] logger.info("Flushing buffer of %d message(s)...", len(self._messages_buffer)) messages_to_send = self._messages_buffer[:self._flush_amount] self._messages_buffer = self._messages_buffer[self._flush_amount:] response = await self._sqs_client.send_message_batch(QueueUrl=self._queue_url, Entries=messages_to_send) retryable_ids = [entry["Id"] for entry in response.get("Failed", []) if entry["SenderFault"] is False] if len(retryable_ids) > 0: logger.info("%d message(s) retryable... putting back.", len(retryable_ids)) retryable_messages = [m for m in messages_to_send if m["Id"] in retryable_ids] self._messages_buffer.extend(retryable_messages) sender_faults = [entry for entry in response.get("Failed", []) if entry["SenderFault"] is True] if len(sender_faults) > 0: sender_fault = sender_faults[0] raise ValueError( "Sending message {} failed: code={}, message={}".format(sender_fault["Id"], sender_fault["Code"], sender_fault["Message"]))
def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) records = event["Records"] with items_table.new_batch_writer() as batch_writer: for record in records: record = json.loads(record["body"]) with trace("Processing {}", json.dumps(record)): index = record["index"] batch_id = record["batchId"] request = record["request"] item_no = request["itemNo"] items_table.put_item( { "itemNo": str(item_no), "updateTimestamp": now_epoch_millis() }, batch_writer) work_bucket.write_task_result(batch_id, index, request, { "success": True, "message": "Ok" }) work_bucket.delete_pending_task(batch_id, index) if not work_bucket.exists_pending_task(batch_id): gather_queue.send_batch_complete_message(batch_id)
async def handle_event(event, lambda_context): logger.info("Event: {}".format(json.dumps(event, indent=2))) s3_object = __get_s3_object_from(event) if s3_object is None: return batch_id = __extract_batch_id(s3_object[1]) async with trace("Scattering {}", batch_id): async with aioaws.resource("s3") as s3_resource, \ aioaws.client("sqs") as sqs_client, \ aioaws.resource("dynamodb") as dynamodb_resource: batch_doc = await input_bucket.read_batch_input( s3_object[0], s3_object[1], s3_resource) validate_input(batch_doc) records = batch_doc.get("records", []) record_batch_started(batch_id) await batch_status_table.put_batch_status(batch_id, len(records), dynamodb_resource) await __write_chunks_and_send_messages(batch_id, records, dynamodb_resource, sqs_client) await input_bucket.delete_batch_input(s3_object[0], s3_object[1], s3_resource) record_scatter_finished(batch_id, len(records))
def __enter__(self): self.start = time.time_ns() logger.info("START \"%s\"...", str(self.message)) return self