async def __process(record, s3_resource, batch_writer): async with trace("Processing {}", json.dumps(record)): validate_pending_task(record) index = record["index"] batch_id = record["batchId"] request = record["request"] item_no = request["itemNo"] await items_table.put_item( { "itemNo": str(item_no), "updateTimestamp": now_epoch_millis() }, batch_writer) processed_task = { "batchId": batch_id, "index": index, "request": request, "response": { "success": True, "message": "Ok" } } validate_processed_task(processed_task) await work_bucket.write_task_result(batch_id, index, processed_task, s3_resource) await work_bucket.delete_pending_task(batch_id, index, s3_resource)
async def __write_tasks_and_send_messages(batch_id, records, s3_resource, sqs_client): async with trace("Writing/sending {} tasks for batch {}", len(records), batch_id): async with S3BatchWriter(s3_resource=s3_resource, flush_amount=100) as batch_writer: for index, record in enumerate(records, start=0): pending_task = { "batchId": batch_id, "index": index, "request": record } validate_pending_task(pending_task) await work_bucket.write_pending_task(batch_id, index, pending_task, batch_writer) async with process_queue.new_batch_sender(sqs_client) as batch_sender: for index, record in enumerate(records, start=0): pending_task = { "batchId": batch_id, "index": index, "request": record } validate_pending_task(pending_task) await batch_sender.send_message( message={ "Id": str(uuid4()), "MessageBody": json.dumps(pending_task) })
async def __write_chunks_and_send_messages(batch_id, records, dynamodb_resource, sqs_client): async with trace("Writing/sending chunks for batch {}", batch_id): async with await batch_tasks_table.new_batch_writer(dynamodb_resource ) as batch_writer: for index, record in enumerate(records): pending_task = { "batchId": batch_id, "index": index, "request": record } validate_pending_task(pending_task) await batch_tasks_table.put_pending_batch_task( pending_task, batch_writer) async with process_queue.new_batch_sender(sqs_client) as batch_sender: for chunk_index, chunk in enumchunks(records, CHUNK_SIZE): async with trace("Sending chunk {} of tasks for batch_id={}", chunk_index, batch_id): chunk = { "batchId": batch_id, "index": chunk_index, "records": [{ "index": chunk_index * CHUNK_SIZE + record_index } for record_index, record in enumerate(chunk)] } await batch_sender.send_message( message={ "Id": str(uuid4()), "MessageBody": json.dumps(chunk) })
def test_happy_doc(self): doc = { "batchId": "abc", "index": 0, "request": { "itemNo": "1", "price": 30 } } validate_pending_task(doc)
def test_empty_is_invalid(self): doc = {} with self.assertRaises(ValidationError): validate_pending_task(doc)
def test_missing_index_invalid(self): doc = {"batch_id": "abcd"} with self.assertRaises(ValidationError): validate_pending_task(doc)
def test_missing_batch_id_invalid(self): doc = {"index": 0} with self.assertRaises(ValidationError): validate_pending_task(doc)