async def __process(record, s3_resource, batch_writer):
async with trace("Processing {}", json.dumps(record)):
validate_pending_task(record)
index = record["index"]
batch_id = record["batchId"]
request = record["request"]
item_no = request["itemNo"]
await items_table.put_item(
{
"itemNo": str(item_no),
"updateTimestamp": now_epoch_millis()
}, batch_writer)
processed_task = {
"batchId": batch_id,
"index": index,
"request": request,
"response": {
"success": True,
"message": "Ok"
}
}
validate_processed_task(processed_task)
await work_bucket.write_task_result(batch_id, index, processed_task,
s3_resource)
await work_bucket.delete_pending_task(batch_id, index, s3_resource)
async def __write_tasks_and_send_messages(batch_id, records, s3_resource,
sqs_client):
async with trace("Writing/sending {} tasks for batch {}", len(records),
batch_id):
async with S3BatchWriter(s3_resource=s3_resource,
flush_amount=100) as batch_writer:
for index, record in enumerate(records, start=0):
pending_task = {
"batchId": batch_id,
"index": index,
"request": record
}
validate_pending_task(pending_task)
await work_bucket.write_pending_task(batch_id, index,
pending_task,
batch_writer)
async with process_queue.new_batch_sender(sqs_client) as batch_sender:
for index, record in enumerate(records, start=0):
pending_task = {
"batchId": batch_id,
"index": index,
"request": record
}
validate_pending_task(pending_task)
await batch_sender.send_message(
message={
"Id": str(uuid4()),
"MessageBody": json.dumps(pending_task)
})
async def __write_chunks_and_send_messages(batch_id, records,
dynamodb_resource, sqs_client):
async with trace("Writing/sending chunks for batch {}", batch_id):
async with await batch_tasks_table.new_batch_writer(dynamodb_resource
) as batch_writer:
for index, record in enumerate(records):
pending_task = {
"batchId": batch_id,
"index": index,
"request": record
}
validate_pending_task(pending_task)
await batch_tasks_table.put_pending_batch_task(
pending_task, batch_writer)
async with process_queue.new_batch_sender(sqs_client) as batch_sender:
for chunk_index, chunk in enumchunks(records, CHUNK_SIZE):
async with trace("Sending chunk {} of tasks for batch_id={}",
chunk_index, batch_id):
chunk = {
"batchId":
batch_id,
"index":
chunk_index,
"records": [{
"index":
chunk_index * CHUNK_SIZE + record_index
} for record_index, record in enumerate(chunk)]
}
await batch_sender.send_message(
message={
"Id": str(uuid4()),
"MessageBody": json.dumps(chunk)
})
def test_happy_doc(self):
doc = {
"batchId": "abc",
"index": 0,
"request": {
"itemNo": "1",
"price": 30
}
}
validate_pending_task(doc)
def test_empty_is_invalid(self):
doc = {}
with self.assertRaises(ValidationError):
validate_pending_task(doc)
def test_missing_index_invalid(self):
doc = {"batch_id": "abcd"}
with self.assertRaises(ValidationError):
validate_pending_task(doc)
def test_missing_batch_id_invalid(self):
doc = {"index": 0}
with self.assertRaises(ValidationError):
validate_pending_task(doc)