def get_records(records, record_type, operation, new_image=True):
return [
deserialize_item(r["dynamodb"].get(
"NewImage" if new_image else "OldImage", {})) for r in records
if is_record_type(r, record_type, new_image)
and is_operation(r, operation)
]
def get_data_mappers():
results = paginate(ddb_client,
ddb_client.scan,
"Items",
TableName=data_mapper_table_name)
for result in results:
yield deserialize_item(result)
def get_deletion_queue():
results = paginate(ddb_client,
ddb_client.scan,
"Items",
TableName=deletion_queue_table_name)
for result in results:
yield deserialize_item(result)
def handler(event, context):
results = paginate(ddb_client, ddb_client.scan, "Items", TableName=event["TableName"])
items = [deserialize_item(result) for result in results]
return {
"Items": items,
"Count": len(items)
}
def handler(event, context):
records = event["Records"]
new_jobs = [
deserialize_item(r["dynamodb"]["NewImage"])
for r in records
if is_record_type(r, "Job") and is_operation(r, "INSERT")
]
events = [
deserialize_item(r["dynamodb"]["NewImage"])
for r in records
if is_record_type(r, "JobEvent") and is_operation(r, "INSERT")
]
grouped_events = groupby(sorted(events, key=itemgetter("Id")), key=itemgetter("Id"))
for job in new_jobs:
process_job(job)
for job_id, group in grouped_events:
group = [i for i in group]
update_stats(job_id, group)
updated_job = update_status(job_id, group)
# Perform cleanup if required
if (
updated_job
and updated_job.get("JobStatus") == "FORGET_COMPLETED_CLEANUP_IN_PROGRESS"
):
try:
clear_deletion_queue(updated_job)
emit_event(
job_id, "CleanupSucceeded", utc_timestamp(), "StreamProcessor"
)
except Exception as e:
emit_event(
job_id,
"CleanupFailed",
{"Error": "Unable to clear deletion queue: {}".format(str(e))},
"StreamProcessor",
)
elif updated_job and updated_job.get("JobStatus") in skip_cleanup_states:
emit_event(job_id, "CleanupSkipped", utc_timestamp(), "StreamProcessor")
def test_it_deserializes_items():
result = deserialize_item({
"DataMappers": {
"L": [{
"S": "test"
}]
},
"MatchId": {
"S": "test"
}
})
assert {"MatchId": "test", "DataMappers": ["test"]} == result
def is_record_type(record, record_type):
new_image = record["dynamodb"].get("NewImage")
if not new_image:
return False
item = deserialize_item(new_image)
return item.get("Type") and item.get("Type") == record_type
