def get_records(records, record_type, operation, new_image=True): return [ deserialize_item(r["dynamodb"].get( "NewImage" if new_image else "OldImage", {})) for r in records if is_record_type(r, record_type, new_image) and is_operation(r, operation) ]
def get_data_mappers(): results = paginate(ddb_client, ddb_client.scan, "Items", TableName=data_mapper_table_name) for result in results: yield deserialize_item(result)
def get_deletion_queue(): results = paginate(ddb_client, ddb_client.scan, "Items", TableName=deletion_queue_table_name) for result in results: yield deserialize_item(result)
def handler(event, context): results = paginate(ddb_client, ddb_client.scan, "Items", TableName=event["TableName"]) items = [deserialize_item(result) for result in results] return { "Items": items, "Count": len(items) }
def handler(event, context): records = event["Records"] new_jobs = [ deserialize_item(r["dynamodb"]["NewImage"]) for r in records if is_record_type(r, "Job") and is_operation(r, "INSERT") ] events = [ deserialize_item(r["dynamodb"]["NewImage"]) for r in records if is_record_type(r, "JobEvent") and is_operation(r, "INSERT") ] grouped_events = groupby(sorted(events, key=itemgetter("Id")), key=itemgetter("Id")) for job in new_jobs: process_job(job) for job_id, group in grouped_events: group = [i for i in group] update_stats(job_id, group) updated_job = update_status(job_id, group) # Perform cleanup if required if ( updated_job and updated_job.get("JobStatus") == "FORGET_COMPLETED_CLEANUP_IN_PROGRESS" ): try: clear_deletion_queue(updated_job) emit_event( job_id, "CleanupSucceeded", utc_timestamp(), "StreamProcessor" ) except Exception as e: emit_event( job_id, "CleanupFailed", {"Error": "Unable to clear deletion queue: {}".format(str(e))}, "StreamProcessor", ) elif updated_job and updated_job.get("JobStatus") in skip_cleanup_states: emit_event(job_id, "CleanupSkipped", utc_timestamp(), "StreamProcessor")
def test_it_deserializes_items(): result = deserialize_item({ "DataMappers": { "L": [{ "S": "test" }] }, "MatchId": { "S": "test" } }) assert {"MatchId": "test", "DataMappers": ["test"]} == result
def is_record_type(record, record_type): new_image = record["dynamodb"].get("NewImage") if not new_image: return False item = deserialize_item(new_image) return item.get("Type") and item.get("Type") == record_type