def get_job_status(event, context):
bundle_fqid = event["bundle"]
version = event["version"]
replica = Replica[event["replica"]]
check_count = 0
if "status" in event:
check_count = event["status"].get("check_count", 0)
complete_count = 0
total_count = 0
for src_key, dst_key in get_manifest_files(bundle_fqid, version, replica):
total_count += 1
if validate_file_dst(get_dst_bucket(event), dst_key, replica):
complete_count += 1
checkout_status = "SUCCESS" if complete_count == total_count else "IN_PROGRESS"
check_count += 1
logger.info(
"Check copy status for checkout jobId %s , check count %d , status %s",
event['execution_name'], check_count, checkout_status)
return {
"complete_count": complete_count,
"total_count": total_count,
"check_count": check_count,
"checkout_status": checkout_status
}
def test_manifest_files(self):
bundle_uuid = "011c7340-9b3c-4d62-bf49-090d79daf198"
version = "2017-06-20T214506.766634Z"
replica = Replica.aws
file_count = 0
with override_bucket_config(BucketConfig.TEST_FIXTURE):
for _ in get_manifest_files(bundle_uuid, version, replica):
file_count += 1
self.assertEqual(file_count, 1)
def schedule_copy(event, context):
bundle_fqid = event["bundle"]
version = event["version"]
dss_bucket = event["dss_bucket"]
dst_bucket = get_dst_bucket(event)
replica = Replica[event["replica"]]
scheduled = 0
for src_key, dst_key in get_manifest_files(bundle_fqid, version, replica):
logger.info("Schedule copying a file %s to bucket %s", dst_key, dss_bucket)
parallel_copy(dss_bucket, src_key, dst_bucket, dst_key, replica)
scheduled += 1
return {"files_scheduled": scheduled,
"dst_location": get_dst_bundle_prefix(bundle_fqid, version),
"wait_time_seconds": 30}