def process_new_s3_syncable_object(event, context):
if event.get("Event") == "s3:TestEvent":
app.log.info("DSS sync daemon received S3 test event")
else:
bucket = resources.s3.Bucket(event['Records'][0]["s3"]["bucket"]["name"])
obj = bucket.Object(unquote(event['Records'][0]["s3"]["object"]["key"]))
sync_blob(source_platform="s3", source_key=obj.key, dest_platform="gs", context=context)
def process_new_gs_syncable_object(event, context):
"""
This handler receives GS events via SNS through the Google Cloud Function deployed from daemons/dss-gs-event-relay.
"""
gs_event = json.loads(event["Records"][0]["Sns"]["Message"])
gs_key_name = gs_event["data"]["name"]
sync_blob(source_platform="gs", source_key=gs_key_name, dest_platform="s3", context=context)
def test_sync_blob(self):
self.cleanup_sync_test_objects()
payload = os.urandom(2**20)
test_metadata = {"metadata-sync-test": str(uuid.uuid4())}
test_key = "hca-dss-sync-test/s3-to-gcs/{}".format(uuid.uuid4())
src_blob = self.s3_bucket.Object(test_key)
gs_dest_blob = self.gs_bucket.blob(test_key)
src_blob.put(Body=payload, Metadata=test_metadata)
sync.sync_blob(source_platform="s3", source_key=test_key, dest_platform="gs", context=None)
self.assertEqual(gs_dest_blob.download_as_string(), payload)
test_key = "hca-dss-sync-test/gcs-to-s3/{}".format(uuid.uuid4())
src_blob = self.gs_bucket.blob(test_key)
dest_blob = self.s3_bucket.Object(test_key)
src_blob.metadata = test_metadata
src_blob.upload_from_string(payload)
sync.sync_blob(source_platform="gs", source_key=test_key, dest_platform="s3", context=None)
self.assertEqual(dest_blob.get()["Body"].read(), payload)
self.assertEqual(dest_blob.metadata, test_metadata)
# GS metadata seems to take a while to propagate, so we wait until the above test completes to read it back
gs_dest_blob.reload()
self.assertEqual(gs_dest_blob.metadata, test_metadata)