def sync_blob(source_platform, source_key, dest_platform, context): gs = Config.get_native_handle(Replica.gcp) logger.info( f"Begin transfer of {source_key} from {source_platform} to {dest_platform}" ) gs_bucket, s3_bucket = gs.bucket( Config.get_gs_bucket()), resources.s3.Bucket(Config.get_s3_bucket()) if source_platform == "s3" and dest_platform == "gs": source = BlobLocation(platform=source_platform, bucket=s3_bucket, blob=s3_bucket.Object(source_key)) dest = BlobLocation(platform=dest_platform, bucket=gs_bucket, blob=gs_bucket.blob(source_key)) elif source_platform == "gs" and dest_platform == "s3": source = BlobLocation(platform=source_platform, bucket=gs_bucket, blob=gs_bucket.blob(source_key)) dest = BlobLocation(platform=dest_platform, bucket=s3_bucket, blob=s3_bucket.Object(source_key)) else: raise NotImplementedError() if source_platform == "s3" and dest_platform == "gs" and use_gsts: sync_s3_to_gcsts(gs.project, source.bucket.name, dest.bucket.name, source_key) elif source_platform == "s3" and dest_platform == "gs": if dest.blob.exists(): logger.info(f"Key {source_key} already exists in GS") return elif source.blob.content_length < part_size["s3"]: sync_s3_to_gs_oneshot(source, dest) else: dispatch_multipart_sync(source, dest, context) elif source_platform == "gs" and dest_platform == "s3": try: dest.blob.load() logger.info(f"Key {source_key} already exists in S3") return except clients.s3.exceptions.ClientError as e: if e.response["Error"].get("Message") != "Not Found": raise source.blob.reload() if source.blob.size < part_size["s3"]: sync_gs_to_s3_oneshot(source, dest) else: dispatch_multipart_sync(source, dest, context) logger.info( f"Completed transfer of {source_key} from {source.bucket} to {dest.bucket}" )
from dss.index.backend import CompositeIndexBackend from dss.index.indexer import Indexer from dss.logging import configure_lambda_logging from dss.util import tracing from dss.util.time import RemainingLambdaContextTime, AdjustedRemainingTime app = domovoi.Domovoi(configure_logs=False) configure_lambda_logging() logger = logging.getLogger(__name__) dss.Config.set_config(dss.BucketConfig.NORMAL) @app.s3_event_handler(bucket=Config.get_s3_bucket(), events=["s3:ObjectCreated:*"], use_sqs=True) def dispatch_s3_indexer_event(event, context) -> None: if event.get("Event") == "s3:TestEvent": logger.info("DSS index daemon received S3 test event") else: for event_record in event["Records"]: key = unquote(event_record['s3']['object']['key']) _handle_event(Replica.aws, key, context) @app.sqs_queue_subscriber("dss-index-" + os.environ["DSS_DEPLOYMENT_STAGE"]) def dispatch_gs_indexer_event(event, context): """ This handler receives GS events via the Google Cloud Function deployed from daemons/dss-gs-event-relay. """ for event_record in event["Records"]:
build_bundle_metadata_document) from dss.events.handlers.notify_v2 import should_notify, notify_or_queue, notify from dss.events.handlers.sync import exists from dss.subscriptions_v2 import get_subscription, get_subscriptions_for_replica configure_lambda_logging() logger = logging.getLogger(__name__) dss.Config.set_config(dss.BucketConfig.NORMAL) app = domovoi.Domovoi() # This entry point is for S3 native events forwarded through SQS. @app.s3_event_handler( bucket=Config.get_s3_bucket(), events=["s3:ObjectCreated:*", "s3:ObjectRemoved:Delete"], use_sqs=True, sqs_queue_attributes=dict( VisibilityTimeout="920"), # Lambda timeout + 20 seconds ) def launch_from_s3_event(event, context): replica = Replica.aws if event.get("Event") == "s3:TestEvent": logger.info("S3 test event received and processed successfully") else: for event_record in event['Records']: bucket = event_record['s3']['bucket']['name'] if bucket != replica.bucket: logger.error( "Received S3 event for bucket %s with no configured replica",
from dss.events.handlers.sync import ( compose_upload, initiate_multipart_upload, complete_multipart_upload, copy_part, exists, get_part_size, get_sync_work_state, parts_per_worker, dependencies_exist, do_oneshot_copy, sync_sfn_dep_wait_sleep_seconds, sync_sfn_num_threads) from dss.storage.identifiers import BLOB_KEY_REGEX, BLOB_PREFIX configure_lambda_logging() logger = logging.getLogger(__name__) dss.Config.set_config(dss.BucketConfig.NORMAL) app = domovoi.Domovoi() # This entry point is for S3 native events forwarded through SQS. @app.s3_event_handler(bucket=Config.get_s3_bucket(), events=["s3:ObjectCreated:*"], use_sqs=True, sqs_queue_attributes=dict(VisibilityTimeout="920")) def launch_from_s3_event(event, context): source_replica = Replica.aws executions = {} if event.get("Event") == "s3:TestEvent": logger.info("S3 test event received and processed successfully") else: for event_record in event["Records"]: bucket = resources.s3.Bucket(event_record["s3"]["bucket"]["name"]) obj = bucket.Object(unquote(event_record["s3"]["object"]["key"])) if obj.key.startswith(BLOB_PREFIX) and not BLOB_KEY_REGEX.match( obj.key): logger.info(
def _parse_event(self, event): assert event['Records'][0]['s3']['bucket'][ 'name'] == Config.get_s3_bucket() key = unquote(event['Records'][0]['s3']['object']['key']) return key