def execute(self, context):
hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
delegate_to=self.delegate_to)
log_message = 'Executing copy of gs://{0}/{1} to gs://{2}/{3}'
if self.wildcard in self.source_object:
prefix, delimiter = self.source_object.split(self.wildcard, 1)
objects = hook.list(self.source_bucket,
prefix=prefix,
delimiter=delimiter)
for source_object in objects:
if self.last_modified_time is not None:
# Check to see if object was modified after last_modified_time
if hook.is_updated_after(self.source_bucket, source_object,
self.last_modified_time):
pass
else:
continue
if self.destination_object is None:
destination_object = source_object
else:
destination_object = source_object.replace(
prefix, self.destination_object, 1)
self.log.info(
log_message.format(self.source_bucket, source_object,
self.destination_bucket,
destination_object))
hook.rewrite(self.source_bucket, source_object,
self.destination_bucket, destination_object)
if self.move_object:
hook.delete(self.source_bucket, source_object)
else:
if self.last_modified_time is not None:
if hook.is_updated_after(self.source_bucket,
self.source_object,
self.last_modified_time):
pass
else:
return
self.log.info(
log_message.format(
self.source_bucket, self.source_object,
self.destination_bucket or self.source_bucket,
self.destination_object or self.source_object))
hook.rewrite(self.source_bucket, self.source_object,
self.destination_bucket, self.destination_object)
if self.move_object:
hook.delete(self.source_bucket, self.source_object)
def execute(self, context):
hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
delegate_to=self.delegate_to
)
log_message = 'Executing copy of gs://{0}/{1} to gs://{2}/{3}'
if self.wildcard in self.source_object:
prefix, delimiter = self.source_object.split(self.wildcard, 1)
objects = hook.list(self.source_bucket, prefix=prefix, delimiter=delimiter)
for source_object in objects:
if self.last_modified_time is not None:
# Check to see if object was modified after last_modified_time
if hook.is_updated_after(self.source_bucket, source_object,
self.last_modified_time):
pass
else:
continue
if self.destination_object is None:
destination_object = source_object
else:
destination_object = source_object.replace(prefix,
self.destination_object, 1)
self.log.info(
log_message.format(self.source_bucket, source_object,
self.destination_bucket, destination_object)
)
hook.rewrite(self.source_bucket, source_object,
self.destination_bucket, destination_object)
if self.move_object:
hook.delete(self.source_bucket, source_object)
else:
if self.last_modified_time is not None:
if hook.is_updated_after(self.source_bucket,
self.source_object,
self.last_modified_time):
pass
else:
return
self.log.info(
log_message.format(self.source_bucket, self.source_object,
self.destination_bucket or self.source_bucket,
self.destination_object or self.source_object)
)
hook.rewrite(self.source_bucket, self.source_object,
self.destination_bucket, self.destination_object)
if self.move_object:
hook.delete(self.source_bucket, self.source_object)
def poke(self, context):
self.log.info('Sensor checks existence of : %s, %s', self.bucket,
self.object)
hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.google_cloud_conn_id,
delegate_to=self.delegate_to)
return hook.is_updated_after(self.bucket, self.object,
self.ts_func(context))
def poke(self, context):
self.log.info('Sensor checks existence of : %s, %s', self.bucket, self.object)
hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.google_cloud_conn_id,
delegate_to=self.delegate_to)
return hook.is_updated_after(self.bucket, self.object, self.ts_func(context))
