def test_execute_wildcard_with_destination_object_retained_prefix(
self, mock_hook):
mock_hook.return_value.list.return_value = SOURCE_FILES_LIST
operator = GCSToGCSOperator(
task_id=TASK_ID,
source_bucket=TEST_BUCKET,
source_object=SOURCE_OBJECT_WILDCARD_FILENAME,
destination_bucket=DESTINATION_BUCKET,
destination_object='{}/{}'.format(
DESTINATION_OBJECT_PREFIX, SOURCE_OBJECT_WILDCARD_SUFFIX[:-1]))
operator.execute(None)
mock_calls_retained = [
mock.call(TEST_BUCKET, 'test_object/file1.txt', DESTINATION_BUCKET,
'foo/bar/test_object/file1.txt'),
mock.call(TEST_BUCKET, 'test_object/file2.txt', DESTINATION_BUCKET,
'foo/bar/test_object/file2.txt'),
]
mock_hook.return_value.rewrite.assert_has_calls(mock_calls_retained)
def test_executes_with_different_delimiter_and_destination_object(
self, mock_hook):
mock_hook.return_value.list.return_value = [
'test_object/file1.txt', 'test_object/file2.txt'
]
operator = GCSToGCSOperator(task_id=TASK_ID,
source_bucket=TEST_BUCKET,
source_objects=SOURCE_OBJECTS_LIST,
destination_bucket=DESTINATION_BUCKET,
destination_object=DESTINATION_OBJECT,
delimiter='.txt')
operator.execute(None)
mock_calls = [
mock.call(TEST_BUCKET, 'test_object/file1.txt', DESTINATION_BUCKET,
DESTINATION_OBJECT),
mock.call(TEST_BUCKET, 'test_object/file2.txt', DESTINATION_BUCKET,
DESTINATION_OBJECT),
]
mock_hook.return_value.rewrite.assert_has_calls(mock_calls)
def test_wc_with_last_modified_time_with_all_true_cond(self, mock_hook):
mock_hook.return_value.list.return_value = SOURCE_FILES_LIST
mock_hook.return_value.is_updated_after.side_effect = [
True, True, True
]
operator = GCSToGCSOperator(
task_id=TASK_ID,
source_bucket=TEST_BUCKET,
source_object=SOURCE_OBJECT_WILDCARD_FILENAME,
destination_bucket=DESTINATION_BUCKET,
last_modified_time=MOD_TIME_1)
operator.execute(None)
mock_calls_none = [
mock.call(TEST_BUCKET, 'test_object/file1.txt', DESTINATION_BUCKET,
'test_object/file1.txt'),
mock.call(TEST_BUCKET, 'test_object/file2.txt', DESTINATION_BUCKET,
'test_object/file2.txt'),
]
mock_hook.return_value.rewrite.assert_has_calls(mock_calls_none)
role=GCS_ACL_OBJECT_ROLE,
task_id="gcs_object_create_acl_entry_task",
)
# [END howto_operator_gcs_object_create_acl_entry_task]
download_file = GCSToLocalOperator(
task_id="download_file",
object_name=BUCKET_FILE_LOCATION,
bucket=BUCKET_1,
filename=PATH_TO_SAVED_FILE,
)
copy_file = GCSToGCSOperator(
task_id="copy_file",
source_bucket=BUCKET_1,
source_object=BUCKET_FILE_LOCATION,
destination_bucket=BUCKET_2,
destination_object=BUCKET_FILE_LOCATION,
)
delete_files = GCSDeleteObjectsOperator(
task_id="delete_files", bucket_name=BUCKET_1, objects=[BUCKET_FILE_LOCATION]
)
# [START howto_operator_gcs_delete_bucket]
delete_bucket_1 = GCSDeleteBucketOperator(task_id="delete_bucket", bucket_name=BUCKET_1)
delete_bucket_2 = GCSDeleteBucketOperator(task_id="delete_bucket", bucket_name=BUCKET_2)
# [END howto_operator_gcs_delete_bucket]
[create_bucket1, create_bucket2] >> list_buckets >> list_buckets_result
[create_bucket1, create_bucket2] >> upload_file
# [END howto_synch_to_subdir]
# [START howto_sync_from_subdir]
sync_from_subdirectory = GCSSynchronizeBucketsOperator(
task_id="sync_from_subdirectory",
source_bucket=BUCKET_1_SRC,
source_object="subdir/",
destination_bucket=BUCKET_1_DST
)
# [END howto_sync_from_subdir]
# [START howto_operator_gcs_to_gcs_single_file]
copy_single_file = GCSToGCSOperator(
task_id="copy_single_gcs_file",
source_bucket=BUCKET_1_SRC,
source_object=OBJECT_1,
destination_bucket=BUCKET_1_DST, # If not supplied the source_bucket value will be used
destination_object="backup_" + OBJECT_1 # If not supplied the source_object value will be used
)
# [END howto_operator_gcs_to_gcs_single_file]
# [START howto_operator_gcs_to_gcs_wildcard]
copy_files_with_wildcard = GCSToGCSOperator(
task_id="copy_files_with_wildcard",
source_bucket=BUCKET_1_SRC,
source_object="data/*.txt",
destination_bucket=BUCKET_1_DST,
destination_object="backup/"
)
# [END howto_operator_gcs_to_gcs_wildcard]
