def __init__(self, do_parallel):
self.bucket_storage_uri_class = BucketStorageUri
support_map = {'gs': ['JSON'], 's3': ['XML']}
default_map = {'gs': 'JSON', 's3': 'XML'}
self.gsutil_api_map = cs_api_map.GsutilApiMapFactory.GetApiMap(
cs_api_map.GsutilApiClassMapFactory, support_map, default_map)
self.logger = CreateGsutilLogger('FakeCommand')
self.parallel_operations = do_parallel
self.failure_count = 0
self.gsutil_api = MockCloudApi()
self.multiprocessing_is_available = (
CheckMultiprocessingAvailableAndInit().is_available)
self.debug = 0
def test_FilterExistingComponentsNonVersioned(self):
"""Tests upload with a variety of component states."""
mock_api = MockCloudApi()
bucket_name = self.MakeTempName('bucket')
tracker_file = self.CreateTempFile(file_name='foo', contents='asdf')
tracker_file_lock = CreateLock()
# dst_obj_metadata used for passing content-type.
empty_object = apitools_messages.Object()
# Already uploaded, contents still match, component still used.
fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1',
contents='1')
fpath_uploaded_correctly_url = StorageUrlFromString(
str(fpath_uploaded_correctly))
object_uploaded_correctly_url = StorageUrlFromString(
'%s://%s/%s' %
(self.default_provider, bucket_name, fpath_uploaded_correctly))
with open(fpath_uploaded_correctly) as f_in:
fpath_uploaded_correctly_md5 = CalculateB64EncodedMd5FromContents(
f_in)
mock_api.MockCreateObjectWithMetadata(apitools_messages.Object(
bucket=bucket_name,
name=fpath_uploaded_correctly,
md5Hash=fpath_uploaded_correctly_md5),
contents='1')
args_uploaded_correctly = PerformParallelUploadFileToObjectArgs(
fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url,
object_uploaded_correctly_url, '', empty_object, tracker_file,
tracker_file_lock, None)
# Not yet uploaded, but needed.
fpath_not_uploaded = self.CreateTempFile(file_name='foo2',
contents='2')
fpath_not_uploaded_url = StorageUrlFromString(str(fpath_not_uploaded))
object_not_uploaded_url = StorageUrlFromString(
'%s://%s/%s' %
(self.default_provider, bucket_name, fpath_not_uploaded))
args_not_uploaded = PerformParallelUploadFileToObjectArgs(
fpath_not_uploaded, 0, 1, fpath_not_uploaded_url,
object_not_uploaded_url, '', empty_object, tracker_file,
tracker_file_lock, None)
# Already uploaded, but contents no longer match. Even though the contents
# differ, we don't delete this since the bucket is not versioned and it
# will be overwritten anyway.
fpath_wrong_contents = self.CreateTempFile(file_name='foo4',
contents='4')
fpath_wrong_contents_url = StorageUrlFromString(
str(fpath_wrong_contents))
object_wrong_contents_url = StorageUrlFromString(
'%s://%s/%s' %
(self.default_provider, bucket_name, fpath_wrong_contents))
with open(self.CreateTempFile(contents='_')) as f_in:
fpath_wrong_contents_md5 = CalculateB64EncodedMd5FromContents(f_in)
mock_api.MockCreateObjectWithMetadata(apitools_messages.Object(
bucket=bucket_name,
name=fpath_wrong_contents,
md5Hash=fpath_wrong_contents_md5),
contents='1')
args_wrong_contents = PerformParallelUploadFileToObjectArgs(
fpath_wrong_contents, 0, 1, fpath_wrong_contents_url,
object_wrong_contents_url, '', empty_object, tracker_file,
tracker_file_lock, None)
# Exists in tracker file, but component object no longer exists.
fpath_remote_deleted = self.CreateTempFile(file_name='foo5',
contents='5')
fpath_remote_deleted_url = StorageUrlFromString(
str(fpath_remote_deleted))
args_remote_deleted = PerformParallelUploadFileToObjectArgs(
fpath_remote_deleted, 0, 1, fpath_remote_deleted_url, '', '',
empty_object, tracker_file, tracker_file_lock, None)
# Exists in tracker file and already uploaded, but no longer needed.
fpath_no_longer_used = self.CreateTempFile(file_name='foo6',
contents='6')
with open(fpath_no_longer_used) as f_in:
file_md5 = CalculateB64EncodedMd5FromContents(f_in)
mock_api.MockCreateObjectWithMetadata(apitools_messages.Object(
bucket=bucket_name, name='foo6', md5Hash=file_md5),
contents='6')
dst_args = {
fpath_uploaded_correctly: args_uploaded_correctly,
fpath_not_uploaded: args_not_uploaded,
fpath_wrong_contents: args_wrong_contents,
fpath_remote_deleted: args_remote_deleted
}
existing_components = [
ObjectFromTracker(fpath_uploaded_correctly, ''),
ObjectFromTracker(fpath_wrong_contents, ''),
ObjectFromTracker(fpath_remote_deleted, ''),
ObjectFromTracker(fpath_no_longer_used, '')
]
bucket_url = StorageUrlFromString('%s://%s' %
(self.default_provider, bucket_name))
(components_to_upload, uploaded_components,
existing_objects_to_delete) = (FilterExistingComponents(
dst_args, existing_components, bucket_url, mock_api))
for arg in [
args_not_uploaded, args_wrong_contents, args_remote_deleted
]:
self.assertTrue(arg in components_to_upload)
self.assertEqual(1, len(uploaded_components))
self.assertEqual(args_uploaded_correctly.dst_url.url_string,
uploaded_components[0].url_string)
self.assertEqual(1, len(existing_objects_to_delete))
no_longer_used_url = StorageUrlFromString(
'%s://%s/%s' %
(self.default_provider, bucket_name, fpath_no_longer_used))
self.assertEqual(no_longer_used_url.url_string,
existing_objects_to_delete[0].url_string)
def test_FilterExistingComponentsVersioned(self):
"""Tests upload with versionined parallel components."""
mock_api = MockCloudApi()
bucket_name = self.MakeTempName('bucket')
mock_api.MockCreateVersionedBucket(bucket_name)
# dst_obj_metadata used for passing content-type.
empty_object = apitools_messages.Object()
tracker_file = self.CreateTempFile(file_name='foo', contents='asdf')
tracker_file_lock = CreateLock()
# Already uploaded, contents still match, component still used.
fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1',
contents='1')
fpath_uploaded_correctly_url = StorageUrlFromString(
str(fpath_uploaded_correctly))
with open(fpath_uploaded_correctly) as f_in:
fpath_uploaded_correctly_md5 = CalculateB64EncodedMd5FromContents(
f_in)
object_uploaded_correctly = mock_api.MockCreateObjectWithMetadata(
apitools_messages.Object(bucket=bucket_name,
name=fpath_uploaded_correctly,
md5Hash=fpath_uploaded_correctly_md5),
contents='1')
object_uploaded_correctly_url = StorageUrlFromString(
'%s://%s/%s#%s' %
(self.default_provider, bucket_name, fpath_uploaded_correctly,
object_uploaded_correctly.generation))
args_uploaded_correctly = PerformParallelUploadFileToObjectArgs(
fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url,
object_uploaded_correctly_url,
object_uploaded_correctly.generation, empty_object, tracker_file,
tracker_file_lock, None)
# Duplicate object name in tracker file, but uploaded correctly.
fpath_duplicate = fpath_uploaded_correctly
fpath_duplicate_url = StorageUrlFromString(str(fpath_duplicate))
duplicate_uploaded_correctly = mock_api.MockCreateObjectWithMetadata(
apitools_messages.Object(bucket=bucket_name,
name=fpath_duplicate,
md5Hash=fpath_uploaded_correctly_md5),
contents='1')
duplicate_uploaded_correctly_url = StorageUrlFromString(
'%s://%s/%s#%s' %
(self.default_provider, bucket_name, fpath_uploaded_correctly,
duplicate_uploaded_correctly.generation))
args_duplicate = PerformParallelUploadFileToObjectArgs(
fpath_duplicate, 0, 1, fpath_duplicate_url,
duplicate_uploaded_correctly_url,
duplicate_uploaded_correctly.generation, empty_object,
tracker_file, tracker_file_lock, None)
# Already uploaded, but contents no longer match.
fpath_wrong_contents = self.CreateTempFile(file_name='foo4',
contents='4')
fpath_wrong_contents_url = StorageUrlFromString(
str(fpath_wrong_contents))
with open(self.CreateTempFile(contents='_')) as f_in:
fpath_wrong_contents_md5 = CalculateB64EncodedMd5FromContents(f_in)
object_wrong_contents = mock_api.MockCreateObjectWithMetadata(
apitools_messages.Object(bucket=bucket_name,
name=fpath_wrong_contents,
md5Hash=fpath_wrong_contents_md5),
contents='_')
wrong_contents_url = StorageUrlFromString(
'%s://%s/%s#%s' %
(self.default_provider, bucket_name, fpath_wrong_contents,
object_wrong_contents.generation))
args_wrong_contents = PerformParallelUploadFileToObjectArgs(
fpath_wrong_contents, 0, 1, fpath_wrong_contents_url,
wrong_contents_url, '', empty_object, tracker_file,
tracker_file_lock, None)
dst_args = {
fpath_uploaded_correctly: args_uploaded_correctly,
fpath_wrong_contents: args_wrong_contents
}
existing_components = [
ObjectFromTracker(fpath_uploaded_correctly,
object_uploaded_correctly_url.generation),
ObjectFromTracker(fpath_duplicate,
duplicate_uploaded_correctly_url.generation),
ObjectFromTracker(fpath_wrong_contents,
wrong_contents_url.generation)
]
bucket_url = StorageUrlFromString('%s://%s' %
(self.default_provider, bucket_name))
(components_to_upload, uploaded_components,
existing_objects_to_delete) = (FilterExistingComponents(
dst_args, existing_components, bucket_url, mock_api))
self.assertEqual([args_wrong_contents], components_to_upload)
self.assertEqual(args_uploaded_correctly.dst_url.url_string,
uploaded_components[0].url_string)
expected_to_delete = [(args_wrong_contents.dst_url.object_name,
args_wrong_contents.dst_url.generation),
(args_duplicate.dst_url.object_name,
args_duplicate.dst_url.generation)]
for uri in existing_objects_to_delete:
self.assertTrue((uri.object_name,
uri.generation) in expected_to_delete)
self.assertEqual(len(expected_to_delete),
len(existing_objects_to_delete))
