def _connect_storage(self):
logging.debug('Loading storage_provider: {}'.format(self._config.storage_provider))
if self._config.storage_provider == Provider.GOOGLE_STORAGE:
google_storage = GoogleStorage(self._config)
google_storage.check_dependencies()
return google_storage
elif self._config.storage_provider == Provider.AZURE_BLOBS:
azure_storage = AzureStorage(self._config)
azure_storage.check_dependencies()
return azure_storage
elif self._config.storage_provider == Provider.S3_RGW:
return S3RGWStorage(self._config)
elif self._config.storage_provider.lower() == "s3_compatible":
s3_storage = S3BaseStorage(self._config)
s3_storage.check_dependencies()
return s3_storage
elif self._config.storage_provider.startswith(Provider.S3):
s3_storage = S3Storage(self._config)
s3_storage.check_dependencies()
return s3_storage
elif self._config.storage_provider == Provider.LOCAL:
return LocalStorage(self._config)
elif self._config.storage_provider.lower() == "ibm_storage":
s3_storage = S3BaseStorage(self._config)
s3_storage.check_dependencies()
return s3_storage
raise NotImplementedError("Unsupported storage provider")
def file_matches_cache(src,
cached_item,
threshold=None,
enable_md5_checks=False):
# for S3RGW, we never set threshold so the S3's multipart never happens
return S3Storage.file_matches_cache(src, cached_item, None,
enable_md5_checks)
def compare_with_manifest(actual_size,
size_in_manifest,
actual_hash=None,
hash_in_manifest=None,
threshold=None):
return S3Storage.compare_with_manifest(actual_size, size_in_manifest,
actual_hash, hash_in_manifest,
None)
def _connect_storage(self):
if self._config.storage_provider == Provider.GOOGLE_STORAGE:
return GoogleStorage(self._config)
elif self._config.storage_provider.startswith(Provider.S3):
return S3Storage(self._config)
elif self._config.storage_provider == Provider.LOCAL:
return LocalStorage(self._config)
raise NotImplementedError("Unsupported storage provider")
def test_multi_part_s3_file(self):
# Multi part hashes have a special structure, with the number of chunks at the end
cached_item = {
'MD5': 'e4344d1ea2b32372db7f7e1c81d154b9-1',
'size': 113651
}
src = pathlib.Path(
__file__).parent / "resources/s3/md-10-big-CompressionInfo.db"
assert S3Storage.file_matches_cache(src, cached_item, 100, True)
def test_single_part_s3_file(self):
cached_item = {
'MD5': '620c203520494bb92811fddc6d88cd65',
'size': 113651
}
src = pathlib.Path(
__file__).parent / "resources/s3/md-10-big-CompressionInfo.db"
assert S3Storage.file_matches_cache(src, cached_item,
100 * 1024 * 1024, True)
def test_multi_part_s3_file_fail(self):
# File size is below the multi part threshold, a single part hash will be computed
cached_item = {
'MD5': 'e4344d1ea2b32372db7f7e1c81d154b9-1',
'size': 113651
}
src = pathlib.Path(
__file__).parent / "resources/s3/md-10-big-CompressionInfo.db"
assert not S3Storage.file_matches_cache(src, cached_item,
100 * 1024 * 1024, True)
def _connect_storage(self):
if self._config.storage_provider == Provider.GOOGLE_STORAGE:
google_storage = GoogleStorage(self._config)
google_storage.check_dependencies()
return google_storage
elif self._config.storage_provider == Provider.S3_RGW:
return S3RGWStorage(self._config)
elif self._config.storage_provider.startswith(Provider.S3):
s3_storage = S3Storage(self._config)
s3_storage.check_dependencies()
return s3_storage
elif self._config.storage_provider == Provider.LOCAL:
return LocalStorage(self._config)
raise NotImplementedError("Unsupported storage provider")
def _connect_storage(self):
if self._config.storage_provider == Provider.GOOGLE_STORAGE:
google_storage = GoogleStorage(self._config)
google_storage.check_dependencies()
return google_storage
elif self._config.storage_provider == Provider.AZURE_BLOBS:
azure_storage = AzureStorage(self._config)
azure_storage.check_dependencies()
return azure_storage
elif self._config.storage_provider == Provider.S3_RGW:
return S3RGWStorage(self._config)
elif self._config.storage_provider.startswith(Provider.S3):
s3_storage = S3Storage(self._config)
s3_storage.check_dependencies()
return s3_storage
elif self._config.storage_provider == Provider.LOCAL:
return LocalStorage(self._config)
elif self._config.storage_provider.lower() == "ibm_storage":
return IBMCloudStorage(self._config)
elif self._config.storage_provider.lower() == "minio":
return MinIOStorage(self._config)
raise NotImplementedError("Unsupported storage provider")
def blob_matches_manifest(blob, object_in_manifest):
return S3Storage.blob_matches_manifest(blob, object_in_manifest)
def blob_matches_manifest(blob,
object_in_manifest,
enable_md5_checks=False):
return S3Storage.blob_matches_manifest(blob, object_in_manifest,
enable_md5_checks)