def _connect_storage(self): logging.debug('Loading storage_provider: {}'.format(self._config.storage_provider)) if self._config.storage_provider == Provider.GOOGLE_STORAGE: google_storage = GoogleStorage(self._config) google_storage.check_dependencies() return google_storage elif self._config.storage_provider == Provider.AZURE_BLOBS: azure_storage = AzureStorage(self._config) azure_storage.check_dependencies() return azure_storage elif self._config.storage_provider == Provider.S3_RGW: return S3RGWStorage(self._config) elif self._config.storage_provider.lower() == "s3_compatible": s3_storage = S3BaseStorage(self._config) s3_storage.check_dependencies() return s3_storage elif self._config.storage_provider.startswith(Provider.S3): s3_storage = S3Storage(self._config) s3_storage.check_dependencies() return s3_storage elif self._config.storage_provider == Provider.LOCAL: return LocalStorage(self._config) elif self._config.storage_provider.lower() == "ibm_storage": s3_storage = S3BaseStorage(self._config) s3_storage.check_dependencies() return s3_storage raise NotImplementedError("Unsupported storage provider")
def file_matches_cache(src, cached_item, threshold=None, enable_md5_checks=False): # for S3RGW, we never set threshold so the S3's multipart never happens return S3Storage.file_matches_cache(src, cached_item, None, enable_md5_checks)
def compare_with_manifest(actual_size, size_in_manifest, actual_hash=None, hash_in_manifest=None, threshold=None): return S3Storage.compare_with_manifest(actual_size, size_in_manifest, actual_hash, hash_in_manifest, None)
def _connect_storage(self): if self._config.storage_provider == Provider.GOOGLE_STORAGE: return GoogleStorage(self._config) elif self._config.storage_provider.startswith(Provider.S3): return S3Storage(self._config) elif self._config.storage_provider == Provider.LOCAL: return LocalStorage(self._config) raise NotImplementedError("Unsupported storage provider")
def test_multi_part_s3_file(self): # Multi part hashes have a special structure, with the number of chunks at the end cached_item = { 'MD5': 'e4344d1ea2b32372db7f7e1c81d154b9-1', 'size': 113651 } src = pathlib.Path( __file__).parent / "resources/s3/md-10-big-CompressionInfo.db" assert S3Storage.file_matches_cache(src, cached_item, 100, True)
def test_single_part_s3_file(self): cached_item = { 'MD5': '620c203520494bb92811fddc6d88cd65', 'size': 113651 } src = pathlib.Path( __file__).parent / "resources/s3/md-10-big-CompressionInfo.db" assert S3Storage.file_matches_cache(src, cached_item, 100 * 1024 * 1024, True)
def test_multi_part_s3_file_fail(self): # File size is below the multi part threshold, a single part hash will be computed cached_item = { 'MD5': 'e4344d1ea2b32372db7f7e1c81d154b9-1', 'size': 113651 } src = pathlib.Path( __file__).parent / "resources/s3/md-10-big-CompressionInfo.db" assert not S3Storage.file_matches_cache(src, cached_item, 100 * 1024 * 1024, True)
def _connect_storage(self): if self._config.storage_provider == Provider.GOOGLE_STORAGE: google_storage = GoogleStorage(self._config) google_storage.check_dependencies() return google_storage elif self._config.storage_provider == Provider.S3_RGW: return S3RGWStorage(self._config) elif self._config.storage_provider.startswith(Provider.S3): s3_storage = S3Storage(self._config) s3_storage.check_dependencies() return s3_storage elif self._config.storage_provider == Provider.LOCAL: return LocalStorage(self._config) raise NotImplementedError("Unsupported storage provider")
def _connect_storage(self): if self._config.storage_provider == Provider.GOOGLE_STORAGE: google_storage = GoogleStorage(self._config) google_storage.check_dependencies() return google_storage elif self._config.storage_provider == Provider.AZURE_BLOBS: azure_storage = AzureStorage(self._config) azure_storage.check_dependencies() return azure_storage elif self._config.storage_provider == Provider.S3_RGW: return S3RGWStorage(self._config) elif self._config.storage_provider.startswith(Provider.S3): s3_storage = S3Storage(self._config) s3_storage.check_dependencies() return s3_storage elif self._config.storage_provider == Provider.LOCAL: return LocalStorage(self._config) elif self._config.storage_provider.lower() == "ibm_storage": return IBMCloudStorage(self._config) elif self._config.storage_provider.lower() == "minio": return MinIOStorage(self._config) raise NotImplementedError("Unsupported storage provider")
def blob_matches_manifest(blob, object_in_manifest): return S3Storage.blob_matches_manifest(blob, object_in_manifest)
def blob_matches_manifest(blob, object_in_manifest, enable_md5_checks=False): return S3Storage.blob_matches_manifest(blob, object_in_manifest, enable_md5_checks)