def delete_package(self, pkg_name: str, top_hash: str = None): package_path = self.pointers_dir(pkg_name) paths = list(list_url(package_path)) if not paths: raise QuiltException( "No such package exists in the given directory.") if top_hash is not None: top_hash = self.resolve_top_hash(pkg_name, top_hash) deleted = [] remaining = [] for path, _ in paths: pkg_hash = get_bytes(self.pointer_pk(pkg_name, path)).decode() (deleted if pkg_hash == top_hash else remaining).append(path) if not deleted: raise QuiltException( "No such package version exists in the given directory.") for path in deleted: delete_url(self.pointer_pk(pkg_name, path)) if 'latest' in deleted and remaining: # Create a new "latest". Technically, we need to compare numerically, # but string comparisons will be fine till year 2286. new_latest = max(remaining) copy_file(self.pointer_pk(pkg_name, new_latest), self.pointer_latest_pk(pkg_name)) else: for path, _ in paths: delete_url(self.pointer_pk(pkg_name, path))
def test_simple_upload(self): path = DATA_DIR / 'small_file.csv' # Unversioned bucket self.s3_stubber.add_response(method='put_object', service_response={'VersionId': 'null'}, expected_params={ 'Body': ANY, 'Bucket': 'example', 'Key': 'foo.csv', }) data_transfer.copy_file(path.as_uri(), 's3://example/foo.csv')
def test_simple_upload(self): path = DATA_DIR / 'small_file.csv' # Unversioned bucket self.s3_stubber.add_response(method='put_object', service_response={}, expected_params={ 'Body': ANY, 'Bucket': 'example', 'Key': 'foo.csv', }) data_transfer.copy_file(PhysicalKey.from_path(path), PhysicalKey.from_url('s3://example/foo.csv'))
def test_body_is_seekable(self): """ No errors if request body.read() or body.seek() are called right before sending request. """ def handler(request, **kwargs): request.body.read(2) request.body.seek(0) raise Success path = DATA_DIR / 'small_file.csv' self.s3_client.meta.events.register_first('before-send.*', handler) with pytest.raises(Success): data_transfer.copy_file(PhysicalKey.from_path(path), PhysicalKey.from_url('s3://example/foo.csv'))
def delete_package_version(self, pkg_name: str, top_hash: str): deleted = [] remaining = [] for path, pkg_hash in self.list_package_versions(pkg_name): (deleted if pkg_hash == top_hash else remaining).append(path) if not deleted: raise QuiltException("No such package version exists in the given directory.") for path in deleted: delete_url(self.pointer_pk(pkg_name, path)) if 'latest' in deleted and remaining: # Create a new "latest". Technically, we need to compare numerically, # but string comparisons will be fine till year 2286. new_latest = max(remaining) copy_file(self.pointer_pk(pkg_name, new_latest), self.pointer_latest_pk(pkg_name))
def test_progress_updateds(self, mocked_update): """ Progress callback is called when calling body.read() or body.seek(). """ def handler(request, **kwargs): request.body.read(2) mocked_update.assert_called_once_with(2) mocked_update.reset_mock() request.body.seek(0) mocked_update.assert_called_once_with(-2) raise Success path = DATA_DIR / 'small_file.csv' self.s3_client.meta.events.register_first('before-send.*', handler) with pytest.raises(Success): data_transfer.copy_file(PhysicalKey.from_path(path), PhysicalKey.from_url('s3://example/foo.csv'))
def test_download_latest_in_versioned_bucket(self): bucket = 'example' key = 'foo.csv' src = PhysicalKey(bucket, key, None) latest_version = '1' latest_size = 3 # Check what is the latest version and size. expected_params = { 'Bucket': bucket, 'Key': key, } self.s3_stubber.add_response( 'head_object', service_response={ 'VersionId': latest_version, 'ContentLength': latest_size, }, expected_params=expected_params, ) for i in range(latest_size): self.s3_stubber.add_response( 'get_object', service_response={ 'Body': io.BytesIO(b'0'), }, expected_params={ **expected_params, 'Range': f'bytes={i}-{i}', # Version must be specified, otherwise we will end with # a truncated file if the file was modified after getting the latest # version/size. 'VersionId': latest_version, }, ) data_transfer.copy_file( src, PhysicalKey.from_path('some-file'), )