def delete_package(self, pkg_name: str, top_hash: str = None):
package_path = self.pointers_dir(pkg_name)
paths = list(list_url(package_path))
if not paths:
raise QuiltException(
"No such package exists in the given directory.")
if top_hash is not None:
top_hash = self.resolve_top_hash(pkg_name, top_hash)
deleted = []
remaining = []
for path, _ in paths:
pkg_hash = get_bytes(self.pointer_pk(pkg_name, path)).decode()
(deleted if pkg_hash == top_hash else remaining).append(path)
if not deleted:
raise QuiltException(
"No such package version exists in the given directory.")
for path in deleted:
delete_url(self.pointer_pk(pkg_name, path))
if 'latest' in deleted and remaining:
# Create a new "latest". Technically, we need to compare numerically,
# but string comparisons will be fine till year 2286.
new_latest = max(remaining)
copy_file(self.pointer_pk(pkg_name, new_latest),
self.pointer_latest_pk(pkg_name))
else:
for path, _ in paths:
delete_url(self.pointer_pk(pkg_name, path))
def test_simple_upload(self):
path = DATA_DIR / 'small_file.csv'
# Unversioned bucket
self.s3_stubber.add_response(method='put_object',
service_response={'VersionId': 'null'},
expected_params={
'Body': ANY,
'Bucket': 'example',
'Key': 'foo.csv',
})
data_transfer.copy_file(path.as_uri(), 's3://example/foo.csv')
def test_simple_upload(self):
path = DATA_DIR / 'small_file.csv'
# Unversioned bucket
self.s3_stubber.add_response(method='put_object',
service_response={},
expected_params={
'Body': ANY,
'Bucket': 'example',
'Key': 'foo.csv',
})
data_transfer.copy_file(PhysicalKey.from_path(path),
PhysicalKey.from_url('s3://example/foo.csv'))
def test_body_is_seekable(self):
"""
No errors if request body.read() or body.seek() are called right before sending request.
"""
def handler(request, **kwargs):
request.body.read(2)
request.body.seek(0)
raise Success
path = DATA_DIR / 'small_file.csv'
self.s3_client.meta.events.register_first('before-send.*', handler)
with pytest.raises(Success):
data_transfer.copy_file(PhysicalKey.from_path(path), PhysicalKey.from_url('s3://example/foo.csv'))
def delete_package_version(self, pkg_name: str, top_hash: str):
deleted = []
remaining = []
for path, pkg_hash in self.list_package_versions(pkg_name):
(deleted if pkg_hash == top_hash else remaining).append(path)
if not deleted:
raise QuiltException("No such package version exists in the given directory.")
for path in deleted:
delete_url(self.pointer_pk(pkg_name, path))
if 'latest' in deleted and remaining:
# Create a new "latest". Technically, we need to compare numerically,
# but string comparisons will be fine till year 2286.
new_latest = max(remaining)
copy_file(self.pointer_pk(pkg_name, new_latest), self.pointer_latest_pk(pkg_name))
def test_progress_updateds(self, mocked_update):
"""
Progress callback is called when calling body.read() or body.seek().
"""
def handler(request, **kwargs):
request.body.read(2)
mocked_update.assert_called_once_with(2)
mocked_update.reset_mock()
request.body.seek(0)
mocked_update.assert_called_once_with(-2)
raise Success
path = DATA_DIR / 'small_file.csv'
self.s3_client.meta.events.register_first('before-send.*', handler)
with pytest.raises(Success):
data_transfer.copy_file(PhysicalKey.from_path(path), PhysicalKey.from_url('s3://example/foo.csv'))
def test_download_latest_in_versioned_bucket(self):
bucket = 'example'
key = 'foo.csv'
src = PhysicalKey(bucket, key, None)
latest_version = '1'
latest_size = 3
# Check what is the latest version and size.
expected_params = {
'Bucket': bucket,
'Key': key,
}
self.s3_stubber.add_response(
'head_object',
service_response={
'VersionId': latest_version,
'ContentLength': latest_size,
},
expected_params=expected_params,
)
for i in range(latest_size):
self.s3_stubber.add_response(
'get_object',
service_response={
'Body': io.BytesIO(b'0'),
},
expected_params={
**expected_params,
'Range': f'bytes={i}-{i}',
# Version must be specified, otherwise we will end with
# a truncated file if the file was modified after getting the latest
# version/size.
'VersionId': latest_version,
},
)
data_transfer.copy_file(
src,
PhysicalKey.from_path('some-file'),
)
