def test_local_package_delete_overlapping(self):
"""
Verify local package delete works when multiple packages reference the
same tophash.
"""
top_hash = Package().build("Quilt/Test1").top_hash
top_hash = Package().build("Quilt/Test2").top_hash
assert 'Quilt/Test1' in quilt3.list_packages()
assert top_hash in [
p.name for p in (LOCAL_REGISTRY / '.quilt/packages').iterdir()
]
quilt3.delete_package('Quilt/Test1')
assert 'Quilt/Test1' not in quilt3.list_packages()
assert top_hash in [
p.name for p in (LOCAL_REGISTRY / '.quilt/packages').iterdir()
]
quilt3.delete_package('Quilt/Test2')
assert 'Quilt/Test2' not in quilt3.list_packages()
assert top_hash not in [
p.name for p in (LOCAL_REGISTRY / '.quilt/packages').iterdir()
]
def test_local_package_delete(self):
"""Verify local package delete works."""
top_hash = Package().build("Quilt/Test").top_hash
assert 'Quilt/Test' in quilt3.list_packages()
quilt3.delete_package('Quilt/Test')
assert 'Quilt/Test' not in quilt3.list_packages()
def exec_module(cls, module):
"""
Module executor.
"""
name_parts = module.__name__.split('.')
registry = get_from_config('default_local_registry')
if module.__name__ == 'quilt3.data':
# __path__ must be set even if the package is virtual. Since __path__ will be
# scanned by all other finders preceding this one in sys.meta_path order, make sure
# it points to someplace lacking importable objects
module.__path__ = MODULE_PATH
return module
elif len(name_parts) == 3: # e.g. module.__name__ == quilt3.data.foo
namespace = name_parts[2]
# we do not know the name the user will ask for, so populate all valid names
for pkg in list_packages():
pkg_user, pkg_name = pkg.split('/')
if pkg_user == namespace:
module.__dict__[pkg_name] = Package.browse(
pkg, registry=registry)
module.__path__ = MODULE_PATH
return module
else:
assert False
def test_list_remote_packages(self):
"""Verify that listing remote packages works as expected."""
self.s3_stubber.add_response(
method='list_objects_v2',
service_response={
'Contents': [{
'Key': '.quilt/named_packages/foo/bar/1549931300',
'Size': 64,
}, {
'Key': '.quilt/named_packages/foo/bar/1549931634',
'Size': 64,
}, {
'Key': '.quilt/named_packages/foo/bar/latest',
'Size': 64,
}]
},
expected_params={
'Bucket': 'my_test_bucket',
'Prefix': '.quilt/named_packages/',
})
pkgs = list(quilt3.list_packages('s3://my_test_bucket/'))
assert len(pkgs) == 1
assert list(pkgs) == ['foo/bar']
def get_packages(self):
""" Get the names of the packages in the S3 bucket
Returns:
:obj:`list` of :obj:`str`: list of package names
"""
packages = quilt3.list_packages(self.get_aws_bucket_uri())
return list(packages)
def test_list_local_packages(self):
"""Verify that list returns packages in the appdirs directory."""
# Build a new package into the local registry.
Package().build("Quilt/Foo")
Package().build("Quilt/Bar")
Package().build("Quilt/Test")
# Verify packages are returned.
pkgs = list(quilt3.list_packages())
assert len(pkgs) == 3
assert "Quilt/Foo" in pkgs
assert "Quilt/Bar" in pkgs
# Verify specifying a local path explicitly works as expected.
assert list(pkgs) == list(
quilt3.list_packages(LOCAL_REGISTRY.as_posix()))
def test_list_local_packages(self):
"""Verify that list returns packages in the appdirs directory."""
# Build a new package into the local registry.
with patch('time.time', return_value=1234567890):
Package().build("Quilt/Foo")
Package().build("Quilt/Bar")
Package().build("Quilt/Test")
# Verify packages are returned.
pkgs = list(quilt3.list_packages())
assert len(pkgs) == 3
assert "Quilt/Foo" in pkgs
assert "Quilt/Bar" in pkgs
versions = set(quilt3.list_package_versions('Quilt/Foo'))
assert versions == {
('latest', '2a5a67156ca9238c14d12042db51c5b52260fdd5511b61ea89b58929d6e1769b'),
('1234567890', '2a5a67156ca9238c14d12042db51c5b52260fdd5511b61ea89b58929d6e1769b'),
}
# Verify specifying a local path explicitly works as expected.
assert list(pkgs) == list(quilt3.list_packages(LOCAL_REGISTRY.as_posix()))
def test_list_remote_packages(self):
"""Verify that listing remote packages works as expected."""
def pseudo_list_objects(bucket, prefix, recursive):
if prefix == '.quilt/named_packages/':
return ([{'Prefix': '.quilt/named_packages/foo/'}], [])
elif prefix == '.quilt/named_packages/foo/':
return ([{'Prefix': '.quilt/named_packages/foo/bar/'}], [])
elif prefix == '.quilt/named_packages/foo/bar/':
import datetime
return ([], [{
'Key':
'.quilt/named_packages/foo/bar/1549931300',
'LastModified':
datetime.datetime.now() - datetime.timedelta(seconds=30)
}, {
'Key': '.quilt/named_packages/foo/bar/1549931634',
'LastModified': datetime.datetime.now()
}, {
'Key': '.quilt/named_packages/foo/bar/latest',
'LastModified': datetime.datetime.now()
}])
else:
raise ValueError
def pseudo_get_bytes(src):
if src.endswith('foo/bar/latest') or src.endswith(
'foo/bar/1549931634'):
return (b'100', None)
elif src.endswith('foo/bar/1549931300'):
return (b'90', None)
else:
raise ValueError
with patch('quilt3.api.list_objects', side_effect=pseudo_list_objects), \
patch('quilt3.api.get_bytes', side_effect=pseudo_get_bytes), \
patch('quilt3.Package.browse', return_value=Package()):
pkgs = quilt3.list_packages('s3://my_test_bucket/')
assert len(pkgs) == 1
assert list(pkgs) == ['foo/bar']
expected = (
'PACKAGE \tTOP HASH \tCREATED \tSIZE \t\n'
'foo/bar:latest \t100 \tnow \t0 Bytes\t\n'
'foo/bar \t90 \t30 seconds ago \t0 Bytes\t\n'
)
assert str(pkgs) == expected
def make_pkg_maps(
all_pkgs=[
"rorydm/2d_segmented_fields",
"rorydm/2d_segmented_fields2",
"rorydm/2d_segmented_fields_actn2",
"rorydm/2d_segmented_fields_actn2_2",
"rorydm/2d_autocontrasted_fields_and_single_cells",
"rorydm/2d_autocontrasted_fields_and_single_cells_part2",
"rorydm/2d_autocontrasted_fields_and_single_cells_actn2",
"rorydm/2d_autocontrasted_fields_and_single_cells_actn2_2",
"tanyasg/2d_autocontrasted_single_cell_features",
"tanyasg/2d_autocontrasted_single_cell_features2",
"tanyasg/2d_autocontrasted_single_cell_features_actn2",
"tanyasg/2d_autocontrasted_single_cell_features_actn2_2",
"matheus/assay_dev_fish_analysis",
"tanyasg/struct_scores_bonus",
"tanyasg/struct_scores_actn2",
"tanyasg/struct_scores_actn2_2",
"tanyasg/struct_scores_actn2_live",
"rorydm/manuscript_plots",
"tanyasg/revised_manuscript_plots",
"calystay/2d_nuclear_masks",
"calystay/3d_actn2_segmentation",
"calystay/probe_localization",
"calystay/probe_structure_classifier",
"tanyasg/2d_nonstructure_fields",
"tanyasg/2d_nonstructure_single_cell_features",
"tanyasg/scrnaseq_data",
"tanyasg/scrnaseq_raw",
],
skip_test_pkgs=[
"tanyasg/scrnaseq_raw_test",
"rorydm/manuscript_plots_test",
"tanyasg/revised_manuscript_plots_test",
"rorydm/2d_segmented_fields2_test",
"rorydm/2d_segmented_fields_actn2_test",
"rorydm/2d_segmented_fields_actn2_2_test",
"rorydm/2d_autocontrasted_fields_and_single_cells_part2_test",
"rorydm/2d_autocontrasted_fields_and_single_cells_actn2_test",
"rorydm/2d_autocontrasted_fields_and_single_cells_actn2_2_test",
"tanyasg/2d_autocontrasted_single_cell_features2_test",
"tanyasg/2d_autocontrasted_single_cell_features_actn2_test",
"tanyasg/2d_autocontrasted_single_cell_features_actn2_2_test",
"tanyasg/struct_scores_bonus_test",
"tanyasg/struct_scores_actn2_test",
"tanyasg/struct_scores_actn2_2_test",
"tanyasg/struct_scores_actn2_live_test",
],
manual_rename={
"rorydm/2d_segmented_fields":
"2d_segmented_fields_fish_1",
"rorydm/2d_segmented_fields2":
"2d_segmented_fields_fish_2",
"rorydm/2d_segmented_fields_actn2":
"2d_segmented_fields_fish_3",
"rorydm/2d_segmented_fields_actn2_2":
"2d_segmented_fields_fish_4",
"rorydm/2d_autocontrasted_fields_and_single_cells":
"2d_autocontrasted_fields_and_single_cells_fish_1",
"rorydm/2d_autocontrasted_fields_and_single_cells_part2":
"2d_autocontrasted_fields_and_single_cells_fish_2",
"rorydm/2d_autocontrasted_fields_and_single_cells_actn2":
"2d_autocontrasted_fields_and_single_cells_fish_3",
"rorydm/2d_autocontrasted_fields_and_single_cells_actn2_2":
"2d_autocontrasted_fields_and_single_cells_fish_4",
"tanyasg/2d_autocontrasted_single_cell_features":
"2d_autocontrasted_single_cell_features_fish_1",
"tanyasg/2d_autocontrasted_single_cell_features2":
"2d_autocontrasted_single_cell_features_fish_2",
"tanyasg/2d_autocontrasted_single_cell_features_actn2":
"2d_autocontrasted_single_cell_features_fish_3",
"tanyasg/2d_autocontrasted_single_cell_features_actn2_2":
"2d_autocontrasted_single_cell_features_fish_4",
"matheus/assay_dev_fish_analysis":
"automated_local_and_global_structure_fish_1",
"tanyasg/struct_scores_bonus":
"automated_local_and_global_structure_fish_2",
"tanyasg/struct_scores_actn2":
"automated_local_and_global_structure_fish_3",
"tanyasg/struct_scores_actn2_2":
"automated_local_and_global_structure_fish_4",
"tanyasg/struct_scores_actn2_live":
"automated_local_and_global_structure_live",
},
internal_s3_url="s3://allencell-internal-quilt",
):
# grab a list of packages that actually exist internally
pkg_list = list(quilt3.list_packages(internal_s3_url))
# create dictionary for original fully qualified name to simple name
pkg_map = {p: p.split("/")[-1] for p in all_pkgs}
# manually fix up some opaque names
for k, v in manual_rename.items():
pkg_map[k] = v
# make the map for test pkgs
pkg_map_test = {f"{k}_test": v for k, v in pkg_map.items()}
# make sure every key has a unique value
assert len(set(pkg_map.values())) == len(pkg_map)
assert len(set(pkg_map_test.values())) == len(pkg_map_test)
# make sure all our keys are in the s3 bucket, skipping some _test versions that don't exist
assert len([k for k, v in pkg_map.items()
if k in pkg_list]) == len(pkg_map)
assert len([k for k, v in pkg_map_test.items()
if k in pkg_list]) == len(pkg_map_test) - len(skip_test_pkgs)
# once checked, filter on only what's there
pkg_map = {k: v for k, v in pkg_map.items() if k in pkg_list}
pkg_map_test = {k: v for k, v in pkg_map_test.items() if k in pkg_list}
return pkg_map, pkg_map_test