def test_local_package_delete_overlapping(self): """ Verify local package delete works when multiple packages reference the same tophash. """ top_hash = Package().build("Quilt/Test1").top_hash top_hash = Package().build("Quilt/Test2").top_hash assert 'Quilt/Test1' in quilt3.list_packages() assert top_hash in [ p.name for p in (LOCAL_REGISTRY / '.quilt/packages').iterdir() ] quilt3.delete_package('Quilt/Test1') assert 'Quilt/Test1' not in quilt3.list_packages() assert top_hash in [ p.name for p in (LOCAL_REGISTRY / '.quilt/packages').iterdir() ] quilt3.delete_package('Quilt/Test2') assert 'Quilt/Test2' not in quilt3.list_packages() assert top_hash not in [ p.name for p in (LOCAL_REGISTRY / '.quilt/packages').iterdir() ]
def test_local_package_delete(self): """Verify local package delete works.""" top_hash = Package().build("Quilt/Test").top_hash assert 'Quilt/Test' in quilt3.list_packages() quilt3.delete_package('Quilt/Test') assert 'Quilt/Test' not in quilt3.list_packages()
def exec_module(cls, module): """ Module executor. """ name_parts = module.__name__.split('.') registry = get_from_config('default_local_registry') if module.__name__ == 'quilt3.data': # __path__ must be set even if the package is virtual. Since __path__ will be # scanned by all other finders preceding this one in sys.meta_path order, make sure # it points to someplace lacking importable objects module.__path__ = MODULE_PATH return module elif len(name_parts) == 3: # e.g. module.__name__ == quilt3.data.foo namespace = name_parts[2] # we do not know the name the user will ask for, so populate all valid names for pkg in list_packages(): pkg_user, pkg_name = pkg.split('/') if pkg_user == namespace: module.__dict__[pkg_name] = Package.browse( pkg, registry=registry) module.__path__ = MODULE_PATH return module else: assert False
def test_list_remote_packages(self): """Verify that listing remote packages works as expected.""" self.s3_stubber.add_response( method='list_objects_v2', service_response={ 'Contents': [{ 'Key': '.quilt/named_packages/foo/bar/1549931300', 'Size': 64, }, { 'Key': '.quilt/named_packages/foo/bar/1549931634', 'Size': 64, }, { 'Key': '.quilt/named_packages/foo/bar/latest', 'Size': 64, }] }, expected_params={ 'Bucket': 'my_test_bucket', 'Prefix': '.quilt/named_packages/', }) pkgs = list(quilt3.list_packages('s3://my_test_bucket/')) assert len(pkgs) == 1 assert list(pkgs) == ['foo/bar']
def get_packages(self): """ Get the names of the packages in the S3 bucket Returns: :obj:`list` of :obj:`str`: list of package names """ packages = quilt3.list_packages(self.get_aws_bucket_uri()) return list(packages)
def test_list_local_packages(self): """Verify that list returns packages in the appdirs directory.""" # Build a new package into the local registry. Package().build("Quilt/Foo") Package().build("Quilt/Bar") Package().build("Quilt/Test") # Verify packages are returned. pkgs = list(quilt3.list_packages()) assert len(pkgs) == 3 assert "Quilt/Foo" in pkgs assert "Quilt/Bar" in pkgs # Verify specifying a local path explicitly works as expected. assert list(pkgs) == list( quilt3.list_packages(LOCAL_REGISTRY.as_posix()))
def test_list_local_packages(self): """Verify that list returns packages in the appdirs directory.""" # Build a new package into the local registry. with patch('time.time', return_value=1234567890): Package().build("Quilt/Foo") Package().build("Quilt/Bar") Package().build("Quilt/Test") # Verify packages are returned. pkgs = list(quilt3.list_packages()) assert len(pkgs) == 3 assert "Quilt/Foo" in pkgs assert "Quilt/Bar" in pkgs versions = set(quilt3.list_package_versions('Quilt/Foo')) assert versions == { ('latest', '2a5a67156ca9238c14d12042db51c5b52260fdd5511b61ea89b58929d6e1769b'), ('1234567890', '2a5a67156ca9238c14d12042db51c5b52260fdd5511b61ea89b58929d6e1769b'), } # Verify specifying a local path explicitly works as expected. assert list(pkgs) == list(quilt3.list_packages(LOCAL_REGISTRY.as_posix()))
def test_list_remote_packages(self): """Verify that listing remote packages works as expected.""" def pseudo_list_objects(bucket, prefix, recursive): if prefix == '.quilt/named_packages/': return ([{'Prefix': '.quilt/named_packages/foo/'}], []) elif prefix == '.quilt/named_packages/foo/': return ([{'Prefix': '.quilt/named_packages/foo/bar/'}], []) elif prefix == '.quilt/named_packages/foo/bar/': import datetime return ([], [{ 'Key': '.quilt/named_packages/foo/bar/1549931300', 'LastModified': datetime.datetime.now() - datetime.timedelta(seconds=30) }, { 'Key': '.quilt/named_packages/foo/bar/1549931634', 'LastModified': datetime.datetime.now() }, { 'Key': '.quilt/named_packages/foo/bar/latest', 'LastModified': datetime.datetime.now() }]) else: raise ValueError def pseudo_get_bytes(src): if src.endswith('foo/bar/latest') or src.endswith( 'foo/bar/1549931634'): return (b'100', None) elif src.endswith('foo/bar/1549931300'): return (b'90', None) else: raise ValueError with patch('quilt3.api.list_objects', side_effect=pseudo_list_objects), \ patch('quilt3.api.get_bytes', side_effect=pseudo_get_bytes), \ patch('quilt3.Package.browse', return_value=Package()): pkgs = quilt3.list_packages('s3://my_test_bucket/') assert len(pkgs) == 1 assert list(pkgs) == ['foo/bar'] expected = ( 'PACKAGE \tTOP HASH \tCREATED \tSIZE \t\n' 'foo/bar:latest \t100 \tnow \t0 Bytes\t\n' 'foo/bar \t90 \t30 seconds ago \t0 Bytes\t\n' ) assert str(pkgs) == expected
def make_pkg_maps( all_pkgs=[ "rorydm/2d_segmented_fields", "rorydm/2d_segmented_fields2", "rorydm/2d_segmented_fields_actn2", "rorydm/2d_segmented_fields_actn2_2", "rorydm/2d_autocontrasted_fields_and_single_cells", "rorydm/2d_autocontrasted_fields_and_single_cells_part2", "rorydm/2d_autocontrasted_fields_and_single_cells_actn2", "rorydm/2d_autocontrasted_fields_and_single_cells_actn2_2", "tanyasg/2d_autocontrasted_single_cell_features", "tanyasg/2d_autocontrasted_single_cell_features2", "tanyasg/2d_autocontrasted_single_cell_features_actn2", "tanyasg/2d_autocontrasted_single_cell_features_actn2_2", "matheus/assay_dev_fish_analysis", "tanyasg/struct_scores_bonus", "tanyasg/struct_scores_actn2", "tanyasg/struct_scores_actn2_2", "tanyasg/struct_scores_actn2_live", "rorydm/manuscript_plots", "tanyasg/revised_manuscript_plots", "calystay/2d_nuclear_masks", "calystay/3d_actn2_segmentation", "calystay/probe_localization", "calystay/probe_structure_classifier", "tanyasg/2d_nonstructure_fields", "tanyasg/2d_nonstructure_single_cell_features", "tanyasg/scrnaseq_data", "tanyasg/scrnaseq_raw", ], skip_test_pkgs=[ "tanyasg/scrnaseq_raw_test", "rorydm/manuscript_plots_test", "tanyasg/revised_manuscript_plots_test", "rorydm/2d_segmented_fields2_test", "rorydm/2d_segmented_fields_actn2_test", "rorydm/2d_segmented_fields_actn2_2_test", "rorydm/2d_autocontrasted_fields_and_single_cells_part2_test", "rorydm/2d_autocontrasted_fields_and_single_cells_actn2_test", "rorydm/2d_autocontrasted_fields_and_single_cells_actn2_2_test", "tanyasg/2d_autocontrasted_single_cell_features2_test", "tanyasg/2d_autocontrasted_single_cell_features_actn2_test", "tanyasg/2d_autocontrasted_single_cell_features_actn2_2_test", "tanyasg/struct_scores_bonus_test", "tanyasg/struct_scores_actn2_test", "tanyasg/struct_scores_actn2_2_test", "tanyasg/struct_scores_actn2_live_test", ], manual_rename={ "rorydm/2d_segmented_fields": "2d_segmented_fields_fish_1", "rorydm/2d_segmented_fields2": "2d_segmented_fields_fish_2", "rorydm/2d_segmented_fields_actn2": "2d_segmented_fields_fish_3", "rorydm/2d_segmented_fields_actn2_2": "2d_segmented_fields_fish_4", "rorydm/2d_autocontrasted_fields_and_single_cells": "2d_autocontrasted_fields_and_single_cells_fish_1", "rorydm/2d_autocontrasted_fields_and_single_cells_part2": "2d_autocontrasted_fields_and_single_cells_fish_2", "rorydm/2d_autocontrasted_fields_and_single_cells_actn2": "2d_autocontrasted_fields_and_single_cells_fish_3", "rorydm/2d_autocontrasted_fields_and_single_cells_actn2_2": "2d_autocontrasted_fields_and_single_cells_fish_4", "tanyasg/2d_autocontrasted_single_cell_features": "2d_autocontrasted_single_cell_features_fish_1", "tanyasg/2d_autocontrasted_single_cell_features2": "2d_autocontrasted_single_cell_features_fish_2", "tanyasg/2d_autocontrasted_single_cell_features_actn2": "2d_autocontrasted_single_cell_features_fish_3", "tanyasg/2d_autocontrasted_single_cell_features_actn2_2": "2d_autocontrasted_single_cell_features_fish_4", "matheus/assay_dev_fish_analysis": "automated_local_and_global_structure_fish_1", "tanyasg/struct_scores_bonus": "automated_local_and_global_structure_fish_2", "tanyasg/struct_scores_actn2": "automated_local_and_global_structure_fish_3", "tanyasg/struct_scores_actn2_2": "automated_local_and_global_structure_fish_4", "tanyasg/struct_scores_actn2_live": "automated_local_and_global_structure_live", }, internal_s3_url="s3://allencell-internal-quilt", ): # grab a list of packages that actually exist internally pkg_list = list(quilt3.list_packages(internal_s3_url)) # create dictionary for original fully qualified name to simple name pkg_map = {p: p.split("/")[-1] for p in all_pkgs} # manually fix up some opaque names for k, v in manual_rename.items(): pkg_map[k] = v # make the map for test pkgs pkg_map_test = {f"{k}_test": v for k, v in pkg_map.items()} # make sure every key has a unique value assert len(set(pkg_map.values())) == len(pkg_map) assert len(set(pkg_map_test.values())) == len(pkg_map_test) # make sure all our keys are in the s3 bucket, skipping some _test versions that don't exist assert len([k for k, v in pkg_map.items() if k in pkg_list]) == len(pkg_map) assert len([k for k, v in pkg_map_test.items() if k in pkg_list]) == len(pkg_map_test) - len(skip_test_pkgs) # once checked, filter on only what's there pkg_map = {k: v for k, v in pkg_map.items() if k in pkg_list} pkg_map_test = {k: v for k, v in pkg_map_test.items() if k in pkg_list} return pkg_map, pkg_map_test