def test_s3_set_dir(self): """ Verify building a package from an S3 directory. """ with patch('quilt3.packages.list_object_versions') as list_object_versions_mock: pkg = Package() list_object_versions_mock.return_value = ([ dict(Key='foo/a.txt', VersionId='xyz', IsLatest=True, Size=10), dict(Key='foo/x/y.txt', VersionId='null', IsLatest=True, Size=10), dict(Key='foo/z.txt', VersionId='123', IsLatest=False, Size=10), ], []) pkg.set_dir('', 's3://bucket/foo/', meta='test_meta') assert pkg['a.txt'].physical_keys[0] == 's3://bucket/foo/a.txt?versionId=xyz' assert pkg['x']['y.txt'].physical_keys[0] == 's3://bucket/foo/x/y.txt' assert pkg.meta == "test_meta" assert pkg['x']['y.txt'].size == 10 # GH368 list_object_versions_mock.assert_called_with('bucket', 'foo/') list_object_versions_mock.reset_mock() pkg.set_dir('bar', 's3://bucket/foo') assert pkg['bar']['a.txt'].physical_keys[0] == 's3://bucket/foo/a.txt?versionId=xyz' assert pkg['bar']['x']['y.txt'].physical_keys[0] == 's3://bucket/foo/x/y.txt' assert pkg['bar']['a.txt'].size == 10 # GH368 list_object_versions_mock.assert_called_with('bucket', 'foo/')
def upload_test_resources(args: Args): # Try running the download pipeline try: # Get test resources dir resources_dir = (Path(__file__).parent.parent / "aicsimageio" / "tests" / "resources").resolve(strict=True) # Report with directory will be used for upload log.info(f"Using contents of directory: {resources_dir}") # Create quilt package package = Package() package.set_dir("resources", resources_dir) # Report package contents log.info(f"Package contents: {package}") # Construct package name package_name = "aicsimageio/test_resources" # Check for dry run if args.dry_run: # Attempt to build the package built = package.build(package_name) # Get resolved save path manifest_save_path = Path("upload_manifest.jsonl").resolve() with open(manifest_save_path, "w") as manifest_write: package.dump(manifest_write) # Report where manifest was saved log.info( f"Dry run generated manifest stored to: {manifest_save_path}") log.info( f"Completed package dry run. Result hash: {built.top_hash}") # Upload else: # Check pre-approved push if args.preapproved: confirmation = True else: # Get upload confirmation confirmation = None while confirmation is None: # Get user input user_input = input("Upload [y]/n? ") # If the user simply pressed enter assume yes if len(user_input) == 0: user_input = "y" # Get first character and lowercase else: user_input = user_input[0].lower() # Set confirmation from None to a value if user_input == "y": confirmation = True elif user_input == "n": confirmation = False # Check confirmation if confirmation: pushed = package.push( package_name, "s3://aics-modeling-packages-test-resources", message= f"Test resources for `aicsimageio` version: {__version__}.", ) log.info( f"Completed package push. Result hash: {pushed.top_hash}") else: log.info(f"Upload canceled.") # Catch any exception except Exception as e: log.error("=============================================") if args.debug: log.error("\n\n" + traceback.format_exc()) log.error("=============================================") log.error("\n\n" + str(e) + "\n") log.error("=============================================") sys.exit(1)
def test_local_set_dir(self): """ Verify building a package from a local directory. """ pkg = Package() # Create some nested example files that contain their names. foodir = pathlib.Path("foo_dir") bazdir = pathlib.Path(foodir, "baz_dir") bazdir.mkdir(parents=True, exist_ok=True) with open('bar', 'w') as fd: fd.write(fd.name) with open('foo', 'w') as fd: fd.write(fd.name) with open(bazdir / 'baz', 'w') as fd: fd.write(fd.name) with open(foodir / 'bar', 'w') as fd: fd.write(fd.name) pkg = pkg.set_dir("/", ".", meta="test_meta") assert pathlib.Path( 'foo').resolve().as_uri() == pkg['foo'].physical_keys[0] assert pathlib.Path( 'bar').resolve().as_uri() == pkg['bar'].physical_keys[0] assert (bazdir / 'baz').resolve().as_uri( ) == pkg['foo_dir/baz_dir/baz'].physical_keys[0] assert ( foodir / 'bar').resolve().as_uri() == pkg['foo_dir/bar'].physical_keys[0] assert pkg.meta == "test_meta" pkg = Package() pkg = pkg.set_dir('/', 'foo_dir/baz_dir/') # todo nested at set_dir site or relative to set_dir path. assert (bazdir / 'baz').resolve().as_uri() == pkg['baz'].physical_keys[0] pkg = Package() pkg = pkg.set_dir('my_keys', 'foo_dir/baz_dir/') # todo nested at set_dir site or relative to set_dir path. assert ( bazdir / 'baz').resolve().as_uri() == pkg['my_keys/baz'].physical_keys[0] # Verify ignoring files in the presence of a dot-quiltignore with open('.quiltignore', 'w') as fd: fd.write('foo\n') fd.write('bar') pkg = Package() pkg = pkg.set_dir("/", ".") assert 'foo_dir' in pkg.keys() assert 'foo' not in pkg.keys() and 'bar' not in pkg.keys() with open('.quiltignore', 'w') as fd: fd.write('foo_dir') pkg = Package() pkg = pkg.set_dir("/", ".") assert 'foo_dir' not in pkg.keys() with open('.quiltignore', 'w') as fd: fd.write('foo_dir\n') fd.write('foo_dir/baz_dir') pkg = Package() pkg = pkg.set_dir("/", ".") assert 'foo_dir/baz_dir' not in pkg.keys( ) and 'foo_dir' not in pkg.keys() pkg = pkg.set_dir("new_dir", ".", meta="new_test_meta") assert pathlib.Path( 'foo').resolve().as_uri() == pkg['new_dir/foo'].physical_keys[0] assert pathlib.Path( 'bar').resolve().as_uri() == pkg['new_dir/bar'].physical_keys[0] assert pkg['new_dir'].meta == "new_test_meta" # verify set_dir logical key shortcut pkg = Package() pkg.set_dir("/") assert pathlib.Path( 'foo').resolve().as_uri() == pkg['foo'].physical_keys[0] assert pathlib.Path( 'bar').resolve().as_uri() == pkg['bar'].physical_keys[0]