def test_s3_set_dir(self):
""" Verify building a package from an S3 directory. """
with patch('quilt3.packages.list_object_versions') as list_object_versions_mock:
pkg = Package()
list_object_versions_mock.return_value = ([
dict(Key='foo/a.txt', VersionId='xyz', IsLatest=True, Size=10),
dict(Key='foo/x/y.txt', VersionId='null', IsLatest=True, Size=10),
dict(Key='foo/z.txt', VersionId='123', IsLatest=False, Size=10),
], [])
pkg.set_dir('', 's3://bucket/foo/', meta='test_meta')
assert pkg['a.txt'].physical_keys[0] == 's3://bucket/foo/a.txt?versionId=xyz'
assert pkg['x']['y.txt'].physical_keys[0] == 's3://bucket/foo/x/y.txt'
assert pkg.meta == "test_meta"
assert pkg['x']['y.txt'].size == 10 # GH368
list_object_versions_mock.assert_called_with('bucket', 'foo/')
list_object_versions_mock.reset_mock()
pkg.set_dir('bar', 's3://bucket/foo')
assert pkg['bar']['a.txt'].physical_keys[0] == 's3://bucket/foo/a.txt?versionId=xyz'
assert pkg['bar']['x']['y.txt'].physical_keys[0] == 's3://bucket/foo/x/y.txt'
assert pkg['bar']['a.txt'].size == 10 # GH368
list_object_versions_mock.assert_called_with('bucket', 'foo/')
def upload_test_resources(args: Args):
# Try running the download pipeline
try:
# Get test resources dir
resources_dir = (Path(__file__).parent.parent / "aicsimageio" /
"tests" / "resources").resolve(strict=True)
# Report with directory will be used for upload
log.info(f"Using contents of directory: {resources_dir}")
# Create quilt package
package = Package()
package.set_dir("resources", resources_dir)
# Report package contents
log.info(f"Package contents: {package}")
# Construct package name
package_name = "aicsimageio/test_resources"
# Check for dry run
if args.dry_run:
# Attempt to build the package
built = package.build(package_name)
# Get resolved save path
manifest_save_path = Path("upload_manifest.jsonl").resolve()
with open(manifest_save_path, "w") as manifest_write:
package.dump(manifest_write)
# Report where manifest was saved
log.info(
f"Dry run generated manifest stored to: {manifest_save_path}")
log.info(
f"Completed package dry run. Result hash: {built.top_hash}")
# Upload
else:
# Check pre-approved push
if args.preapproved:
confirmation = True
else:
# Get upload confirmation
confirmation = None
while confirmation is None:
# Get user input
user_input = input("Upload [y]/n? ")
# If the user simply pressed enter assume yes
if len(user_input) == 0:
user_input = "y"
# Get first character and lowercase
else:
user_input = user_input[0].lower()
# Set confirmation from None to a value
if user_input == "y":
confirmation = True
elif user_input == "n":
confirmation = False
# Check confirmation
if confirmation:
pushed = package.push(
package_name,
"s3://aics-modeling-packages-test-resources",
message=
f"Test resources for `aicsimageio` version: {__version__}.",
)
log.info(
f"Completed package push. Result hash: {pushed.top_hash}")
else:
log.info(f"Upload canceled.")
# Catch any exception
except Exception as e:
log.error("=============================================")
if args.debug:
log.error("\n\n" + traceback.format_exc())
log.error("=============================================")
log.error("\n\n" + str(e) + "\n")
log.error("=============================================")
sys.exit(1)
def test_local_set_dir(self):
""" Verify building a package from a local directory. """
pkg = Package()
# Create some nested example files that contain their names.
foodir = pathlib.Path("foo_dir")
bazdir = pathlib.Path(foodir, "baz_dir")
bazdir.mkdir(parents=True, exist_ok=True)
with open('bar', 'w') as fd:
fd.write(fd.name)
with open('foo', 'w') as fd:
fd.write(fd.name)
with open(bazdir / 'baz', 'w') as fd:
fd.write(fd.name)
with open(foodir / 'bar', 'w') as fd:
fd.write(fd.name)
pkg = pkg.set_dir("/", ".", meta="test_meta")
assert pathlib.Path(
'foo').resolve().as_uri() == pkg['foo'].physical_keys[0]
assert pathlib.Path(
'bar').resolve().as_uri() == pkg['bar'].physical_keys[0]
assert (bazdir / 'baz').resolve().as_uri(
) == pkg['foo_dir/baz_dir/baz'].physical_keys[0]
assert (
foodir /
'bar').resolve().as_uri() == pkg['foo_dir/bar'].physical_keys[0]
assert pkg.meta == "test_meta"
pkg = Package()
pkg = pkg.set_dir('/', 'foo_dir/baz_dir/')
# todo nested at set_dir site or relative to set_dir path.
assert (bazdir /
'baz').resolve().as_uri() == pkg['baz'].physical_keys[0]
pkg = Package()
pkg = pkg.set_dir('my_keys', 'foo_dir/baz_dir/')
# todo nested at set_dir site or relative to set_dir path.
assert (
bazdir /
'baz').resolve().as_uri() == pkg['my_keys/baz'].physical_keys[0]
# Verify ignoring files in the presence of a dot-quiltignore
with open('.quiltignore', 'w') as fd:
fd.write('foo\n')
fd.write('bar')
pkg = Package()
pkg = pkg.set_dir("/", ".")
assert 'foo_dir' in pkg.keys()
assert 'foo' not in pkg.keys() and 'bar' not in pkg.keys()
with open('.quiltignore', 'w') as fd:
fd.write('foo_dir')
pkg = Package()
pkg = pkg.set_dir("/", ".")
assert 'foo_dir' not in pkg.keys()
with open('.quiltignore', 'w') as fd:
fd.write('foo_dir\n')
fd.write('foo_dir/baz_dir')
pkg = Package()
pkg = pkg.set_dir("/", ".")
assert 'foo_dir/baz_dir' not in pkg.keys(
) and 'foo_dir' not in pkg.keys()
pkg = pkg.set_dir("new_dir", ".", meta="new_test_meta")
assert pathlib.Path(
'foo').resolve().as_uri() == pkg['new_dir/foo'].physical_keys[0]
assert pathlib.Path(
'bar').resolve().as_uri() == pkg['new_dir/bar'].physical_keys[0]
assert pkg['new_dir'].meta == "new_test_meta"
# verify set_dir logical key shortcut
pkg = Package()
pkg.set_dir("/")
assert pathlib.Path(
'foo').resolve().as_uri() == pkg['foo'].physical_keys[0]
assert pathlib.Path(
'bar').resolve().as_uri() == pkg['bar'].physical_keys[0]
