async def test_exclude_with_leading_slash() -> None:
ff = FileFilter()
ff.exclude("/spam")
assert not await ff.match("spam")
assert not await ff.match("spam/")
assert await ff.match("ham")
assert await ff.match("dir/spam")
async def test_exclude_with_slash() -> None:
ff = FileFilter()
ff.exclude("dir/*.txt")
assert await ff.match("spam.txt")
assert not await ff.match("dir/spam.txt")
assert not await ff.match("dir/spam.txt/")
assert not await ff.match("dir/.txt")
assert await ff.match("parent/dir/spam.txt")
async def test_exclude() -> None:
ff = FileFilter()
ff.exclude("*.txt")
assert await ff.match("spam")
assert not await ff.match("spam.txt")
assert not await ff.match(".txt")
assert not await ff.match("dir/spam.txt")
assert not await ff.match("dir/.txt")
assert await ff.match("dir.txt/spam")
assert await ff.match("dir/child.txt/spam")
async def test_read_from_buffer() -> None:
ff = FileFilter()
ff.read_from_buffer(
codecs.BOM_UTF8 + b"*.txt \r\n" # CRLF and trailing spaces
b"\n" # empty line
b"# comment\n" # comment
b"!s*", # negation
prefix="base/",
)
assert len(ff.filters) == 2
assert await ff.match("base/spam.txt")
assert not await ff.match("base/ham.txt")
assert await ff.match("ham.txt")
async def test_exclude_include_with_prefix() -> None:
ff = FileFilter()
ff.exclude("*.txt", "parent/")
ff.include("s*", "parent/child/")
assert await ff.match("spam.txt")
assert await ff.match("ham.txt")
assert not await ff.match("parent/spam.txt")
assert not await ff.match("parent/ham.txt")
assert await ff.match("other/spam.txt")
assert await ff.match("other/ham.txt")
assert await ff.match("parent/child/spam.txt")
assert not await ff.match("parent/child/ham.txt")
async def test_exclude_recursive() -> None:
ff = FileFilter()
ff.exclude("**/dir/*.txt")
assert await ff.match("spam.txt")
assert not await ff.match("dir/spam.txt")
assert await ff.match("dir/spam")
assert not await ff.match("parent/dir/spam.txt")
assert await ff.match("parent/dir/spam")
ff = FileFilter()
ff.exclude("dir/**/*.txt")
assert await ff.match("spam.txt")
assert not await ff.match("dir/spam.txt")
assert await ff.match("dir/spam")
assert not await ff.match("dir/child/spam.txt")
assert await ff.match("dir/child/spam")
ff = FileFilter()
ff.exclude("dir/**")
assert await ff.match("spam")
assert not await ff.match("dir/")
assert await ff.match("dir")
assert not await ff.match("dir/child")
assert not await ff.match("dir/child/")
assert not await ff.match("dir/child/spam")
ff = FileFilter()
ff.exclude("dir/**/")
assert await ff.match("spam")
assert not await ff.match("dir/")
assert await ff.match("dir")
assert not await ff.match("dir/child/")
assert await ff.match("dir/child")
assert not await ff.match("dir/child/")
assert not await ff.match("dir/child/spam/")
assert await ff.match("dir/child/spam")
async def test_exclude_crosscomponent() -> None:
ff = FileFilter()
ff.exclude("a?b")
assert not await ff.match("a-b")
assert await ff.match("a/b")
ff = FileFilter()
ff.exclude("a*b")
assert not await ff.match("ab")
assert not await ff.match("a-b")
assert not await ff.match("arab")
assert await ff.match("a/b")
assert await ff.match("alice/bob")
ff = FileFilter()
ff.exclude("a[!0-9]b")
assert await ff.match("a0b")
assert not await ff.match("a-b")
assert await ff.match("a/b")
async def test_exclude_with_trailing_slash() -> None:
ff = FileFilter()
ff.exclude("spam/")
assert await ff.match("spam")
assert not await ff.match("spam/")
async def test_empty_filter() -> None:
ff = FileFilter()
assert await ff.match("spam")
assert await ff.match(".spam")
assert await ff.match("spam/ham")
async def test_exclude_all() -> None:
ff = FileFilter()
ff.exclude("*")
assert not await ff.match("spam")
assert not await ff.match(".spam")
async def cp(
root: Root,
sources: Sequence[str],
destination: Optional[str],
recursive: bool,
glob: bool,
target_directory: Optional[str],
no_target_directory: bool,
filters: Optional[Tuple[Tuple[bool, str], ...]],
exclude_from_files: str,
progress: bool,
) -> None:
"""
Simple utility to copy files and directories into and from Blob Storage.
Either SOURCES or DESTINATION should have `blob://` scheme.
If scheme is omitted, file:// scheme is assumed. It is currently not possible to
copy files between Blob Storage (`blob://`) destination, nor with `storage://`
scheme paths.
Use `/dev/stdin` and `/dev/stdout` file names to upload a file from standard input
or output to stdout.
Any number of --exclude and --include options can be passed. The
filters that appear later in the command take precedence over filters
that appear earlier in the command. If neither --exclude nor
--include options are specified the default can be changed using the
storage.cp-exclude configuration variable documented in
"neuro help user-config".
File permissions, modification times and other attributes will not be passed to
Blob Storage metadata during upload.
"""
target_dir: Optional[URL]
dst: Optional[URL]
if target_directory:
if no_target_directory:
raise click.UsageError(
"Cannot combine --target-directory (-t) and --no-target-directory (-T)"
)
if destination is None:
raise click.MissingParameter(param_type="argument",
param_hint='"SOURCES..."')
sources = *sources, destination
target_dir = parse_blob_or_file_resource(target_directory, root)
dst = None
else:
if destination is None:
raise click.MissingParameter(param_type="argument",
param_hint='"DESTINATION"')
if not sources:
raise click.MissingParameter(param_type="argument",
param_hint='"SOURCES..."')
dst = parse_blob_or_file_resource(destination, root)
# From gsutil:
#
# There's an additional wrinkle when working with subdirectories: the resulting
# names depend on whether the destination subdirectory exists. For example,
# if gs://my-bucket/subdir exists as a subdirectory, the command:
# gsutil cp -r dir1/dir2 gs://my-bucket/subdir
# will create the blob gs://my-bucket/subdir/dir2/a/b/c. In contrast, if
# gs://my-bucket/subdir does not exist, this same gsutil cp command will create
# the blob gs://my-bucket/subdir/a/b/c.
if no_target_directory or not await _is_dir(root, dst):
target_dir = None
else:
target_dir = dst
dst = None
ignore_file_names = await calc_ignore_file_names(root.client,
exclude_from_files)
filters = await calc_filters(root.client, filters)
srcs = await _expand(sources, root, glob, allow_file=True)
if no_target_directory and len(srcs) > 1:
raise click.UsageError(f"Extra operand after {str(srcs[1])!r}")
file_filter = FileFilter()
for exclude, pattern in filters:
log.debug("%s %s", "Exclude" if exclude else "Include", pattern)
file_filter.append(exclude, pattern)
show_progress = root.tty and progress
errors = False
for src in srcs:
# `src.name` will return empty string if URL has trailing slash, ie.:
# `neuro blob cp data/ blob:my_bucket` -> dst == blob:my_bucket/file.txt
# `neuro blob cp data blob:my_bucket` -> dst == blob:my_bucket/data/file.txt
# `neuro blob cp blob:my_bucket data` -> dst == data/my_bucket/file.txt
# `neuro blob cp blob:my_bucket/ data` -> dst == data/file.txt
if target_dir:
dst = target_dir / src.name
assert dst
progress_blob = create_storage_progress(root, show_progress)
progress_blob.begin(src, dst)
try:
if src.scheme == "file" and dst.scheme == "blob":
if recursive and await _is_dir(root, src):
await root.client.blob_storage.upload_dir(
src,
dst,
filter=file_filter.match,
ignore_file_names=frozenset(ignore_file_names),
progress=progress_blob,
)
else:
await root.client.blob_storage.upload_file(
src, dst, progress=progress_blob)
elif src.scheme == "blob" and dst.scheme == "file":
if recursive and await _is_dir(root, src):
await root.client.blob_storage.download_dir(
src,
dst,
filter=file_filter.match,
progress=progress_blob)
else:
await root.client.blob_storage.download_file(
src, dst, progress=progress_blob)
else:
raise RuntimeError(
f"Copy operation of the file with scheme '{src.scheme}'"
f" to the file with scheme '{dst.scheme}'"
f" is not supported")
except (OSError, ResourceNotFound, IllegalArgumentError) as error:
log.error(f"cannot copy {src} to {dst}: {error}")
errors = True
progress_blob.end()
if errors:
sys.exit(EX_OSFILE)
async def cp(
root: Root,
sources: Sequence[str],
destination: Optional[str],
recursive: bool,
glob: bool,
target_directory: Optional[str],
no_target_directory: bool,
update: bool,
filters: Optional[Tuple[Tuple[bool, str], ...]],
exclude_from_files: str,
progress: bool,
) -> None:
"""
Copy files and directories.
Either SOURCES or DESTINATION should have storage:// scheme.
If scheme is omitted, file:// scheme is assumed.
Use /dev/stdin and /dev/stdout file names to copy a file from terminal
and print the content of file on the storage to console.
Any number of --exclude and --include options can be passed. The
filters that appear later in the command take precedence over filters
that appear earlier in the command. If neither --exclude nor
--include options are specified the default can be changed using the
storage.cp-exclude configuration variable documented in
"neuro help user-config".
Examples:
# copy local files into remote storage root
neuro cp foo.txt bar/baz.dat storage:
neuro cp foo.txt bar/baz.dat -t storage:
# copy local directory `foo` into existing remote directory `bar`
neuro cp -r foo -t storage:bar
# copy the content of local directory `foo` into existing remote
# directory `bar`
neuro cp -r -T storage:foo storage:bar
# download remote file `foo.txt` into local file `/tmp/foo.txt` with
# explicit file:// scheme set
neuro cp storage:foo.txt file:///tmp/foo.txt
neuro cp -T storage:foo.txt file:///tmp/foo.txt
neuro cp storage:foo.txt file:///tmp
neuro cp storage:foo.txt -t file:///tmp
# download other user's remote file into the current directory
neuro cp storage://{username}/foo.txt .
# download only files with extension `.out` into the current directory
neuro cp storage:results/*.out .
"""
target_dir: Optional[URL]
dst: Optional[URL]
if target_directory:
if no_target_directory:
raise click.UsageError(
"Cannot combine --target-directory (-t) and --no-target-directory (-T)"
)
if destination is None:
raise click.MissingParameter(param_type="argument",
param_hint='"SOURCES..."')
sources = *sources, destination
target_dir = parse_file_resource(target_directory, root)
dst = None
else:
if destination is None:
raise click.MissingParameter(param_type="argument",
param_hint='"DESTINATION"')
if not sources:
raise click.MissingParameter(param_type="argument",
param_hint='"SOURCES..."')
dst = parse_file_resource(destination, root)
if no_target_directory or not await _is_dir(root, dst):
target_dir = None
else:
target_dir = dst
dst = None
ignore_file_names = await calc_ignore_file_names(root.client,
exclude_from_files)
filters = await calc_filters(root.client, filters)
srcs = await _expand(sources, root, glob, allow_file=True)
if no_target_directory and len(srcs) > 1:
raise click.UsageError(f"Extra operand after {str(srcs[1])!r}")
file_filter = FileFilter()
for exclude, pattern in filters:
log.debug("%s %s", "Exclude" if exclude else "Include", pattern)
file_filter.append(exclude, pattern)
show_progress = root.tty and progress
errors = False
for src in srcs:
if target_dir:
dst = target_dir / src.name
assert dst
progress_obj = create_storage_progress(root, show_progress)
progress_obj.begin(src, dst)
try:
if src.scheme == "file" and dst.scheme == "storage":
if recursive and await _is_dir(root, src):
await root.client.storage.upload_dir(
src,
dst,
update=update,
filter=file_filter.match,
ignore_file_names=frozenset(ignore_file_names),
progress=progress_obj,
)
else:
await root.client.storage.upload_file(
src, dst, update=update, progress=progress_obj)
elif src.scheme == "storage" and dst.scheme == "file":
if recursive and await _is_dir(root, src):
await root.client.storage.download_dir(
src,
dst,
update=update,
filter=file_filter.match,
progress=progress_obj,
)
else:
await root.client.storage.download_file(
src, dst, update=update, progress=progress_obj)
else:
raise RuntimeError(
f"Copy operation of the file with scheme '{src.scheme}'"
f" to the file with scheme '{dst.scheme}'"
f" is not supported")
except (OSError, ResourceNotFound, IllegalArgumentError) as error:
log.error(f"cannot copy {src} to {dst}: {error}")
errors = True
progress_obj.end()
if errors:
sys.exit(EX_OSFILE)