def test_uri_from_cli_absolute_storage_uri() -> None:
uri = uri_from_cli(
"storage://otheruser/path/to/file.txt", "testuser", "test-cluster"
)
assert str(uri) == "storage://otheruser/path/to/file.txt"
uri = uri_from_cli("storage:///path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == "storage://test-cluster/path/to/file.txt"
def parse_file_resource(uri: str, root: Root) -> URL:
""" Parses the neuromation resource URI string.
Available schemes: file, storage.
"""
return uri_from_cli(uri,
root.username,
allowed_schemes=("file", "storage"))
def test_uri_from_cli_absolute_storage_uri_special_chars() -> None:
uri = uri_from_cli(
"storage://cluster/user/path/to/file%23%25%3f:@~%C3%9F",
"testuser",
"test-cluster",
)
assert uri.path == "/user/path/to/file#%?:@~ß"
def parse_secret_resource(uri: str, root: Root) -> URL:
return uri_from_cli(
uri,
root.client.username,
root.client.cluster_name,
allowed_schemes=("secret"),
)
def parse_blob_or_file_resource(uri: str, root: Root) -> URL:
# Username will not be used, just part of the signature
return uri_from_cli(
uri,
root.client.username,
root.client.cluster_name,
allowed_schemes=("blob", "file"),
)
def parse_resource_for_sharing(uri: str, root: Root) -> URL:
""" Parses the neuromation resource URI string.
Available schemes: storage, image, job. For image URIs, tags are not allowed.
"""
if uri.startswith("image:"):
parser = _ImageNameParser(root.username, root.registry_url)
image = parser.parse_as_neuro_image(uri, allow_tag=False)
uri = str(image)
return uri_from_cli(uri,
root.username,
allowed_schemes=("storage", "image", "job"))
async def resolve_job(
id_or_name_or_uri: str, *, client: Client, status: Set[JobStatus]
) -> str:
default_user = client.username
default_cluster = client.cluster_name
if id_or_name_or_uri.startswith("job:"):
uri = uri_from_cli(
id_or_name_or_uri,
username=default_user,
cluster_name=default_cluster,
allowed_schemes=("job",),
)
if uri.host != default_cluster:
raise ValueError(f"Invalid job URI: cluster_name != '{default_cluster}'")
owner, _, id_or_name = uri.path.lstrip("/").partition("/")
if not owner:
raise ValueError(f"Invalid job URI: missing owner")
if not id_or_name:
raise ValueError(
f"Invalid job URI: owner='{owner}', missing job-id or job-name"
)
else:
id_or_name = id_or_name_or_uri
owner = default_user
# Temporary fast path.
if re.fullmatch(JOB_ID_PATTERN, id_or_name):
return id_or_name
try:
async for job in client.jobs.list(
name=id_or_name, owners={owner}, reverse=True, limit=1
):
log.debug(f"Job name '{id_or_name}' resolved to job ID '{job.id}'")
return job.id
except asyncio.CancelledError:
raise
except Exception as e:
log.error(
f"Failed to resolve job-name {id_or_name_or_uri} resolved as "
f"name={id_or_name}, owner={owner} to a job-ID: {e}"
)
if owner != default_user:
raise ValueError(f"Failed to resolve job {id_or_name_or_uri}")
return id_or_name
def parse_resource_for_sharing(uri: str, root: Root) -> URL:
"""Parses the neuromation resource URI string.
Available schemes: storage, image, job. For image URIs, tags are not allowed.
"""
if uri.startswith("image:"):
image = root.client.parse.remote_image(uri, tag_option=TagOption.DENY)
uri = str(image)
uri_res = uri_from_cli(
uri,
root.client.username,
root.client.cluster_name,
allowed_schemes=SHARE_SCHEMES,
)
# URI's for object storage can only operate on bucket level
if uri_res.scheme == "blob" and "/" in uri_res.path.strip("/"):
raise ValueError("Only bucket level permissions are supported for Blob Storage")
return uri_res
def test_uri_from_cli_numberic_path() -> None:
uri = uri_from_cli("256", "testuser", "test-cluster")
assert str(uri) == Path("256").absolute().as_uri()
uri = uri_from_cli("123456", "testuser", "test-cluster")
assert str(uri) == Path("123456").absolute().as_uri()
uri = uri_from_cli("file:256", "testuser", "test-cluster")
assert str(uri) == Path("256").absolute().as_uri()
uri = uri_from_cli("file:123456", "testuser", "test-cluster")
assert str(uri) == Path("123456").absolute().as_uri()
uri = uri_from_cli("storage:256", "testuser", "test-cluster")
assert str(uri) == "storage://test-cluster/testuser/256"
uri = uri_from_cli("storage:123456", "testuser", "test-cluster")
assert str(uri) == "storage://test-cluster/testuser/123456"
async def test_uri_from_cli__blob__fail(uri: str) -> None:
with pytest.raises(ValueError):
uri_from_cli(uri, "u", "c", allowed_schemes=("blob", ))
def test_uri_from_cli_relative_storage_uri() -> None:
uri = uri_from_cli("storage:path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == "storage://test-cluster/testuser/path/to/file.txt"
uri = uri_from_cli("storage:/path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == "storage://test-cluster/path/to/file.txt"
def test_uri_from_cli_absolute_file_uri() -> None:
uri = uri_from_cli("file:/path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == Path("/path/to/file.txt").absolute().as_uri()
uri = uri_from_cli("file:///path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == Path("/path/to/file.txt").absolute().as_uri()
def test_uri_from_cli_tilde_only(fake_homedir: Path) -> None:
uri = uri_from_cli("~", "testuser", "test-cluster")
assert str(uri) == fake_homedir.as_uri()
def test_uri_from_cli_path_with_tilde_unknown_user() -> None:
with pytest.raises(ValueError, match=r"Cannot expand user for "):
uri_from_cli("~unknownuser/path/to/file.txt", "testuser", "test-cluster")
def test_uri_from_cli_absolute_path_special_chars() -> None:
uri = uri_from_cli("/path/to/file#%23:?@~", "testuser", "test-cluster")
assert _extract_path(uri) == Path("/path/to/file#%23:?@~").absolute()
def test_uri_from_cli_relative_path() -> None:
uri = uri_from_cli("path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == Path("path/to/file.txt").absolute().as_uri()
async def test_uri_from_cli__file__fail(path_or_uri: str) -> None:
with pytest.raises(ValueError):
uri_from_cli(path_or_uri, "u", "c", allowed_schemes=("file", ))
def test_uri_from_cli_absolute_file_uri_special_chars() -> None:
uri = uri_from_cli("file:/path/to/file%23%25%3f:@~%C3%9F", "testuser",
"test-cluster")
assert uri.path.endswith("/path/to/file#%?:@~ß")
def test_uri_from_cli_path_with_tilde(fake_homedir: Path) -> None:
uri = uri_from_cli("~/path/to/file.txt", "testuser", "test-cluster")
assert str(uri) == (fake_homedir / "path/to/file.txt").as_uri()
def test_uri_from_cli_absolute_path() -> None:
uri = uri_from_cli("/path/to/file.txt", "testuser")
assert str(uri) == Path("/path/to/file.txt").absolute().as_uri()
def test_uri_from_cli_relative_path_special_chars() -> None:
uri = uri_from_cli("path/to/file#%23:?@~", "testuser", "test-cluster")
assert uri.path.endswith("/path/to/file#%23:?@~")
