def test_uri_from_cli_absolute_storage_uri() -> None: uri = uri_from_cli( "storage://otheruser/path/to/file.txt", "testuser", "test-cluster" ) assert str(uri) == "storage://otheruser/path/to/file.txt" uri = uri_from_cli("storage:///path/to/file.txt", "testuser", "test-cluster") assert str(uri) == "storage://test-cluster/path/to/file.txt"
def parse_file_resource(uri: str, root: Root) -> URL: """ Parses the neuromation resource URI string. Available schemes: file, storage. """ return uri_from_cli(uri, root.username, allowed_schemes=("file", "storage"))
def test_uri_from_cli_absolute_storage_uri_special_chars() -> None: uri = uri_from_cli( "storage://cluster/user/path/to/file%23%25%3f:@~%C3%9F", "testuser", "test-cluster", ) assert uri.path == "/user/path/to/file#%?:@~ß"
def parse_secret_resource(uri: str, root: Root) -> URL: return uri_from_cli( uri, root.client.username, root.client.cluster_name, allowed_schemes=("secret"), )
def parse_blob_or_file_resource(uri: str, root: Root) -> URL: # Username will not be used, just part of the signature return uri_from_cli( uri, root.client.username, root.client.cluster_name, allowed_schemes=("blob", "file"), )
def parse_resource_for_sharing(uri: str, root: Root) -> URL: """ Parses the neuromation resource URI string. Available schemes: storage, image, job. For image URIs, tags are not allowed. """ if uri.startswith("image:"): parser = _ImageNameParser(root.username, root.registry_url) image = parser.parse_as_neuro_image(uri, allow_tag=False) uri = str(image) return uri_from_cli(uri, root.username, allowed_schemes=("storage", "image", "job"))
async def resolve_job( id_or_name_or_uri: str, *, client: Client, status: Set[JobStatus] ) -> str: default_user = client.username default_cluster = client.cluster_name if id_or_name_or_uri.startswith("job:"): uri = uri_from_cli( id_or_name_or_uri, username=default_user, cluster_name=default_cluster, allowed_schemes=("job",), ) if uri.host != default_cluster: raise ValueError(f"Invalid job URI: cluster_name != '{default_cluster}'") owner, _, id_or_name = uri.path.lstrip("/").partition("/") if not owner: raise ValueError(f"Invalid job URI: missing owner") if not id_or_name: raise ValueError( f"Invalid job URI: owner='{owner}', missing job-id or job-name" ) else: id_or_name = id_or_name_or_uri owner = default_user # Temporary fast path. if re.fullmatch(JOB_ID_PATTERN, id_or_name): return id_or_name try: async for job in client.jobs.list( name=id_or_name, owners={owner}, reverse=True, limit=1 ): log.debug(f"Job name '{id_or_name}' resolved to job ID '{job.id}'") return job.id except asyncio.CancelledError: raise except Exception as e: log.error( f"Failed to resolve job-name {id_or_name_or_uri} resolved as " f"name={id_or_name}, owner={owner} to a job-ID: {e}" ) if owner != default_user: raise ValueError(f"Failed to resolve job {id_or_name_or_uri}") return id_or_name
def parse_resource_for_sharing(uri: str, root: Root) -> URL: """Parses the neuromation resource URI string. Available schemes: storage, image, job. For image URIs, tags are not allowed. """ if uri.startswith("image:"): image = root.client.parse.remote_image(uri, tag_option=TagOption.DENY) uri = str(image) uri_res = uri_from_cli( uri, root.client.username, root.client.cluster_name, allowed_schemes=SHARE_SCHEMES, ) # URI's for object storage can only operate on bucket level if uri_res.scheme == "blob" and "/" in uri_res.path.strip("/"): raise ValueError("Only bucket level permissions are supported for Blob Storage") return uri_res
def test_uri_from_cli_numberic_path() -> None: uri = uri_from_cli("256", "testuser", "test-cluster") assert str(uri) == Path("256").absolute().as_uri() uri = uri_from_cli("123456", "testuser", "test-cluster") assert str(uri) == Path("123456").absolute().as_uri() uri = uri_from_cli("file:256", "testuser", "test-cluster") assert str(uri) == Path("256").absolute().as_uri() uri = uri_from_cli("file:123456", "testuser", "test-cluster") assert str(uri) == Path("123456").absolute().as_uri() uri = uri_from_cli("storage:256", "testuser", "test-cluster") assert str(uri) == "storage://test-cluster/testuser/256" uri = uri_from_cli("storage:123456", "testuser", "test-cluster") assert str(uri) == "storage://test-cluster/testuser/123456"
async def test_uri_from_cli__blob__fail(uri: str) -> None: with pytest.raises(ValueError): uri_from_cli(uri, "u", "c", allowed_schemes=("blob", ))
def test_uri_from_cli_relative_storage_uri() -> None: uri = uri_from_cli("storage:path/to/file.txt", "testuser", "test-cluster") assert str(uri) == "storage://test-cluster/testuser/path/to/file.txt" uri = uri_from_cli("storage:/path/to/file.txt", "testuser", "test-cluster") assert str(uri) == "storage://test-cluster/path/to/file.txt"
def test_uri_from_cli_absolute_file_uri() -> None: uri = uri_from_cli("file:/path/to/file.txt", "testuser", "test-cluster") assert str(uri) == Path("/path/to/file.txt").absolute().as_uri() uri = uri_from_cli("file:///path/to/file.txt", "testuser", "test-cluster") assert str(uri) == Path("/path/to/file.txt").absolute().as_uri()
def test_uri_from_cli_tilde_only(fake_homedir: Path) -> None: uri = uri_from_cli("~", "testuser", "test-cluster") assert str(uri) == fake_homedir.as_uri()
def test_uri_from_cli_path_with_tilde_unknown_user() -> None: with pytest.raises(ValueError, match=r"Cannot expand user for "): uri_from_cli("~unknownuser/path/to/file.txt", "testuser", "test-cluster")
def test_uri_from_cli_absolute_path_special_chars() -> None: uri = uri_from_cli("/path/to/file#%23:?@~", "testuser", "test-cluster") assert _extract_path(uri) == Path("/path/to/file#%23:?@~").absolute()
def test_uri_from_cli_relative_path() -> None: uri = uri_from_cli("path/to/file.txt", "testuser", "test-cluster") assert str(uri) == Path("path/to/file.txt").absolute().as_uri()
async def test_uri_from_cli__file__fail(path_or_uri: str) -> None: with pytest.raises(ValueError): uri_from_cli(path_or_uri, "u", "c", allowed_schemes=("file", ))
def test_uri_from_cli_absolute_file_uri_special_chars() -> None: uri = uri_from_cli("file:/path/to/file%23%25%3f:@~%C3%9F", "testuser", "test-cluster") assert uri.path.endswith("/path/to/file#%?:@~ß")
def test_uri_from_cli_path_with_tilde(fake_homedir: Path) -> None: uri = uri_from_cli("~/path/to/file.txt", "testuser", "test-cluster") assert str(uri) == (fake_homedir / "path/to/file.txt").as_uri()
def test_uri_from_cli_absolute_path() -> None: uri = uri_from_cli("/path/to/file.txt", "testuser") assert str(uri) == Path("/path/to/file.txt").absolute().as_uri()
def test_uri_from_cli_relative_path_special_chars() -> None: uri = uri_from_cli("path/to/file#%23:?@~", "testuser", "test-cluster") assert uri.path.endswith("/path/to/file#%23:?@~")