def get_artifacts_metadata(self, project_name, experiment_id=None):
"""Retrieve all artifacts' metadata from the configured
filesystem that belong to the specified object.
Parameters
----------
project_name : str
The name of the project to retrieve all artifacts
from.
experiment_id : str, optional
The ID of the experiment to retrieve all artifacts
from. Artifacts do not need to belong to an
experiment.
Returns
-------
list of rubicon.domain.Artifact
The artifacts logged to the specified object.
"""
artifact_metadata_root = self._get_artifact_metadata_root(
project_name, experiment_id)
try:
artifact_metadata_paths = self._ls_directories_only(
artifact_metadata_root)
artifacts = [
domain.Artifact(**json.loads(data)) for data in
self.filesystem.cat(artifact_metadata_paths).values()
]
except FileNotFoundError:
return []
return artifacts
async def get_artifacts_metadata(self, project_name, experiment_id=None):
"""Overrides `rubicon.repository.BaseRepository.get_artifacts_metadata`
to asynchronously retrieve all artifacts' metadata from the configured
filesystem that belong to the specified object.
Parameters
----------
project_name : str
The name of the project to retrieve all artifacts
from.
experiment_id : str, optional
The ID of the experiment to retrieve all artifacts
from. Artifacts do not need to belong to an
experiment.
Returns
-------
list of rubicon.domain.Artifact
The artifacts logged to the specified object.
"""
artifact_metadata_root = self._get_artifact_metadata_root(project_name, experiment_id)
try:
artifact_metadata_paths = await self._ls_directories_only(artifact_metadata_root)
artifacts = [
domain.Artifact(**json.loads(data))
for data in await asyncio.gather(
*[self.filesystem._cat_file(path) for path in artifact_metadata_paths]
)
]
except FileNotFoundError:
return []
return artifacts
def get_artifact_metadata(self, project_name, artifact_id, experiment_id=None):
"""Retrieve an artifact's metadata from the configured filesystem.
Parameters
----------
project_name : str
The name of the project the artifact with ID
`artifact_id` is logged to.
artifact_id : str
The ID of the artifact to retrieve.
experiment_id : str, optional
The ID of the experiment the artifact with ID
`artifact_id` is logged to. Artifacts do not
need to belong to an experiment.
Returns
-------
rubicon.domain.Artifact
The artifact with ID `artifact_id`.
"""
artifact_metadata_path = self._get_artifact_metadata_path(
project_name, experiment_id, artifact_id
)
try:
open_file = self.filesystem.open(artifact_metadata_path)
except FileNotFoundError:
raise RubiconException(f"No artifact with id `{artifact_id}` found.")
with open_file as f:
artifact = json.load(f)
return domain.Artifact(**artifact)
async def get_artifact_metadata(self, project_name, artifact_id, experiment_id=None):
"""Overrides `rubicon.repository.BaseRepository.get_artifact_metadata`
to asynchronously retrieve an artifact's metadata from the configured
filesystem.
Parameters
----------
project_name : str
The name of the project the artifact with ID
`artifact_id` is logged to.
artifact_id : str
The ID of the artifact to retrieve.
experiment_id : str, optional
The ID of the experiment the artifact with ID
`artifact_id` is logged to. Artifacts do not
need to belong to an experiment.
Returns
-------
rubicon.domain.Artifact
The artifact with ID `artifact_id`.
"""
artifact_metadata_path = self._get_artifact_metadata_path(
project_name, experiment_id, artifact_id
)
try:
artifact = json.loads(await self.filesystem._cat_file(artifact_metadata_path))
except FileNotFoundError:
raise RubiconException(f"No artifact with id `{artifact_id}` found.")
return domain.Artifact(**artifact)
def test_properties(project_client):
parent = project_client
domain_artifact = domain.Artifact(name="test.txt")
artifact = Artifact(domain_artifact, parent)
assert artifact.id == domain_artifact.id
assert artifact.name == domain_artifact.name
assert artifact.description == domain_artifact.description
assert artifact.created_at == domain_artifact.created_at
assert artifact.parent == parent
def _create_artifact(repository, project=None, artifact_data=None):
if project is None:
project = _create_project(repository)
if artifact_data is None:
artifact_data = b"test artifact data"
artifact = domain.Artifact(name=f"{uuid.uuid4}.txt", parent_id=project.id)
repository.create_artifact(artifact, artifact_data, project.name)
return artifact
def test_delete_artifacts(asyn_client_w_mock_repo):
rubicon = asyn_client_w_mock_repo
project_name = f"Test Project {uuid.uuid4()}"
project = asyncio.run(rubicon.create_project(project_name))
artifact_domains = [
domain.Artifact(parent_id=project.id, name=f"Test Artifact {uuid.uuid4()}")
for _ in range(0, 3)
]
artifact_ids = [a.id for a in artifact_domains]
asyncio.run(ArtifactMixin.delete_artifacts(project, artifact_ids))
expected = [
call.delete_artifact(project.name, artifact_id, experiment_id=None)
for artifact_id in artifact_ids
]
assert rubicon.repository.mock_calls[1:] == expected
def test_get_artifacts(asyn_client_w_mock_repo):
rubicon = asyn_client_w_mock_repo
project_name = f"Test Project {uuid.uuid4()}"
project = asyncio.run(rubicon.create_project(project_name))
artifact_domains = [
domain.Artifact(parent_id=project.id, name=f"Test Artifact {uuid.uuid4()}")
for _ in range(0, 3)
]
rubicon.repository.get_artifacts_metadata.return_value = artifact_domains
artifacts = asyncio.run(ArtifactMixin.artifacts(project))
expected = [call.get_artifacts_metadata(project.name, experiment_id=None)]
artifact_ids = [a.id for a in artifacts]
for artifact_id in [a.id for a in artifact_domains]:
assert artifact_id in artifact_ids
artifact_ids.remove(artifact_id)
assert len(artifact_ids) == 0
assert rubicon.repository.mock_calls[1:] == expected
def _create_artifact_domain(project=None, tags=[]):
if project is None:
project = domain.Project(f"Test Project {uuid.uuid4()}")
return project, domain.Artifact(name=f"Test Artifact {uuid.uuid4()}", parent_id=project.id)
async def log_artifact(
self, data_bytes=None, data_file=None, data_path=None, name=None, description=None
):
"""Overrides `rubicon.client.ArtifactMixin.log_artifact` to
asynchronously log an artifact to this client object.
Parameters
----------
data_bytes : bytes, optional
The raw bytes to log as an artifact.
data_file : TextIOWrapper, optional
The open file to log as an artifact.
data_path : str, optional
The absolute or relative local path or S3 path
to the data to log as an artifact. S3 paths
must be prepended with 's3://'.
name : str, optional
The name of the artifact file. Required if
`data_path` is not provided.
description : str, optional
A description of the artifact. Use to provide
additional context.
Notes
-----
Only one of `data_bytes`, `data_file`, and `data_path`
should be provided. If more than one is given, the order
of precedence is `data_bytes`, `data_file`, `data_path`.
Returns
-------
rubicon.client.Artifact
The new artifact.
Examples
--------
>>> # Log with bytes
>>> await experiment.log_artifact(
... data_bytes=b'hello rubicon!', name='bytes_artifact', description="log artifact from bytes"
... )
>>> # Log with file
>>> with open('some_relevant_file', 'rb') as f:
>>> await project.log_artifact(
... data_file=f, name='file_artifact', description="log artifact from file"
... )
>>> # Log with file path
>>> await experiment.log_artifact(
... data_path="./path/to/artifact.pkl", description="log artifact from file path"
... )
"""
data_bytes, name = self._validate_data(data_bytes, data_file, data_path, name)
artifact = domain.Artifact(name=name, description=description, parent_id=self._domain.id)
project_name, experiment_id = self._get_parent_identifiers()
await self.repository.create_artifact(
artifact, data_bytes, project_name, experiment_id=experiment_id
)
return client.Artifact(artifact, self)