def store_path(self,
artifact,
path,
name=None,
checksum=True,
max_objects=None):
# Resolve the reference until the result is a concrete asset
# so that we don't have multiple hops.
artifact_id, artifact_file_path = WBArtifactHandler.parse_path(path)
target_artifact = PublicApi().artifact_from_id(
util.hex_to_b64_id(artifact_id))
entry = target_artifact._manifest.get_entry_by_path(artifact_file_path)
while entry.ref is not None and urlparse(
entry.ref).scheme == self._scheme:
artifact_id, artifact_file_path = WBArtifactHandler.parse_path(
entry.ref)
target_artifact = PublicApi().artifact_from_id(
util.hex_to_b64_id(artifact_id))
entry = target_artifact._manifest.get_entry_by_path(
artifact_file_path)
path = "wandb-artifact://{}/{}".format(
util.b64_to_hex_id(target_artifact.id), str(entry.path))
size = 0
return [
ArtifactManifestEntry(
name or os.path.basename(path),
path,
size=size,
digest=path,
)
]
def load_path(self, artifact, manifest_entry, local=False):
"""
Loads the file within the specified artifact given its
corresponding entry. In this case, the referenced artifact is downloaded
and a new symlink is created and returned to the caller.
Arguments:
manifest_entry (ArtifactManifestEntry): The index entry to load
Returns:
(os.PathLike): A path to the file represented by `index_entry`
"""
# We don't check for cache hits here. Since we have 0 for size (since this
# is a cross-artifact reference which and we've made the choice to store 0
# in the size field), we can't confirm if the file is complete. So we just
# rely on the dep_artifact entry's download() method to do its own cache
# check.
# Parse the reference path and download the artifact if needed
artifact_id = util.host_from_path(manifest_entry.ref)
artifact_file_path = util.uri_from_path(manifest_entry.ref)
dep_artifact = PublicArtifact.from_id(util.hex_to_b64_id(artifact_id),
self.client)
link_target_path = dep_artifact.get_path(artifact_file_path).download()
return link_target_path
def store_path(self,
artifact,
path,
name=None,
checksum=True,
max_objects=None):
"""
Stores the file or directory at the given path within the specified artifact. In this
case we recursively resolve the reference until the result is a concrete asset so that
we don't have multiple hops. TODO-This resolution could be done in the server for
performance improvements.
Arguments:
artifact: The artifact doing the storing
path (str): The path to store
name (str): If specified, the logical name that should map to `path`
Returns:
(list[ArtifactManifestEntry]): A list of manifest entries to store within the artifact
"""
# Recursively resolve the reference until a concrete asset is found
while path is not None and urlparse(path).scheme == self._scheme:
artifact_id = util.host_from_path(path)
artifact_file_path = util.uri_from_path(path)
target_artifact = PublicArtifact.from_id(
util.hex_to_b64_id(artifact_id), self.client)
# this should only have an effect if the user added the reference by url
# string directly (in other words they did not already load the artifact into ram.)
target_artifact._load_manifest()
entry = target_artifact._manifest.get_entry_by_path(
artifact_file_path)
path = entry.ref
# Create the path reference
path = "wandb-artifact://{}/{}".format(
util.b64_to_hex_id(target_artifact.id), artifact_file_path)
# Return the new entry
return [
ArtifactManifestEntry(
name or os.path.basename(path),
path,
size=0,
digest=entry.digest,
)
]
def load_path(self, artifact, manifest_entry, local=False):
path, hit = self._cache.check_etag_obj_path(manifest_entry.digest,
manifest_entry.size)
if hit:
return path
artifact_id, artifact_file_path = WBArtifactHandler.parse_path(
manifest_entry.ref)
artifact = PublicApi().artifact_from_id(
util.hex_to_b64_id(artifact_id))
artifact_path = artifact.download()
link_target_path = os.path.join(artifact_path, artifact_file_path)
link_creation_path = os.path.join(self._cache._cache_dir, "tmp",
link_target_path)
filesystem._safe_makedirs(os.path.dirname(link_creation_path))
if os.path.islink(link_creation_path):
os.unlink(link_creation_path)
os.symlink(os.path.abspath(link_target_path), link_creation_path)
return link_creation_path
