def ls(path, type):
public_api = PublicApi()
if type is not None:
types = [public_api.artifact_type(type, path)]
else:
types = public_api.artifact_types(path)
def human_size(bytes, units=None):
units = units or ["", "KB", "MB", "GB", "TB", "PB", "EB"]
return (
str(bytes) + units[0]
if bytes < 1024
else human_size(bytes >> 10, units[1:])
)
for kind in types:
for collection in kind.collections():
versions = public_api.artifact_versions(
kind.type,
"/".join([kind.entity, kind.project, collection.name]),
per_page=1,
)
latest = next(versions)
print(
"{:<15s}{:<15s}{:>15s} {:<20s}".format(
kind.type, latest.updated_at, human_size(latest.size), latest.name
)
)
def store_path(self,
artifact,
path,
name=None,
checksum=True,
max_objects=None):
# Resolve the reference until the result is a concrete asset
# so that we don't have multiple hops.
artifact_id, artifact_file_path = WBArtifactHandler.parse_path(path)
target_artifact = PublicApi().artifact_from_id(
util.hex_to_b64_id(artifact_id))
entry = target_artifact._manifest.get_entry_by_path(artifact_file_path)
while entry.ref is not None and urlparse(
entry.ref).scheme == self._scheme:
artifact_id, artifact_file_path = WBArtifactHandler.parse_path(
entry.ref)
target_artifact = PublicApi().artifact_from_id(
util.hex_to_b64_id(artifact_id))
entry = target_artifact._manifest.get_entry_by_path(
artifact_file_path)
path = "wandb-artifact://{}/{}".format(
util.b64_to_hex_id(target_artifact.id), str(entry.path))
size = 0
return [
ArtifactManifestEntry(
name or os.path.basename(path),
path,
size=size,
digest=path,
)
]
def get(path, root, type):
public_api = PublicApi()
entity, project, artifact_name = public_api._parse_artifact_path(path)
if project is None:
project = click.prompt("Enter the name of the project you want to use")
try:
artifact_parts = artifact_name.split(":")
if len(artifact_parts) > 1:
version = artifact_parts[1]
artifact_name = artifact_parts[0]
else:
version = "latest"
full_path = "{entity}/{project}/{artifact}:{version}".format(
entity=entity, project=project, artifact=artifact_name, version=version
)
wandb.termlog(
"Downloading {type} artifact {full_path}".format(
type=type or "dataset", full_path=full_path
)
)
artifact = public_api.artifact(full_path, type=type)
path = artifact.download(root=root)
wandb.termlog("Artifact downloaded to %s" % path)
except ValueError:
raise ClickException("Unable to download artifact")
def use_artifact(self, artifact, type=None, aliases=None):
""" Declare an artifact as an input to a run, call `download` or `file` on \
the returned object to get the contents locally.
Args:
artifact_or_name (str or Artifact): An artifact name. May be prefixed with entity/project. Valid names
can be in the following forms:
name:version
name:alias
digest
You can also pass an Artifact object created by calling `wandb.Artifact`
type (str, optional): The type of artifact to use.
aliases (list, optional): Aliases to apply to this artifact
Returns:
A :obj:`Artifact` object.
"""
if self.mode == "dryrun":
wandb.termwarn("Using artifacts in dryrun mode is currently unsupported.")
return artifact
self.history.ensure_jupyter_started()
if isinstance(artifact, six.string_types):
# Ensure we have an entity (_load_entity puts the result into api.settings())
# then initialize PublicApi with it.
self._load_entity(self.api, True)
public_api = PublicApi(self.api.settings())
artifact = public_api.artifact(name=artifact)
if type is not None and type != artifact.type:
raise ValueError('Supplied type {} does not match type {} of artifact {}'.format(
type, artifact.type, artifact.name))
self.api.use_artifact(artifact.id)
return artifact
else:
if isinstance(aliases, six.string_types):
aliases = [aliases]
if isinstance(artifact, wandb.Artifact):
artifact.finalize()
self.send_message({
'use_artifact': {
'type': artifact.type,
'name': artifact.name,
'server_manifest_entries': artifact.server_manifest.entries,
'manifest': artifact.manifest.to_manifest_json(include_local=True),
'digest': artifact.digest,
'metadata': artifact.metadata,
'aliases': aliases
}
})
return artifact
elif isinstance(artifact, ApiArtifact):
self.api.use_artifact(artifact.id)
return artifact
else:
raise ValueError('You must pass an artifact name (e.g. "pedestrian-dataset:v1"), an instance of wandb.Artifact, or wandb.Api().artifact() to use_artifact')
def test_run_in_launch_context_with_artifacts_api(runner, live_mock_server,
test_settings, capsys):
live_mock_server.set_ctx({"swappable_artifacts": True})
arti = {
"name": "test:v0",
"project": "test",
"entity": "test",
"_version": "v0",
"_type": "artifactVersion",
"id": "QXJ0aWZhY3Q6NTI1MDk4",
}
overrides = {
"overrides": {
"run_config": {
"epochs": 10
},
"artifacts": {
"old_name:v0": arti
},
},
}
with runner.isolated_filesystem():
path = _project_spec.DEFAULT_LAUNCH_METADATA_PATH
with open(path, "w") as fp:
json.dump(overrides, fp)
test_settings.update(launch=True,
source=wandb.sdk.wandb_settings.Source.INIT)
test_settings.update(launch_config_path=path,
source=wandb.sdk.wandb_settings.Source.INIT)
run = wandb.init(settings=test_settings,
config={
"epochs": 2,
"lr": 0.004
})
public_api = PublicApi()
art = public_api.artifact("old_name:v0")
arti_inst = run.use_artifact(art)
assert run.config.epochs == 10
assert run.config.lr == 0.004
run.finish()
assert arti_inst.name == "old_name:v0"
arti_info = live_mock_server.get_ctx()["used_artifact_info"]
assert arti_info["used_name"] == "old_name:v0"
_, err = capsys.readouterr()
assert (
"Swapping artifacts does not support swapping artifacts used as an instance of"
in err)
def use_artifact(self, artifact, type=None, aliases=None):
if self.mode == "dryrun":
wandb.termwarn(
"Using artifacts in dryrun mode is currently unsupported.")
return artifact
self.history.ensure_jupyter_started()
if isinstance(artifact, str):
if type is None:
raise ValueError('type required')
public_api = PublicApi()
artifact = public_api.artifact(type=type, name=artifact)
self.api.use_artifact(artifact.id)
return artifact
else:
if type is not None:
raise ValueError(
'cannot specify type when passing Artifact object')
if isinstance(aliases, str):
aliases = [aliases]
if isinstance(artifact, wandb.Artifact):
artifact.finalize()
self.send_message({
'use_artifact': {
'type':
artifact.type,
'name':
artifact.name,
'server_manifest_entries':
artifact.server_manifest.entries,
'manifest':
artifact.manifest.to_manifest_json(include_local=True),
'digest':
artifact.digest,
'metadata':
artifact.metadata,
'aliases':
aliases
}
})
elif isinstance(artifact, ApiArtifact):
self.api.use_artifact(artifact.id)
return artifact
else:
raise ValueError(
'You must pass an artifact name (e.g. "pedestrian-dataset:v1"), an instance of wandb.Artifact, or wandb.Api().artifact() to use_artifact'
)
def put(path, name, description, type, alias):
if name is None:
name = os.path.basename(path)
public_api = PublicApi()
entity, project, artifact_name = public_api._parse_artifact_path(name)
if project is None:
project = click.prompt("Enter the name of the project you want to use")
# TODO: settings nightmare...
api = InternalApi()
api.set_setting("entity", entity)
api.set_setting("project", project)
artifact = wandb.Artifact(name=artifact_name, type=type, description=description)
artifact_path = "{entity}/{project}/{name}:{alias}".format(entity=entity,
project=project, name=artifact_name, alias=alias[0])
if os.path.isdir(path):
wandb.termlog("Uploading directory {path} to: \"{artifact_path}\" ({type})".format(
path=path, type=type, artifact_path=artifact_path))
artifact.add_dir(path)
elif os.path.isfile(path):
wandb.termlog("Uploading file {path} to: \"{artifact_path}\" ({type})".format(
path=path, type=type, artifact_path=artifact_path))
artifact.add_file(path)
elif "://" in path:
wandb.termlog("Logging reference artifact from {path} to: \"{artifact_path}\" ({type})".format(
path=path, type=type, artifact_path=artifact_path))
artifact.add_reference(path)
else:
raise ClickException("Path argument must be a file or directory")
run = wandb.init(entity=entity, project=project, config={"path": path}, job_type="cli_put")
# We create the artifact manually to get the current version
res = api.create_artifact(type, artifact_name, artifact.digest,
entity_name=entity, project_name=project, run_name=run.id, description=description,
aliases=[{"artifactCollectionName": artifact_name, "alias": a} for a in alias])
artifact_path = artifact_path.split(":")[0] + ":" + res.get("version", "latest")
# Re-create the artifact and actually upload any files needed
run.log_artifact(artifact, aliases=alias)
wandb.termlog("Artifact uploaded, use this artifact in a run by adding:\n", prefix=False)
wandb.termlog(" artifact = run.use_artifact(\"{path}\")\n".format(
path=artifact_path,
), prefix=False)
def load_path(self, artifact, manifest_entry, local=False):
path, hit = self._cache.check_etag_obj_path(manifest_entry.digest,
manifest_entry.size)
if hit:
return path
artifact_id, artifact_file_path = WBArtifactHandler.parse_path(
manifest_entry.ref)
artifact = PublicApi().artifact_from_id(
util.hex_to_b64_id(artifact_id))
artifact_path = artifact.download()
link_target_path = os.path.join(artifact_path, artifact_file_path)
link_creation_path = os.path.join(self._cache._cache_dir, "tmp",
link_target_path)
filesystem._safe_makedirs(os.path.dirname(link_creation_path))
if os.path.islink(link_creation_path):
os.unlink(link_creation_path)
os.symlink(os.path.abspath(link_target_path), link_creation_path)
return link_creation_path
def use_artifact(self, artifact, type=None, aliases=None):
if self.mode == "dryrun":
wandb.termwarn("Using artifacts in dryrun mode is currently unsupported.")
return artifact
self.history.ensure_jupyter_started()
if isinstance(artifact, str):
# Ensure we have an entity (_load_entity puts the result into api.settings())
# then initialize PublicApi with it.
self._load_entity(self.api, True)
public_api = PublicApi(self.api.settings())
artifact = public_api.artifact(name=artifact)
if type is not None and type != artifact.type:
raise ValueError('Supplied type {} does not match type {} of artifact {}'.format(
type, artifact.type, artifact.name))
self.api.use_artifact(artifact.id)
return artifact
else:
if isinstance(aliases, str):
aliases = [aliases]
if isinstance(artifact, wandb.Artifact):
artifact.finalize()
self.send_message({
'use_artifact': {
'type': artifact.type,
'name': artifact.name,
'server_manifest_entries': artifact.server_manifest.entries,
'manifest': artifact.manifest.to_manifest_json(include_local=True),
'digest': artifact.digest,
'metadata': artifact.metadata,
'aliases': aliases
}
})
elif isinstance(artifact, ApiArtifact):
self.api.use_artifact(artifact.id)
return artifact
else:
raise ValueError('You must pass an artifact name (e.g. "pedestrian-dataset:v1"), an instance of wandb.Artifact, or wandb.Api().artifact() to use_artifact')
def client(self):
if self._client is None:
self._client = PublicApi()
return self._client