def ls(path, type): public_api = PublicApi() if type is not None: types = [public_api.artifact_type(type, path)] else: types = public_api.artifact_types(path) def human_size(bytes, units=None): units = units or ["", "KB", "MB", "GB", "TB", "PB", "EB"] return ( str(bytes) + units[0] if bytes < 1024 else human_size(bytes >> 10, units[1:]) ) for kind in types: for collection in kind.collections(): versions = public_api.artifact_versions( kind.type, "/".join([kind.entity, kind.project, collection.name]), per_page=1, ) latest = next(versions) print( "{:<15s}{:<15s}{:>15s} {:<20s}".format( kind.type, latest.updated_at, human_size(latest.size), latest.name ) )
def store_path(self, artifact, path, name=None, checksum=True, max_objects=None): # Resolve the reference until the result is a concrete asset # so that we don't have multiple hops. artifact_id, artifact_file_path = WBArtifactHandler.parse_path(path) target_artifact = PublicApi().artifact_from_id( util.hex_to_b64_id(artifact_id)) entry = target_artifact._manifest.get_entry_by_path(artifact_file_path) while entry.ref is not None and urlparse( entry.ref).scheme == self._scheme: artifact_id, artifact_file_path = WBArtifactHandler.parse_path( entry.ref) target_artifact = PublicApi().artifact_from_id( util.hex_to_b64_id(artifact_id)) entry = target_artifact._manifest.get_entry_by_path( artifact_file_path) path = "wandb-artifact://{}/{}".format( util.b64_to_hex_id(target_artifact.id), str(entry.path)) size = 0 return [ ArtifactManifestEntry( name or os.path.basename(path), path, size=size, digest=path, ) ]
def get(path, root, type): public_api = PublicApi() entity, project, artifact_name = public_api._parse_artifact_path(path) if project is None: project = click.prompt("Enter the name of the project you want to use") try: artifact_parts = artifact_name.split(":") if len(artifact_parts) > 1: version = artifact_parts[1] artifact_name = artifact_parts[0] else: version = "latest" full_path = "{entity}/{project}/{artifact}:{version}".format( entity=entity, project=project, artifact=artifact_name, version=version ) wandb.termlog( "Downloading {type} artifact {full_path}".format( type=type or "dataset", full_path=full_path ) ) artifact = public_api.artifact(full_path, type=type) path = artifact.download(root=root) wandb.termlog("Artifact downloaded to %s" % path) except ValueError: raise ClickException("Unable to download artifact")
def use_artifact(self, artifact, type=None, aliases=None): """ Declare an artifact as an input to a run, call `download` or `file` on \ the returned object to get the contents locally. Args: artifact_or_name (str or Artifact): An artifact name. May be prefixed with entity/project. Valid names can be in the following forms: name:version name:alias digest You can also pass an Artifact object created by calling `wandb.Artifact` type (str, optional): The type of artifact to use. aliases (list, optional): Aliases to apply to this artifact Returns: A :obj:`Artifact` object. """ if self.mode == "dryrun": wandb.termwarn("Using artifacts in dryrun mode is currently unsupported.") return artifact self.history.ensure_jupyter_started() if isinstance(artifact, six.string_types): # Ensure we have an entity (_load_entity puts the result into api.settings()) # then initialize PublicApi with it. self._load_entity(self.api, True) public_api = PublicApi(self.api.settings()) artifact = public_api.artifact(name=artifact) if type is not None and type != artifact.type: raise ValueError('Supplied type {} does not match type {} of artifact {}'.format( type, artifact.type, artifact.name)) self.api.use_artifact(artifact.id) return artifact else: if isinstance(aliases, six.string_types): aliases = [aliases] if isinstance(artifact, wandb.Artifact): artifact.finalize() self.send_message({ 'use_artifact': { 'type': artifact.type, 'name': artifact.name, 'server_manifest_entries': artifact.server_manifest.entries, 'manifest': artifact.manifest.to_manifest_json(include_local=True), 'digest': artifact.digest, 'metadata': artifact.metadata, 'aliases': aliases } }) return artifact elif isinstance(artifact, ApiArtifact): self.api.use_artifact(artifact.id) return artifact else: raise ValueError('You must pass an artifact name (e.g. "pedestrian-dataset:v1"), an instance of wandb.Artifact, or wandb.Api().artifact() to use_artifact')
def test_run_in_launch_context_with_artifacts_api(runner, live_mock_server, test_settings, capsys): live_mock_server.set_ctx({"swappable_artifacts": True}) arti = { "name": "test:v0", "project": "test", "entity": "test", "_version": "v0", "_type": "artifactVersion", "id": "QXJ0aWZhY3Q6NTI1MDk4", } overrides = { "overrides": { "run_config": { "epochs": 10 }, "artifacts": { "old_name:v0": arti }, }, } with runner.isolated_filesystem(): path = _project_spec.DEFAULT_LAUNCH_METADATA_PATH with open(path, "w") as fp: json.dump(overrides, fp) test_settings.update(launch=True, source=wandb.sdk.wandb_settings.Source.INIT) test_settings.update(launch_config_path=path, source=wandb.sdk.wandb_settings.Source.INIT) run = wandb.init(settings=test_settings, config={ "epochs": 2, "lr": 0.004 }) public_api = PublicApi() art = public_api.artifact("old_name:v0") arti_inst = run.use_artifact(art) assert run.config.epochs == 10 assert run.config.lr == 0.004 run.finish() assert arti_inst.name == "old_name:v0" arti_info = live_mock_server.get_ctx()["used_artifact_info"] assert arti_info["used_name"] == "old_name:v0" _, err = capsys.readouterr() assert ( "Swapping artifacts does not support swapping artifacts used as an instance of" in err)
def use_artifact(self, artifact, type=None, aliases=None): if self.mode == "dryrun": wandb.termwarn( "Using artifacts in dryrun mode is currently unsupported.") return artifact self.history.ensure_jupyter_started() if isinstance(artifact, str): if type is None: raise ValueError('type required') public_api = PublicApi() artifact = public_api.artifact(type=type, name=artifact) self.api.use_artifact(artifact.id) return artifact else: if type is not None: raise ValueError( 'cannot specify type when passing Artifact object') if isinstance(aliases, str): aliases = [aliases] if isinstance(artifact, wandb.Artifact): artifact.finalize() self.send_message({ 'use_artifact': { 'type': artifact.type, 'name': artifact.name, 'server_manifest_entries': artifact.server_manifest.entries, 'manifest': artifact.manifest.to_manifest_json(include_local=True), 'digest': artifact.digest, 'metadata': artifact.metadata, 'aliases': aliases } }) elif isinstance(artifact, ApiArtifact): self.api.use_artifact(artifact.id) return artifact else: raise ValueError( 'You must pass an artifact name (e.g. "pedestrian-dataset:v1"), an instance of wandb.Artifact, or wandb.Api().artifact() to use_artifact' )
def put(path, name, description, type, alias): if name is None: name = os.path.basename(path) public_api = PublicApi() entity, project, artifact_name = public_api._parse_artifact_path(name) if project is None: project = click.prompt("Enter the name of the project you want to use") # TODO: settings nightmare... api = InternalApi() api.set_setting("entity", entity) api.set_setting("project", project) artifact = wandb.Artifact(name=artifact_name, type=type, description=description) artifact_path = "{entity}/{project}/{name}:{alias}".format(entity=entity, project=project, name=artifact_name, alias=alias[0]) if os.path.isdir(path): wandb.termlog("Uploading directory {path} to: \"{artifact_path}\" ({type})".format( path=path, type=type, artifact_path=artifact_path)) artifact.add_dir(path) elif os.path.isfile(path): wandb.termlog("Uploading file {path} to: \"{artifact_path}\" ({type})".format( path=path, type=type, artifact_path=artifact_path)) artifact.add_file(path) elif "://" in path: wandb.termlog("Logging reference artifact from {path} to: \"{artifact_path}\" ({type})".format( path=path, type=type, artifact_path=artifact_path)) artifact.add_reference(path) else: raise ClickException("Path argument must be a file or directory") run = wandb.init(entity=entity, project=project, config={"path": path}, job_type="cli_put") # We create the artifact manually to get the current version res = api.create_artifact(type, artifact_name, artifact.digest, entity_name=entity, project_name=project, run_name=run.id, description=description, aliases=[{"artifactCollectionName": artifact_name, "alias": a} for a in alias]) artifact_path = artifact_path.split(":")[0] + ":" + res.get("version", "latest") # Re-create the artifact and actually upload any files needed run.log_artifact(artifact, aliases=alias) wandb.termlog("Artifact uploaded, use this artifact in a run by adding:\n", prefix=False) wandb.termlog(" artifact = run.use_artifact(\"{path}\")\n".format( path=artifact_path, ), prefix=False)
def load_path(self, artifact, manifest_entry, local=False): path, hit = self._cache.check_etag_obj_path(manifest_entry.digest, manifest_entry.size) if hit: return path artifact_id, artifact_file_path = WBArtifactHandler.parse_path( manifest_entry.ref) artifact = PublicApi().artifact_from_id( util.hex_to_b64_id(artifact_id)) artifact_path = artifact.download() link_target_path = os.path.join(artifact_path, artifact_file_path) link_creation_path = os.path.join(self._cache._cache_dir, "tmp", link_target_path) filesystem._safe_makedirs(os.path.dirname(link_creation_path)) if os.path.islink(link_creation_path): os.unlink(link_creation_path) os.symlink(os.path.abspath(link_target_path), link_creation_path) return link_creation_path
def use_artifact(self, artifact, type=None, aliases=None): if self.mode == "dryrun": wandb.termwarn("Using artifacts in dryrun mode is currently unsupported.") return artifact self.history.ensure_jupyter_started() if isinstance(artifact, str): # Ensure we have an entity (_load_entity puts the result into api.settings()) # then initialize PublicApi with it. self._load_entity(self.api, True) public_api = PublicApi(self.api.settings()) artifact = public_api.artifact(name=artifact) if type is not None and type != artifact.type: raise ValueError('Supplied type {} does not match type {} of artifact {}'.format( type, artifact.type, artifact.name)) self.api.use_artifact(artifact.id) return artifact else: if isinstance(aliases, str): aliases = [aliases] if isinstance(artifact, wandb.Artifact): artifact.finalize() self.send_message({ 'use_artifact': { 'type': artifact.type, 'name': artifact.name, 'server_manifest_entries': artifact.server_manifest.entries, 'manifest': artifact.manifest.to_manifest_json(include_local=True), 'digest': artifact.digest, 'metadata': artifact.metadata, 'aliases': aliases } }) elif isinstance(artifact, ApiArtifact): self.api.use_artifact(artifact.id) return artifact else: raise ValueError('You must pass an artifact name (e.g. "pedestrian-dataset:v1"), an instance of wandb.Artifact, or wandb.Api().artifact() to use_artifact')
def client(self): if self._client is None: self._client = PublicApi() return self._client