def __init__(self, load_settings=True, root_dir=None):
self._global_settings = Settings._settings()
self._local_settings = Settings._settings()
self.root_dir = root_dir
if load_settings:
self._global_settings.read([Settings._global_path()])
# Only attempt to read if there is a directory existing
if os.path.isdir(core.wandb_dir(self.root_dir)):
self._local_settings.read(
[Settings._local_path(self.root_dir)])
def download_write_file(self, metadata, out_dir=None):
"""Download a file from a run and write it to wandb/
Args:
metadata (obj): The metadata object for the file to download. Comes from Api.download_urls().
Returns:
A tuple of the file's local path and the streaming response. The streaming response is None if the file already existed and was up to date.
"""
fileName = metadata['name']
path = os.path.join(out_dir or wandb_dir(), fileName)
if self.file_current(fileName, metadata['md5']):
return path, None
size, response = self.download_file(metadata['url'])
with open(path, "wb") as file:
for data in response.iter_content(chunk_size=1024):
file.write(data)
return path, response
def _local_path(root_dir=None):
util.mkdir_exists_ok(core.wandb_dir(root_dir))
return os.path.join(core.wandb_dir(root_dir), "settings")
def init(ctx, project, entity, reset):
from wandb.old.core import _set_stage_dir, __stage_dir__, wandb_dir
if __stage_dir__ is None:
_set_stage_dir("wandb")
# non interactive init
if reset or project or entity:
api = InternalApi()
if reset:
api.clear_setting("entity", persist=True)
api.clear_setting("project", persist=True)
# TODO(jhr): clear more settings?
if entity:
api.set_setting("entity", entity, persist=True)
if project:
api.set_setting("project", project, persist=True)
return
if os.path.isdir(wandb_dir()) and os.path.exists(
os.path.join(wandb_dir(), "settings")
):
click.confirm(
click.style(
"This directory has been configured previously, should we re-configure it?",
bold=True,
),
abort=True,
)
else:
click.echo(
click.style("Let's setup this directory for W&B!", fg="green", bold=True)
)
api = InternalApi()
if api.api_key is None:
ctx.invoke(login)
viewer = api.viewer()
# Viewer can be `None` in case your API information became invalid, or
# in testing if you switch hosts.
if not viewer:
click.echo(
click.style(
"Your login information seems to be invalid: can you log in again please?",
fg="red",
bold=True,
)
)
ctx.invoke(login)
# This shouldn't happen.
viewer = api.viewer()
if not viewer:
click.echo(
click.style(
"We're sorry, there was a problem logging you in. Please send us a note at [email protected] and tell us how this happened.",
fg="red",
bold=True,
)
)
sys.exit(1)
# At this point we should be logged in successfully.
if len(viewer["teams"]["edges"]) > 1:
team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]]
question = {
"type": "list",
"name": "team_name",
"message": "Which team should we use?",
"choices": team_names
# TODO(jhr): disabling manual entry for cling
# 'choices': team_names + ["Manual Entry"]
}
result = whaaaaat.prompt([question])
# result can be empty on click
if result:
entity = result["team_name"]
else:
entity = "Manual Entry"
if entity == "Manual Entry":
entity = click.prompt("Enter the name of the team you want to use")
else:
entity = viewer.get("entity") or click.prompt(
"What username or team should we use?"
)
# TODO: this error handling sucks and the output isn't pretty
try:
project = prompt_for_project(ctx, entity)
except ClickWandbException:
raise ClickException("Could not find team: %s" % entity)
api.set_setting("entity", entity, persist=True)
api.set_setting("project", project, persist=True)
api.set_setting("base_url", api.settings().get("base_url"), persist=True)
util.mkdir_exists_ok(wandb_dir())
with open(os.path.join(wandb_dir(), ".gitignore"), "w") as file:
file.write("*\n!settings")
click.echo(
click.style("This directory is configured! Next, track a run:\n", fg="green")
+ textwrap.dedent(
"""\
* In your training script:
{code1}
{code2}
* then `{run}`.
"""
).format(
code1=click.style("import wandb", bold=True),
code2=click.style('wandb.init(project="%s")' % project, bold=True),
run=click.style("python <train.py>", bold=True),
)
)
def restore(ctx, run, no_git, branch, project, entity):
from wandb.old.core import wandb_dir
api = _get_cling_api()
if ":" in run:
if "/" in run:
entity, rest = run.split("/", 1)
else:
rest = run
project, run = rest.split(":", 1)
elif run.count("/") > 1:
entity, run = run.split("/", 1)
project, run = api.parse_slug(run, project=project)
commit, json_config, patch_content, metadata = api.run_config(
project, run=run, entity=entity
)
print(metadata)
repo = metadata.get("git", {}).get("repo")
image = metadata.get("docker")
restore_message = (
"""`wandb restore` needs to be run from the same git repository as the original run.
Run `git clone %s` and restore from there or pass the --no-git flag."""
% repo
)
if no_git:
commit = None
elif not api.git.enabled:
if repo:
raise ClickException(restore_message)
elif image:
wandb.termlog(
"Original run has no git history. Just restoring config and docker"
)
if commit and api.git.enabled:
subprocess.check_call(["git", "fetch", "--all"])
try:
api.git.repo.commit(commit)
except ValueError:
wandb.termlog("Couldn't find original commit: {}".format(commit))
commit = None
files = api.download_urls(project, run=run, entity=entity)
for filename in files:
if filename.startswith("upstream_diff_") and filename.endswith(
".patch"
):
commit = filename[len("upstream_diff_") : -len(".patch")]
try:
api.git.repo.commit(commit)
except ValueError:
commit = None
else:
break
if commit:
wandb.termlog("Falling back to upstream commit: {}".format(commit))
patch_path, _ = api.download_write_file(files[filename])
else:
raise ClickException(restore_message)
else:
if patch_content:
patch_path = os.path.join(wandb_dir(), "diff.patch")
with open(patch_path, "w") as f:
f.write(patch_content)
else:
patch_path = None
branch_name = "wandb/%s" % run
if branch and branch_name not in api.git.repo.branches:
api.git.repo.git.checkout(commit, b=branch_name)
wandb.termlog("Created branch %s" % click.style(branch_name, bold=True))
elif branch:
wandb.termlog(
"Using existing branch, run `git branch -D %s` from master for a clean checkout"
% branch_name
)
api.git.repo.git.checkout(branch_name)
else:
wandb.termlog("Checking out %s in detached mode" % commit)
api.git.repo.git.checkout(commit)
if patch_path:
# we apply the patch from the repository root so git doesn't exclude
# things outside the current directory
root = api.git.root
patch_rel_path = os.path.relpath(patch_path, start=root)
# --reject is necessary or else this fails any time a binary file
# occurs in the diff
# we use .call() instead of .check_call() for the same reason
# TODO(adrian): this means there is no error checking here
subprocess.call(["git", "apply", "--reject", patch_rel_path], cwd=root)
wandb.termlog("Applied patch")
util.mkdir_exists_ok(wandb_dir())
config_path = os.path.join(wandb_dir(), "config.yaml")
config = Config()
for k, v in json_config.items():
if k not in ("_wandb", "wandb_version"):
config[k] = v
s = b"wandb_version: 1"
s += b"\n\n" + yaml.dump(
config._as_dict(),
Dumper=yaml.SafeDumper,
default_flow_style=False,
allow_unicode=True,
encoding="utf-8",
)
s = s.decode("utf-8")
with open(config_path, "w") as f:
f.write(s)
wandb.termlog("Restored config variables to %s" % config_path)
if image:
if not metadata["program"].startswith("<") and metadata.get("args") is not None:
# TODO: we may not want to default to python here.
runner = util.find_runner(metadata["program"]) or ["python"]
command = runner + [metadata["program"]] + metadata["args"]
cmd = " ".join(command)
else:
wandb.termlog("Couldn't find original command, just restoring environment")
cmd = None
wandb.termlog("Docker image found, attempting to start")
ctx.invoke(docker, docker_run_args=[image], cmd=cmd)
return commit, json_config, patch_content, repo, metadata