def run_user_process(self, program, args, env):
"""Launch a user process, capture its output, and sync its files to the backend.
This returns after the process has ended and syncing is done.
Captures ctrl-c's, signals, etc.
"""
stdout_streams, stderr_streams = self._get_stdout_stderr_streams()
if sys.platform == "win32":
# PTYs don't work in windows so we use pipes.
self._stdout_tee = io_wrap.Tee.pipe(*stdout_streams)
self._stderr_tee = io_wrap.Tee.pipe(*stderr_streams)
# Seems like the following actually isn't necessary on Windows
# TODO(adrian): we may need to do the following if we use pipes instead of PTYs
# because Python on Unix doesn't like writing UTF-8 to files
# tell child python interpreters we accept utf-8
# env['PYTHONIOENCODING'] = 'UTF-8'
else:
self._stdout_tee = io_wrap.Tee.pty(*stdout_streams)
self._stderr_tee = io_wrap.Tee.pty(*stderr_streams)
self._stdout_stream.write_string(
" ".join(psutil.Process(os.getpid()).cmdline()) + "\n\n")
command = [program] + list(args)
runner = util.find_runner(program)
if runner:
command = runner + command
command = ' '.join(six.moves.shlex_quote(arg) for arg in command)
try:
self.proc = subprocess.Popen(
command,
env=env,
stdout=self._stdout_tee.tee_file,
stderr=self._stderr_tee.tee_file,
shell=True,
)
except (OSError, IOError):
raise Exception('Could not find program: %s' % command)
self._sync_etc()
def test_find_runner():
res = util.find_runner(__file__)
assert "python" in res[0]
def restore(ctx, run, no_git, branch, project, entity):
if ":" in run:
if "/" in run:
entity, rest = run.split("/", 1)
else:
rest = run
project, run = rest.split(":", 1)
elif run.count("/") > 1:
entity, run = run.split("/", 1)
project, run = api.parse_slug(run, project=project)
commit, json_config, patch_content, metadata = api.run_config(
project, run=run, entity=entity)
repo = metadata.get("git", {}).get("repo")
image = metadata.get("docker")
RESTORE_MESSAGE = """`wandb restore` needs to be run from the same git repository as the original run.
Run `git clone %s` and restore from there or pass the --no-git flag.""" % repo
if no_git:
commit = None
elif not api.git.enabled:
if repo:
raise ClickException(RESTORE_MESSAGE)
elif image:
wandb.termlog(
"Original run has no git history. Just restoring config and docker"
)
if commit and api.git.enabled:
subprocess.check_call(['git', 'fetch', '--all'])
try:
api.git.repo.commit(commit)
except ValueError:
wandb.termlog("Couldn't find original commit: {}".format(commit))
commit = None
files = api.download_urls(project, run=run, entity=entity)
for filename in files:
if filename.startswith('upstream_diff_') and filename.endswith(
'.patch'):
commit = filename[len('upstream_diff_'):-len('.patch')]
try:
api.git.repo.commit(commit)
except ValueError:
commit = None
else:
break
if commit:
wandb.termlog(
"Falling back to upstream commit: {}".format(commit))
patch_path, _ = api.download_write_file(files[filename])
else:
raise ClickException(RESTORE_MESSAGE)
else:
if patch_content:
patch_path = os.path.join(wandb.wandb_dir(), 'diff.patch')
with open(patch_path, "w") as f:
f.write(patch_content)
else:
patch_path = None
branch_name = "wandb/%s" % run
if branch and branch_name not in api.git.repo.branches:
api.git.repo.git.checkout(commit, b=branch_name)
wandb.termlog("Created branch %s" %
click.style(branch_name, bold=True))
elif branch:
wandb.termlog(
"Using existing branch, run `git branch -D %s` from master for a clean checkout"
% branch_name)
api.git.repo.git.checkout(branch_name)
else:
wandb.termlog("Checking out %s in detached mode" % commit)
api.git.repo.git.checkout(commit)
if patch_path:
# we apply the patch from the repository root so git doesn't exclude
# things outside the current directory
root = api.git.root
patch_rel_path = os.path.relpath(patch_path, start=root)
# --reject is necessary or else this fails any time a binary file
# occurs in the diff
# we use .call() instead of .check_call() for the same reason
# TODO(adrian): this means there is no error checking here
subprocess.call(['git', 'apply', '--reject', patch_rel_path],
cwd=root)
wandb.termlog("Applied patch")
# TODO: we should likely respect WANDB_DIR here.
util.mkdir_exists_ok("wandb")
config = Config(run_dir="wandb")
config.load_json(json_config)
config.persist()
wandb.termlog("Restored config variables to %s" % config._config_path())
if image:
if not metadata["program"].startswith("<") and metadata.get(
"args") is not None:
# TODO: we may not want to default to python here.
runner = util.find_runner(metadata["program"]) or ["python"]
command = runner + [metadata["program"]] + metadata["args"]
cmd = " ".join(command)
else:
wandb.termlog(
"Couldn't find original command, just restoring environment")
cmd = None
wandb.termlog("Docker image found, attempting to start")
ctx.invoke(docker, docker_run_args=[image], cmd=cmd)
return commit, json_config, patch_content, repo, metadata
def restore(ctx, run, no_git, branch, project, entity):
from wandb.old.core import wandb_dir
api = _get_cling_api()
if ":" in run:
if "/" in run:
entity, rest = run.split("/", 1)
else:
rest = run
project, run = rest.split(":", 1)
elif run.count("/") > 1:
entity, run = run.split("/", 1)
project, run = api.parse_slug(run, project=project)
commit, json_config, patch_content, metadata = api.run_config(
project, run=run, entity=entity
)
print(metadata)
repo = metadata.get("git", {}).get("repo")
image = metadata.get("docker")
restore_message = (
"""`wandb restore` needs to be run from the same git repository as the original run.
Run `git clone %s` and restore from there or pass the --no-git flag."""
% repo
)
if no_git:
commit = None
elif not api.git.enabled:
if repo:
raise ClickException(restore_message)
elif image:
wandb.termlog(
"Original run has no git history. Just restoring config and docker"
)
if commit and api.git.enabled:
subprocess.check_call(["git", "fetch", "--all"])
try:
api.git.repo.commit(commit)
except ValueError:
wandb.termlog("Couldn't find original commit: {}".format(commit))
commit = None
files = api.download_urls(project, run=run, entity=entity)
for filename in files:
if filename.startswith("upstream_diff_") and filename.endswith(
".patch"
):
commit = filename[len("upstream_diff_") : -len(".patch")]
try:
api.git.repo.commit(commit)
except ValueError:
commit = None
else:
break
if commit:
wandb.termlog("Falling back to upstream commit: {}".format(commit))
patch_path, _ = api.download_write_file(files[filename])
else:
raise ClickException(restore_message)
else:
if patch_content:
patch_path = os.path.join(wandb_dir(), "diff.patch")
with open(patch_path, "w") as f:
f.write(patch_content)
else:
patch_path = None
branch_name = "wandb/%s" % run
if branch and branch_name not in api.git.repo.branches:
api.git.repo.git.checkout(commit, b=branch_name)
wandb.termlog("Created branch %s" % click.style(branch_name, bold=True))
elif branch:
wandb.termlog(
"Using existing branch, run `git branch -D %s` from master for a clean checkout"
% branch_name
)
api.git.repo.git.checkout(branch_name)
else:
wandb.termlog("Checking out %s in detached mode" % commit)
api.git.repo.git.checkout(commit)
if patch_path:
# we apply the patch from the repository root so git doesn't exclude
# things outside the current directory
root = api.git.root
patch_rel_path = os.path.relpath(patch_path, start=root)
# --reject is necessary or else this fails any time a binary file
# occurs in the diff
# we use .call() instead of .check_call() for the same reason
# TODO(adrian): this means there is no error checking here
subprocess.call(["git", "apply", "--reject", patch_rel_path], cwd=root)
wandb.termlog("Applied patch")
util.mkdir_exists_ok(wandb_dir())
config_path = os.path.join(wandb_dir(), "config.yaml")
config = Config()
for k, v in json_config.items():
if k not in ("_wandb", "wandb_version"):
config[k] = v
s = b"wandb_version: 1"
s += b"\n\n" + yaml.dump(
config._as_dict(),
Dumper=yaml.SafeDumper,
default_flow_style=False,
allow_unicode=True,
encoding="utf-8",
)
s = s.decode("utf-8")
with open(config_path, "w") as f:
f.write(s)
wandb.termlog("Restored config variables to %s" % config_path)
if image:
if not metadata["program"].startswith("<") and metadata.get("args") is not None:
# TODO: we may not want to default to python here.
runner = util.find_runner(metadata["program"]) or ["python"]
command = runner + [metadata["program"]] + metadata["args"]
cmd = " ".join(command)
else:
wandb.termlog("Couldn't find original command, just restoring environment")
cmd = None
wandb.termlog("Docker image found, attempting to start")
ctx.invoke(docker, docker_run_args=[image], cmd=cmd)
return commit, json_config, patch_content, repo, metadata
