def run_user_process(self, program, args, env): """Launch a user process, capture its output, and sync its files to the backend. This returns after the process has ended and syncing is done. Captures ctrl-c's, signals, etc. """ stdout_streams, stderr_streams = self._get_stdout_stderr_streams() if sys.platform == "win32": # PTYs don't work in windows so we use pipes. self._stdout_tee = io_wrap.Tee.pipe(*stdout_streams) self._stderr_tee = io_wrap.Tee.pipe(*stderr_streams) # Seems like the following actually isn't necessary on Windows # TODO(adrian): we may need to do the following if we use pipes instead of PTYs # because Python on Unix doesn't like writing UTF-8 to files # tell child python interpreters we accept utf-8 # env['PYTHONIOENCODING'] = 'UTF-8' else: self._stdout_tee = io_wrap.Tee.pty(*stdout_streams) self._stderr_tee = io_wrap.Tee.pty(*stderr_streams) self._stdout_stream.write_string( " ".join(psutil.Process(os.getpid()).cmdline()) + "\n\n") command = [program] + list(args) runner = util.find_runner(program) if runner: command = runner + command command = ' '.join(six.moves.shlex_quote(arg) for arg in command) try: self.proc = subprocess.Popen( command, env=env, stdout=self._stdout_tee.tee_file, stderr=self._stderr_tee.tee_file, shell=True, ) except (OSError, IOError): raise Exception('Could not find program: %s' % command) self._sync_etc()
def test_find_runner(): res = util.find_runner(__file__) assert "python" in res[0]
def restore(ctx, run, no_git, branch, project, entity): if ":" in run: if "/" in run: entity, rest = run.split("/", 1) else: rest = run project, run = rest.split(":", 1) elif run.count("/") > 1: entity, run = run.split("/", 1) project, run = api.parse_slug(run, project=project) commit, json_config, patch_content, metadata = api.run_config( project, run=run, entity=entity) repo = metadata.get("git", {}).get("repo") image = metadata.get("docker") RESTORE_MESSAGE = """`wandb restore` needs to be run from the same git repository as the original run. Run `git clone %s` and restore from there or pass the --no-git flag.""" % repo if no_git: commit = None elif not api.git.enabled: if repo: raise ClickException(RESTORE_MESSAGE) elif image: wandb.termlog( "Original run has no git history. Just restoring config and docker" ) if commit and api.git.enabled: subprocess.check_call(['git', 'fetch', '--all']) try: api.git.repo.commit(commit) except ValueError: wandb.termlog("Couldn't find original commit: {}".format(commit)) commit = None files = api.download_urls(project, run=run, entity=entity) for filename in files: if filename.startswith('upstream_diff_') and filename.endswith( '.patch'): commit = filename[len('upstream_diff_'):-len('.patch')] try: api.git.repo.commit(commit) except ValueError: commit = None else: break if commit: wandb.termlog( "Falling back to upstream commit: {}".format(commit)) patch_path, _ = api.download_write_file(files[filename]) else: raise ClickException(RESTORE_MESSAGE) else: if patch_content: patch_path = os.path.join(wandb.wandb_dir(), 'diff.patch') with open(patch_path, "w") as f: f.write(patch_content) else: patch_path = None branch_name = "wandb/%s" % run if branch and branch_name not in api.git.repo.branches: api.git.repo.git.checkout(commit, b=branch_name) wandb.termlog("Created branch %s" % click.style(branch_name, bold=True)) elif branch: wandb.termlog( "Using existing branch, run `git branch -D %s` from master for a clean checkout" % branch_name) api.git.repo.git.checkout(branch_name) else: wandb.termlog("Checking out %s in detached mode" % commit) api.git.repo.git.checkout(commit) if patch_path: # we apply the patch from the repository root so git doesn't exclude # things outside the current directory root = api.git.root patch_rel_path = os.path.relpath(patch_path, start=root) # --reject is necessary or else this fails any time a binary file # occurs in the diff # we use .call() instead of .check_call() for the same reason # TODO(adrian): this means there is no error checking here subprocess.call(['git', 'apply', '--reject', patch_rel_path], cwd=root) wandb.termlog("Applied patch") # TODO: we should likely respect WANDB_DIR here. util.mkdir_exists_ok("wandb") config = Config(run_dir="wandb") config.load_json(json_config) config.persist() wandb.termlog("Restored config variables to %s" % config._config_path()) if image: if not metadata["program"].startswith("<") and metadata.get( "args") is not None: # TODO: we may not want to default to python here. runner = util.find_runner(metadata["program"]) or ["python"] command = runner + [metadata["program"]] + metadata["args"] cmd = " ".join(command) else: wandb.termlog( "Couldn't find original command, just restoring environment") cmd = None wandb.termlog("Docker image found, attempting to start") ctx.invoke(docker, docker_run_args=[image], cmd=cmd) return commit, json_config, patch_content, repo, metadata
def restore(ctx, run, no_git, branch, project, entity): from wandb.old.core import wandb_dir api = _get_cling_api() if ":" in run: if "/" in run: entity, rest = run.split("/", 1) else: rest = run project, run = rest.split(":", 1) elif run.count("/") > 1: entity, run = run.split("/", 1) project, run = api.parse_slug(run, project=project) commit, json_config, patch_content, metadata = api.run_config( project, run=run, entity=entity ) print(metadata) repo = metadata.get("git", {}).get("repo") image = metadata.get("docker") restore_message = ( """`wandb restore` needs to be run from the same git repository as the original run. Run `git clone %s` and restore from there or pass the --no-git flag.""" % repo ) if no_git: commit = None elif not api.git.enabled: if repo: raise ClickException(restore_message) elif image: wandb.termlog( "Original run has no git history. Just restoring config and docker" ) if commit and api.git.enabled: subprocess.check_call(["git", "fetch", "--all"]) try: api.git.repo.commit(commit) except ValueError: wandb.termlog("Couldn't find original commit: {}".format(commit)) commit = None files = api.download_urls(project, run=run, entity=entity) for filename in files: if filename.startswith("upstream_diff_") and filename.endswith( ".patch" ): commit = filename[len("upstream_diff_") : -len(".patch")] try: api.git.repo.commit(commit) except ValueError: commit = None else: break if commit: wandb.termlog("Falling back to upstream commit: {}".format(commit)) patch_path, _ = api.download_write_file(files[filename]) else: raise ClickException(restore_message) else: if patch_content: patch_path = os.path.join(wandb_dir(), "diff.patch") with open(patch_path, "w") as f: f.write(patch_content) else: patch_path = None branch_name = "wandb/%s" % run if branch and branch_name not in api.git.repo.branches: api.git.repo.git.checkout(commit, b=branch_name) wandb.termlog("Created branch %s" % click.style(branch_name, bold=True)) elif branch: wandb.termlog( "Using existing branch, run `git branch -D %s` from master for a clean checkout" % branch_name ) api.git.repo.git.checkout(branch_name) else: wandb.termlog("Checking out %s in detached mode" % commit) api.git.repo.git.checkout(commit) if patch_path: # we apply the patch from the repository root so git doesn't exclude # things outside the current directory root = api.git.root patch_rel_path = os.path.relpath(patch_path, start=root) # --reject is necessary or else this fails any time a binary file # occurs in the diff # we use .call() instead of .check_call() for the same reason # TODO(adrian): this means there is no error checking here subprocess.call(["git", "apply", "--reject", patch_rel_path], cwd=root) wandb.termlog("Applied patch") util.mkdir_exists_ok(wandb_dir()) config_path = os.path.join(wandb_dir(), "config.yaml") config = Config() for k, v in json_config.items(): if k not in ("_wandb", "wandb_version"): config[k] = v s = b"wandb_version: 1" s += b"\n\n" + yaml.dump( config._as_dict(), Dumper=yaml.SafeDumper, default_flow_style=False, allow_unicode=True, encoding="utf-8", ) s = s.decode("utf-8") with open(config_path, "w") as f: f.write(s) wandb.termlog("Restored config variables to %s" % config_path) if image: if not metadata["program"].startswith("<") and metadata.get("args") is not None: # TODO: we may not want to default to python here. runner = util.find_runner(metadata["program"]) or ["python"] command = runner + [metadata["program"]] + metadata["args"] cmd = " ".join(command) else: wandb.termlog("Couldn't find original command, just restoring environment") cmd = None wandb.termlog("Docker image found, attempting to start") ctx.invoke(docker, docker_run_args=[image], cmd=cmd) return commit, json_config, patch_content, repo, metadata