def write_settings(entity, project, url):
if not os.path.isdir(wandb_dir()):
os.mkdir(wandb_dir())
with open(os.path.join(wandb_dir(), 'settings'), "w") as file:
print('[default]', file=file)
print('entity: {}'.format(entity), file=file)
print('project: {}'.format(project), file=file)
print('base_url: {}'.format(url), file=file)
def tag_and_push(self, name, description, force=True):
if self.git.enabled and not self.tagged:
self.tagged = True
# TODO: this is getting called twice...
print("Tagging your git repo...")
if not force and self.git.dirty:
raise CommError(
"You have un-committed changes. Use the force flag or commit your changes.")
elif self.git.dirty and os.path.exists(wandb_dir()):
self.git.repo.git.execute(['git', 'diff'], output_stream=open(
os.path.join(wandb_dir(), 'diff.patch'), 'wb'))
self.git.tag(name, description)
result = self.git.push(name)
if(result is None or len(result) is None):
print("Unable to push git tag.")
def write_settings(entity=None, project=None, settings=None):
if not os.path.isdir(wandb_dir()):
os.mkdir(wandb_dir())
with open(os.path.join(wandb_dir(), 'settings'), "w") as file:
print('[default]', file=file)
if entity is not None:
print('entity: {}'.format(entity))
print('entity: {}'.format(entity), file=file)
if project is not None:
print('project: {}'.format(project), file=file)
print('project: {}'.format(project))
if settings is not None:
for key, value in settings.items():
if key == 'base_url' or key == 'anonymous':
print('{}: {}'.format(key, value), file=file)
def gc(ctx, keep):
"""Garbage collector, cleans up your local run directory"""
directory = wandb.wandb_dir()
if not os.path.exists(directory):
raise ClickException('No wandb directory found at %s' % directory)
paths = glob.glob(directory + "/*run*")
dates = [
datetime.datetime.strptime(p.split("-")[1], '%Y%m%d_%H%M%S')
for p in paths
]
since = datetime.datetime.utcnow() - datetime.timedelta(hours=keep)
bad_paths = [paths[i] for i, d, in enumerate(dates) if d < since]
if len(bad_paths) > 0:
click.echo("Found {} runs, {} are older than {} hours".format(
len(paths), len(bad_paths), keep))
click.confirm(click.style("Are you sure you want to remove %i runs?" %
len(bad_paths),
bold=True),
abort=True)
for path in bad_paths:
shutil.rmtree(path)
click.echo(click.style("Success!", fg="green"))
else:
click.echo(
click.style("No runs older than %i hours found" % keep, fg="red"))
def run(ctx, program, args, id, resume, dir, configs, message, name, notes,
show, tags, run_group, job_type):
wandb.ensure_configured()
if configs:
config_paths = configs.split(',')
else:
config_paths = []
config = Config(config_paths=config_paths,
wandb_dir=dir or wandb.wandb_dir())
tags = [tag for tag in tags.split(",") if tag] if tags else None
# populate run parameters from env if not specified
id = id or os.environ.get(env.RUN_ID)
message = message or os.environ.get(env.DESCRIPTION)
tags = tags or env.get_tags()
run_group = run_group or os.environ.get(env.RUN_GROUP)
job_type = job_type or os.environ.get(env.JOB_TYPE)
name = name or os.environ.get(env.NAME)
notes = notes or os.environ.get(env.NOTES)
resume = resume or os.environ.get(env.RESUME)
run = wandb_run.Run(run_id=id,
mode='clirun',
config=config,
description=message,
program=program,
tags=tags,
group=run_group,
job_type=job_type,
name=name,
notes=notes,
resume=resume)
run.enable_logging()
environ = dict(os.environ)
if configs:
environ[env.CONFIG_PATHS] = configs
if show:
environ[env.SHOW_RUN] = 'True'
if not run.api.api_key:
util.prompt_api_key(run.api, input_callback=click.prompt)
try:
rm = run_manager.RunManager(run)
rm.init_run(environ)
except run_manager.Error:
exc_type, exc_value, exc_traceback = sys.exc_info()
wandb.termerror(
'An Exception was raised during setup, see %s for full traceback.'
% util.get_log_file_path())
wandb.termerror(str(exc_value))
if 'permission' in str(exc_value):
wandb.termerror(
'Are you sure you provided the correct API key to "wandb login"?'
)
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error('\n'.join(lines))
sys.exit(1)
rm.run_user_process(program, args, environ)
def run_dir_path(run_id, dry=False):
if dry:
prefix = 'dryrun'
else:
prefix = 'run'
time_str = datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S')
return os.path.join(wandb.wandb_dir(), '{}-{}-{}'.format(prefix, time_str, run_id))
def from_environment_or_defaults(cls):
conf_paths = os.environ.get(env.CONFIG_PATHS, [])
run_dir = os.environ.get(env.RUN_DIR)
if conf_paths:
conf_paths = conf_paths.split(',')
return Config(config_paths=conf_paths,
wandb_dir=wandb.wandb_dir(),
run_dir=run_dir)
def __init__(self, load_settings=True):
config_dir = os.environ.get(
env.CONFIG_DIR,
os.path.join(os.path.expanduser("~"), ".config", "wandb"))
# Ensure the config directory and settings file both exist.
util.mkdir_exists_ok(config_dir)
util.mkdir_exists_ok(wandb_dir())
self._global_settings_path = os.path.join(config_dir, 'settings')
self._global_settings = Settings._settings_wth_defaults({})
self._local_settings_path = os.path.join(wandb_dir(), 'settings')
self._local_settings = Settings._settings_wth_defaults({})
if load_settings:
self._global_settings.read([self._global_settings_path])
self._local_settings.read([self._local_settings_path])
def __init__(self,
default_settings=None,
load_settings=True,
retry_timedelta=datetime.timedelta(days=1),
environ=os.environ):
self._environ = environ
self.default_settings = {
'section': "default",
'run': "latest",
'git_remote': "origin",
'ignore_globs': [],
'base_url': "https://api.wandb.ai"
}
self.retry_timedelta = retry_timedelta
self.default_settings.update(default_settings or {})
self._settings = None
self.retry_uploads = 10
self.settings_parser = configparser.ConfigParser()
if load_settings:
potential_settings_paths = [
os.path.expanduser('~/.wandb/settings')
]
potential_settings_paths.append(
os.path.join(wandb_dir(), 'settings'))
files = self.settings_parser.read(potential_settings_paths)
self.settings_file = files[0] if len(files) > 0 else "Not found"
else:
self.settings_file = "Not found"
self.git = GitRepo(remote=self.settings("git_remote"))
# Mutable settings set by the _file_stream_api
self.dynamic_settings = {
'system_sample_seconds': 2,
'system_samples': 15,
'heartbeat_seconds': 30,
}
self.client = Client(transport=RequestsHTTPTransport(
headers={
'User-Agent': self.user_agent,
'X-WANDB-USERNAME': env.get_username(env=self._environ)
},
use_json=True,
# this timeout won't apply when the DNS lookup fails. in that case, it will be 60s
# https://bugs.python.org/issue22889
timeout=self.HTTP_TIMEOUT,
auth=("api", self.api_key or ""),
url='%s/graphql' % self.settings('base_url')))
self.gql = retry.Retry(
self.execute,
retry_timedelta=retry_timedelta,
check_retry_fn=util.no_retry_auth,
retryable_exceptions=(RetryError, requests.RequestException))
self._current_run_id = None
self._file_stream_api = None
def run(ctx, program, args, id, resume, dir, configs, message, name, notes,
show, tags, run_group, job_type):
wandb.ensure_configured()
if configs:
config_paths = configs.split(',')
else:
config_paths = []
config = Config(config_paths=config_paths,
wandb_dir=dir or wandb.wandb_dir())
tags = [tag for tag in tags.split(",") if tag] if tags else None
run = wandb_run.Run(run_id=id,
mode='clirun',
config=config,
description=message,
program=program,
tags=tags,
group=run_group,
job_type=job_type,
name=name,
notes=notes,
resume=resume)
run.enable_logging()
environ = dict(os.environ)
if configs:
environ[env.CONFIG_PATHS] = configs
if show:
environ[env.SHOW_RUN] = 'True'
run.check_anonymous()
try:
rm = run_manager.RunManager(run)
rm.init_run(environ)
except run_manager.Error:
exc_type, exc_value, exc_traceback = sys.exc_info()
wandb.termerror(
'An Exception was raised during setup, see %s for full traceback.'
% util.get_log_file_path())
wandb.termerror(str(exc_value))
if 'permission' in str(exc_value):
wandb.termerror(
'Are you sure you provided the correct API key to "wandb login"?'
)
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error('\n'.join(lines))
sys.exit(1)
rm.run_user_process(program, args, environ)
def signup(ctx):
import webbrowser
server = LocalServer()
url = api.app_url + "/login?invited"
launched = webbrowser.open_new_tab(url + "&{}".format(server.qs()))
if launched:
signal.signal(signal.SIGINT, server.stop)
click.echo('Opened [{0}] in your default browser'.format(url))
server.start(blocking=False)
key = ctx.invoke(login, server=server, browser=False)
if key:
# Only init if we aren't pre-configured
if not os.path.isdir(wandb_dir()):
ctx.invoke(init)
else:
click.echo("Signup with this url in your browser: {0}".format(url))
click.echo("Then run wandb login")
def __init__(self,
default_settings=None,
load_settings=True,
retry_timedelta=datetime.timedelta(days=1)):
self.default_settings = {
'section': "default",
'entity': "models",
'run': "latest",
'git_remote': "origin",
'git_tag': False,
'base_url': "https://api.wandb.ai"
}
self.retry_timedelta = retry_timedelta
self.default_settings.update(default_settings or {})
self._settings = None
self.retries = 3
self._settings_parser = configparser.ConfigParser()
self.tagged = False
if load_settings:
potential_settings_paths = [
os.path.expanduser('~/.wandb/settings')
]
potential_settings_paths.append(
os.path.join(wandb_dir(), 'settings'))
files = self._settings_parser.read(potential_settings_paths)
self.settings_file = files[0] if len(files) > 0 else "Not found"
else:
self.settings_file = "Not found"
self.git = GitRepo(remote=self.settings("git_remote"))
client = Client(retries=1,
transport=RequestsHTTPTransport(
headers={'User-Agent': self.user_agent},
use_json=True,
timeout=self.HTTP_TIMEOUT,
auth=("api", self.api_key),
url='%s/graphql' % self.settings('base_url')))
# 1-day worth of retry
self.gql = retry.Retry(client.execute,
retry_timedelta=retry_timedelta,
retryable_exceptions=(RetryError,
requests.HTTPError))
self._current_run_id = None
self._file_stream_api = None
def status(run, settings, project):
logged_in = bool(api.api_key)
if not os.path.isdir(wandb_dir()):
if logged_in:
msg = "Directory not initialized. Please run %s to get started." % click.style(
"wandb init", bold=True)
else:
msg = "You are not logged in. Please run %s to get started." % click.style(
"wandb login", bold=True)
termlog(msg)
elif settings:
click.echo(click.style("Logged in?", bold=True) + " %s" % logged_in)
click.echo(click.style("Current Settings", bold=True))
settings = api.settings()
click.echo(
json.dumps(settings,
sort_keys=True,
indent=2,
separators=(',', ': ')))
def run(ctx, program, args, id, resume, dir, configs, message, show):
api.ensure_configured()
if configs:
config_paths = configs.split(',')
else:
config_paths = []
config = Config(config_paths=config_paths,
wandb_dir=dir or wandb.wandb_dir())
run = wandb_run.Run(run_id=id,
mode='clirun',
config=config,
description=message,
program=program,
resume=resume)
api.set_current_run_id(run.id)
env = dict(os.environ)
if configs:
env['WANDB_CONFIG_PATHS'] = configs
if show:
env['WANDB_SHOW_RUN'] = 'True'
try:
rm = run_manager.RunManager(api, run)
rm.init_run(env)
except run_manager.Error:
exc_type, exc_value, exc_traceback = sys.exc_info()
wandb.termerror(
'An Exception was raised during setup, see %s for full traceback.'
% util.get_log_file_path())
wandb.termerror(str(exc_value))
if 'permission' in str(exc_value):
wandb.termerror(
'Are you sure you provided the correct API key to "wandb login"?'
)
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error('\n'.join(lines))
sys.exit(1)
rm.run_user_process(program, args, env)
def download_write_file(self, metadata, out_dir=None):
"""Download a file from a run and write it to wandb/
Args:
metadata (obj): The metadata object for the file to download. Comes from Api.download_urls().
Returns:
A tuple of the file's local path and the streaming response. The streaming response is None if the file already existed and was up to date.
"""
fileName = metadata['name']
path = os.path.join(out_dir or wandb_dir(), fileName)
if self.file_current(fileName, metadata['md5']):
return path, None
size, response = self.download_file(metadata['url'])
with open(path, "wb") as file:
for data in response.iter_content(chunk_size=1024):
file.write(data)
return path, response
def __init__(self,
run_id=None,
mode=None,
dir=None,
group=None,
job_type=None,
config=None,
sweep_id=None,
storage_id=None,
description=None,
resume=None,
program=None,
args=None,
wandb_dir=None,
tags=None,
name=None,
notes=None,
api=None):
"""Create a Run.
Arguments:
description (str): This is the old, deprecated style of description: the run's
name followed by a newline, followed by multiline notes.
"""
# self.storage_id is "id" in GQL.
self.storage_id = storage_id
# self.id is "name" in GQL.
self.id = run_id if run_id else util.generate_id()
# self._name is "display_name" in GQL.
self._name = None
self.notes = None
self.resume = resume if resume else 'never'
self.mode = mode if mode else 'run'
self.group = group
self.job_type = job_type
self.pid = os.getpid()
self.resumed = False # we set resume when history is first accessed
if api:
if api.current_run_id and api.current_run_id != self.id:
raise RuntimeError(
'Api object passed to run {} is already being used by run {}'
.format(self.id, api.current_run_id))
else:
api.set_current_run_id(self.id)
self._api = api
if dir is None:
self._dir = run_dir_path(self.id, dry=self.mode == 'dryrun')
else:
self._dir = os.path.abspath(dir)
self._mkdir()
# self.name and self.notes used to be combined into a single field.
# Now if name and notes don't have their own values, we get them from
# self._name_and_description, but we don't update description.md
# if they're changed. This is to discourage relying on self.description
# and self._name_and_description so that we can drop them later.
#
# This needs to be set before name and notes because name and notes may
# influence it. They have higher precedence.
self._name_and_description = None
if description:
wandb.termwarn(
'Run.description is deprecated. Please use wandb.init(notes="long notes") instead.'
)
self._name_and_description = description
elif os.path.exists(self.description_path):
with open(self.description_path) as d_file:
self._name_and_description = d_file.read()
if name is not None:
self.name = name
if notes is not None:
self.notes = notes
self.program = program
if not self.program:
try:
import __main__
self.program = __main__.__file__
except (ImportError, AttributeError):
# probably `python -c`, an embedded interpreter or something
self.program = '<python with no main file>'
self.args = args
if self.args is None:
self.args = sys.argv[1:]
self.wandb_dir = wandb_dir
with configure_scope() as scope:
self.project = self.api.settings("project")
scope.set_tag("project", self.project)
scope.set_tag("entity", self.entity)
try:
scope.set_tag("url", self.get_url(self.api, network=False)
) # TODO: Move this somewhere outside of init
except CommError:
pass
if self.resume == "auto":
util.mkdir_exists_ok(wandb.wandb_dir())
resume_path = os.path.join(wandb.wandb_dir(), RESUME_FNAME)
with open(resume_path, "w") as f:
f.write(json.dumps({"run_id": self.id}))
if config is None:
self.config = Config()
else:
self.config = config
# socket server, currently only available in headless mode
self.socket = None
self.tags = tags if tags else []
self.sweep_id = sweep_id
self._history = None
self._events = None
self._summary = None
self._meta = None
self._run_manager = None
self._jupyter_agent = None
def __init__(self,
run_id=None,
mode=None,
dir=None,
group=None,
job_type=None,
config=None,
sweep_id=None,
storage_id=None,
description=None,
resume=None,
program=None,
args=None,
wandb_dir=None,
tags=None):
# self.id is actually stored in the "name" attribute in GQL
self.id = run_id if run_id else util.generate_id()
self.display_name = self.id
self.resume = resume if resume else 'never'
self.mode = mode if mode else 'run'
self.group = group
self.job_type = job_type
self.pid = os.getpid()
self.resumed = False # we set resume when history is first accessed
self.program = program
if not self.program:
try:
import __main__
self.program = __main__.__file__
except (ImportError, AttributeError):
# probably `python -c`, an embedded interpreter or something
self.program = '<python with no main file>'
self.args = args
if self.args is None:
self.args = sys.argv[1:]
self.wandb_dir = wandb_dir
with configure_scope() as scope:
api = InternalApi()
self.project = api.settings("project")
self.entity = api.settings("entity")
scope.set_tag("project", self.project)
scope.set_tag("entity", self.entity)
scope.set_tag("url", self.get_url(api))
if dir is None:
self._dir = run_dir_path(self.id, dry=self.mode == 'dryrun')
else:
self._dir = os.path.abspath(dir)
self._mkdir()
if self.resume == "auto":
util.mkdir_exists_ok(wandb.wandb_dir())
resume_path = os.path.join(wandb.wandb_dir(), RESUME_FNAME)
with open(resume_path, "w") as f:
f.write(json.dumps({"run_id": self.id}))
if config is None:
self.config = Config()
else:
self.config = config
# this is the GQL ID:
self.storage_id = storage_id
# socket server, currently only available in headless mode
self.socket = None
self.name_and_description = ""
if description is not None:
self.name_and_description = description
elif os.path.exists(self.description_path):
with open(self.description_path) as d_file:
self.name_and_description = d_file.read()
self.tags = tags if tags else []
self.sweep_id = sweep_id
self._history = None
self._events = None
self._summary = None
self._meta = None
self._run_manager = None
self._jupyter_agent = None
def test_auto_resume_remove(wandb_init_run):
assert not os.path.exists(os.path.join(
wandb.wandb_dir(), wandb_run.RESUME_FNAME))
def test_auto_resume_first(wandb_init_run):
assert json.load(open(os.path.join(wandb.wandb_dir(), wandb_run.RESUME_FNAME)))[
"run_id"] == wandb_init_run.id
assert not wandb_init_run.resumed
def init(ctx):
from wandb import _set_stage_dir, __stage_dir__, wandb_dir
if __stage_dir__ is None:
_set_stage_dir('wandb')
if os.path.isdir(wandb_dir()) and os.path.exists(
os.path.join(wandb_dir(), "settings")):
click.confirm(click.style(
"This directory has been configured previously, should we re-configure it?",
bold=True),
abort=True)
else:
click.echo(
click.style("Let's setup this directory for W&B!",
fg="green",
bold=True))
if api.api_key is None:
ctx.invoke(login)
viewer = api.viewer()
# Viewer can be `None` in case your API information became invalid, or
# in testing if you switch hosts.
if not viewer:
click.echo(
click.style(
"Your login information seems to be invalid: can you log in again please?",
fg="red",
bold=True))
ctx.invoke(login)
# This shouldn't happen.
viewer = api.viewer()
if not viewer:
click.echo(
click.style(
"We're sorry, there was a problem logging you in. Please send us a note at [email protected] and tell us how this happened.",
fg="red",
bold=True))
sys.exit(1)
# At this point we should be logged in successfully.
if len(viewer["teams"]["edges"]) > 1:
team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]]
question = {
'type': 'list',
'name': 'team_name',
'message': "Which team should we use?",
'choices': team_names + ["Manual Entry"]
}
result = whaaaaat.prompt([question])
# result can be empty on click
if result:
entity = result['team_name']
else:
entity = "Manual Entry"
if entity == "Manual Entry":
entity = click.prompt("Enter the name of the team you want to use")
else:
entity = click.prompt("What username or team should we use?",
default=viewer.get('entity', 'models'))
# TODO: this error handling sucks and the output isn't pretty
try:
project = prompt_for_project(ctx, entity)
except wandb.cli.ClickWandbException:
raise ClickException('Could not find team: %s' % entity)
util.write_settings(entity, project, api.settings())
with open(os.path.join(wandb_dir(), '.gitignore'), "w") as file:
file.write("*\n!settings")
click.echo(
click.style("This directory is configured! Next, track a run:\n",
fg="green") +
textwrap.dedent("""\
* In your training script:
{code1}
{code2}
* then `{run}`.
""").format(
code1=click.style("import wandb", bold=True),
code2=click.style("wandb.init()", bold=True),
run=click.style("python <train.py>", bold=True),
# saving this here so I can easily put it back when we re-enable
# push/pull
# """
# * Run `{push}` to manually add a file.
# * Pull popular models into your project with: `{pull}`.
# """
# push=click.style("wandb push run_id weights.h5", bold=True),
# pull=click.style("wandb pull models/inception-v4", bold=True)
))
def restore(ctx, run, no_git, branch, project, entity):
if ":" in run:
if "/" in run:
entity, rest = run.split("/", 1)
else:
rest = run
project, run = rest.split(":", 1)
elif run.count("/") > 1:
entity, run = run.split("/", 1)
project, run = api.parse_slug(run, project=project)
commit, json_config, patch_content, metadata = api.run_config(
project, run=run, entity=entity)
repo = metadata.get("git", {}).get("repo")
image = metadata.get("docker")
RESTORE_MESSAGE = """`wandb restore` needs to be run from the same git repository as the original run.
Run `git clone %s` and restore from there or pass the --no-git flag.""" % repo
if no_git:
commit = None
elif not api.git.enabled:
if repo:
raise ClickException(RESTORE_MESSAGE)
elif image:
wandb.termlog(
"Original run has no git history. Just restoring config and docker"
)
if commit and api.git.enabled:
subprocess.check_call(['git', 'fetch', '--all'])
try:
api.git.repo.commit(commit)
except ValueError:
wandb.termlog("Couldn't find original commit: {}".format(commit))
commit = None
files = api.download_urls(project, run=run, entity=entity)
for filename in files:
if filename.startswith('upstream_diff_') and filename.endswith(
'.patch'):
commit = filename[len('upstream_diff_'):-len('.patch')]
try:
api.git.repo.commit(commit)
except ValueError:
commit = None
else:
break
if commit:
wandb.termlog(
"Falling back to upstream commit: {}".format(commit))
patch_path, _ = api.download_write_file(files[filename])
else:
raise ClickException(RESTORE_MESSAGE)
else:
if patch_content:
patch_path = os.path.join(wandb.wandb_dir(), 'diff.patch')
with open(patch_path, "w") as f:
f.write(patch_content)
else:
patch_path = None
branch_name = "wandb/%s" % run
if branch and branch_name not in api.git.repo.branches:
api.git.repo.git.checkout(commit, b=branch_name)
wandb.termlog("Created branch %s" %
click.style(branch_name, bold=True))
elif branch:
wandb.termlog(
"Using existing branch, run `git branch -D %s` from master for a clean checkout"
% branch_name)
api.git.repo.git.checkout(branch_name)
else:
wandb.termlog("Checking out %s in detached mode" % commit)
api.git.repo.git.checkout(commit)
if patch_path:
# we apply the patch from the repository root so git doesn't exclude
# things outside the current directory
root = api.git.root
patch_rel_path = os.path.relpath(patch_path, start=root)
# --reject is necessary or else this fails any time a binary file
# occurs in the diff
# we use .call() instead of .check_call() for the same reason
# TODO(adrian): this means there is no error checking here
subprocess.call(['git', 'apply', '--reject', patch_rel_path],
cwd=root)
wandb.termlog("Applied patch")
# TODO: we should likely respect WANDB_DIR here.
util.mkdir_exists_ok("wandb")
config = Config(run_dir="wandb")
config.load_json(json_config)
config.persist()
wandb.termlog("Restored config variables to %s" % config._config_path())
if image:
if not metadata["program"].startswith("<") and metadata.get(
"args") is not None:
# TODO: we may not want to default to python here.
runner = util.find_runner(metadata["program"]) or ["python"]
command = runner + [metadata["program"]] + metadata["args"]
cmd = " ".join(command)
else:
wandb.termlog(
"Couldn't find original command, just restoring environment")
cmd = None
wandb.termlog("Docker image found, attempting to start")
ctx.invoke(docker, docker_run_args=[image], cmd=cmd)
return commit, json_config, patch_content, repo, metadata
def restore(run, branch, project, entity):
project, run = api.parse_slug(run, project=project)
commit, json_config, patch_content = api.run_config(project,
run=run,
entity=entity)
subprocess.check_call(['git', 'fetch', '--all'])
if commit:
try:
api.git.repo.commit(commit)
except ValueError:
click.echo("Couldn't find original commit: {}".format(commit))
commit = None
files = api.download_urls(project, run=run, entity=entity)
for filename in files:
if filename.startswith('upstream_diff_') and filename.endswith(
'.patch'):
commit = filename[len('upstream_diff_'):-len('.patch')]
try:
api.git.repo.commit(commit)
except ValueError:
commit = None
else:
break
if commit:
click.echo(
"Falling back to upstream commit: {}".format(commit))
patch_path, _ = api.download_write_file(files[filename])
else:
raise ClickException(
"Can't find commit from which to restore code")
else:
if patch_content:
patch_path = os.path.join(wandb.wandb_dir(), 'diff.patch')
with open(patch_path, "w") as f:
f.write(patch_content)
else:
patch_path = None
branch_name = "wandb/%s" % run
if branch and branch_name not in api.git.repo.branches:
api.git.repo.git.checkout(commit, b=branch_name)
click.echo("Created branch %s" %
click.style(branch_name, bold=True))
elif branch:
click.secho(
"Using existing branch, run `git branch -D %s` from master for a clean checkout"
% branch_name,
fg="red")
api.git.repo.git.checkout(branch_name)
else:
click.secho("Checking out %s in detached mode" % commit)
api.git.repo.git.checkout(commit)
if patch_path:
# we apply the patch from the repository root so git doesn't exclude
# things outside the current directory
root = api.git.root
patch_rel_path = os.path.relpath(patch_path, start=root)
# --reject is necessary or else this fails any time a binary file
# occurs in the diff
# we use .call() instead of .check_call() for the same reason
# TODO(adrian): this means there is no error checking here
subprocess.call(['git', 'apply', '--reject', patch_rel_path],
cwd=root)
click.echo("Applied patch")
config = Config()
config.load_json(json_config)
config.persist()
click.echo("Restored config variables")
def test_remove_auto_resume(mocker, run_manager):
resume_path = os.path.join(wandb.wandb_dir(), RESUME_FNAME)
with open(resume_path, "w") as f:
f.write("{}")
run_manager.test_shutdown()
assert not os.path.exists(resume_path)
from wandb import wandb_dir
import glob
import os
import json
import re
import logging
import sys
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from wandb.board.app.models import Dir, Settings, RunMutator
from wandb.board.app.util.errors import NotFoundError
from wandb import Error
base_path = wandb_dir()
data = {
'Runs': []
}
settings = Settings(base_path)
def load(path_override=None):
global data
data['Runs'] = []
root = os.path.abspath(path_override or base_path)
if os.path.exists(root):
print("Loading %s ..." % root)
else:
raise Error("Directory does not exist: %s" % root)
settings.path = root
for path in sorted(glob.glob(root + "/*run-*"), key=lambda p: p.split("run-")[1], reverse=True):
run_dir = Dir(path)
def init(ctx):
from wandb import _set_stage_dir, __stage_dir__, wandb_dir
if __stage_dir__ is None:
_set_stage_dir('wandb')
if os.path.isdir(wandb_dir()):
click.confirm(click.style(
"This directory has been configured previously, should we re-configure it?",
bold=True),
abort=True)
else:
click.echo(
click.style("Let's setup this directory for W&B!",
fg="green",
bold=True))
global api, IS_INIT
if api.api_key is None:
ctx.invoke(login)
api = Api()
IS_INIT = True
viewer = api.viewer()
if len(viewer["teams"]["edges"]) > 1:
team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]]
question = {
'type': 'list',
'name': 'team_name',
'message': "Which team should we use?",
'choices': team_names + ["Manual Entry"]
}
entity = whaaaaat.prompt([question])['team_name']
if entity == "Manual Entry":
entity = click.prompt("Enter the name of the team you want to use")
else:
entity = click.prompt("What username or team should we use?",
default=viewer.get('entity', 'models'))
# TODO: this error handling sucks and the output isn't pretty
try:
project = prompt_for_project(ctx, entity)
except wandb.cli.ClickWandbException:
raise ClickException('Could not find team: %s' % entity)
if not os.path.isdir(wandb_dir()):
os.mkdir(wandb_dir())
with open(os.path.join(wandb_dir(), 'settings'), "w") as file:
print('[default]', file=file)
print('entity: {}'.format(entity), file=file)
print('project: {}'.format(project), file=file)
print('base_url: {}'.format(api.settings()['base_url']), file=file)
with open(os.path.join(wandb_dir(), '.gitignore'), "w") as file:
file.write("*\n!settings")
click.echo(
click.style("This directory is configured! Next, track a run:\n",
fg="green") +
textwrap.dedent("""\
* In your training script:
{code1}
{code2}
* then `{run}`.
""").format(
code1=click.style("import wandb", bold=True),
code2=click.style("wandb.init()", bold=True),
run=click.style("wandb run <train.py>", bold=True),
# saving this here so I can easily put it back when we re-enable
# push/pull
#"""
#* Run `{push}` to manually add a file.
#* Pull popular models into your project with: `{pull}`.
#"""
# push=click.style("wandb push run_id weights.h5", bold=True),
# pull=click.style("wandb pull models/inception-v4", bold=True)
))
def from_environment_or_defaults(cls):
conf_paths = os.environ.get('WANDB_CONFIG_PATHS', [])
if conf_paths:
conf_paths = conf_paths.split(',')
return Config(config_paths=conf_paths, wandb_dir=wandb.wandb_dir())
def _local_path():
util.mkdir_exists_ok(wandb_dir())
return os.path.join(wandb_dir(), 'settings')
def wandb_init_run(request, tmpdir, request_mocker, mock_server, monkeypatch,
mocker, capsys, local_netrc):
"""Fixture that calls wandb.init(), yields a run (or an exception) that
gets created, then cleans up afterward. This is meant to test the logic
in wandb.init, it should generally not spawn a run_manager. If you need
to test run_manager logic use that fixture.
"""
# save the environment so we can restore it later. pytest
# may actually do this itself. didn't check.
orig_environ = dict(os.environ)
orig_namespace = None
run = None
# Reset the tensorboard and pytest state
wandb.tensorboard.reset_state()
wandb._global_watch_idx = 0
try:
with CliRunner().isolated_filesystem():
if request.node.get_closest_marker('jupyter'):
def fake_ipython():
class Jupyter(object):
__module__ = "jupyter"
def __init__(self):
class Hook(object):
def register(self, what, where):
pass
class Pub(object):
def publish(self, **kwargs):
pass
class Hist(object):
def get_range(self, **kwargs):
return [[None, 1, ('#source code', None)]]
self.events = Hook()
self.display_pub = Pub()
self.history_manager = Hist()
def register_magics(self, magic):
pass
return Jupyter()
wandb.get_ipython = fake_ipython
wandb.jupyter.get_ipython = fake_ipython
# no i/o wrapping - it breaks pytest
os.environ['WANDB_MODE'] = 'clirun'
if request.node.get_closest_marker('headless'):
mocker.patch('subprocess.Popen')
else:
def mock_headless(run, cloud=True):
print("_init_headless called with cloud=%s" % cloud)
mocker.patch('wandb._init_headless', mock_headless)
if not request.node.get_closest_marker('unconfigured'):
os.environ['WANDB_API_KEY'] = 'test'
os.environ['WANDB_ENTITY'] = 'test'
os.environ['WANDB_PROJECT'] = 'unit-test-project'
else:
# when unconfigured we enable run mode to test missing creds
os.environ['WANDB_MODE'] = 'run'
monkeypatch.setattr('wandb.apis.InternalApi.api_key', None)
monkeypatch.setattr(
'getpass.getpass',
lambda x: "0123456789012345678901234567890123456789")
assert InternalApi().api_key == None
os.environ['WANDB_RUN_DIR'] = str(tmpdir)
if request.node.get_closest_marker('silent'):
os.environ['WANDB_SILENT'] = "true"
orig_namespace = vars(wandb)
assert wandb.run is None
# Mock out run_manager, we add it to run to access state in tests
orig_rm = wandb.run_manager.RunManager
mock = mocker.patch('wandb.run_manager.RunManager')
def fake_init(run, port=None, output=None, cloud=True):
print("Initialized fake run manager")
rm = fake_run_manager(mocker,
run,
cloud=cloud,
rm_class=orig_rm)
rm._block_file_observer()
run.run_manager = rm
return rm
mock.side_effect = fake_init
if request.node.get_closest_marker('args'):
kwargs = request.node.get_closest_marker('args').kwargs
# Unfortunate to enable the test to work
if kwargs.get("dir"):
del os.environ['WANDB_RUN_DIR']
if kwargs.get("tensorboard"):
# The test uses tensorboardX so we need to be sure it's imported
# we use get_module because tensorboardX isn't available in py2
wandb.util.get_module("tensorboardX")
if kwargs.get("error"):
err = kwargs["error"]
del kwargs['error']
if err == "io":
@classmethod
def error(cls):
raise IOError
monkeypatch.setattr(
'wandb.wandb_run.Run.from_environment_or_defaults',
error)
elif err == "socket":
class Error(object):
@property
def port(self):
return 123
def listen(self, secs):
return False, None
monkeypatch.setattr("wandb.wandb_socket.Server", Error)
if kwargs.get('k8s') is not None:
token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token"
crt_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
orig_exist = os.path.exists
def exists(path):
return True if path in token_path else orig_exist(path)
def magic(path, *args, **kwargs):
if path == token_path:
return six.StringIO('token')
mocker.patch('wandb.util.open', magic, create=True)
mocker.patch('wandb.util.os.path.exists', exists)
os.environ["KUBERNETES_SERVICE_HOST"] = "k8s"
os.environ["KUBERNETES_PORT_443_TCP_PORT"] = "123"
os.environ["HOSTNAME"] = "test"
if kwargs["k8s"]:
request_mocker.register_uri(
"GET",
"https://*****:*****@sha256:1234"}]}}'
)
else:
request_mocker.register_uri(
"GET",
"https://k8s:123/api/v1/namespaces/default/pods/test",
content=b'{}',
status_code=500)
del kwargs["k8s"]
if kwargs.get('sagemaker'):
del kwargs['sagemaker']
config_path = "/opt/ml/input/config/hyperparameters.json"
resource_path = "/opt/ml/input/config/resourceconfig.json"
secrets_path = "secrets.env"
os.environ['TRAINING_JOB_NAME'] = 'sage'
os.environ['CURRENT_HOST'] = 'maker'
orig_exist = os.path.exists
def exists(path):
return True if path in (
config_path, secrets_path,
resource_path) else orig_exist(path)
mocker.patch('wandb.os.path.exists', exists)
def magic(path, *args, **kwargs):
if path == config_path:
return six.StringIO('{"f****n": "A"}')
elif path == resource_path:
return six.StringIO('{"hosts":["a", "b"]}')
elif path == secrets_path:
return six.StringIO('WANDB_TEST_SECRET=TRUE')
else:
return six.StringIO()
mocker.patch('wandb.open', magic, create=True)
mocker.patch('wandb.util.open', magic, create=True)
elif kwargs.get("tf_config"):
os.environ['TF_CONFIG'] = json.dumps(kwargs['tf_config'])
del kwargs['tf_config']
elif kwargs.get("env"):
for k, v in six.iteritems(kwargs["env"]):
os.environ[k] = v
del kwargs["env"]
else:
kwargs = {}
if request.node.get_closest_marker('resume'):
# env was leaking when running the whole suite...
if os.getenv(env.RUN_ID):
del os.environ[env.RUN_ID]
os.mkdir(wandb.wandb_dir())
with open(
os.path.join(wandb.wandb_dir(),
wandb_run.RESUME_FNAME), "w") as f:
f.write(json.dumps({"run_id": "test"}))
try:
print("Initializing with", kwargs)
run = wandb.init(**kwargs)
if request.node.get_closest_marker(
'resume') or request.node.get_closest_marker(
'mocked_run_manager'):
# Reset history
run._history = None
rm = wandb.run_manager.RunManager(run)
rm.init_run(os.environ)
if request.node.get_closest_marker('mock_socket'):
run.socket = mocker.MagicMock()
assert run is wandb.run
assert run.config is wandb.config
except wandb.LaunchError as e:
print("!!! wandb LaunchError raised")
run = e
yield run
if hasattr(run, "run_manager"):
print("Shutting down run manager")
run.run_manager.test_shutdown()
finally:
# restore the original environment
os.environ.clear()
os.environ.update(orig_environ)
wandb.uninit()
wandb.get_ipython = lambda: None
assert vars(wandb) == orig_namespace
