def write_settings(entity, project, url): if not os.path.isdir(wandb_dir()): os.mkdir(wandb_dir()) with open(os.path.join(wandb_dir(), 'settings'), "w") as file: print('[default]', file=file) print('entity: {}'.format(entity), file=file) print('project: {}'.format(project), file=file) print('base_url: {}'.format(url), file=file)
def tag_and_push(self, name, description, force=True): if self.git.enabled and not self.tagged: self.tagged = True # TODO: this is getting called twice... print("Tagging your git repo...") if not force and self.git.dirty: raise CommError( "You have un-committed changes. Use the force flag or commit your changes.") elif self.git.dirty and os.path.exists(wandb_dir()): self.git.repo.git.execute(['git', 'diff'], output_stream=open( os.path.join(wandb_dir(), 'diff.patch'), 'wb')) self.git.tag(name, description) result = self.git.push(name) if(result is None or len(result) is None): print("Unable to push git tag.")
def write_settings(entity=None, project=None, settings=None): if not os.path.isdir(wandb_dir()): os.mkdir(wandb_dir()) with open(os.path.join(wandb_dir(), 'settings'), "w") as file: print('[default]', file=file) if entity is not None: print('entity: {}'.format(entity)) print('entity: {}'.format(entity), file=file) if project is not None: print('project: {}'.format(project), file=file) print('project: {}'.format(project)) if settings is not None: for key, value in settings.items(): if key == 'base_url' or key == 'anonymous': print('{}: {}'.format(key, value), file=file)
def gc(ctx, keep): """Garbage collector, cleans up your local run directory""" directory = wandb.wandb_dir() if not os.path.exists(directory): raise ClickException('No wandb directory found at %s' % directory) paths = glob.glob(directory + "/*run*") dates = [ datetime.datetime.strptime(p.split("-")[1], '%Y%m%d_%H%M%S') for p in paths ] since = datetime.datetime.utcnow() - datetime.timedelta(hours=keep) bad_paths = [paths[i] for i, d, in enumerate(dates) if d < since] if len(bad_paths) > 0: click.echo("Found {} runs, {} are older than {} hours".format( len(paths), len(bad_paths), keep)) click.confirm(click.style("Are you sure you want to remove %i runs?" % len(bad_paths), bold=True), abort=True) for path in bad_paths: shutil.rmtree(path) click.echo(click.style("Success!", fg="green")) else: click.echo( click.style("No runs older than %i hours found" % keep, fg="red"))
def run(ctx, program, args, id, resume, dir, configs, message, name, notes, show, tags, run_group, job_type): wandb.ensure_configured() if configs: config_paths = configs.split(',') else: config_paths = [] config = Config(config_paths=config_paths, wandb_dir=dir or wandb.wandb_dir()) tags = [tag for tag in tags.split(",") if tag] if tags else None # populate run parameters from env if not specified id = id or os.environ.get(env.RUN_ID) message = message or os.environ.get(env.DESCRIPTION) tags = tags or env.get_tags() run_group = run_group or os.environ.get(env.RUN_GROUP) job_type = job_type or os.environ.get(env.JOB_TYPE) name = name or os.environ.get(env.NAME) notes = notes or os.environ.get(env.NOTES) resume = resume or os.environ.get(env.RESUME) run = wandb_run.Run(run_id=id, mode='clirun', config=config, description=message, program=program, tags=tags, group=run_group, job_type=job_type, name=name, notes=notes, resume=resume) run.enable_logging() environ = dict(os.environ) if configs: environ[env.CONFIG_PATHS] = configs if show: environ[env.SHOW_RUN] = 'True' if not run.api.api_key: util.prompt_api_key(run.api, input_callback=click.prompt) try: rm = run_manager.RunManager(run) rm.init_run(environ) except run_manager.Error: exc_type, exc_value, exc_traceback = sys.exc_info() wandb.termerror( 'An Exception was raised during setup, see %s for full traceback.' % util.get_log_file_path()) wandb.termerror(str(exc_value)) if 'permission' in str(exc_value): wandb.termerror( 'Are you sure you provided the correct API key to "wandb login"?' ) lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.error('\n'.join(lines)) sys.exit(1) rm.run_user_process(program, args, environ)
def run_dir_path(run_id, dry=False): if dry: prefix = 'dryrun' else: prefix = 'run' time_str = datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S') return os.path.join(wandb.wandb_dir(), '{}-{}-{}'.format(prefix, time_str, run_id))
def from_environment_or_defaults(cls): conf_paths = os.environ.get(env.CONFIG_PATHS, []) run_dir = os.environ.get(env.RUN_DIR) if conf_paths: conf_paths = conf_paths.split(',') return Config(config_paths=conf_paths, wandb_dir=wandb.wandb_dir(), run_dir=run_dir)
def __init__(self, load_settings=True): config_dir = os.environ.get( env.CONFIG_DIR, os.path.join(os.path.expanduser("~"), ".config", "wandb")) # Ensure the config directory and settings file both exist. util.mkdir_exists_ok(config_dir) util.mkdir_exists_ok(wandb_dir()) self._global_settings_path = os.path.join(config_dir, 'settings') self._global_settings = Settings._settings_wth_defaults({}) self._local_settings_path = os.path.join(wandb_dir(), 'settings') self._local_settings = Settings._settings_wth_defaults({}) if load_settings: self._global_settings.read([self._global_settings_path]) self._local_settings.read([self._local_settings_path])
def __init__(self, default_settings=None, load_settings=True, retry_timedelta=datetime.timedelta(days=1), environ=os.environ): self._environ = environ self.default_settings = { 'section': "default", 'run': "latest", 'git_remote': "origin", 'ignore_globs': [], 'base_url': "https://api.wandb.ai" } self.retry_timedelta = retry_timedelta self.default_settings.update(default_settings or {}) self._settings = None self.retry_uploads = 10 self.settings_parser = configparser.ConfigParser() if load_settings: potential_settings_paths = [ os.path.expanduser('~/.wandb/settings') ] potential_settings_paths.append( os.path.join(wandb_dir(), 'settings')) files = self.settings_parser.read(potential_settings_paths) self.settings_file = files[0] if len(files) > 0 else "Not found" else: self.settings_file = "Not found" self.git = GitRepo(remote=self.settings("git_remote")) # Mutable settings set by the _file_stream_api self.dynamic_settings = { 'system_sample_seconds': 2, 'system_samples': 15, 'heartbeat_seconds': 30, } self.client = Client(transport=RequestsHTTPTransport( headers={ 'User-Agent': self.user_agent, 'X-WANDB-USERNAME': env.get_username(env=self._environ) }, use_json=True, # this timeout won't apply when the DNS lookup fails. in that case, it will be 60s # https://bugs.python.org/issue22889 timeout=self.HTTP_TIMEOUT, auth=("api", self.api_key or ""), url='%s/graphql' % self.settings('base_url'))) self.gql = retry.Retry( self.execute, retry_timedelta=retry_timedelta, check_retry_fn=util.no_retry_auth, retryable_exceptions=(RetryError, requests.RequestException)) self._current_run_id = None self._file_stream_api = None
def run(ctx, program, args, id, resume, dir, configs, message, name, notes, show, tags, run_group, job_type): wandb.ensure_configured() if configs: config_paths = configs.split(',') else: config_paths = [] config = Config(config_paths=config_paths, wandb_dir=dir or wandb.wandb_dir()) tags = [tag for tag in tags.split(",") if tag] if tags else None run = wandb_run.Run(run_id=id, mode='clirun', config=config, description=message, program=program, tags=tags, group=run_group, job_type=job_type, name=name, notes=notes, resume=resume) run.enable_logging() environ = dict(os.environ) if configs: environ[env.CONFIG_PATHS] = configs if show: environ[env.SHOW_RUN] = 'True' run.check_anonymous() try: rm = run_manager.RunManager(run) rm.init_run(environ) except run_manager.Error: exc_type, exc_value, exc_traceback = sys.exc_info() wandb.termerror( 'An Exception was raised during setup, see %s for full traceback.' % util.get_log_file_path()) wandb.termerror(str(exc_value)) if 'permission' in str(exc_value): wandb.termerror( 'Are you sure you provided the correct API key to "wandb login"?' ) lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.error('\n'.join(lines)) sys.exit(1) rm.run_user_process(program, args, environ)
def signup(ctx): import webbrowser server = LocalServer() url = api.app_url + "/login?invited" launched = webbrowser.open_new_tab(url + "&{}".format(server.qs())) if launched: signal.signal(signal.SIGINT, server.stop) click.echo('Opened [{0}] in your default browser'.format(url)) server.start(blocking=False) key = ctx.invoke(login, server=server, browser=False) if key: # Only init if we aren't pre-configured if not os.path.isdir(wandb_dir()): ctx.invoke(init) else: click.echo("Signup with this url in your browser: {0}".format(url)) click.echo("Then run wandb login")
def __init__(self, default_settings=None, load_settings=True, retry_timedelta=datetime.timedelta(days=1)): self.default_settings = { 'section': "default", 'entity': "models", 'run': "latest", 'git_remote': "origin", 'git_tag': False, 'base_url': "https://api.wandb.ai" } self.retry_timedelta = retry_timedelta self.default_settings.update(default_settings or {}) self._settings = None self.retries = 3 self._settings_parser = configparser.ConfigParser() self.tagged = False if load_settings: potential_settings_paths = [ os.path.expanduser('~/.wandb/settings') ] potential_settings_paths.append( os.path.join(wandb_dir(), 'settings')) files = self._settings_parser.read(potential_settings_paths) self.settings_file = files[0] if len(files) > 0 else "Not found" else: self.settings_file = "Not found" self.git = GitRepo(remote=self.settings("git_remote")) client = Client(retries=1, transport=RequestsHTTPTransport( headers={'User-Agent': self.user_agent}, use_json=True, timeout=self.HTTP_TIMEOUT, auth=("api", self.api_key), url='%s/graphql' % self.settings('base_url'))) # 1-day worth of retry self.gql = retry.Retry(client.execute, retry_timedelta=retry_timedelta, retryable_exceptions=(RetryError, requests.HTTPError)) self._current_run_id = None self._file_stream_api = None
def status(run, settings, project): logged_in = bool(api.api_key) if not os.path.isdir(wandb_dir()): if logged_in: msg = "Directory not initialized. Please run %s to get started." % click.style( "wandb init", bold=True) else: msg = "You are not logged in. Please run %s to get started." % click.style( "wandb login", bold=True) termlog(msg) elif settings: click.echo(click.style("Logged in?", bold=True) + " %s" % logged_in) click.echo(click.style("Current Settings", bold=True)) settings = api.settings() click.echo( json.dumps(settings, sort_keys=True, indent=2, separators=(',', ': ')))
def run(ctx, program, args, id, resume, dir, configs, message, show): api.ensure_configured() if configs: config_paths = configs.split(',') else: config_paths = [] config = Config(config_paths=config_paths, wandb_dir=dir or wandb.wandb_dir()) run = wandb_run.Run(run_id=id, mode='clirun', config=config, description=message, program=program, resume=resume) api.set_current_run_id(run.id) env = dict(os.environ) if configs: env['WANDB_CONFIG_PATHS'] = configs if show: env['WANDB_SHOW_RUN'] = 'True' try: rm = run_manager.RunManager(api, run) rm.init_run(env) except run_manager.Error: exc_type, exc_value, exc_traceback = sys.exc_info() wandb.termerror( 'An Exception was raised during setup, see %s for full traceback.' % util.get_log_file_path()) wandb.termerror(str(exc_value)) if 'permission' in str(exc_value): wandb.termerror( 'Are you sure you provided the correct API key to "wandb login"?' ) lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.error('\n'.join(lines)) sys.exit(1) rm.run_user_process(program, args, env)
def download_write_file(self, metadata, out_dir=None): """Download a file from a run and write it to wandb/ Args: metadata (obj): The metadata object for the file to download. Comes from Api.download_urls(). Returns: A tuple of the file's local path and the streaming response. The streaming response is None if the file already existed and was up to date. """ fileName = metadata['name'] path = os.path.join(out_dir or wandb_dir(), fileName) if self.file_current(fileName, metadata['md5']): return path, None size, response = self.download_file(metadata['url']) with open(path, "wb") as file: for data in response.iter_content(chunk_size=1024): file.write(data) return path, response
def __init__(self, run_id=None, mode=None, dir=None, group=None, job_type=None, config=None, sweep_id=None, storage_id=None, description=None, resume=None, program=None, args=None, wandb_dir=None, tags=None, name=None, notes=None, api=None): """Create a Run. Arguments: description (str): This is the old, deprecated style of description: the run's name followed by a newline, followed by multiline notes. """ # self.storage_id is "id" in GQL. self.storage_id = storage_id # self.id is "name" in GQL. self.id = run_id if run_id else util.generate_id() # self._name is "display_name" in GQL. self._name = None self.notes = None self.resume = resume if resume else 'never' self.mode = mode if mode else 'run' self.group = group self.job_type = job_type self.pid = os.getpid() self.resumed = False # we set resume when history is first accessed if api: if api.current_run_id and api.current_run_id != self.id: raise RuntimeError( 'Api object passed to run {} is already being used by run {}' .format(self.id, api.current_run_id)) else: api.set_current_run_id(self.id) self._api = api if dir is None: self._dir = run_dir_path(self.id, dry=self.mode == 'dryrun') else: self._dir = os.path.abspath(dir) self._mkdir() # self.name and self.notes used to be combined into a single field. # Now if name and notes don't have their own values, we get them from # self._name_and_description, but we don't update description.md # if they're changed. This is to discourage relying on self.description # and self._name_and_description so that we can drop them later. # # This needs to be set before name and notes because name and notes may # influence it. They have higher precedence. self._name_and_description = None if description: wandb.termwarn( 'Run.description is deprecated. Please use wandb.init(notes="long notes") instead.' ) self._name_and_description = description elif os.path.exists(self.description_path): with open(self.description_path) as d_file: self._name_and_description = d_file.read() if name is not None: self.name = name if notes is not None: self.notes = notes self.program = program if not self.program: try: import __main__ self.program = __main__.__file__ except (ImportError, AttributeError): # probably `python -c`, an embedded interpreter or something self.program = '<python with no main file>' self.args = args if self.args is None: self.args = sys.argv[1:] self.wandb_dir = wandb_dir with configure_scope() as scope: self.project = self.api.settings("project") scope.set_tag("project", self.project) scope.set_tag("entity", self.entity) try: scope.set_tag("url", self.get_url(self.api, network=False) ) # TODO: Move this somewhere outside of init except CommError: pass if self.resume == "auto": util.mkdir_exists_ok(wandb.wandb_dir()) resume_path = os.path.join(wandb.wandb_dir(), RESUME_FNAME) with open(resume_path, "w") as f: f.write(json.dumps({"run_id": self.id})) if config is None: self.config = Config() else: self.config = config # socket server, currently only available in headless mode self.socket = None self.tags = tags if tags else [] self.sweep_id = sweep_id self._history = None self._events = None self._summary = None self._meta = None self._run_manager = None self._jupyter_agent = None
def __init__(self, run_id=None, mode=None, dir=None, group=None, job_type=None, config=None, sweep_id=None, storage_id=None, description=None, resume=None, program=None, args=None, wandb_dir=None, tags=None): # self.id is actually stored in the "name" attribute in GQL self.id = run_id if run_id else util.generate_id() self.display_name = self.id self.resume = resume if resume else 'never' self.mode = mode if mode else 'run' self.group = group self.job_type = job_type self.pid = os.getpid() self.resumed = False # we set resume when history is first accessed self.program = program if not self.program: try: import __main__ self.program = __main__.__file__ except (ImportError, AttributeError): # probably `python -c`, an embedded interpreter or something self.program = '<python with no main file>' self.args = args if self.args is None: self.args = sys.argv[1:] self.wandb_dir = wandb_dir with configure_scope() as scope: api = InternalApi() self.project = api.settings("project") self.entity = api.settings("entity") scope.set_tag("project", self.project) scope.set_tag("entity", self.entity) scope.set_tag("url", self.get_url(api)) if dir is None: self._dir = run_dir_path(self.id, dry=self.mode == 'dryrun') else: self._dir = os.path.abspath(dir) self._mkdir() if self.resume == "auto": util.mkdir_exists_ok(wandb.wandb_dir()) resume_path = os.path.join(wandb.wandb_dir(), RESUME_FNAME) with open(resume_path, "w") as f: f.write(json.dumps({"run_id": self.id})) if config is None: self.config = Config() else: self.config = config # this is the GQL ID: self.storage_id = storage_id # socket server, currently only available in headless mode self.socket = None self.name_and_description = "" if description is not None: self.name_and_description = description elif os.path.exists(self.description_path): with open(self.description_path) as d_file: self.name_and_description = d_file.read() self.tags = tags if tags else [] self.sweep_id = sweep_id self._history = None self._events = None self._summary = None self._meta = None self._run_manager = None self._jupyter_agent = None
def test_auto_resume_remove(wandb_init_run): assert not os.path.exists(os.path.join( wandb.wandb_dir(), wandb_run.RESUME_FNAME))
def test_auto_resume_first(wandb_init_run): assert json.load(open(os.path.join(wandb.wandb_dir(), wandb_run.RESUME_FNAME)))[ "run_id"] == wandb_init_run.id assert not wandb_init_run.resumed
def init(ctx): from wandb import _set_stage_dir, __stage_dir__, wandb_dir if __stage_dir__ is None: _set_stage_dir('wandb') if os.path.isdir(wandb_dir()) and os.path.exists( os.path.join(wandb_dir(), "settings")): click.confirm(click.style( "This directory has been configured previously, should we re-configure it?", bold=True), abort=True) else: click.echo( click.style("Let's setup this directory for W&B!", fg="green", bold=True)) if api.api_key is None: ctx.invoke(login) viewer = api.viewer() # Viewer can be `None` in case your API information became invalid, or # in testing if you switch hosts. if not viewer: click.echo( click.style( "Your login information seems to be invalid: can you log in again please?", fg="red", bold=True)) ctx.invoke(login) # This shouldn't happen. viewer = api.viewer() if not viewer: click.echo( click.style( "We're sorry, there was a problem logging you in. Please send us a note at [email protected] and tell us how this happened.", fg="red", bold=True)) sys.exit(1) # At this point we should be logged in successfully. if len(viewer["teams"]["edges"]) > 1: team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]] question = { 'type': 'list', 'name': 'team_name', 'message': "Which team should we use?", 'choices': team_names + ["Manual Entry"] } result = whaaaaat.prompt([question]) # result can be empty on click if result: entity = result['team_name'] else: entity = "Manual Entry" if entity == "Manual Entry": entity = click.prompt("Enter the name of the team you want to use") else: entity = click.prompt("What username or team should we use?", default=viewer.get('entity', 'models')) # TODO: this error handling sucks and the output isn't pretty try: project = prompt_for_project(ctx, entity) except wandb.cli.ClickWandbException: raise ClickException('Could not find team: %s' % entity) util.write_settings(entity, project, api.settings()) with open(os.path.join(wandb_dir(), '.gitignore'), "w") as file: file.write("*\n!settings") click.echo( click.style("This directory is configured! Next, track a run:\n", fg="green") + textwrap.dedent("""\ * In your training script: {code1} {code2} * then `{run}`. """).format( code1=click.style("import wandb", bold=True), code2=click.style("wandb.init()", bold=True), run=click.style("python <train.py>", bold=True), # saving this here so I can easily put it back when we re-enable # push/pull # """ # * Run `{push}` to manually add a file. # * Pull popular models into your project with: `{pull}`. # """ # push=click.style("wandb push run_id weights.h5", bold=True), # pull=click.style("wandb pull models/inception-v4", bold=True) ))
def restore(ctx, run, no_git, branch, project, entity): if ":" in run: if "/" in run: entity, rest = run.split("/", 1) else: rest = run project, run = rest.split(":", 1) elif run.count("/") > 1: entity, run = run.split("/", 1) project, run = api.parse_slug(run, project=project) commit, json_config, patch_content, metadata = api.run_config( project, run=run, entity=entity) repo = metadata.get("git", {}).get("repo") image = metadata.get("docker") RESTORE_MESSAGE = """`wandb restore` needs to be run from the same git repository as the original run. Run `git clone %s` and restore from there or pass the --no-git flag.""" % repo if no_git: commit = None elif not api.git.enabled: if repo: raise ClickException(RESTORE_MESSAGE) elif image: wandb.termlog( "Original run has no git history. Just restoring config and docker" ) if commit and api.git.enabled: subprocess.check_call(['git', 'fetch', '--all']) try: api.git.repo.commit(commit) except ValueError: wandb.termlog("Couldn't find original commit: {}".format(commit)) commit = None files = api.download_urls(project, run=run, entity=entity) for filename in files: if filename.startswith('upstream_diff_') and filename.endswith( '.patch'): commit = filename[len('upstream_diff_'):-len('.patch')] try: api.git.repo.commit(commit) except ValueError: commit = None else: break if commit: wandb.termlog( "Falling back to upstream commit: {}".format(commit)) patch_path, _ = api.download_write_file(files[filename]) else: raise ClickException(RESTORE_MESSAGE) else: if patch_content: patch_path = os.path.join(wandb.wandb_dir(), 'diff.patch') with open(patch_path, "w") as f: f.write(patch_content) else: patch_path = None branch_name = "wandb/%s" % run if branch and branch_name not in api.git.repo.branches: api.git.repo.git.checkout(commit, b=branch_name) wandb.termlog("Created branch %s" % click.style(branch_name, bold=True)) elif branch: wandb.termlog( "Using existing branch, run `git branch -D %s` from master for a clean checkout" % branch_name) api.git.repo.git.checkout(branch_name) else: wandb.termlog("Checking out %s in detached mode" % commit) api.git.repo.git.checkout(commit) if patch_path: # we apply the patch from the repository root so git doesn't exclude # things outside the current directory root = api.git.root patch_rel_path = os.path.relpath(patch_path, start=root) # --reject is necessary or else this fails any time a binary file # occurs in the diff # we use .call() instead of .check_call() for the same reason # TODO(adrian): this means there is no error checking here subprocess.call(['git', 'apply', '--reject', patch_rel_path], cwd=root) wandb.termlog("Applied patch") # TODO: we should likely respect WANDB_DIR here. util.mkdir_exists_ok("wandb") config = Config(run_dir="wandb") config.load_json(json_config) config.persist() wandb.termlog("Restored config variables to %s" % config._config_path()) if image: if not metadata["program"].startswith("<") and metadata.get( "args") is not None: # TODO: we may not want to default to python here. runner = util.find_runner(metadata["program"]) or ["python"] command = runner + [metadata["program"]] + metadata["args"] cmd = " ".join(command) else: wandb.termlog( "Couldn't find original command, just restoring environment") cmd = None wandb.termlog("Docker image found, attempting to start") ctx.invoke(docker, docker_run_args=[image], cmd=cmd) return commit, json_config, patch_content, repo, metadata
def restore(run, branch, project, entity): project, run = api.parse_slug(run, project=project) commit, json_config, patch_content = api.run_config(project, run=run, entity=entity) subprocess.check_call(['git', 'fetch', '--all']) if commit: try: api.git.repo.commit(commit) except ValueError: click.echo("Couldn't find original commit: {}".format(commit)) commit = None files = api.download_urls(project, run=run, entity=entity) for filename in files: if filename.startswith('upstream_diff_') and filename.endswith( '.patch'): commit = filename[len('upstream_diff_'):-len('.patch')] try: api.git.repo.commit(commit) except ValueError: commit = None else: break if commit: click.echo( "Falling back to upstream commit: {}".format(commit)) patch_path, _ = api.download_write_file(files[filename]) else: raise ClickException( "Can't find commit from which to restore code") else: if patch_content: patch_path = os.path.join(wandb.wandb_dir(), 'diff.patch') with open(patch_path, "w") as f: f.write(patch_content) else: patch_path = None branch_name = "wandb/%s" % run if branch and branch_name not in api.git.repo.branches: api.git.repo.git.checkout(commit, b=branch_name) click.echo("Created branch %s" % click.style(branch_name, bold=True)) elif branch: click.secho( "Using existing branch, run `git branch -D %s` from master for a clean checkout" % branch_name, fg="red") api.git.repo.git.checkout(branch_name) else: click.secho("Checking out %s in detached mode" % commit) api.git.repo.git.checkout(commit) if patch_path: # we apply the patch from the repository root so git doesn't exclude # things outside the current directory root = api.git.root patch_rel_path = os.path.relpath(patch_path, start=root) # --reject is necessary or else this fails any time a binary file # occurs in the diff # we use .call() instead of .check_call() for the same reason # TODO(adrian): this means there is no error checking here subprocess.call(['git', 'apply', '--reject', patch_rel_path], cwd=root) click.echo("Applied patch") config = Config() config.load_json(json_config) config.persist() click.echo("Restored config variables")
def test_remove_auto_resume(mocker, run_manager): resume_path = os.path.join(wandb.wandb_dir(), RESUME_FNAME) with open(resume_path, "w") as f: f.write("{}") run_manager.test_shutdown() assert not os.path.exists(resume_path)
from wandb import wandb_dir import glob import os import json import re import logging import sys from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler from wandb.board.app.models import Dir, Settings, RunMutator from wandb.board.app.util.errors import NotFoundError from wandb import Error base_path = wandb_dir() data = { 'Runs': [] } settings = Settings(base_path) def load(path_override=None): global data data['Runs'] = [] root = os.path.abspath(path_override or base_path) if os.path.exists(root): print("Loading %s ..." % root) else: raise Error("Directory does not exist: %s" % root) settings.path = root for path in sorted(glob.glob(root + "/*run-*"), key=lambda p: p.split("run-")[1], reverse=True): run_dir = Dir(path)
def init(ctx): from wandb import _set_stage_dir, __stage_dir__, wandb_dir if __stage_dir__ is None: _set_stage_dir('wandb') if os.path.isdir(wandb_dir()): click.confirm(click.style( "This directory has been configured previously, should we re-configure it?", bold=True), abort=True) else: click.echo( click.style("Let's setup this directory for W&B!", fg="green", bold=True)) global api, IS_INIT if api.api_key is None: ctx.invoke(login) api = Api() IS_INIT = True viewer = api.viewer() if len(viewer["teams"]["edges"]) > 1: team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]] question = { 'type': 'list', 'name': 'team_name', 'message': "Which team should we use?", 'choices': team_names + ["Manual Entry"] } entity = whaaaaat.prompt([question])['team_name'] if entity == "Manual Entry": entity = click.prompt("Enter the name of the team you want to use") else: entity = click.prompt("What username or team should we use?", default=viewer.get('entity', 'models')) # TODO: this error handling sucks and the output isn't pretty try: project = prompt_for_project(ctx, entity) except wandb.cli.ClickWandbException: raise ClickException('Could not find team: %s' % entity) if not os.path.isdir(wandb_dir()): os.mkdir(wandb_dir()) with open(os.path.join(wandb_dir(), 'settings'), "w") as file: print('[default]', file=file) print('entity: {}'.format(entity), file=file) print('project: {}'.format(project), file=file) print('base_url: {}'.format(api.settings()['base_url']), file=file) with open(os.path.join(wandb_dir(), '.gitignore'), "w") as file: file.write("*\n!settings") click.echo( click.style("This directory is configured! Next, track a run:\n", fg="green") + textwrap.dedent("""\ * In your training script: {code1} {code2} * then `{run}`. """).format( code1=click.style("import wandb", bold=True), code2=click.style("wandb.init()", bold=True), run=click.style("wandb run <train.py>", bold=True), # saving this here so I can easily put it back when we re-enable # push/pull #""" #* Run `{push}` to manually add a file. #* Pull popular models into your project with: `{pull}`. #""" # push=click.style("wandb push run_id weights.h5", bold=True), # pull=click.style("wandb pull models/inception-v4", bold=True) ))
def from_environment_or_defaults(cls): conf_paths = os.environ.get('WANDB_CONFIG_PATHS', []) if conf_paths: conf_paths = conf_paths.split(',') return Config(config_paths=conf_paths, wandb_dir=wandb.wandb_dir())
def _local_path(): util.mkdir_exists_ok(wandb_dir()) return os.path.join(wandb_dir(), 'settings')
def wandb_init_run(request, tmpdir, request_mocker, mock_server, monkeypatch, mocker, capsys, local_netrc): """Fixture that calls wandb.init(), yields a run (or an exception) that gets created, then cleans up afterward. This is meant to test the logic in wandb.init, it should generally not spawn a run_manager. If you need to test run_manager logic use that fixture. """ # save the environment so we can restore it later. pytest # may actually do this itself. didn't check. orig_environ = dict(os.environ) orig_namespace = None run = None # Reset the tensorboard and pytest state wandb.tensorboard.reset_state() wandb._global_watch_idx = 0 try: with CliRunner().isolated_filesystem(): if request.node.get_closest_marker('jupyter'): def fake_ipython(): class Jupyter(object): __module__ = "jupyter" def __init__(self): class Hook(object): def register(self, what, where): pass class Pub(object): def publish(self, **kwargs): pass class Hist(object): def get_range(self, **kwargs): return [[None, 1, ('#source code', None)]] self.events = Hook() self.display_pub = Pub() self.history_manager = Hist() def register_magics(self, magic): pass return Jupyter() wandb.get_ipython = fake_ipython wandb.jupyter.get_ipython = fake_ipython # no i/o wrapping - it breaks pytest os.environ['WANDB_MODE'] = 'clirun' if request.node.get_closest_marker('headless'): mocker.patch('subprocess.Popen') else: def mock_headless(run, cloud=True): print("_init_headless called with cloud=%s" % cloud) mocker.patch('wandb._init_headless', mock_headless) if not request.node.get_closest_marker('unconfigured'): os.environ['WANDB_API_KEY'] = 'test' os.environ['WANDB_ENTITY'] = 'test' os.environ['WANDB_PROJECT'] = 'unit-test-project' else: # when unconfigured we enable run mode to test missing creds os.environ['WANDB_MODE'] = 'run' monkeypatch.setattr('wandb.apis.InternalApi.api_key', None) monkeypatch.setattr( 'getpass.getpass', lambda x: "0123456789012345678901234567890123456789") assert InternalApi().api_key == None os.environ['WANDB_RUN_DIR'] = str(tmpdir) if request.node.get_closest_marker('silent'): os.environ['WANDB_SILENT'] = "true" orig_namespace = vars(wandb) assert wandb.run is None # Mock out run_manager, we add it to run to access state in tests orig_rm = wandb.run_manager.RunManager mock = mocker.patch('wandb.run_manager.RunManager') def fake_init(run, port=None, output=None, cloud=True): print("Initialized fake run manager") rm = fake_run_manager(mocker, run, cloud=cloud, rm_class=orig_rm) rm._block_file_observer() run.run_manager = rm return rm mock.side_effect = fake_init if request.node.get_closest_marker('args'): kwargs = request.node.get_closest_marker('args').kwargs # Unfortunate to enable the test to work if kwargs.get("dir"): del os.environ['WANDB_RUN_DIR'] if kwargs.get("tensorboard"): # The test uses tensorboardX so we need to be sure it's imported # we use get_module because tensorboardX isn't available in py2 wandb.util.get_module("tensorboardX") if kwargs.get("error"): err = kwargs["error"] del kwargs['error'] if err == "io": @classmethod def error(cls): raise IOError monkeypatch.setattr( 'wandb.wandb_run.Run.from_environment_or_defaults', error) elif err == "socket": class Error(object): @property def port(self): return 123 def listen(self, secs): return False, None monkeypatch.setattr("wandb.wandb_socket.Server", Error) if kwargs.get('k8s') is not None: token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" crt_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" orig_exist = os.path.exists def exists(path): return True if path in token_path else orig_exist(path) def magic(path, *args, **kwargs): if path == token_path: return six.StringIO('token') mocker.patch('wandb.util.open', magic, create=True) mocker.patch('wandb.util.os.path.exists', exists) os.environ["KUBERNETES_SERVICE_HOST"] = "k8s" os.environ["KUBERNETES_PORT_443_TCP_PORT"] = "123" os.environ["HOSTNAME"] = "test" if kwargs["k8s"]: request_mocker.register_uri( "GET", "https://*****:*****@sha256:1234"}]}}' ) else: request_mocker.register_uri( "GET", "https://k8s:123/api/v1/namespaces/default/pods/test", content=b'{}', status_code=500) del kwargs["k8s"] if kwargs.get('sagemaker'): del kwargs['sagemaker'] config_path = "/opt/ml/input/config/hyperparameters.json" resource_path = "/opt/ml/input/config/resourceconfig.json" secrets_path = "secrets.env" os.environ['TRAINING_JOB_NAME'] = 'sage' os.environ['CURRENT_HOST'] = 'maker' orig_exist = os.path.exists def exists(path): return True if path in ( config_path, secrets_path, resource_path) else orig_exist(path) mocker.patch('wandb.os.path.exists', exists) def magic(path, *args, **kwargs): if path == config_path: return six.StringIO('{"f****n": "A"}') elif path == resource_path: return six.StringIO('{"hosts":["a", "b"]}') elif path == secrets_path: return six.StringIO('WANDB_TEST_SECRET=TRUE') else: return six.StringIO() mocker.patch('wandb.open', magic, create=True) mocker.patch('wandb.util.open', magic, create=True) elif kwargs.get("tf_config"): os.environ['TF_CONFIG'] = json.dumps(kwargs['tf_config']) del kwargs['tf_config'] elif kwargs.get("env"): for k, v in six.iteritems(kwargs["env"]): os.environ[k] = v del kwargs["env"] else: kwargs = {} if request.node.get_closest_marker('resume'): # env was leaking when running the whole suite... if os.getenv(env.RUN_ID): del os.environ[env.RUN_ID] os.mkdir(wandb.wandb_dir()) with open( os.path.join(wandb.wandb_dir(), wandb_run.RESUME_FNAME), "w") as f: f.write(json.dumps({"run_id": "test"})) try: print("Initializing with", kwargs) run = wandb.init(**kwargs) if request.node.get_closest_marker( 'resume') or request.node.get_closest_marker( 'mocked_run_manager'): # Reset history run._history = None rm = wandb.run_manager.RunManager(run) rm.init_run(os.environ) if request.node.get_closest_marker('mock_socket'): run.socket = mocker.MagicMock() assert run is wandb.run assert run.config is wandb.config except wandb.LaunchError as e: print("!!! wandb LaunchError raised") run = e yield run if hasattr(run, "run_manager"): print("Shutting down run manager") run.run_manager.test_shutdown() finally: # restore the original environment os.environ.clear() os.environ.update(orig_environ) wandb.uninit() wandb.get_ipython = lambda: None assert vars(wandb) == orig_namespace