def login(key, host, cloud, relogin, anonymously, no_offline=False): # TODO: handle no_offline anon_mode = "must" if anonymously else "never" wandb.setup(settings=wandb.Settings( _cli_only_mode=True, anonymous=anon_mode, base_url=host)) api = _get_cling_api() if host == "https://api.wandb.ai" or (host is None and cloud): api.clear_setting("base_url", globally=True, persist=True) # To avoid writing an empty local settings file, we only clear if it exists if os.path.exists(Settings._local_path()): api.clear_setting("base_url", persist=True) elif host: if not host.startswith("http"): raise ClickException("host must start with http(s)://") api.set_setting("base_url", host.strip("/"), globally=True, persist=True) key = key[0] if len(key) > 0 else None wandb.login(relogin=relogin, key=key, anonymous=anon_mode, host=host, force=True)
def run_job(self, job): run_id = job.run_id config_file = os.path.join("wandb", "sweep-" + self._sweep_id, "config-" + run_id + ".yaml") config_util.save_config_file_from_dict(config_file, job.config) os.environ[wandb.env.RUN_ID] = run_id os.environ[wandb.env.CONFIG_PATHS] = config_file os.environ[wandb.env.SWEEP_ID] = self._sweep_id wandb.setup(_reset=True) print("wandb: Agent Starting Run: {} with config:\n".format(run_id) + "\n".join([ "\t{}: {}".format(k, v["value"]) for k, v in job.config.items() ])) try: self._function() if wandb.run: wandb.join() except KeyboardInterrupt as e: print("Keyboard interrupt", e) return True except Exception as e: print("Problem", e) return True
def test_setup_offline(live_mock_server, test_settings): # this is to increase coverage login_settings = test_settings.copy() login_settings.update(mode="offline") assert wandb.setup(settings=login_settings)._instance._get_entity() is None assert wandb.setup( settings=login_settings)._instance._load_viewer() is None
def _get_cling_api(): """Get a reference to the internal api with cling settings.""" global _api if _api is None: # TODO(jhr): make a settings object that is better for non runs. wandb.setup(settings=wandb.Settings(_cli_only_mode=True)) _api = InternalApi() return _api
def main(): wandb.require("service") wandb.setup() num_proc = 4 pool = mp.Pool(processes=num_proc) result = pool.map_async(do_run, range(num_proc)) data = result.get(60) print(f"DEBUG: {data}") assert len(data) == 4
def setup(self, kwargs): self.kwargs = kwargs # built up login settings login_settings: Settings = wandb.Settings() settings_param = kwargs.pop("_settings", None) if settings_param: login_settings._apply_settings(settings_param) _logger = wandb.setup()._get_logger() login_settings._apply_login(kwargs, _logger=_logger) # make sure they are applied globally self._wl = wandb.setup(settings=login_settings) self._settings = self._wl._settings
def api_key(settings=None): if not settings: settings = wandb.setup().settings if settings.api_key: return settings.api_key if auth := requests.utils.get_netrc_auth(settings.base_url): return auth[-1]
def login(anonymous=None, key=None, relogin=None, host=None, force=None): """ Log in to W&B. Arguments: anonymous: (string, optional) Can be "must", "allow", or "never". If set to "must" we'll always login anonymously, if set to "allow" we'll only create an anonymous user if the user isn't already logged in. key: (string, optional) authentication key. relogin: (bool, optional) If true, will re-prompt for API key. host: (string, optional) The host to connect to. Returns: bool: if key is configured Raises: UsageError - if api_key can not configured and no tty """ _handle_host_wandb_setting(host) if wandb.setup()._settings._noop: return True kwargs = dict(locals()) configured = _login(**kwargs) return True if configured else False
def login(settings=None, relogin=None): wl = wandb.setup(settings=settings) settings = wl.settings() active_entity = None if is_logged_in(): active_entity = wl._get_entity() if active_entity and not relogin: login_state_str = "Currently logged in as:" login_info_str = "(use `wandb login --relogin` to force relogin)" wandb.termlog("{} {} {}".format( login_state_str, click.style(active_entity, fg="yellow"), login_info_str)) return app_url = settings.base_url.replace("//api.", "//app.") # TODO(jhr): use settings object in_jupyter = _get_python_type() != "python" authorize_str = "Go to this URL in a browser" authorize_link_str = "{}/authorize".format(app_url) if in_jupyter: print("{}: {}\n".format(authorize_str, authorize_link_str)) key = getpass.getpass("Enter your authorization code:\n") else: wandb.termlog("{}: {}".format( authorize_str, click.style(authorize_link_str, fg="blue"))) key = prompt(u"Enter api key: ", is_password=True) apikey.write_key(settings, key) return
def login( anonymous: Optional[Literal["must", "allow", "never"]] = None, key: Optional[str] = None, relogin: Optional[bool] = None, host: Optional[str] = None, force: Optional[bool] = None, timeout: Optional[int] = None, ) -> bool: """ Log in to W&B. Arguments: anonymous: (string, optional) Can be "must", "allow", or "never". If set to "must" we'll always login anonymously, if set to "allow" we'll only create an anonymous user if the user isn't already logged in. key: (string, optional) authentication key. relogin: (bool, optional) If true, will re-prompt for API key. host: (string, optional) The host to connect to. force: (bool, optional) If true, will force a relogin. timeout: (int, optional) Number of seconds to wait for user input. Returns: bool: if key is configured Raises: UsageError - if api_key can not configured and no tty """ _handle_host_wandb_setting(host) if wandb.setup()._settings._noop: return True kwargs = dict(locals()) configured = _login(**kwargs) return True if configured else False
def __init__(self): self._done = False self.record_q = None self.result_q = None self.wandb_process = None self.interface = None self._internal_pid = None self._wl = wandb.setup()
def test_login_timeout_choose(mock_tty): mock_tty("3\n") start_time = time.time() ret = wandb.login(timeout=8) elapsed = time.time() - start_time assert elapsed < 15 assert ret is False assert wandb.api.api_key is None assert wandb.setup().settings.mode == "offline"
def test_login_timeout(mock_tty): mock_tty("junk\nmore\n") start_time = time.time() ret = wandb.login(timeout=4) elapsed = time.time() - start_time assert 2 < elapsed < 15 assert ret is False assert wandb.api.api_key is None assert wandb.setup().settings.mode == "disabled"
def test_login_timeout_env_blank(mock_tty, reset_login_timeout): mock_tty("\n\n\n") os.environ["WANDB_LOGIN_TIMEOUT"] = "4" start_time = time.time() ret = wandb.login() elapsed = time.time() - start_time assert elapsed < 15 assert ret is False assert wandb.api.api_key is None assert wandb.setup().settings.mode == "disabled"
def update_session(self, key): _logger = wandb.setup()._get_logger() settings: Settings = wandb.Settings() login_settings = dict(api_key=key) if key else dict(mode="offline") settings._apply_source_login(login_settings, _logger=_logger) self._wl._update(settings=settings) # Whenever the key changes, make sure to pull in user settings # from server. if not self._wl.settings._offline: self._wl._update_user_settings()
def api_key(settings=None): if not settings: wl = wandb.setup() settings = wl.settings() if settings.api_key: return settings.api_key auth = requests.utils.get_netrc_auth(settings.base_url) if auth: return auth[-1] return None
def setup(self, kwargs): self.kwargs = kwargs # built up login settings login_settings: Settings = wandb.Settings() settings_param = kwargs.pop("_settings", None) # note that this case does not come up anywhere except for the tests if settings_param is not None: if isinstance(settings_param, Settings): login_settings._apply_settings(settings_param) elif isinstance(settings_param, dict): login_settings.update(settings_param, source=Source.LOGIN) _logger = wandb.setup()._get_logger() # Do not save relogin into settings as we just want to relogin once self._relogin = kwargs.pop("relogin", None) login_settings._apply_login(kwargs, _logger=_logger) # make sure they are applied globally self._wl = wandb.setup(settings=login_settings) self._settings = self._wl.settings
class _WandbLogin(object): def __init__(self): self.kwargs: Optional[Dict] = None self._settings: Optional[Settings] = None self._backend = None self._silent = None self._wl = None self._key = None def setup(self, kwargs): self.kwargs = kwargs # built up login settings login_settings: Settings = wandb.Settings() if settings_param := kwargs.pop("_settings", None): login_settings._apply_settings(settings_param) _logger = wandb.setup()._get_logger() login_settings._apply_login(kwargs, _logger=_logger) # make sure they are applied globally self._wl = wandb.setup(settings=login_settings) self._settings = self._wl._settings
def run(self): wandb.require("service") wandb.init(*self.args, **self.kwargs) wandb.setup() while True: item_type, item_content = self.queue.get() if item_type == _QueueItem.END: break if item_type == _QueueItem.CHECKPOINT: self._handle_checkpoint(item_content) continue assert item_type == _QueueItem.RESULT log, config_update = self._handle_result(item_content) try: wandb.config.update(config_update, allow_val_change=True) wandb.log(log) except urllib.error.HTTPError as e: # Ignore HTTPError. Missing a few data points is not a # big issue, as long as things eventually recover. logger.warn("Failed to log result to w&b: {}".format(str(e))) wandb.finish()
def __init__(self, mode=None): self.wandb_process = None self.fd_pipe_parent = None self.process_queue = None # self.fd_request_queue = None # self.fd_response_queue = None self.req_queue = None self.resp_queue = None self.cancel_queue = None self.notify_queue = None # notify activity on ... self._done = False self._wl = wandb.setup() self.interface = None
def __init__(self, mode=None): self.calls = {} self._run = None self._done = True self._wl = wandb.setup() self.record_q = self._wl._multiprocessing.Queue() self.result_q = self._wl._multiprocessing.Queue() self.interface = None self.last_queued = None self.history = [] self.summary = {} self.config = {} self.files = {} self.mocker = _get_mock_module(get_config()) self._internal_pid = None
def update_session(self, key: Optional[str], status: ApiKeyStatus = ApiKeyStatus.VALID) -> None: _logger = wandb.setup()._get_logger() login_settings = dict() if status == ApiKeyStatus.OFFLINE: login_settings = dict(mode="offline") elif status == ApiKeyStatus.DISABLED: login_settings = dict(mode="disabled") elif key: login_settings = dict(api_key=key) self._wl._settings._apply_login(login_settings, _logger=_logger) # Whenever the key changes, make sure to pull in user settings # from server. if not self._wl.settings._offline: self._wl._update_user_settings()
def __init__(self, mode=None): self.calls = {} self._run = None self._done = True self._wl = wandb.setup() self.process_queue = self._wl._multiprocessing.Queue() self.req_queue = self._wl._multiprocessing.Queue() self.resp_queue = self._wl._multiprocessing.Queue() self.cancel_queue = self._wl._multiprocessing.Queue() self.notify_queue = self._wl._multiprocessing.Queue() self.interface = None self.last_queued = None self.history = [] self.summary = {} self.config = {} self.files = {} self.mocker = _get_mock_module(get_config())
def login(settings=None): if is_logged_in(): return if not settings: wl = wandb.setup() settings = wl.settings() app_url = settings.base_url.replace("//api.", "//app.") in_jupyter = _get_python_type() != "python" if in_jupyter: print("Go to this URL in a browser: {}/authorize\n".format(app_url)) key = getpass.getpass("Enter your authorization code:\n") else: print("Go to this URL in a browser: {}/authorize\n".format(app_url)) key = prompt(u'Enter api key: ', is_password=True) apikey.write_key(settings, key) return
def setup(self, kwargs): self.kwargs = kwargs wl = wandb.setup() settings: Settings = wl.settings( dict(kwargs.pop("settings", None) or tuple())) self._reporter = reporting.setup_reporter( settings=settings.duplicate().freeze()) # Remove parameters that are not part of settings self.config = kwargs.pop("config", None) # Temporarily unsupported parameters unsupported = ( "magic", "config_exclude_keys", "config_include_keys", "reinit", "anonymous", "dir", "allow_val_change", "resume", "force", "tensorboard", "sync_tensorboard", "monitor_gym", ) for key in unsupported: val = kwargs.pop(key, None) if val: self._reporter.warning("unsupported wandb.init() arg: %s", key) settings.apply_init(kwargs) # TODO(jhr): should this be moved? probably. d = dict(start_time=time.time()) settings.update(d) self.wl = wl self.settings = settings.freeze()
def sagemaker_auth(overrides=None, path=".", api_key=None): """Write a secrets.env file with the W&B ApiKey and any additional secrets passed. Args: overrides (dict, optional): Additional environment variables to write to secrets.env path (str, optional): The path to write the secrets file. """ settings = wandb.setup().settings current_api_key = apikey.api_key(settings=settings) overrides = overrides or dict() api_key = overrides.get(env.API_KEY, api_key or current_api_key) if api_key is None: raise ValueError( "Can't find W&B ApiKey, set the WANDB_API_KEY env variable " "or run `wandb login`") overrides[env.API_KEY] = api_key with open(os.path.join(path, "secrets.env"), "w") as file: for k, v in six.iteritems(overrides): file.write("{}={}\n".format(k, v))
def is_logged_in(settings=None): wl = wandb.setup() wl_settings = wl.settings() wl_settings._apply_settings(settings=settings) return apikey.api_key(settings=wl_settings) is not None
def setup(self, kwargs): """ Complete setup for wandb.init(). This includes parsing all arguments, applying them with settings and enabling logging. """ self.kwargs = kwargs self._wl = wandb_setup._setup() # Make sure we have a logger setup (might be an early logger) _set_logger(self._wl._get_logger()) # Start with settings from wandb library singleton settings = self._wl._clone_settings() settings_param = kwargs.pop("settings", None) if settings_param: settings._apply_settings(settings_param) self._reporter = reporting.setup_reporter( settings=settings.duplicate().freeze() ) sm_config = sagemaker.parse_sm_config() if sm_config: sm_api_key = sm_config.get("wandb_api_key", None) sm_run, sm_env = sagemaker.parse_sm_resources() if sm_env: if sm_api_key: sm_env["WANDB_API_KEY"] = sm_api_key settings._apply_environ(sm_env) wandb.setup(settings=settings) for k, v in six.iteritems(sm_run): kwargs.setdefault(k, v) # Remove parameters that are not part of settings init_config = kwargs.pop("config", None) or dict() config_include_keys = kwargs.pop("config_include_keys", None) config_exclude_keys = kwargs.pop("config_exclude_keys", None) # Add deprecation message once we can better track it and document alternatives # if config_include_keys or config_exclude_keys: # wandb.termwarn( # "config_include_keys and config_exclude_keys are deprecated:" # " use config=wandb.helper.parse_config(config_object, include=('key',))" # " or config=wandb.helper.parse_config(config_object, exclude=('key',))" # ) init_config = parse_config( init_config, include=config_include_keys, exclude=config_exclude_keys ) # merge config with sweep or sm (or config file) self.config = sm_config or self._wl._config or dict() for k, v in init_config.items(): self.config.setdefault(k, v) monitor_gym = kwargs.pop("monitor_gym", None) if monitor_gym and len(wandb.patched["gym"]) == 0: wandb.gym.monitor() tensorboard = kwargs.pop("tensorboard", None) sync_tensorboard = kwargs.pop("sync_tensorboard", None) if tensorboard or sync_tensorboard and len(wandb.patched["tensorboard"]) == 0: wandb.tensorboard.patch() magic = kwargs.get("magic") if magic not in (None, False): magic_install(kwargs) # handle login related parameters as these are applied to global state anonymous = kwargs.pop("anonymous", None) force = kwargs.pop("force", None) # TODO: move above parameters into apply_init_login settings._apply_init_login(kwargs) if not settings._offline and not settings._noop: wandb_login._login(anonymous=anonymous, force=force, _disable_warning=True) # apply updated global state after login was handled settings._apply_settings(wandb.setup()._settings) settings._apply_init(kwargs) if not settings._offline and not settings._noop: user_settings = self._wl._load_user_settings() settings._apply_user(user_settings) # TODO(jhr): should this be moved? probably. d = dict(_start_time=time.time(), _start_datetime=datetime.datetime.now(),) settings.update(d) if settings._jupyter: self._jupyter_setup(settings) self._log_setup(settings) self.settings = settings.freeze()
help="For testing, make sure resumed") args = parser.parse_args() defaults = dict( dropout=0.2, hidden_layer_size=128, layer_1_size=16, layer_2_size=32, learn_rate=0.01, decay=1e-6, momentum=0.9, epochs=10, ) # For testing purposes only, make sure resume can handle empty config if args.test_no_config and wandb.setup().settings.mode != "disabled": defaults = None run = wandb.init(config=defaults, resume=True) config = run.config # For testing purposes only, assert if we didnt resume if args.test_must_resume and not wandb.run.disabled: assert wandb.run.resumed (X_train, y_train), (X_test, y_test) = fashion_mnist.load_data() labels = [ "T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot" ]
def _login( anonymous=None, key=None, relogin=None, force=None, host=None, _backend=None, _disable_warning=None, _settings=None, ): """Log in to W&B. Args: settings (dict, optional): Override settings. relogin (bool, optional): If true, will re-prompt for API key. host (string, optional): The host to connect to anonymous (string, optional): Can be "must", "allow", or "never". If set to "must" we'll always login anonymously, if set to "allow" we'll only create an anonymous user if the user isn't already logged in. Returns: bool: if key is configured Raises: UsageError - if api_key can not configured and no tty """ if wandb.run is not None: if not _disable_warning: wandb.termwarn( "Calling wandb.login() after wandb.init() is a no-op.") return True settings_dict = {} api = Api() if anonymous is not None: # TODO: Move this check into wandb_settings probably. if not _validate_anonymous_setting(anonymous): wandb.termwarn( "Invalid value passed for argument `anonymous` to " "wandb.login(). Can be 'must', 'allow', or 'never'.") return False settings_dict.update({"anonymous": anonymous}) if host is not None: settings_dict.update({"base_url": host}) if key: settings_dict.update({"api_key": key}) # Note: This won't actually do anything if called from a codepath where # wandb.setup was previously called. If wandb.setup is called further up, # you must make sure the anonymous setting (and any other settings) are # already properly set up there. wl = wandb.setup(settings=wandb.Settings(**settings_dict)) wl_settings = wl.settings() if _settings: wl_settings._apply_settings(settings=_settings) settings = wl_settings if settings._offline: return False active_entity = None logged_in = is_logged_in(settings=settings) if logged_in: # TODO: do we want to move all login logic to the backend? if _backend: res = _backend.interface.communicate_login(key, anonymous) active_entity = res.active_entity else: active_entity = wl._get_entity() if active_entity and not relogin: login_state_str = "Currently logged in as:" login_info_str = "(use `wandb login --relogin` to force relogin)" wandb.termlog( "{} {} {}".format(login_state_str, click.style(active_entity, fg="yellow"), login_info_str), repeat=False, ) return True jupyter = settings._jupyter or False if key: if jupyter: wandb.termwarn( ("If you're specifying your api key in code, ensure this " "code is not shared publically.\nConsider setting the " "WANDB_API_KEY environment variable, or running " "`wandb login` from the command line.")) apikey.write_key(settings, key) else: key = apikey.prompt_api_key(settings, api=api, no_offline=force, no_create=force) if key is False: raise UsageError( "api_key not configured (no-tty). Run wandb login") if _backend and not logged_in: # TODO: calling this twice is gross, this deserves a refactor # Make sure our backend picks up the new creds _ = _backend.interface.communicate_login(key, anonymous) return key or False