def login(key, host, cloud, relogin, anonymously, no_offline=False):
# TODO: handle no_offline
anon_mode = "must" if anonymously else "never"
wandb.setup(settings=wandb.Settings(
_cli_only_mode=True, anonymous=anon_mode, base_url=host))
api = _get_cling_api()
if host == "https://api.wandb.ai" or (host is None and cloud):
api.clear_setting("base_url", globally=True, persist=True)
# To avoid writing an empty local settings file, we only clear if it exists
if os.path.exists(Settings._local_path()):
api.clear_setting("base_url", persist=True)
elif host:
if not host.startswith("http"):
raise ClickException("host must start with http(s)://")
api.set_setting("base_url",
host.strip("/"),
globally=True,
persist=True)
key = key[0] if len(key) > 0 else None
wandb.login(relogin=relogin,
key=key,
anonymous=anon_mode,
host=host,
force=True)
def run_job(self, job):
run_id = job.run_id
config_file = os.path.join("wandb", "sweep-" + self._sweep_id,
"config-" + run_id + ".yaml")
config_util.save_config_file_from_dict(config_file, job.config)
os.environ[wandb.env.RUN_ID] = run_id
os.environ[wandb.env.CONFIG_PATHS] = config_file
os.environ[wandb.env.SWEEP_ID] = self._sweep_id
wandb.setup(_reset=True)
print("wandb: Agent Starting Run: {} with config:\n".format(run_id) +
"\n".join([
"\t{}: {}".format(k, v["value"])
for k, v in job.config.items()
]))
try:
self._function()
if wandb.run:
wandb.join()
except KeyboardInterrupt as e:
print("Keyboard interrupt", e)
return True
except Exception as e:
print("Problem", e)
return True
def test_setup_offline(live_mock_server, test_settings):
# this is to increase coverage
login_settings = test_settings.copy()
login_settings.update(mode="offline")
assert wandb.setup(settings=login_settings)._instance._get_entity() is None
assert wandb.setup(
settings=login_settings)._instance._load_viewer() is None
def _get_cling_api():
"""Get a reference to the internal api with cling settings."""
global _api
if _api is None:
# TODO(jhr): make a settings object that is better for non runs.
wandb.setup(settings=wandb.Settings(_cli_only_mode=True))
_api = InternalApi()
return _api
def main():
wandb.require("service")
wandb.setup()
num_proc = 4
pool = mp.Pool(processes=num_proc)
result = pool.map_async(do_run, range(num_proc))
data = result.get(60)
print(f"DEBUG: {data}")
assert len(data) == 4
def setup(self, kwargs):
self.kwargs = kwargs
# built up login settings
login_settings: Settings = wandb.Settings()
settings_param = kwargs.pop("_settings", None)
if settings_param:
login_settings._apply_settings(settings_param)
_logger = wandb.setup()._get_logger()
login_settings._apply_login(kwargs, _logger=_logger)
# make sure they are applied globally
self._wl = wandb.setup(settings=login_settings)
self._settings = self._wl._settings
def api_key(settings=None):
if not settings:
settings = wandb.setup().settings
if settings.api_key:
return settings.api_key
if auth := requests.utils.get_netrc_auth(settings.base_url):
return auth[-1]
def login(anonymous=None, key=None, relogin=None, host=None, force=None):
"""
Log in to W&B.
Arguments:
anonymous: (string, optional) Can be "must", "allow", or "never".
If set to "must" we'll always login anonymously, if set to
"allow" we'll only create an anonymous user if the user
isn't already logged in.
key: (string, optional) authentication key.
relogin: (bool, optional) If true, will re-prompt for API key.
host: (string, optional) The host to connect to.
Returns:
bool: if key is configured
Raises:
UsageError - if api_key can not configured and no tty
"""
_handle_host_wandb_setting(host)
if wandb.setup()._settings._noop:
return True
kwargs = dict(locals())
configured = _login(**kwargs)
return True if configured else False
def login(settings=None, relogin=None):
wl = wandb.setup(settings=settings)
settings = wl.settings()
active_entity = None
if is_logged_in():
active_entity = wl._get_entity()
if active_entity and not relogin:
login_state_str = "Currently logged in as:"
login_info_str = "(use `wandb login --relogin` to force relogin)"
wandb.termlog("{} {} {}".format(
login_state_str, click.style(active_entity, fg="yellow"),
login_info_str))
return
app_url = settings.base_url.replace("//api.", "//app.")
# TODO(jhr): use settings object
in_jupyter = _get_python_type() != "python"
authorize_str = "Go to this URL in a browser"
authorize_link_str = "{}/authorize".format(app_url)
if in_jupyter:
print("{}: {}\n".format(authorize_str, authorize_link_str))
key = getpass.getpass("Enter your authorization code:\n")
else:
wandb.termlog("{}: {}".format(
authorize_str, click.style(authorize_link_str, fg="blue")))
key = prompt(u"Enter api key: ", is_password=True)
apikey.write_key(settings, key)
return
def login(
anonymous: Optional[Literal["must", "allow", "never"]] = None,
key: Optional[str] = None,
relogin: Optional[bool] = None,
host: Optional[str] = None,
force: Optional[bool] = None,
timeout: Optional[int] = None,
) -> bool:
"""
Log in to W&B.
Arguments:
anonymous: (string, optional) Can be "must", "allow", or "never".
If set to "must" we'll always login anonymously, if set to
"allow" we'll only create an anonymous user if the user
isn't already logged in.
key: (string, optional) authentication key.
relogin: (bool, optional) If true, will re-prompt for API key.
host: (string, optional) The host to connect to.
force: (bool, optional) If true, will force a relogin.
timeout: (int, optional) Number of seconds to wait for user input.
Returns:
bool: if key is configured
Raises:
UsageError - if api_key can not configured and no tty
"""
_handle_host_wandb_setting(host)
if wandb.setup()._settings._noop:
return True
kwargs = dict(locals())
configured = _login(**kwargs)
return True if configured else False
def __init__(self):
self._done = False
self.record_q = None
self.result_q = None
self.wandb_process = None
self.interface = None
self._internal_pid = None
self._wl = wandb.setup()
def test_login_timeout_choose(mock_tty):
mock_tty("3\n")
start_time = time.time()
ret = wandb.login(timeout=8)
elapsed = time.time() - start_time
assert elapsed < 15
assert ret is False
assert wandb.api.api_key is None
assert wandb.setup().settings.mode == "offline"
def test_login_timeout(mock_tty):
mock_tty("junk\nmore\n")
start_time = time.time()
ret = wandb.login(timeout=4)
elapsed = time.time() - start_time
assert 2 < elapsed < 15
assert ret is False
assert wandb.api.api_key is None
assert wandb.setup().settings.mode == "disabled"
def test_login_timeout_env_blank(mock_tty, reset_login_timeout):
mock_tty("\n\n\n")
os.environ["WANDB_LOGIN_TIMEOUT"] = "4"
start_time = time.time()
ret = wandb.login()
elapsed = time.time() - start_time
assert elapsed < 15
assert ret is False
assert wandb.api.api_key is None
assert wandb.setup().settings.mode == "disabled"
def update_session(self, key):
_logger = wandb.setup()._get_logger()
settings: Settings = wandb.Settings()
login_settings = dict(api_key=key) if key else dict(mode="offline")
settings._apply_source_login(login_settings, _logger=_logger)
self._wl._update(settings=settings)
# Whenever the key changes, make sure to pull in user settings
# from server.
if not self._wl.settings._offline:
self._wl._update_user_settings()
def api_key(settings=None):
if not settings:
wl = wandb.setup()
settings = wl.settings()
if settings.api_key:
return settings.api_key
auth = requests.utils.get_netrc_auth(settings.base_url)
if auth:
return auth[-1]
return None
def setup(self, kwargs):
self.kwargs = kwargs
# built up login settings
login_settings: Settings = wandb.Settings()
settings_param = kwargs.pop("_settings", None)
# note that this case does not come up anywhere except for the tests
if settings_param is not None:
if isinstance(settings_param, Settings):
login_settings._apply_settings(settings_param)
elif isinstance(settings_param, dict):
login_settings.update(settings_param, source=Source.LOGIN)
_logger = wandb.setup()._get_logger()
# Do not save relogin into settings as we just want to relogin once
self._relogin = kwargs.pop("relogin", None)
login_settings._apply_login(kwargs, _logger=_logger)
# make sure they are applied globally
self._wl = wandb.setup(settings=login_settings)
self._settings = self._wl.settings
class _WandbLogin(object):
def __init__(self):
self.kwargs: Optional[Dict] = None
self._settings: Optional[Settings] = None
self._backend = None
self._silent = None
self._wl = None
self._key = None
def setup(self, kwargs):
self.kwargs = kwargs
# built up login settings
login_settings: Settings = wandb.Settings()
if settings_param := kwargs.pop("_settings", None):
login_settings._apply_settings(settings_param)
_logger = wandb.setup()._get_logger()
login_settings._apply_login(kwargs, _logger=_logger)
# make sure they are applied globally
self._wl = wandb.setup(settings=login_settings)
self._settings = self._wl._settings
def run(self):
wandb.require("service")
wandb.init(*self.args, **self.kwargs)
wandb.setup()
while True:
item_type, item_content = self.queue.get()
if item_type == _QueueItem.END:
break
if item_type == _QueueItem.CHECKPOINT:
self._handle_checkpoint(item_content)
continue
assert item_type == _QueueItem.RESULT
log, config_update = self._handle_result(item_content)
try:
wandb.config.update(config_update, allow_val_change=True)
wandb.log(log)
except urllib.error.HTTPError as e:
# Ignore HTTPError. Missing a few data points is not a
# big issue, as long as things eventually recover.
logger.warn("Failed to log result to w&b: {}".format(str(e)))
wandb.finish()
def __init__(self, mode=None):
self.wandb_process = None
self.fd_pipe_parent = None
self.process_queue = None
# self.fd_request_queue = None
# self.fd_response_queue = None
self.req_queue = None
self.resp_queue = None
self.cancel_queue = None
self.notify_queue = None # notify activity on ...
self._done = False
self._wl = wandb.setup()
self.interface = None
def __init__(self, mode=None):
self.calls = {}
self._run = None
self._done = True
self._wl = wandb.setup()
self.record_q = self._wl._multiprocessing.Queue()
self.result_q = self._wl._multiprocessing.Queue()
self.interface = None
self.last_queued = None
self.history = []
self.summary = {}
self.config = {}
self.files = {}
self.mocker = _get_mock_module(get_config())
self._internal_pid = None
def update_session(self,
key: Optional[str],
status: ApiKeyStatus = ApiKeyStatus.VALID) -> None:
_logger = wandb.setup()._get_logger()
login_settings = dict()
if status == ApiKeyStatus.OFFLINE:
login_settings = dict(mode="offline")
elif status == ApiKeyStatus.DISABLED:
login_settings = dict(mode="disabled")
elif key:
login_settings = dict(api_key=key)
self._wl._settings._apply_login(login_settings, _logger=_logger)
# Whenever the key changes, make sure to pull in user settings
# from server.
if not self._wl.settings._offline:
self._wl._update_user_settings()
def __init__(self, mode=None):
self.calls = {}
self._run = None
self._done = True
self._wl = wandb.setup()
self.process_queue = self._wl._multiprocessing.Queue()
self.req_queue = self._wl._multiprocessing.Queue()
self.resp_queue = self._wl._multiprocessing.Queue()
self.cancel_queue = self._wl._multiprocessing.Queue()
self.notify_queue = self._wl._multiprocessing.Queue()
self.interface = None
self.last_queued = None
self.history = []
self.summary = {}
self.config = {}
self.files = {}
self.mocker = _get_mock_module(get_config())
def login(settings=None):
if is_logged_in():
return
if not settings:
wl = wandb.setup()
settings = wl.settings()
app_url = settings.base_url.replace("//api.", "//app.")
in_jupyter = _get_python_type() != "python"
if in_jupyter:
print("Go to this URL in a browser: {}/authorize\n".format(app_url))
key = getpass.getpass("Enter your authorization code:\n")
else:
print("Go to this URL in a browser: {}/authorize\n".format(app_url))
key = prompt(u'Enter api key: ', is_password=True)
apikey.write_key(settings, key)
return
def setup(self, kwargs):
self.kwargs = kwargs
wl = wandb.setup()
settings: Settings = wl.settings(
dict(kwargs.pop("settings", None) or tuple()))
self._reporter = reporting.setup_reporter(
settings=settings.duplicate().freeze())
# Remove parameters that are not part of settings
self.config = kwargs.pop("config", None)
# Temporarily unsupported parameters
unsupported = (
"magic",
"config_exclude_keys",
"config_include_keys",
"reinit",
"anonymous",
"dir",
"allow_val_change",
"resume",
"force",
"tensorboard",
"sync_tensorboard",
"monitor_gym",
)
for key in unsupported:
val = kwargs.pop(key, None)
if val:
self._reporter.warning("unsupported wandb.init() arg: %s", key)
settings.apply_init(kwargs)
# TODO(jhr): should this be moved? probably.
d = dict(start_time=time.time())
settings.update(d)
self.wl = wl
self.settings = settings.freeze()
def sagemaker_auth(overrides=None, path=".", api_key=None):
"""Write a secrets.env file with the W&B ApiKey and any additional secrets passed.
Args:
overrides (dict, optional): Additional environment variables to write
to secrets.env
path (str, optional): The path to write the secrets file.
"""
settings = wandb.setup().settings
current_api_key = apikey.api_key(settings=settings)
overrides = overrides or dict()
api_key = overrides.get(env.API_KEY, api_key or current_api_key)
if api_key is None:
raise ValueError(
"Can't find W&B ApiKey, set the WANDB_API_KEY env variable "
"or run `wandb login`")
overrides[env.API_KEY] = api_key
with open(os.path.join(path, "secrets.env"), "w") as file:
for k, v in six.iteritems(overrides):
file.write("{}={}\n".format(k, v))
def is_logged_in(settings=None):
wl = wandb.setup()
wl_settings = wl.settings()
wl_settings._apply_settings(settings=settings)
return apikey.api_key(settings=wl_settings) is not None
def setup(self, kwargs):
"""
Complete setup for wandb.init(). This includes parsing all arguments,
applying them with settings and enabling logging.
"""
self.kwargs = kwargs
self._wl = wandb_setup._setup()
# Make sure we have a logger setup (might be an early logger)
_set_logger(self._wl._get_logger())
# Start with settings from wandb library singleton
settings = self._wl._clone_settings()
settings_param = kwargs.pop("settings", None)
if settings_param:
settings._apply_settings(settings_param)
self._reporter = reporting.setup_reporter(
settings=settings.duplicate().freeze()
)
sm_config = sagemaker.parse_sm_config()
if sm_config:
sm_api_key = sm_config.get("wandb_api_key", None)
sm_run, sm_env = sagemaker.parse_sm_resources()
if sm_env:
if sm_api_key:
sm_env["WANDB_API_KEY"] = sm_api_key
settings._apply_environ(sm_env)
wandb.setup(settings=settings)
for k, v in six.iteritems(sm_run):
kwargs.setdefault(k, v)
# Remove parameters that are not part of settings
init_config = kwargs.pop("config", None) or dict()
config_include_keys = kwargs.pop("config_include_keys", None)
config_exclude_keys = kwargs.pop("config_exclude_keys", None)
# Add deprecation message once we can better track it and document alternatives
# if config_include_keys or config_exclude_keys:
# wandb.termwarn(
# "config_include_keys and config_exclude_keys are deprecated:"
# " use config=wandb.helper.parse_config(config_object, include=('key',))"
# " or config=wandb.helper.parse_config(config_object, exclude=('key',))"
# )
init_config = parse_config(
init_config, include=config_include_keys, exclude=config_exclude_keys
)
# merge config with sweep or sm (or config file)
self.config = sm_config or self._wl._config or dict()
for k, v in init_config.items():
self.config.setdefault(k, v)
monitor_gym = kwargs.pop("monitor_gym", None)
if monitor_gym and len(wandb.patched["gym"]) == 0:
wandb.gym.monitor()
tensorboard = kwargs.pop("tensorboard", None)
sync_tensorboard = kwargs.pop("sync_tensorboard", None)
if tensorboard or sync_tensorboard and len(wandb.patched["tensorboard"]) == 0:
wandb.tensorboard.patch()
magic = kwargs.get("magic")
if magic not in (None, False):
magic_install(kwargs)
# handle login related parameters as these are applied to global state
anonymous = kwargs.pop("anonymous", None)
force = kwargs.pop("force", None)
# TODO: move above parameters into apply_init_login
settings._apply_init_login(kwargs)
if not settings._offline and not settings._noop:
wandb_login._login(anonymous=anonymous, force=force, _disable_warning=True)
# apply updated global state after login was handled
settings._apply_settings(wandb.setup()._settings)
settings._apply_init(kwargs)
if not settings._offline and not settings._noop:
user_settings = self._wl._load_user_settings()
settings._apply_user(user_settings)
# TODO(jhr): should this be moved? probably.
d = dict(_start_time=time.time(), _start_datetime=datetime.datetime.now(),)
settings.update(d)
if settings._jupyter:
self._jupyter_setup(settings)
self._log_setup(settings)
self.settings = settings.freeze()
help="For testing, make sure resumed")
args = parser.parse_args()
defaults = dict(
dropout=0.2,
hidden_layer_size=128,
layer_1_size=16,
layer_2_size=32,
learn_rate=0.01,
decay=1e-6,
momentum=0.9,
epochs=10,
)
# For testing purposes only, make sure resume can handle empty config
if args.test_no_config and wandb.setup().settings.mode != "disabled":
defaults = None
run = wandb.init(config=defaults, resume=True)
config = run.config
# For testing purposes only, assert if we didnt resume
if args.test_must_resume and not wandb.run.disabled:
assert wandb.run.resumed
(X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()
labels = [
"T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt",
"Sneaker", "Bag", "Ankle boot"
]
def _login(
anonymous=None,
key=None,
relogin=None,
force=None,
host=None,
_backend=None,
_disable_warning=None,
_settings=None,
):
"""Log in to W&B.
Args:
settings (dict, optional): Override settings.
relogin (bool, optional): If true, will re-prompt for API key.
host (string, optional): The host to connect to
anonymous (string, optional): Can be "must", "allow", or "never".
If set to "must" we'll always login anonymously, if set to
"allow" we'll only create an anonymous user if the user
isn't already logged in.
Returns:
bool: if key is configured
Raises:
UsageError - if api_key can not configured and no tty
"""
if wandb.run is not None:
if not _disable_warning:
wandb.termwarn(
"Calling wandb.login() after wandb.init() is a no-op.")
return True
settings_dict = {}
api = Api()
if anonymous is not None:
# TODO: Move this check into wandb_settings probably.
if not _validate_anonymous_setting(anonymous):
wandb.termwarn(
"Invalid value passed for argument `anonymous` to "
"wandb.login(). Can be 'must', 'allow', or 'never'.")
return False
settings_dict.update({"anonymous": anonymous})
if host is not None:
settings_dict.update({"base_url": host})
if key:
settings_dict.update({"api_key": key})
# Note: This won't actually do anything if called from a codepath where
# wandb.setup was previously called. If wandb.setup is called further up,
# you must make sure the anonymous setting (and any other settings) are
# already properly set up there.
wl = wandb.setup(settings=wandb.Settings(**settings_dict))
wl_settings = wl.settings()
if _settings:
wl_settings._apply_settings(settings=_settings)
settings = wl_settings
if settings._offline:
return False
active_entity = None
logged_in = is_logged_in(settings=settings)
if logged_in:
# TODO: do we want to move all login logic to the backend?
if _backend:
res = _backend.interface.communicate_login(key, anonymous)
active_entity = res.active_entity
else:
active_entity = wl._get_entity()
if active_entity and not relogin:
login_state_str = "Currently logged in as:"
login_info_str = "(use `wandb login --relogin` to force relogin)"
wandb.termlog(
"{} {} {}".format(login_state_str,
click.style(active_entity, fg="yellow"),
login_info_str),
repeat=False,
)
return True
jupyter = settings._jupyter or False
if key:
if jupyter:
wandb.termwarn(
("If you're specifying your api key in code, ensure this "
"code is not shared publically.\nConsider setting the "
"WANDB_API_KEY environment variable, or running "
"`wandb login` from the command line."))
apikey.write_key(settings, key)
else:
key = apikey.prompt_api_key(settings,
api=api,
no_offline=force,
no_create=force)
if key is False:
raise UsageError(
"api_key not configured (no-tty). Run wandb login")
if _backend and not logged_in:
# TODO: calling this twice is gross, this deserves a refactor
# Make sure our backend picks up the new creds
_ = _backend.interface.communicate_login(key, anonymous)
return key or False
