def _set_api_key(api_key_file: Optional[str] = None,
api_key: Optional[str] = None):
"""Set WandB API key from `wandb_config`. Will pop the
`api_key_file` and `api_key` keys from `wandb_config` parameter"""
if api_key_file:
if api_key:
raise ValueError("Both WandB `api_key_file` and `api_key` set.")
with open(api_key_file, "rt") as fp:
api_key = fp.readline().strip()
# Try to get API key from external hook
if not api_key and WANDB_SETUP_API_KEY_HOOK in os.environ:
try:
api_key = _load_class(os.environ[WANDB_SETUP_API_KEY_HOOK])()
except Exception as e:
logger.exception(
f"Error executing {WANDB_SETUP_API_KEY_HOOK} to setup API key: {e}",
exc_info=e,
)
if api_key:
os.environ[WANDB_ENV_VAR] = api_key
elif not os.environ.get(WANDB_ENV_VAR):
try:
# Check if user is already logged into wandb.
wandb.ensure_configured()
if wandb.api.api_key:
logger.info("Already logged into W&B.")
return
except AttributeError:
pass
raise ValueError(
"No WandB API key found. Either set the {} environment "
"variable, pass `api_key` or `api_key_file` to the"
"`WandbLoggerCallback` class as arguments, "
"or run `wandb login` from the command line".format(WANDB_ENV_VAR))
def _set_api_key(api_key_file: Optional[str] = None, api_key: Optional[str] = None):
"""Set WandB API key from `wandb_config`. Will pop the
`api_key_file` and `api_key` keys from `wandb_config` parameter"""
if api_key_file:
if api_key:
raise ValueError("Both WandB `api_key_file` and `api_key` set.")
with open(api_key_file, "rt") as fp:
api_key = fp.readline().strip()
if api_key:
os.environ[WANDB_ENV_VAR] = api_key
elif not os.environ.get(WANDB_ENV_VAR):
try:
# Check if user is already logged into wandb.
wandb.ensure_configured()
if wandb.api.api_key:
logger.info("Already logged into W&B.")
return
except AttributeError:
pass
raise ValueError(
"No WandB API key found. Either set the {} environment "
"variable, pass `api_key` or `api_key_file` to the"
"`WandbLoggerCallback` class as arguments, "
"or run `wandb login` from the command line".format(WANDB_ENV_VAR)
)
def _set_api_key(wandb_config):
"""Set WandB API key from `wandb_config`. Will pop the
`api_key_file` and `api_key` keys from `wandb_config` parameter"""
api_key_file = os.path.expanduser(wandb_config.pop("api_key_file", ""))
api_key = wandb_config.pop("api_key", None)
if api_key_file:
if api_key:
raise ValueError("Both WandB `api_key_file` and `api_key` set.")
with open(api_key_file, "rt") as fp:
api_key = fp.readline().strip()
if api_key:
os.environ[WANDB_ENV_VAR] = api_key
elif not os.environ.get(WANDB_ENV_VAR):
try:
# Check if user is already logged into wandb.
wandb.ensure_configured()
if wandb.api.api_key:
logger.info("Already logged into W&B.")
return
except AttributeError:
pass
raise ValueError(
"No WandB API key found. Either set the {} environment "
"variable, pass `api_key` or `api_key_file` in the config, "
"or run `wandb login` from the command line".format(WANDB_ENV_VAR))
def run(ctx, program, args, id, resume, dir, configs, message, name, notes,
show, tags, run_group, job_type):
wandb.ensure_configured()
if configs:
config_paths = configs.split(',')
else:
config_paths = []
config = Config(config_paths=config_paths,
wandb_dir=dir or wandb.wandb_dir())
tags = [tag for tag in tags.split(",") if tag] if tags else None
# populate run parameters from env if not specified
id = id or os.environ.get(env.RUN_ID)
message = message or os.environ.get(env.DESCRIPTION)
tags = tags or env.get_tags()
run_group = run_group or os.environ.get(env.RUN_GROUP)
job_type = job_type or os.environ.get(env.JOB_TYPE)
name = name or os.environ.get(env.NAME)
notes = notes or os.environ.get(env.NOTES)
resume = resume or os.environ.get(env.RESUME)
run = wandb_run.Run(run_id=id,
mode='clirun',
config=config,
description=message,
program=program,
tags=tags,
group=run_group,
job_type=job_type,
name=name,
notes=notes,
resume=resume)
run.enable_logging()
environ = dict(os.environ)
if configs:
environ[env.CONFIG_PATHS] = configs
if show:
environ[env.SHOW_RUN] = 'True'
if not run.api.api_key:
util.prompt_api_key(run.api, input_callback=click.prompt)
try:
rm = run_manager.RunManager(run)
rm.init_run(environ)
except run_manager.Error:
exc_type, exc_value, exc_traceback = sys.exc_info()
wandb.termerror(
'An Exception was raised during setup, see %s for full traceback.'
% util.get_log_file_path())
wandb.termerror(str(exc_value))
if 'permission' in str(exc_value):
wandb.termerror(
'Are you sure you provided the correct API key to "wandb login"?'
)
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error('\n'.join(lines))
sys.exit(1)
rm.run_user_process(program, args, environ)
def __init__(self,
samples,
mrr_at_k: int = 10,
name: str = '',
write_csv: bool = True,
similarity_fct=cos_sim,
batch_size: int = 64,
show_progress_bar: bool = False):
self.samples = samples
self.name = name
self.mrr_at_k = mrr_at_k
self.similarity_fct = similarity_fct
self.batch_size = batch_size
self.show_progress_bar = show_progress_bar
wandb.ensure_configured()
if wandb.api.api_key is None:
self._has_wandb = False
else:
self._has_wandb = False if os.getenv("WANDB_DISABLED") else True
if isinstance(self.samples, dict):
self.samples = list(self.samples.values())
### Remove sample with empty positive / negative set
self.samples = [
sample for sample in self.samples
if len(sample['positive']) > 0 and len(sample['negative']) > 0
]
self.csv_file = "RerankingEvaluator" + ("_" + name if name else
'') + "_results.csv"
self.csv_headers = ["epoch", "steps", "MAP", "MRR@{}".format(mrr_at_k)]
self.write_csv = write_csv
def on():
wandb.ensure_configured()
api = InternalApi()
try:
api.clear_setting('disabled')
except configparser.Error:
pass
click.echo(
"W&B enabled, running your script from this directory will now sync to the cloud.")
def __init__(self,
model,
train_loader,
val_loader,
test_loader,
num_ns_eval,
task_type,
tokenizer,
validate_every_epochs,
num_validation_batches,
num_epochs,
lr,
sacred_ex,
validate_every_steps=-1,
max_grad_norm=0.5,
validation_metric='ndcg_cut_10',
num_training_instances=-1,
optimizer_class=optim.Adam):
self.best_model_path = "./best_model"
self.num_ns_eval = num_ns_eval
self.task_type = task_type
self.tokenizer = tokenizer
self.validation_metric = validation_metric
self.validate_every_epochs = validate_every_epochs
self.validate_every_steps = validate_every_steps
self.num_validation_batches = num_validation_batches
self.num_epochs = num_epochs
self.lr = lr
self.sacred_ex = sacred_ex
self.num_training_instances = num_training_instances
self.num_gpu = torch.cuda.device_count()
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
logging.info("Device {}".format(self.device))
logging.info("Num GPU {}".format(self.num_gpu))
self.model = model.to(self.device)
if self.num_gpu > 1:
devices = [v for v in range(self.num_gpu)]
self.model = nn.DataParallel(self.model, device_ids=devices)
self.best_eval_metric = 0
self.train_loader = train_loader
self.val_loader = val_loader
self.test_loader = test_loader
self.optimizer = optimizer_class(self.model.parameters(), lr=self.lr)
self.max_grad_norm = max_grad_norm
#copied from huggingface transformer trainer
wandb.ensure_configured()
if wandb.api.api_key is None:
self._has_wandb = False
else:
self._has_wandb = False if os.getenv("WANDB_DISABLED") else True
def off():
wandb.ensure_configured()
api = InternalApi()
try:
api.set_setting('disabled', 'true')
click.echo(
"W&B disabled, running your script from this directory will only write metadata locally.")
except configparser.Error as e:
click.echo(
'Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=dryrun')
def on():
wandb.ensure_configured()
api = InternalApi()
parser = api.settings_parser
try:
parser.remove_option('default', 'disabled')
with open(api.settings_file, "w") as f:
parser.write(f)
except configparser.Error:
pass
click.echo(
"W&B enabled, running your script from this directory will now sync to the cloud.")
def off():
wandb.ensure_configured()
api = Api()
parser = api.settings_parser
try:
parser.set('default', 'disabled', 'true')
with open(api.settings_file, "w") as f:
parser.write(f)
click.echo(
"W&B disabled, running your script from this directory will only write metadata locally.")
except configparser.Error as e:
click.echo(
'Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=dryrun')
def test_login_no_key(local_netrc, mocker):
stdin_mock = mocker.patch("wandb.util.sys.stdin.isatty")
stdin_mock.return_value = True
stdout_mock = mocker.patch("wandb.util.sys.stdout.isatty")
stdout_mock.return_value = True
inp = mocker.patch("wandb.util.six.moves.input")
inp.return_value = "2"
getpass = mocker.patch("wandb.util.getpass.getpass")
getpass.return_value = "C" * 40
wandb.ensure_configured()
assert wandb.api.api_key == None
wandb.login()
assert wandb.api.api_key == "C" * 40
def __init__(self):
has_wandb = is_wandb_available()
assert has_wandb, "WandbCallback requires wandb to be installed. Run `pip install wandb`."
if has_wandb:
import wandb
wandb.ensure_configured()
if wandb.api.api_key is None:
has_wandb = False
logger.warning(
"W&B installed but not logged in. Run `wandb login` or set the WANDB_API_KEY env variable."
)
self._wandb = wandb
else:
self._wandb = None
self._initialized = False
def run(ctx, program, args, id, resume, dir, configs, message, name, notes,
show, tags, run_group, job_type):
wandb.ensure_configured()
if configs:
config_paths = configs.split(',')
else:
config_paths = []
config = Config(config_paths=config_paths,
wandb_dir=dir or wandb.wandb_dir())
tags = [tag for tag in tags.split(",") if tag] if tags else None
run = wandb_run.Run(run_id=id,
mode='clirun',
config=config,
description=message,
program=program,
tags=tags,
group=run_group,
job_type=job_type,
name=name,
notes=notes,
resume=resume)
run.enable_logging()
environ = dict(os.environ)
if configs:
environ[env.CONFIG_PATHS] = configs
if show:
environ[env.SHOW_RUN] = 'True'
run.check_anonymous()
try:
rm = run_manager.RunManager(run)
rm.init_run(environ)
except run_manager.Error:
exc_type, exc_value, exc_traceback = sys.exc_info()
wandb.termerror(
'An Exception was raised during setup, see %s for full traceback.'
% util.get_log_file_path())
wandb.termerror(str(exc_value))
if 'permission' in str(exc_value):
wandb.termerror(
'Are you sure you provided the correct API key to "wandb login"?'
)
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.error('\n'.join(lines))
sys.exit(1)
rm.run_user_process(program, args, environ)
def __init__(self):
has_wandb = is_wandb_available()
assert has_wandb, "WandbCallback requires wandb to be installed. Run `pip install wandb`."
if has_wandb:
import wandb
wandb.ensure_configured()
if wandb.api.api_key is None:
has_wandb = False
logger.warning(
"W&B installed but not logged in. Run `wandb login` or set the WANDB_API_KEY env variable."
)
self._wandb = None
else:
self._wandb = wandb
self._initialized = False
# log outputs
self._log_model = os.getenv("WANDB_LOG_MODEL", "FALSE").upper() in ENV_VARS_TRUE_VALUES.union({"TRUE"})
import os
import random
from typing import Dict, NamedTuple, Optional
import numpy as np
try:
import wandb
wandb.ensure_configured()
if wandb.api.api_key is None:
_has_wandb = False
wandb.termwarn(
"W&B installed but not logged in. Run `wandb login` or set the WANDB_API_KEY env variable."
)
else:
_has_wandb = False if os.getenv("WANDB_DISABLED") else True
except (ImportError, AttributeError):
_has_wandb = False
def is_wandb_available():
return _has_wandb
def set_seed(seed: int):
"""
Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch`` and/or ``tf``
(if installed).
Args:
def test_login_invalid_key():
os.environ["WANDB_API_KEY"] = "B" * 40
wandb.ensure_configured()
with pytest.raises(wandb.UsageError):
wandb.login()
del os.environ["WANDB_API_KEY"]
def test_login_existing_key(local_netrc):
os.environ["WANDB_API_KEY"] = "B" * 40
wandb.ensure_configured()
wandb.login()
assert wandb.api.api_key == "B" * 40
