def __init__(self, config_paths=[], wandb_dir=None, run_dir=None):
object.__setattr__(self, '_wandb_dir', wandb_dir)
# OrderedDict to make writing unit tests easier. (predictable order for
# .key())
object.__setattr__(self, '_items', OrderedDict())
object.__setattr__(self, '_descriptions', {})
self._load_defaults()
self._load_wandb()
for conf_path in config_paths:
self._load_file(conf_path)
self._set_wandb('cli_version', wandb.__version__)
self._set_wandb('python_version', platform.python_version())
self._set_wandb('is_jupyter_run', wandb._get_python_type() != "python")
self._set_wandb('is_kaggle_kernel', is_kaggle())
hf_version = huggingface_version()
if hf_version:
self._set_wandb('huggingface_version', hf_version)
# Do this after defaults because it triggers loading of pre-existing
# config.yaml (if it exists)
self.set_run_dir(run_dir)
self.persist()
def sweep(sweep, entity=None, project=None):
from wandb.sweeps.config import SweepConfig
import types
if isinstance(sweep, types.FunctionType):
sweep = sweep()
if isinstance(sweep, SweepConfig):
sweep = dict(sweep)
"""Sweep create for controller api and jupyter (eventually for cli)."""
in_jupyter = wandb._get_python_type() != "python"
if in_jupyter:
os.environ[env.JUPYTER] = "true"
_api0 = InternalApi()
if not _api0.api_key:
wandb._jupyter_login(api=_api0)
if entity:
env.set_entity(entity)
if project:
env.set_project(project)
api = InternalApi()
sweep_id = api.upsert_sweep(sweep)
print('Create sweep with ID:', sweep_id)
sweep_url = _get_sweep_url(api, sweep_id)
if sweep_url:
print('Sweep URL:', sweep_url)
return sweep_id
def agent(sweep_id, function=None, entity=None, project=None, count=None):
"""Generic agent entrypoint, used for CLI or jupyter.
Args:
sweep_id (dict): Sweep ID generated by CLI or sweep API
function (func, optional): A function to call instead of the "program" specifed in the config
entity (str, optional): W&B Entity
project (str, optional): W&B Project
count (int, optional): the number of trials to run.
"""
in_jupyter = wandb._get_python_type() != "python"
if in_jupyter:
os.environ[wandb.env.JUPYTER] = "true"
_api0 = InternalApi()
if not _api0.api_key:
wandb._jupyter_login(api=_api0)
_ = wandb.Settings()
return run_agent(
sweep_id,
function=function,
in_jupyter=in_jupyter,
entity=entity,
project=project,
count=count,
)
def agent(sweep_id, function=None, entity=None, project=None, count=None):
"""Generic agent entrypoint, used for CLI or jupyter.
Args:
sweep_id (dict): Sweep ID generated by CLI or sweep API
function (func, optional): A function to call instead of the "program" specifed in the config
entity (str, optional): W&B Entity
project (str, optional): W&B Project
count (int, optional): the number of trials to run.
"""
global _INSTANCES
_INSTANCES += 1
try:
# make sure we are logged in
wandb_sdk.wandb_login._login(_silent=True)
if function:
return pyagent(sweep_id, function, entity, project, count)
in_jupyter = wandb._get_python_type() != "python"
return run_agent(
sweep_id,
function=function,
in_jupyter=in_jupyter,
entity=entity,
project=project,
count=count,
)
finally:
_INSTANCES -= 1
def agent(sweep_id, function=None, entity=None, project=None):
"""Generic agent entrypoint, used for CLI or jupyter.
Args:
sweep_id (dict): Sweep ID generated by CLI or sweep API
entity (str, optional): W&B Entity
project (str, optional): W&B Project
function (dict, optional): Configure sweep function
"""
in_jupyter = wandb._get_python_type() != "python"
if in_jupyter:
os.environ[env.JUPYTER] = "true"
_api0 = InternalApi()
if not _api0.api_key:
wandb._jupyter_login(api=_api0)
return run_agent(sweep_id, function=function, in_jupyter=in_jupyter, entity=entity, project=project)
def agent(sweep_id, function=None, entity=None, project=None, count=None):
"""Generic agent entrypoint, used for CLI or jupyter.
Will run a function or program with configuration parameters specified
by server.
Args:
sweep_id (dict): Sweep ID generated by CLI or sweep API
function (func, optional): A function to call instead of the "program"
specifed in the config.
entity (str, optional): W&B Entity
project (str, optional): W&B Project
count (int, optional): the number of trials to run.
Examples:
Run a sample sweep over a function:
def train():
with wandb.init() as run:
print("config:", dict(run.config))
for epoch in range(35):
print("running", epoch)
wandb.log({"metric": run.config.param1, "epoch": epoch})
time.sleep(1)
wandb.agent(sweep_id, function=train)
"""
global _INSTANCES
_INSTANCES += 1
try:
# make sure we are logged in
wandb_sdk.wandb_login._login(_silent=True)
if function:
return pyagent(sweep_id, function, entity, project, count)
in_jupyter = wandb._get_python_type() != "python"
return run_agent(
sweep_id,
function=function,
in_jupyter=in_jupyter,
entity=entity,
project=project,
count=count,
)
finally:
_INSTANCES -= 1
def sweep(sweep, entity=None, project=None):
"""Sweep create for controller api and jupyter (eventually for cli)."""
in_jupyter = wandb._get_python_type() != "python"
if in_jupyter:
os.environ[env.JUPYTER] = "true"
_api0 = InternalApi()
if not _api0.api_key:
wandb.jupyter_login(api=_api0)
if entity:
env.set_entity(entity)
if project:
env.set_project(project)
api = InternalApi()
sweep_id = api.upsert_sweep(sweep)
print('Create sweep with ID:', sweep_id)
sweep_url = _get_sweep_url(api, sweep_id)
if sweep_url:
print('Sweep URL:', sweep_url)
return sweep_id
def magic_install():
global _run_once
if _run_once:
return
_run_once = True
# process system args
_process_system_args()
# install argparse wrapper
in_jupyter_or_ipython = wandb._get_python_type() != "python"
if not in_jupyter_or_ipython:
_monkey_argparse()
# track init calls
trigger.register('on_init', _magic_init)
wandb.init(magic=True)
global _magic_config
global _import_hook
_magic_config = _parse_magic(wandb.env.get_magic())
if 'tensorflow.python.keras' in sys.modules:
_monkey_tfkeras(sys.modules.get('tensorflow.python.keras'))
elif 'keras' in sys.modules:
_monkey_keras(sys.modules.get('keras'))
else:
if not _import_hook:
_import_hook = ImportMetaHook()
_import_hook.install()
_import_hook.add(lastname='keras', on_import=_on_import_keras)
if 'absl.app' in sys.modules:
_monkey_absl(sys.modules.get('absl.app'))
else:
if not _import_hook:
_import_hook = ImportMetaHook()
_import_hook.install()
_import_hook.add(fullname='absl.app', on_import=_on_import_absl)
# update wandb.config on fit or program finish
trigger.register('on_fit', _magic_update_config)
trigger.register('on_finished', _magic_update_config)
def magic_install():
global _run_once
if _run_once:
return
_run_once = True
global _magic_config
global _import_hook
# parse config early, before we have wandb.config overrides
_magic_config, magic_set = _parse_magic(wandb.env.get_magic())
# we are implicitly enabling magic
if _magic_config.get('enable') is None:
_magic_config['enable'] = True
magic_set['enable'] = True
# allow early config to disable magic
if not _magic_config.get('enable'):
return
# process system args
_process_system_args()
# install argparse wrapper
in_jupyter_or_ipython = wandb._get_python_type() != "python"
if not in_jupyter_or_ipython:
_monkey_argparse()
# track init calls
trigger.register('on_init', _magic_init)
# if wandb.init has already been called, this call is ignored
wandb.init(magic=True)
# parse magic from wandb.config (from flattened to dict)
magic_from_config = {}
MAGIC_KEY = "wandb_magic"
for k, v in wandb.config.user_items():
if not k.startswith(MAGIC_KEY + "."):
continue
d = _dict_from_keyval(k, v, json_parse=False)
_merge_dicts(d, magic_from_config)
magic_from_config = magic_from_config.get(MAGIC_KEY, {})
_merge_dicts(magic_from_config, _magic_config)
# allow late config to disable magic
if not _magic_config.get('enable'):
return
# store magic_set into config
if magic_set:
wandb.config._set_wandb('magic', magic_set)
if 'tensorflow.python.keras' in sys.modules:
_monkey_tfkeras(sys.modules.get('tensorflow.python.keras'))
elif 'keras' in sys.modules:
_monkey_keras(sys.modules.get('keras'))
else:
if not _import_hook:
_import_hook = ImportMetaHook()
_import_hook.install()
_import_hook.add(lastname='keras', on_import=_on_import_keras)
if 'absl.app' in sys.modules:
_monkey_absl(sys.modules.get('absl.app'))
else:
if not _import_hook:
_import_hook = ImportMetaHook()
_import_hook.install()
_import_hook.add(fullname='absl.app', on_import=_on_import_absl)
# update wandb.config on fit or program finish
trigger.register('on_fit', _magic_update_config)
trigger.register('on_finished', _magic_update_config)
def setup(self):
class TimeOutException(Exception):
pass
def alarm_handler(signum, frame):
raise TimeOutException()
self.data["root"] = os.getcwd()
program = os.getenv(env.PROGRAM) or util.get_program()
if program:
self.data["program"] = program
else:
self.data["program"] = '<python with no main file>'
if wandb._get_python_type() != "python":
if os.getenv(env.NOTEBOOK_NAME):
self.data["program"] = os.getenv(env.NOTEBOOK_NAME)
else:
meta = wandb.jupyter.notebook_metadata()
if meta.get("path"):
if "fileId=" in meta["path"]:
self.data[
"colab"] = "https://colab.research.google.com/drive/" + meta[
"path"].split("fileId=")[1]
self.data["program"] = meta["name"]
else:
self.data["program"] = meta["path"]
self.data["root"] = meta["root"]
if not os.getenv(env.DISABLE_CODE):
logger.debug("code probe starting")
in_jupyter = wandb._get_python_type() != "python"
# windows doesn't support alarm() and jupyter could call this in a thread context
if platform.system() == "Windows" or not hasattr(
signal, 'SIGALRM') or in_jupyter:
logger.debug("non time limited probe of code")
self._setup_code_git()
self._setup_code_program()
else:
old_alarm = None
try:
try:
old_alarm = signal.signal(signal.SIGALRM,
alarm_handler)
signal.alarm(25)
self._setup_code_git()
self._setup_code_program()
finally:
signal.alarm(0)
except TimeOutException:
logger.debug("timeout waiting for setup_code")
finally:
if old_alarm:
signal.signal(signal.SIGALRM, old_alarm)
logger.debug("code probe done")
self.data["startedAt"] = datetime.utcfromtimestamp(
wandb.START_TIME).isoformat()
try:
username = getpass.getuser()
except KeyError:
# getuser() could raise KeyError in restricted environments like
# chroot jails or docker containers. Return user id in these cases.
username = str(os.getuid())
# Host names, usernames, emails, the root directory, and executable paths are sensitive for anonymous users.
if self._api.settings().get('anonymous') != 'true':
self.data["host"] = os.environ.get(env.HOST, socket.gethostname())
self.data["username"] = os.getenv(env.USERNAME, username)
self.data["executable"] = sys.executable
else:
self.data.pop("email", None)
self.data.pop("root", None)
self.data["os"] = platform.platform(aliased=True)
self.data["python"] = platform.python_version()
if env.get_docker():
self.data["docker"] = env.get_docker()
try:
pynvml.nvmlInit()
self.data["gpu"] = pynvml.nvmlDeviceGetName(
pynvml.nvmlDeviceGetHandleByIndex(0)).decode("utf8")
self.data["gpu_count"] = pynvml.nvmlDeviceGetCount()
except pynvml.NVMLError:
pass
try:
self.data["cpu_count"] = multiprocessing.cpu_count()
except NotImplementedError:
pass
# TODO: we should use the cuda library to collect this
if os.path.exists("/usr/local/cuda/version.txt"):
with open("/usr/local/cuda/version.txt") as f:
self.data["cuda"] = f.read().split(" ")[-1].strip()
self.data["args"] = sys.argv[1:]
self.data["state"] = "running"
def setup(self):
self.data["root"] = os.getcwd()
try:
import __main__
self.data["program"] = __main__.__file__
except (ImportError, AttributeError):
self.data["program"] = '<python with no main file>'
if wandb._get_python_type() != "python":
if os.getenv(env.NOTEBOOK_NAME):
self.data["program"] = os.getenv(env.NOTEBOOK_NAME)
else:
meta = wandb.jupyter.notebook_metadata()
if meta.get("path"):
if "fileId=" in meta["path"]:
self.data[
"colab"] = "https://colab.research.google.com/drive/" + meta[
"path"].split("fileId=")[1]
self.data["program"] = meta["name"]
else:
self.data["program"] = meta["path"]
self.data["root"] = meta["root"]
program = os.path.join(self.data["root"], self.data["program"])
if not os.getenv(env.DISABLE_CODE):
if self._api.git.enabled:
self.data["git"] = {
"remote": self._api.git.remote_url,
"commit": self._api.git.last_commit
}
self.data["email"] = self._api.git.email
self.data["root"] = self._api.git.root or self.data["root"]
if os.path.exists(program) and self._api.git.is_untracked(
self.data["program"]):
util.mkdir_exists_ok(
os.path.join(self.out_dir, "code",
os.path.dirname(self.data["program"])))
saved_program = os.path.join(self.out_dir, "code",
self.data["program"])
if not os.path.exists(saved_program):
self.data["codeSaved"] = True
copyfile(program, saved_program)
self.data["startedAt"] = datetime.utcfromtimestamp(
wandb.START_TIME).isoformat()
self.data["host"] = os.environ.get(env.HOST, socket.gethostname())
try:
username = getpass.getuser()
except KeyError:
# getuser() could raise KeyError in restricted environments like
# chroot jails or docker containers. Return user id in these cases.
username = str(os.getuid())
self.data["username"] = os.getenv(env.USERNAME, username)
self.data["os"] = platform.platform(aliased=True)
self.data["python"] = platform.python_version()
self.data["executable"] = sys.executable
if env.get_docker():
self.data["docker"] = env.get_docker()
try:
pynvml.nvmlInit()
self.data["gpu"] = pynvml.nvmlDeviceGetName(
pynvml.nvmlDeviceGetHandleByIndex(0)).decode("utf8")
self.data["gpu_count"] = pynvml.nvmlDeviceGetCount()
except pynvml.NVMLError:
pass
try:
self.data["cpu_count"] = multiprocessing.cpu_count()
except NotImplementedError:
pass
# TODO: we should use the cuda library to collect this
if os.path.exists("/usr/local/cuda/version.txt"):
self.data["cuda"] = open(
"/usr/local/cuda/version.txt").read().split(" ")[-1].strip()
self.data["args"] = sys.argv[1:]
self.data["state"] = "running"
def magic_install(init_args=None):
global _run_once
if _run_once:
return
_run_once = True
global _magic_config
global _import_hook
from wandb.integration.keras import WandbCallback # add keras import hooks first
# parse config early, before we have wandb.config overrides
_magic_config, magic_set = _parse_magic(wandb.env.get_magic())
# we are implicitly enabling magic
if _magic_config.get("enable") is None:
_magic_config["enable"] = True
magic_set["enable"] = True
# allow early config to disable magic
if not _magic_config.get("enable"):
return
# process system args
_process_system_args()
# install argparse wrapper
in_jupyter_or_ipython = wandb._get_python_type() != "python"
if not in_jupyter_or_ipython:
_monkey_argparse()
# track init calls
trigger.register("on_init", _magic_init)
# if wandb.init has already been called, this call is ignored
init_args = init_args or {}
init_args["magic"] = True
wandb.init(**init_args)
# parse magic from wandb.config (from flattened to dict)
magic_from_config = {}
MAGIC_KEY = "wandb_magic"
for k in wandb.config.keys():
if not k.startswith(MAGIC_KEY + "."):
continue
d = _dict_from_keyval(k, wandb.config[k], json_parse=False)
_merge_dicts(d, magic_from_config)
magic_from_config = magic_from_config.get(MAGIC_KEY, {})
_merge_dicts(magic_from_config, _magic_config)
# allow late config to disable magic
if not _magic_config.get("enable"):
return
# store magic_set into config
if magic_set:
wandb.config["magic"] = magic_set
wandb.config.persist()
# Monkey patch tf.keras
if "tensorflow.python.keras" in sys.modules or "keras" in sys.modules:
_monkey_tfkeras()
# Always setup import hooks looking for keras or tf.keras
add_import_hook(fullname="keras", on_import=_monkey_tfkeras)
add_import_hook(fullname="tensorflow.python.keras",
on_import=_monkey_tfkeras)
if "absl.app" in sys.modules:
_monkey_absl()
else:
add_import_hook(fullname="absl.app", on_import=_monkey_absl)
# update wandb.config on fit or program finish
trigger.register("on_fit", _magic_update_config)
trigger.register("on_finished", _magic_update_config)
# "val_support_micro": ret['metric_classifier_support_micro'],
})
if space["fmin"]["params"]["wandb_enable"]:
wandb.join()
results["attachments"]["models"] = pickle.dumps(
[x["attachments"] for x in rets])
return results
def nested_set(dic, keys, value):
for key in keys[:-1]:
dic = dic.setdefault(key, {})
dic[keys[-1]] = value
if __name__ == "__main__" and wandb._get_python_type() == "python":
# # https://app.wandb.ai/jpizarrom/pan20apml/runs/3qlicko2/files/output.log
parser = argparse.ArgumentParser()
# parser.add_argument('--max_evals', type=int, default=2)
parser.add_argument("--op", type=str, default="run_config")
parser.add_argument("--conf", type=str, default="conf_0")
parser.add_argument("--save-to", type=str)
# parser.add_argument('--use_mongodb', action='store_true')
args = parser.parse_args()
print(args)
if args.save_to is not None and os.path.exists(args.save_to):
raise Exception("save_to exists")
