def _setup(self):
logger.debug("Agent._setup()")
self._init()
parts = dict(entity=self._entity,
project=self._project,
name=self._sweep_path)
if err := util.parse_sweep_id(parts):
wandb.termerror(err)
return
def run_agent(sweep_id,
function=None,
in_jupyter=None,
entity=None,
project=None,
count=None):
parts = dict(entity=entity, project=project, name=sweep_id)
if err := util.parse_sweep_id(parts):
wandb.termerror(err)
return
def run_agent(sweep_id, function=None, in_jupyter=None, entity=None, project=None, count=None):
parts = dict(entity=entity, project=project, name=sweep_id)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or entity
project = parts.get("project") or project
sweep_id = parts.get("name") or sweep_id
if entity:
env.set_entity(entity)
if project:
env.set_project(project)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
log_level = logging.DEBUG
if in_jupyter:
log_level = logging.ERROR
ch.setLevel(log_level)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
try:
logger.addHandler(ch)
api = InternalApi()
queue = multiprocessing.Queue()
agent = Agent(api, queue, sweep_id=sweep_id, function=function, in_jupyter=in_jupyter, count=count)
agent.run()
finally:
# make sure we remove the logging handler (important for jupyter notebooks)
logger.removeHandler(ch)
def setup(self):
parts = dict(entity=self._entity,
project=self._project,
name=self._sweep_path)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or self._entity
project = parts.get("project") or self._project
sweep_id = parts.get("name") or self._sweep_id
if entity:
wandb.env.set_entity(entity)
if project:
wandb.env.set_project(project)
if sweep_id:
self._sweep_id = sweep_id
self.register()
def _setup(self):
logger.debug("Agent._setup()")
self._init()
parts = dict(entity=self._entity, project=self._project, name=self._sweep_path)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or self._entity
project = parts.get("project") or self._project
sweep_id = parts.get("name") or self._sweep_id
if sweep_id:
os.environ[wandb.env.SWEEP_ID] = sweep_id
if entity:
wandb.env.set_entity(entity)
if project:
wandb.env.set_project(project)
if sweep_id:
self._sweep_id = sweep_id
self._register()
def test_parse_sweep_id():
parts = {"name": "test/test/test"}
util.parse_sweep_id(parts)
assert parts == {"name": "test", "entity": "test", "project": "test"}
def sweep(
ctx,
project,
entity,
controller,
verbose,
name,
program,
settings,
update,
config_yaml,
): # noqa: C901
def _parse_settings(settings):
"""settings could be json or comma seperated assignments."""
ret = {}
# TODO(jhr): merge with magic:_parse_magic
if settings.find("=") > 0:
for item in settings.split(","):
kv = item.split("=")
if len(kv) != 2:
wandb.termwarn(
"Unable to parse sweep settings key value pair", repeat=False
)
ret.update(dict([kv]))
return ret
wandb.termwarn("Unable to parse settings parameter", repeat=False)
return ret
api = InternalApi()
if api.api_key is None:
wandb.termlog("Login to W&B to use the sweep feature")
ctx.invoke(login, no_offline=True)
sweep_obj_id = None
if update:
parts = dict(entity=entity, project=project, name=update)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or entity
project = parts.get("project") or project
sweep_id = parts.get("name") or update
found = api.sweep(sweep_id, "{}", entity=entity, project=project)
if not found:
wandb.termerror(
"Could not find sweep {}/{}/{}".format(entity, project, sweep_id)
)
return
sweep_obj_id = found["id"]
wandb.termlog(
"{} sweep from: {}".format(
"Updating" if sweep_obj_id else "Creating", config_yaml
)
)
try:
yaml_file = open(config_yaml)
except OSError:
wandb.termerror("Couldn't open sweep file: %s" % config_yaml)
return
try:
config = util.load_yaml(yaml_file)
except yaml.YAMLError as err:
wandb.termerror("Error in configuration file: %s" % err)
return
if config is None:
wandb.termerror("Configuration file is empty")
return
# Set or override parameters
if name:
config["name"] = name
if program:
config["program"] = program
if settings:
settings = _parse_settings(settings)
if settings:
config.setdefault("settings", {})
config["settings"].update(settings)
if controller:
config.setdefault("controller", {})
config["controller"]["type"] = "local"
is_local = config.get("controller", {}).get("type") == "local"
if is_local:
tuner = wandb_controller.controller()
err = tuner._validate(config)
if err:
wandb.termerror("Error in sweep file: %s" % err)
return
env = os.environ
entity = entity or env.get("WANDB_ENTITY") or config.get("entity")
project = (
project
or env.get("WANDB_PROJECT")
or config.get("project")
or util.auto_project_name(config.get("program"))
)
sweep_id = api.upsert_sweep(
config, project=project, entity=entity, obj_id=sweep_obj_id
)
wandb.termlog(
"{} sweep with ID: {}".format(
"Updated" if sweep_obj_id else "Created", click.style(sweep_id, fg="yellow")
)
)
sweep_url = wandb_controller._get_sweep_url(api, sweep_id)
if sweep_url:
wandb.termlog(
"View sweep at: {}".format(
click.style(sweep_url, underline=True, fg="blue")
)
)
# reprobe entity and project if it was autodetected by upsert_sweep
entity = entity or env.get("WANDB_ENTITY")
project = project or env.get("WANDB_PROJECT")
if entity and project:
sweep_path = "{}/{}/{}".format(entity, project, sweep_id)
elif project:
sweep_path = "{}/{}".format(project, sweep_id)
else:
sweep_path = sweep_id
if sweep_path.find(" ") >= 0:
sweep_path = '"{}"'.format(sweep_path)
wandb.termlog(
"Run sweep agent with: {}".format(
click.style("wandb agent %s" % sweep_path, fg="yellow")
)
)
if controller:
wandb.termlog("Starting wandb controller...")
tuner = wandb_controller.controller(sweep_id)
tuner.run(verbose=verbose)
def sweep(ctx, project, entity, controller, verbose, name, program, settings, update, config_yaml):
def _parse_settings(settings):
"""settings could be json or comma seperated assignments."""
ret = {}
# TODO(jhr): merge with magic_impl:_parse_magic
if settings.find('=') > 0:
for item in settings.split(","):
kv = item.split("=")
if len(kv) != 2:
wandb.termwarn("Unable to parse sweep settings key value pair", repeat=False)
ret.update(dict([kv]))
return ret
wandb.termwarn("Unable to parse settings parameter", repeat=False)
return ret
if api.api_key is None:
termlog("Login to W&B to use the sweep feature")
ctx.invoke(login, no_offline=True)
sweep_obj_id = None
if update:
parts = dict(entity=entity, project=project, name=update)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or entity
project = parts.get("project") or project
sweep_id = parts.get("name") or update
found = api.sweep(sweep_id, '{}', entity=entity, project=project)
if not found:
wandb.termerror('Could not find sweep {}/{}/{}'.format(entity, project, sweep_id))
return
sweep_obj_id = found['id']
wandb.termlog('{} sweep from: {}'.format(
'Updating' if sweep_obj_id else 'Creating',
config_yaml))
try:
yaml_file = open(config_yaml)
except (OSError, IOError):
wandb.termerror('Couldn\'t open sweep file: %s' % config_yaml)
return
try:
config = util.load_yaml(yaml_file)
except yaml.YAMLError as err:
wandb.termerror('Error in configuration file: %s' % err)
return
if config is None:
wandb.termerror('Configuration file is empty')
return
# Set or override parameters
if name:
config["name"] = name
if program:
config["program"] = program
if settings:
settings = _parse_settings(settings)
if settings:
config.setdefault("settings", {})
config["settings"].update(settings)
if controller:
config.setdefault("controller", {})
config["controller"]["type"] = "local"
is_local = config.get('controller', {}).get('type') == 'local'
if is_local:
tuner = wandb_controller.controller()
err = tuner._validate(config)
if err:
wandb.termerror('Error in sweep file: %s' % err)
return
entity = entity or env.get_entity() or config.get('entity')
project = project or env.get_project() or config.get('project') or util.auto_project_name(
config.get("program"), api)
sweep_id = api.upsert_sweep(config, project=project, entity=entity, obj_id=sweep_obj_id)
wandb.termlog('{} sweep with ID: {}'.format(
'Updated' if sweep_obj_id else 'Created',
click.style(sweep_id, fg="yellow")))
sweep_url = wandb_controller._get_sweep_url(api, sweep_id)
if sweep_url:
wandb.termlog("View sweep at: {}".format(
click.style(sweep_url, underline=True, fg='blue')))
# reprobe entity and project if it was autodetected by upsert_sweep
entity = entity or env.get_entity()
project = project or env.get_project()
if entity and project:
sweep_path = "{}/{}/{}".format(entity, project, sweep_id)
elif project:
sweep_path = "{}/{}".format(project, sweep_id)
else:
sweep_path = sweep_id
wandb.termlog("Run sweep agent with: {}".format(
click.style("wandb agent %s" % sweep_path, fg="yellow")))
if controller:
wandb.termlog('Starting wandb controller...')
tuner = wandb_controller.controller(sweep_id)
tuner.run(verbose=verbose)
