def send_run(self, data):
run = data.run
error = None
is_wandb_init = self._run is None
# build config dict
config_dict = None
config_path = os.path.join(self._settings.files_dir,
filenames.CONFIG_FNAME)
if run.config:
config_dict = config_util.dict_from_proto_list(run.config.update)
config_util.save_config_file_from_dict(config_path, config_dict)
if is_wandb_init:
# Ensure we have a project to query for status
if run.project == "":
run.project = util.auto_project_name(self._settings.program)
# Only check resume status on `wandb.init`
error = self._maybe_setup_resume(run)
if error is not None:
if data.control.req_resp:
resp = wandb_internal_pb2.Result(uuid=data.uuid)
resp.run_result.run.CopyFrom(run)
resp.run_result.error.CopyFrom(error)
self._result_q.put(resp)
else:
logger.error("Got error in async mode: %s", error.message)
return
# Save the resumed config
if self._resume_state["config"] is not None:
# TODO: should we merge this with resumed config?
config_override = config_dict or {}
config_dict = self._resume_state["config"]
config_dict.update(config_override)
config_util.save_config_file_from_dict(config_path, config_dict)
self._init_run(run, config_dict)
if data.control.req_resp:
resp = wandb_internal_pb2.Result(uuid=data.uuid)
# TODO: we could do self._interface.publish_defer(resp) to notify
# the handler not to actually perform server updates for this uuid
# because the user process will send a summary update when we resume
resp.run_result.run.CopyFrom(self._run)
self._result_q.put(resp)
# Only spin up our threads on the first run message
if is_wandb_init:
self._start_run_threads()
else:
logger.info("updated run: %s", self._run.run_id)
def auto_project_name(self, api):
return util.auto_project_name(self.program, api)
def send_run(self, data, file_dir=None) -> None:
run = data.run
error = None
is_wandb_init = self._run is None
# update telemetry
if run.telemetry:
self._telemetry_obj.MergeFrom(run.telemetry)
# build config dict
config_value_dict: Optional[DictWithValues] = None
if run.config:
config_util.update_from_proto(self._consolidated_config,
run.config)
config_value_dict = self._config_format(self._consolidated_config)
self._config_save(config_value_dict)
if is_wandb_init:
# Ensure we have a project to query for status
if run.project == "":
run.project = util.auto_project_name(self._settings.program)
# Only check resume status on `wandb.init`
error = self._maybe_setup_resume(run)
if error is not None:
if data.control.req_resp:
resp = wandb_internal_pb2.Result(uuid=data.uuid)
resp.run_result.run.CopyFrom(run)
resp.run_result.error.CopyFrom(error)
self._result_q.put(resp)
else:
logger.error("Got error in async mode: %s", error.message)
return
# Save the resumed config
if self._resume_state["config"] is not None:
# TODO: should we merge this with resumed config?
config_override = self._consolidated_config
config_dict = self._resume_state["config"]
config_dict = config_util.dict_strip_value_dict(config_dict)
config_dict.update(config_override)
self._consolidated_config.update(config_dict)
config_value_dict = self._config_format(self._consolidated_config)
self._config_save(config_value_dict)
# handle empty config
# TODO(jhr): consolidate the 4 ways config is built:
# (passed config, empty config, resume config, send_config)
if not config_value_dict:
config_value_dict = self._config_format(None)
self._config_save(config_value_dict)
self._init_run(run, config_value_dict)
if data.control.req_resp:
resp = wandb_internal_pb2.Result(uuid=data.uuid)
# TODO: we could do self._interface.publish_defer(resp) to notify
# the handler not to actually perform server updates for this uuid
# because the user process will send a summary update when we resume
resp.run_result.run.CopyFrom(self._run)
self._result_q.put(resp)
# Only spin up our threads on the first run message
if is_wandb_init:
self._start_run_threads(file_dir)
else:
logger.info("updated run: %s", self._run.run_id)
def sweep(
ctx,
project,
entity,
controller,
verbose,
name,
program,
settings,
update,
config_yaml,
): # noqa: C901
def _parse_settings(settings):
"""settings could be json or comma seperated assignments."""
ret = {}
# TODO(jhr): merge with magic:_parse_magic
if settings.find("=") > 0:
for item in settings.split(","):
kv = item.split("=")
if len(kv) != 2:
wandb.termwarn(
"Unable to parse sweep settings key value pair", repeat=False
)
ret.update(dict([kv]))
return ret
wandb.termwarn("Unable to parse settings parameter", repeat=False)
return ret
api = InternalApi()
if api.api_key is None:
wandb.termlog("Login to W&B to use the sweep feature")
ctx.invoke(login, no_offline=True)
sweep_obj_id = None
if update:
parts = dict(entity=entity, project=project, name=update)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or entity
project = parts.get("project") or project
sweep_id = parts.get("name") or update
found = api.sweep(sweep_id, "{}", entity=entity, project=project)
if not found:
wandb.termerror(
"Could not find sweep {}/{}/{}".format(entity, project, sweep_id)
)
return
sweep_obj_id = found["id"]
wandb.termlog(
"{} sweep from: {}".format(
"Updating" if sweep_obj_id else "Creating", config_yaml
)
)
try:
yaml_file = open(config_yaml)
except OSError:
wandb.termerror("Couldn't open sweep file: %s" % config_yaml)
return
try:
config = util.load_yaml(yaml_file)
except yaml.YAMLError as err:
wandb.termerror("Error in configuration file: %s" % err)
return
if config is None:
wandb.termerror("Configuration file is empty")
return
# Set or override parameters
if name:
config["name"] = name
if program:
config["program"] = program
if settings:
settings = _parse_settings(settings)
if settings:
config.setdefault("settings", {})
config["settings"].update(settings)
if controller:
config.setdefault("controller", {})
config["controller"]["type"] = "local"
is_local = config.get("controller", {}).get("type") == "local"
if is_local:
tuner = wandb_controller.controller()
err = tuner._validate(config)
if err:
wandb.termerror("Error in sweep file: %s" % err)
return
env = os.environ
entity = entity or env.get("WANDB_ENTITY") or config.get("entity")
project = (
project
or env.get("WANDB_PROJECT")
or config.get("project")
or util.auto_project_name(config.get("program"))
)
sweep_id = api.upsert_sweep(
config, project=project, entity=entity, obj_id=sweep_obj_id
)
wandb.termlog(
"{} sweep with ID: {}".format(
"Updated" if sweep_obj_id else "Created", click.style(sweep_id, fg="yellow")
)
)
sweep_url = wandb_controller._get_sweep_url(api, sweep_id)
if sweep_url:
wandb.termlog(
"View sweep at: {}".format(
click.style(sweep_url, underline=True, fg="blue")
)
)
# reprobe entity and project if it was autodetected by upsert_sweep
entity = entity or env.get("WANDB_ENTITY")
project = project or env.get("WANDB_PROJECT")
if entity and project:
sweep_path = "{}/{}/{}".format(entity, project, sweep_id)
elif project:
sweep_path = "{}/{}".format(project, sweep_id)
else:
sweep_path = sweep_id
if sweep_path.find(" ") >= 0:
sweep_path = '"{}"'.format(sweep_path)
wandb.termlog(
"Run sweep agent with: {}".format(
click.style("wandb agent %s" % sweep_path, fg="yellow")
)
)
if controller:
wandb.termlog("Starting wandb controller...")
tuner = wandb_controller.controller(sweep_id)
tuner.run(verbose=verbose)
def sweep(ctx, project, entity, controller, verbose, name, program, settings, update, config_yaml):
def _parse_settings(settings):
"""settings could be json or comma seperated assignments."""
ret = {}
# TODO(jhr): merge with magic_impl:_parse_magic
if settings.find('=') > 0:
for item in settings.split(","):
kv = item.split("=")
if len(kv) != 2:
wandb.termwarn("Unable to parse sweep settings key value pair", repeat=False)
ret.update(dict([kv]))
return ret
wandb.termwarn("Unable to parse settings parameter", repeat=False)
return ret
if api.api_key is None:
termlog("Login to W&B to use the sweep feature")
ctx.invoke(login, no_offline=True)
sweep_obj_id = None
if update:
parts = dict(entity=entity, project=project, name=update)
err = util.parse_sweep_id(parts)
if err:
wandb.termerror(err)
return
entity = parts.get("entity") or entity
project = parts.get("project") or project
sweep_id = parts.get("name") or update
found = api.sweep(sweep_id, '{}', entity=entity, project=project)
if not found:
wandb.termerror('Could not find sweep {}/{}/{}'.format(entity, project, sweep_id))
return
sweep_obj_id = found['id']
wandb.termlog('{} sweep from: {}'.format(
'Updating' if sweep_obj_id else 'Creating',
config_yaml))
try:
yaml_file = open(config_yaml)
except (OSError, IOError):
wandb.termerror('Couldn\'t open sweep file: %s' % config_yaml)
return
try:
config = util.load_yaml(yaml_file)
except yaml.YAMLError as err:
wandb.termerror('Error in configuration file: %s' % err)
return
if config is None:
wandb.termerror('Configuration file is empty')
return
# Set or override parameters
if name:
config["name"] = name
if program:
config["program"] = program
if settings:
settings = _parse_settings(settings)
if settings:
config.setdefault("settings", {})
config["settings"].update(settings)
if controller:
config.setdefault("controller", {})
config["controller"]["type"] = "local"
is_local = config.get('controller', {}).get('type') == 'local'
if is_local:
tuner = wandb_controller.controller()
err = tuner._validate(config)
if err:
wandb.termerror('Error in sweep file: %s' % err)
return
entity = entity or env.get_entity() or config.get('entity')
project = project or env.get_project() or config.get('project') or util.auto_project_name(
config.get("program"), api)
sweep_id = api.upsert_sweep(config, project=project, entity=entity, obj_id=sweep_obj_id)
wandb.termlog('{} sweep with ID: {}'.format(
'Updated' if sweep_obj_id else 'Created',
click.style(sweep_id, fg="yellow")))
sweep_url = wandb_controller._get_sweep_url(api, sweep_id)
if sweep_url:
wandb.termlog("View sweep at: {}".format(
click.style(sweep_url, underline=True, fg='blue')))
# reprobe entity and project if it was autodetected by upsert_sweep
entity = entity or env.get_entity()
project = project or env.get_project()
if entity and project:
sweep_path = "{}/{}/{}".format(entity, project, sweep_id)
elif project:
sweep_path = "{}/{}".format(project, sweep_id)
else:
sweep_path = sweep_id
wandb.termlog("Run sweep agent with: {}".format(
click.style("wandb agent %s" % sweep_path, fg="yellow")))
if controller:
wandb.termlog('Starting wandb controller...')
tuner = wandb_controller.controller(sweep_id)
tuner.run(verbose=verbose)
