def send_run(self, data): run = data.run error = None is_wandb_init = self._run is None # build config dict config_dict = None config_path = os.path.join(self._settings.files_dir, filenames.CONFIG_FNAME) if run.config: config_dict = config_util.dict_from_proto_list(run.config.update) config_util.save_config_file_from_dict(config_path, config_dict) if is_wandb_init: # Ensure we have a project to query for status if run.project == "": run.project = util.auto_project_name(self._settings.program) # Only check resume status on `wandb.init` error = self._maybe_setup_resume(run) if error is not None: if data.control.req_resp: resp = wandb_internal_pb2.Result(uuid=data.uuid) resp.run_result.run.CopyFrom(run) resp.run_result.error.CopyFrom(error) self._result_q.put(resp) else: logger.error("Got error in async mode: %s", error.message) return # Save the resumed config if self._resume_state["config"] is not None: # TODO: should we merge this with resumed config? config_override = config_dict or {} config_dict = self._resume_state["config"] config_dict.update(config_override) config_util.save_config_file_from_dict(config_path, config_dict) self._init_run(run, config_dict) if data.control.req_resp: resp = wandb_internal_pb2.Result(uuid=data.uuid) # TODO: we could do self._interface.publish_defer(resp) to notify # the handler not to actually perform server updates for this uuid # because the user process will send a summary update when we resume resp.run_result.run.CopyFrom(self._run) self._result_q.put(resp) # Only spin up our threads on the first run message if is_wandb_init: self._start_run_threads() else: logger.info("updated run: %s", self._run.run_id)
def auto_project_name(self, api): return util.auto_project_name(self.program, api)
def send_run(self, data, file_dir=None) -> None: run = data.run error = None is_wandb_init = self._run is None # update telemetry if run.telemetry: self._telemetry_obj.MergeFrom(run.telemetry) # build config dict config_value_dict: Optional[DictWithValues] = None if run.config: config_util.update_from_proto(self._consolidated_config, run.config) config_value_dict = self._config_format(self._consolidated_config) self._config_save(config_value_dict) if is_wandb_init: # Ensure we have a project to query for status if run.project == "": run.project = util.auto_project_name(self._settings.program) # Only check resume status on `wandb.init` error = self._maybe_setup_resume(run) if error is not None: if data.control.req_resp: resp = wandb_internal_pb2.Result(uuid=data.uuid) resp.run_result.run.CopyFrom(run) resp.run_result.error.CopyFrom(error) self._result_q.put(resp) else: logger.error("Got error in async mode: %s", error.message) return # Save the resumed config if self._resume_state["config"] is not None: # TODO: should we merge this with resumed config? config_override = self._consolidated_config config_dict = self._resume_state["config"] config_dict = config_util.dict_strip_value_dict(config_dict) config_dict.update(config_override) self._consolidated_config.update(config_dict) config_value_dict = self._config_format(self._consolidated_config) self._config_save(config_value_dict) # handle empty config # TODO(jhr): consolidate the 4 ways config is built: # (passed config, empty config, resume config, send_config) if not config_value_dict: config_value_dict = self._config_format(None) self._config_save(config_value_dict) self._init_run(run, config_value_dict) if data.control.req_resp: resp = wandb_internal_pb2.Result(uuid=data.uuid) # TODO: we could do self._interface.publish_defer(resp) to notify # the handler not to actually perform server updates for this uuid # because the user process will send a summary update when we resume resp.run_result.run.CopyFrom(self._run) self._result_q.put(resp) # Only spin up our threads on the first run message if is_wandb_init: self._start_run_threads(file_dir) else: logger.info("updated run: %s", self._run.run_id)
def sweep( ctx, project, entity, controller, verbose, name, program, settings, update, config_yaml, ): # noqa: C901 def _parse_settings(settings): """settings could be json or comma seperated assignments.""" ret = {} # TODO(jhr): merge with magic:_parse_magic if settings.find("=") > 0: for item in settings.split(","): kv = item.split("=") if len(kv) != 2: wandb.termwarn( "Unable to parse sweep settings key value pair", repeat=False ) ret.update(dict([kv])) return ret wandb.termwarn("Unable to parse settings parameter", repeat=False) return ret api = InternalApi() if api.api_key is None: wandb.termlog("Login to W&B to use the sweep feature") ctx.invoke(login, no_offline=True) sweep_obj_id = None if update: parts = dict(entity=entity, project=project, name=update) err = util.parse_sweep_id(parts) if err: wandb.termerror(err) return entity = parts.get("entity") or entity project = parts.get("project") or project sweep_id = parts.get("name") or update found = api.sweep(sweep_id, "{}", entity=entity, project=project) if not found: wandb.termerror( "Could not find sweep {}/{}/{}".format(entity, project, sweep_id) ) return sweep_obj_id = found["id"] wandb.termlog( "{} sweep from: {}".format( "Updating" if sweep_obj_id else "Creating", config_yaml ) ) try: yaml_file = open(config_yaml) except OSError: wandb.termerror("Couldn't open sweep file: %s" % config_yaml) return try: config = util.load_yaml(yaml_file) except yaml.YAMLError as err: wandb.termerror("Error in configuration file: %s" % err) return if config is None: wandb.termerror("Configuration file is empty") return # Set or override parameters if name: config["name"] = name if program: config["program"] = program if settings: settings = _parse_settings(settings) if settings: config.setdefault("settings", {}) config["settings"].update(settings) if controller: config.setdefault("controller", {}) config["controller"]["type"] = "local" is_local = config.get("controller", {}).get("type") == "local" if is_local: tuner = wandb_controller.controller() err = tuner._validate(config) if err: wandb.termerror("Error in sweep file: %s" % err) return env = os.environ entity = entity or env.get("WANDB_ENTITY") or config.get("entity") project = ( project or env.get("WANDB_PROJECT") or config.get("project") or util.auto_project_name(config.get("program")) ) sweep_id = api.upsert_sweep( config, project=project, entity=entity, obj_id=sweep_obj_id ) wandb.termlog( "{} sweep with ID: {}".format( "Updated" if sweep_obj_id else "Created", click.style(sweep_id, fg="yellow") ) ) sweep_url = wandb_controller._get_sweep_url(api, sweep_id) if sweep_url: wandb.termlog( "View sweep at: {}".format( click.style(sweep_url, underline=True, fg="blue") ) ) # reprobe entity and project if it was autodetected by upsert_sweep entity = entity or env.get("WANDB_ENTITY") project = project or env.get("WANDB_PROJECT") if entity and project: sweep_path = "{}/{}/{}".format(entity, project, sweep_id) elif project: sweep_path = "{}/{}".format(project, sweep_id) else: sweep_path = sweep_id if sweep_path.find(" ") >= 0: sweep_path = '"{}"'.format(sweep_path) wandb.termlog( "Run sweep agent with: {}".format( click.style("wandb agent %s" % sweep_path, fg="yellow") ) ) if controller: wandb.termlog("Starting wandb controller...") tuner = wandb_controller.controller(sweep_id) tuner.run(verbose=verbose)
def sweep(ctx, project, entity, controller, verbose, name, program, settings, update, config_yaml): def _parse_settings(settings): """settings could be json or comma seperated assignments.""" ret = {} # TODO(jhr): merge with magic_impl:_parse_magic if settings.find('=') > 0: for item in settings.split(","): kv = item.split("=") if len(kv) != 2: wandb.termwarn("Unable to parse sweep settings key value pair", repeat=False) ret.update(dict([kv])) return ret wandb.termwarn("Unable to parse settings parameter", repeat=False) return ret if api.api_key is None: termlog("Login to W&B to use the sweep feature") ctx.invoke(login, no_offline=True) sweep_obj_id = None if update: parts = dict(entity=entity, project=project, name=update) err = util.parse_sweep_id(parts) if err: wandb.termerror(err) return entity = parts.get("entity") or entity project = parts.get("project") or project sweep_id = parts.get("name") or update found = api.sweep(sweep_id, '{}', entity=entity, project=project) if not found: wandb.termerror('Could not find sweep {}/{}/{}'.format(entity, project, sweep_id)) return sweep_obj_id = found['id'] wandb.termlog('{} sweep from: {}'.format( 'Updating' if sweep_obj_id else 'Creating', config_yaml)) try: yaml_file = open(config_yaml) except (OSError, IOError): wandb.termerror('Couldn\'t open sweep file: %s' % config_yaml) return try: config = util.load_yaml(yaml_file) except yaml.YAMLError as err: wandb.termerror('Error in configuration file: %s' % err) return if config is None: wandb.termerror('Configuration file is empty') return # Set or override parameters if name: config["name"] = name if program: config["program"] = program if settings: settings = _parse_settings(settings) if settings: config.setdefault("settings", {}) config["settings"].update(settings) if controller: config.setdefault("controller", {}) config["controller"]["type"] = "local" is_local = config.get('controller', {}).get('type') == 'local' if is_local: tuner = wandb_controller.controller() err = tuner._validate(config) if err: wandb.termerror('Error in sweep file: %s' % err) return entity = entity or env.get_entity() or config.get('entity') project = project or env.get_project() or config.get('project') or util.auto_project_name( config.get("program"), api) sweep_id = api.upsert_sweep(config, project=project, entity=entity, obj_id=sweep_obj_id) wandb.termlog('{} sweep with ID: {}'.format( 'Updated' if sweep_obj_id else 'Created', click.style(sweep_id, fg="yellow"))) sweep_url = wandb_controller._get_sweep_url(api, sweep_id) if sweep_url: wandb.termlog("View sweep at: {}".format( click.style(sweep_url, underline=True, fg='blue'))) # reprobe entity and project if it was autodetected by upsert_sweep entity = entity or env.get_entity() project = project or env.get_project() if entity and project: sweep_path = "{}/{}/{}".format(entity, project, sweep_id) elif project: sweep_path = "{}/{}".format(project, sweep_id) else: sweep_path = sweep_id wandb.termlog("Run sweep agent with: {}".format( click.style("wandb agent %s" % sweep_path, fg="yellow"))) if controller: wandb.termlog('Starting wandb controller...') tuner = wandb_controller.controller(sweep_id) tuner.run(verbose=verbose)