def _calculate_namespace(self, logdir, rootdir): # namespace: "Optional[str]" dirs = list(self._logdirs) + [logdir] if os.path.isfile(logdir): filename = os.path.basename(logdir) else: filename = "" if rootdir == "": rootdir = util.to_forward_slash_path( os.path.dirname(os.path.commonprefix(dirs)) ) # Tensorboard loads all tfevents files in a directory and prepends # their values with the path. Passing namespace to log allows us # to nest the values in wandb # Note that we strip '/' instead of os.sep, because elsewhere we've # converted paths to forward slash. namespace = logdir.replace(filename, "").replace(rootdir, "").strip("/") # TODO: revisit this heuristic, it exists because we don't know the # root log directory until more than one tfevents file is written to if len(dirs) == 1 and namespace not in ["train", "validation"]: namespace = None else: namespace = logdir.replace(filename, "").replace(rootdir, "").strip("/") return namespace
def __init__(self, file_path, save_name, api, file_pusher, *args, **kwargs): self.file_path = file_path # Convert windows paths to unix paths save_name = util.to_forward_slash_path(save_name) self.save_name = save_name self._file_pusher = file_pusher self._last_sync = None self._api = api
def add(self, logdir: str, save: bool, root_dir: str) -> None: logdir = util.to_forward_slash_path(logdir) root_dir = util.to_forward_slash_path(root_dir) if logdir in self._logdirs: return namespace = self._calculate_namespace(logdir, root_dir) # TODO(jhr): implement the deferred tbdirwatcher to find namespace if not self._consumer: self._consumer = TBEventConsumer(self, self._watcher_queue, self._run_proto, self._settings) self._consumer.start() tbdir_watcher = TBDirWatcher(self, logdir, save, namespace, self._watcher_queue, self._force) self._logdirs[logdir] = tbdir_watcher tbdir_watcher.start()
def __init__( self, path, ref, digest, birth_artifact_id=None, size=None, extra=None, local_path=None, ): if local_path is not None and size is None: raise AssertionError( "programming error, size required when local_path specified") self.path = util.to_forward_slash_path(path) self.ref = ref # This is None for files stored in the artifact. self.digest = digest self.birth_artifact_id = birth_artifact_id self.size = size self.extra = extra or {} # This is not stored in the manifest json, it's only used in the process # of saving self.local_path = local_path
def __init__(self, config=None, settings=None): self._config = wandb_config.Config() self._config._set_callback(self._config_callback) self.summary = wandb_summary.Summary() self.summary._set_callback(self._summary_callback) self.history = wandb_history.History(self) self.history._set_callback(self._history_callback) _datatypes_set_callback(self._datatypes_callback) self._settings = settings self._wl = None self._backend = None self._reporter = None self._data = dict() self._entity = None self._project = None self._group = None self._job_type = None self._run_id = settings.run_id self._start_time = time.time() self._starting_step = 0 self._name = None self._notes = None self._tags = None self._hooks = None self._redirect_cb = None self._out_redir = None self._err_redir = None self.stdout_redirector = None self.stderr_redirector = None self._save_stdout = None self._save_stderr = None self._stdout_slave_fd = None self._stderr_slave_fd = None self._exit_code = None self._exit_result = None self._final_summary = None # Pull info from settings self._init_from_settings(settings) # Initial scope setup for sentry. This might get changed when the # actual run comes back. sentry_set_scope("user", self._entity, self._project) # Returned from backend send_run_sync, set from wandb_init? self._run_obj = None # Created when the run "starts". self._run_status_checker = None config = config or dict() wandb_key = "_wandb" config.setdefault(wandb_key, dict()) config[wandb_key]["cli_version"] = wandb.__version__ if settings.save_code and settings.program_relpath: config[wandb_key]["code_path"] = to_forward_slash_path( os.path.join("code", settings.program_relpath)) self._config._update(config) self._atexit_cleanup_called = None self._use_redirect = True