def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> None: # TODO: HACK figure out where this is being set to true metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) self.experiment.debug = self.debug self.experiment.log(metrics, global_step=step)
def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> None: metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) self.experiment.log_metrics(metrics, step) if step is not None and (step + 1) % self._flush_logs_every_n_steps == 0: self.save()
def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> None: assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0" metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) if step is not None: self.experiment.log({**metrics, "trainer/global_step": step}) else: self.experiment.log(metrics)
def test_add_prefix(): """Verify add_prefix modifies the dict keys correctly.""" metrics = {"metric1": 1, "metric2": 2} metrics = _add_prefix(metrics, "prefix", "-") assert "prefix-metric1" in metrics assert "prefix-metric2" in metrics assert "metric1" not in metrics assert "metric2" not in metrics metrics = _add_prefix(metrics, "prefix2", "_") assert "prefix2_prefix-metric1" in metrics assert "prefix2_prefix-metric2" in metrics assert "prefix-metric1" not in metrics assert "prefix-metric2" not in metrics assert metrics["prefix2_prefix-metric1"] == 1 assert metrics["prefix2_prefix-metric2"] == 2
def log_metrics(self, metrics: Dict[str, Union[torch.Tensor, float]], step: Optional[int] = None) -> None: assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0" # Comet.ml expects metrics to be a dictionary of detached tensors on CPU metrics_without_epoch = metrics.copy() for key, val in metrics_without_epoch.items(): if is_tensor(val): metrics_without_epoch[key] = val.cpu().detach() epoch = metrics_without_epoch.pop("epoch", None) metrics_without_epoch = _add_prefix(metrics_without_epoch, self._prefix, self.LOGGER_JOIN_CHAR) self.experiment.log_metrics(metrics_without_epoch, step=step, epoch=epoch)
def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): assert (rank_zero_only.rank == 0 ), "experiment tried to log from global_rank != 0" metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) for metric_name, metric_val in metrics.items(): if is_tensor(metric_val): metric_val = metric_val.cpu().detach().item() self.experiment.log(name=metric_name, val=metric_val) self.experiment.next_step()
def log_metrics(self, metrics: Dict[str, Union[Tensor, float]], step: Optional[int] = None) -> None: """Log metrics (numeric values) in Neptune runs. Args: metrics: Dictionary with metric names as keys and measured quantities as values. step: Step number at which the metrics should be recorded, currently ignored. """ if rank_zero_only.rank != 0: raise ValueError("run tried to log from global_rank != 0") metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) for key, val in metrics.items(): # `step` is ignored because Neptune expects strictly increasing step values which # Lightning does not always guarantee. self.run[key].log(val)
def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> None: assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0" metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) for k, v in metrics.items(): if isinstance(v, Tensor): v = v.item() if isinstance(v, dict): self.experiment.add_scalars(k, v, step) else: try: self.experiment.add_scalar(k, v, step) # todo: specify the possible exception except Exception as ex: m = f"\n you tried to log {v} which is currently not supported. Try a dict or a scalar/tensor." raise ValueError(m) from ex
def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> None: assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0" metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR) timestamp_ms = int(time() * 1000) for k, v in metrics.items(): if isinstance(v, str): log.warning(f"Discarding metric with string value {k}={v}.") continue new_k = re.sub("[^a-zA-Z0-9_/. -]+", "", k) if k != new_k: rank_zero_warn( "MLFlow only allows '_', '/', '.' and ' ' special characters in metric name." f" Replacing {k} with {new_k}.", category=RuntimeWarning, ) k = new_k self.experiment.log_metric(self.run_id, k, v, timestamp_ms, step)