def _log_tensors(self, tensor_values):
"""Update timer, log tensors, send to MLFlow and Graphite"""
self._timer.update_last_triggered_step(self._iter_count)
global_step = tensor_values["global_step"]
if self.skip_after_step is not None and global_step >= self.skip_after_step:
return
# Log tensor and function values
ord_tensor_values = [(tag, tensor_values[tag])
for tag in self._tag_order]
ord_function_values = [
(tag, self.functions[tag]()) for tag in self._fn_order
] if self.functions else []
LOGGER.info(", ".join(
self.formatter(tag, value)
for tag, value in ord_tensor_values + ord_function_values))
# Send to MLFlow and Graphite
for tag, value in ord_tensor_values + ord_function_values:
if self.use_graphite:
graphite.log_metric(tag, value, postfix=self.name)
if self.use_mlflow:
tag = tag if self.name is None else f"{self.name}_{tag}"
mlflow.log_metric(sanitize_metric_name(tag),
value,
step=global_step)
def _log_and_record(self, elapsed_steps, elapsed_time, global_step):
"""Log Steps per second and write summary"""
if self.skip_after_step is not None and global_step >= self.skip_after_step:
return
# Compute steps and number of examples per second
metrics = {
"steps_per_sec": elapsed_steps / elapsed_time,
"examples_per_sec": self.batch_size * elapsed_steps / elapsed_time,
}
# Log tensor values
LOGGER.info(", ".join(f"{tag} = {value:.2f}"
for tag, value in metrics.items()))
# Send to MLFlow and Graphite
for tag, value in metrics.items():
if self.use_graphite:
graphite.log_metric(tag, value, postfix=self.name)
if self.use_mlflow:
tag = tag if self.name is None else f"{self.name}_{tag}"
mlflow.log_metric(tag, value, step=global_step)