def log_histogram( self, name, values, bins, max_bins=None, step=None, timestamp=None ): """Logs a histogram. ```python >>> log_histogram(name="histo", values=np.arange(np.prod((1024,)), dtype=float).reshape((1024,)), bins="auto", step=1) # noqa ``` Args: name: str, name values: np.array bins: int or str max_bins: int, optional step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.histogram( values=values, bins=bins, max_bins=max_bins ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HISTOGRAM, event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value), ) self._add_event(logged_event)
def log_artifact( self, path, name=None, artifact_kind=None, step=None, timestamp=None ): self._log_has_events() name = name or os.path.basename(name) ext = get_path_extension(filepath=path) artifact_kind = artifact_kind or V1ArtifactKind.FILE asset_path = get_asset_path( run_path=self.artifacts_path, kind=artifact_kind, name=name, step=step, ext=ext, ) asset_rel_path = os.path.relpath(asset_path, self.artifacts_path) artifact = events_processors.artifact_path( from_path=path, asset_path=asset_path, kind=artifact_kind, asset_rel_path=asset_rel_path, ) logged_event = LoggedEventSpec( name=name, kind=artifact_kind, event=V1Event.make(timestamp=timestamp, step=step, artifact=artifact), ) self._event_logger.add_event(logged_event)
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None): """Logs a custom curve. ```python >>> log_curve("pr_value", x, y, annotation="more=info", step=10) ``` Args: name: str, name of the curve x: List[float] or numpy.array y: List[float] or numpy.array annotation: str, optional step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.curve( x=x, y=y, annotation=annotation, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._add_event(logged_event)
def log_dataframe( self, path, name=None, content_type=None, step=None, timestamp=None ): self._log_has_events() name = name or os.path.basename(path) ext = get_path_extension(filepath=path) asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.DATAFRAME, name=name, step=step, ext=ext, ) asset_rel_path = os.path.relpath(asset_path, self.artifacts_path) df = events_processors.dataframe_path( from_path=path, asset_path=asset_path, content_type=content_type, asset_rel_path=asset_rel_path, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.DATAFRAME, event=V1Event.make(timestamp=timestamp, step=step, dataframe=df), ) self._event_logger.add_event(logged_event)
def log_metric(self, name: str, value: float, step: int = None, timestamp=None): """Logs a metric datapoint. ```python >>> log_metric(name="loss", value=0.01, step=10) ``` > It's very important to log `step` as one of your metrics if you want to compare experiments on the dashboard and use the steps in x-axis instead of timestamps. Args: name: str, metric name value: float, metric value step: int, optional timestamp: datetime, optional """ self._log_has_events() events = [] event_value = events_processors.metric(value) if event_value == UNKNOWN: return events.append( LoggedEventSpec( name=name, kind=V1ArtifactKind.METRIC, event=V1Event.make(timestamp=timestamp, step=step, metric=event_value), ) ) if events: self._add_events(events) self._results[name] = event_value
def log_model( self, path, name=None, framework=None, spec=None, step=None, timestamp=None ): self._log_has_model() name = name or os.path.basename(path) ext = None if os.path.isfile(path): ext = get_path_extension(filepath=path) asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.MODEL, name=name, step=step, ext=ext, ) asset_rel_path = os.path.relpath(asset_path, self.artifacts_path) model = events_processors.model_path( from_path=path, asset_path=asset_path, framework=framework, spec=spec, asset_rel_path=asset_rel_path, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.MODEL, event=V1Event.make(timestamp=timestamp, step=step, model=model), ) self._event_logger.add_event(logged_event)
def test_log_line_has_iso_datetime(self): parsed = V1Event.make(timestamp="2018-12-11T08:49:07.163495183Z", step=12) expected = V1Event( timestamp=dt_parser.parse("2018-12-11T08:49:07.163495+00:00"), step=12) assert parsed == expected
def log_pr_curve(self, name, precision, recall, average_precision=None, step=None, timestamp=None): """Logs PR curve. This method expects an already processed values. ```python >>> log_pr_curve("pr_value", precision, recall, step=10) ``` Args: name: str, name of the curve y_preds: List[float] or numpy.array y_targets: List[float] or numpy.array step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.pr_curve( precision=precision, recall=recall, average_precision=average_precision, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_sklearn_pr_curve(self, name, y_preds, y_targets, step=None, timestamp=None): """Calculates and logs PR curve using sklearn. ```python >>> log_sklearn_pr_curve("pr_value", y_preds, y_targets, step=10) ``` Args: name: str, name of the event y_preds: List[float] or numpy.array y_targets: List[float] or numpy.array step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.sklearn_pr_curve( y_preds=y_preds, y_targets=y_targets, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_metrics(self, step=None, timestamp=None, **metrics): """Logs multiple metrics. ```python >>> log_metrics(step=123, loss=0.023, accuracy=0.91) ``` > It's very important to log `step` as one of your metrics if you want to compare experiments on the dashboard and use the steps in x-axis instead of timestamps. Args: step: int, optional timestamp: datetime, optional **metrics: **kwargs, key: value """ self._log_has_events() events = [] for metric in metrics: event_value = events_processors.metric(metrics[metric]) if event_value == UNKNOWN: continue events.append( LoggedEventSpec( name=metric, kind=V1ArtifactKind.METRIC, event=V1Event.make(timestamp=timestamp, step=step, metric=event_value), )) if events: self._event_logger.add_events(events)
def log_np_histogram(self, name, values, counts, step=None, timestamp=None): """Logs a numpy histogram. ```python >>> values, counts = np.histogram(np.random.randint(255, size=(1000,))) >>> log_np_histogram(name="histo1", values=values, counts=counts, step=1) ``` Args: name: str, name values: np.array counts: np.array step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.np_histogram(values=values, counts=counts) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HISTOGRAM, event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value), ) self._add_event(logged_event)
def log_roc_auc_curve(self, name, fpr, tpr, auc=None, step=None, timestamp=None): """Logs ROC/AUC curve. This method expects an already processed values. ```python >>> log_roc_auc_curve("roc_value", fpr, tpr, auc=0.6, step=1) ``` Args: name: str, name of the curve fpr: List[float] or numpy.array, false positive rate tpr: List[float] or numpy.array, true positive rate auc: float, optional, calculated area under curve step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.roc_auc_curve( fpr=fpr, tpr=tpr, auc=auc, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_image_with_boxes( self, tensor_image, tensor_boxes, name=None, step=None, timestamp=None, rescale=1, dataformats="CHW", ): """Logs an image with bounding boxes. ```python >>> log_image_with_boxes( >>> name="my_image", >>> tensor_image=np.arange(np.prod((3, 32, 32)), dtype=float).reshape((3, 32, 32)), >>> tensor_boxes=np.array([[10, 10, 40, 40]]), >>> ) ``` Args: tensor_image: numpy.array or str: Image data or file name tensor_boxes: numpy.array or str: Box data (for detected objects) box should be represented as [x1, y1, x2, y2] name: str, name of the image step: int, optional timestamp: datetime, optional rescale: int, optional dataformats: str, optional """ self._log_has_events() name = name or "figure" asset_path = get_asset_path( run_path=self._artifacts_path, kind=V1ArtifactKind.IMAGE, name=name, step=step, ) asset_rel_path = os.path.relpath(asset_path, self._artifacts_path) event_value = events_processors.image_boxes( asset_path=asset_path, tensor_image=tensor_image, tensor_boxes=tensor_boxes, rescale=rescale, dataformats=dataformats, asset_rel_path=asset_rel_path, ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.IMAGE, event=V1Event.make(timestamp=timestamp, step=step, image=event_value), ) self._event_logger.add_event(logged_event)
def log_text(self, name, text, step=None, timestamp=None): self._log_has_events() logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.TEXT, event=V1Event.make(timestamp=timestamp, step=step, text=text), ) self._event_logger.add_event(logged_event)
def metrics_dict_to_list(metrics: Dict) -> List: results = [] for k, v in metrics.items(): results.append( LoggedEventSpec( name=k, kind=V1ArtifactKind.METRIC, event=V1Event.make(metric=v), ) ) return results
def log_html(self, name, html, step=None, timestamp=None): self._log_has_events() logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HTML, event=V1Event.make(timestamp=timestamp, step=step, html=html), ) self._event_logger.add_event(logged_event)
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None): self._log_has_events() chart = events_processors.mpl_plotly_chart(figure=figure) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CHART, event=V1Event.make(timestamp=timestamp, step=step, chart=chart), ) self._event_logger.add_event(logged_event)
def log_roc_auc_curve(self, name, fpr, tpr, auc=None, step=None, timestamp=None): self._log_has_events() event_value = events_processors.roc_auc_curve(fpr=fpr, tpr=tpr, auc=auc,) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None): self._log_has_events() event_value = events_processors.curve(x=x, y=y, annotation=annotation,) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_audio( self, data, name=None, sample_rate=44100, step=None, timestamp=None, content_type=None, ): self._log_dashboard() is_file = isinstance(data, str) and os.path.exists(data) ext = content_type or "wav" if is_file: name = name or os.path.basename(data) ext = get_path_extension(filepath=data) or ext else: name = name or "audio" asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.AUDIO, name=name, step=step, ext=ext, ) asset_rel_path = os.path.relpath(asset_path, self.artifacts_path) if is_file: event_value = events_processors.audio_path( from_path=data, asset_path=asset_path, content_type=content_type, asset_rel_path=asset_rel_path, ) else: event_value = events_processors.audio( asset_path=asset_path, tensor=data, sample_rate=sample_rate, asset_rel_path=asset_rel_path, ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.AUDIO, event=V1Event.make(timestamp=timestamp, step=step, audio=event_value), ) self._event_logger.add_event(logged_event)
def log_image(self, data, name=None, step=None, timestamp=None, rescale=1, dataformats="CHW"): self._log_dashboard() is_file = isinstance(data, str) and os.path.exists(data) ext = "png" if is_file: name = name or os.path.basename(data) ext = get_path_extension(filepath=data) or ext else: name = name or "image" asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.IMAGE, name=name, step=step, ext=ext, ) asset_rel_path = os.path.relpath(asset_path, self.artifacts_path) if is_file: event_value = events_processors.image_path(from_path=data, asset_path=asset_path) elif hasattr(data, "encoded_image_string"): event_value = events_processors.encoded_image( asset_path=asset_path, data=data, asset_rel_path=asset_rel_path, ) else: event_value = events_processors.image( asset_path=asset_path, data=data, rescale=rescale, dataformats=dataformats, asset_rel_path=asset_rel_path, ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.IMAGE, event=V1Event.make(timestamp=timestamp, step=step, image=event_value), ) self._event_logger.add_event(logged_event)
def log_sklearn_pr_curve(self, name, y_preds, y_targets, step=None, timestamp=None): self._log_has_events() event_value = events_processors.sklearn_pr_curve( y_preds=y_preds, y_targets=y_targets, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_video(self, data, name=None, fps=4, step=None, timestamp=None, content_type=None): self._log_dashboard() is_file = isinstance(data, str) and os.path.exists(data) content_type = content_type or "gif" if is_file: name = name or os.path.basename(data) content_type = get_path_extension(filepath=data) or content_type else: name = name or "video" asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.VIDEO, name=name, step=step, ext=content_type, ) asset_rel_path = os.path.relpath(asset_path, self.artifacts_path) if is_file: event_value = events_processors.video_path( from_path=data, asset_path=asset_path, content_type=content_type, asset_rel_path=asset_rel_path, ) else: event_value = events_processors.video( asset_path=asset_path, tensor=data, fps=fps, content_type=content_type, asset_rel_path=asset_rel_path, ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.VIDEO, event=V1Event.make(timestamp=timestamp, step=step, video=event_value), ) self._event_logger.add_event(logged_event)
def log_pr_curve( self, name, precision, recall, average_precision=None, step=None, timestamp=None ): self._log_has_events() event_value = events_processors.pr_curve( precision=precision, recall=recall, average_precision=average_precision, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_np_histogram(self, name, values, counts, step=None, timestamp=None): self._log_has_events() event_value = events_processors.np_histogram(values=values, counts=counts) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HISTOGRAM, event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value), ) self._event_logger.add_event(logged_event)
def log_metric(self, name, value, step=None, timestamp=None): events = [] event_value = events_processors.metric(value) if event_value == UNKNOWN: return events.append( LoggedEventSpec( name=name, kind=V1ArtifactKind.METRIC, event=V1Event.make(timestamp=timestamp, step=step, metric=event_value), ) ) if events: self._event_logger.add_events(events) self._results[name] = event_value
def create_tmp_events(self): text1 = LoggedEventListSpec( name="text1", kind=V1ArtifactKind.TEXT, events=[ V1Event.make(step=1, text="foo1"), V1Event.make(step=2, text="boo2"), ], ) self.create_kind_events(name="text1", kind=V1ArtifactKind.TEXT, events=text1) text2 = LoggedEventListSpec( name="text2", kind=V1ArtifactKind.TEXT, events=[ V1Event.make(step=1, text="foo2"), V1Event.make(step=2, text="boo2"), ], ) self.create_kind_events(name="text2", kind=V1ArtifactKind.TEXT, events=text2) html1 = LoggedEventListSpec( name="html1", kind=V1ArtifactKind.HTML, events=[ V1Event.make(step=1, html="foo1"), V1Event.make(step=2, html="boo2"), ], ) self.create_kind_events(name="html1", kind=V1ArtifactKind.HTML, events=html1) html2 = LoggedEventListSpec( name="htm2", kind=V1ArtifactKind.HTML, events=[ V1Event.make(step=1, html="foo2"), V1Event.make(step=2, html="boo2"), ], ) self.create_kind_events(name="html2", kind=V1ArtifactKind.HTML, events=html2)
def log_metrics(self, step=None, timestamp=None, **metrics): events = [] for metric in metrics: event_value = events_processors.metric(metrics[metric]) if event_value == UNKNOWN: continue events.append( LoggedEventSpec( name=metric, kind=V1ArtifactKind.METRIC, event=V1Event.make(timestamp=timestamp, step=step, metric=event_value), )) if events: self._event_logger.add_events(events)
def log_model(self, path, name=None, framework=None, spec=None, step=None, timestamp=None): """Logs a model. Args: path: str, path to the model to log name: str, name framework: str, optional ,name of the framework spec: Dict, optional, key, value information about the model step: int, optional timestamp: datetime, optional """ self._log_has_model() name = name or os.path.basename(path) ext = None if os.path.isfile(path): ext = get_path_extension(filepath=path) asset_path = get_asset_path( run_path=self._artifacts_path, kind=V1ArtifactKind.MODEL, name=name, step=step, ext=ext, ) asset_rel_path = os.path.relpath(asset_path, self._artifacts_path) model = events_processors.model_path( from_path=path, asset_path=asset_path, framework=framework, spec=spec, asset_rel_path=asset_rel_path, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.MODEL, event=V1Event.make(timestamp=timestamp, step=step, model=model), ) self._event_logger.add_event(logged_event)
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None): """Logs a matplotlib figure to plotly figure. Args: name: str, name of the figure figure: figure step: int, optional timestamp: datetime, optional """ self._log_has_events() chart = events_processors.mpl_plotly_chart(figure=figure) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CHART, event=V1Event.make(timestamp=timestamp, step=step, chart=chart), ) self._event_logger.add_event(logged_event)