def test_dataframes_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), dataframe=V1EventDataframe(path="path1", content_type="parquet"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), dataframe=V1EventDataframe(path="path2", content_type="pickle"), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), dataframe=V1EventDataframe(path="path3"), step=14, ), ] events = V1Events.read( name="foo", kind="dataframe", data=os.path.abspath( "tests/fixtures/polyboard/dataframe/dataframe_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_artifacts_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), artifact=V1EventArtifact(kind="dataframe", path="path1"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), artifact=V1EventArtifact(kind="tsv", path="path2"), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), artifact=V1EventArtifact(kind="csv", path="path3"), step=14, ), ] events = V1Events.read( name="foo", kind="artifact", data=os.path.abspath( "tests/fixtures/polyboard/artifact/artifact_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_log_line_has_iso_datetime(self): parsed = V1Event.make(timestamp="2018-12-11T08:49:07.163495183Z", step=12) expected = V1Event( timestamp=dt_parser.parse("2018-12-11T08:49:07.163495+00:00"), step=12) assert parsed == expected
def test_htmls_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), html="<div>1</div>", step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), html="<div>2</div>", step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), html="<div>3</div>", step=14, ), ] events = V1Events.read( name="foo", kind="html", data=os.path.abspath( "tests/fixtures/polyboard/html/html_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_charts_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), chart=V1EventChart(kind="plotly", figure={"foo": "bar"}), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), chart=V1EventChart(kind="vega", figure={"foo2": "bar2"}), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), chart=V1EventChart(kind="bokeh", figure={"foo3": "bar3"}), step=14, ), ] events = V1Events.read( name="foo", kind="chart", data=os.path.abspath( "tests/fixtures/polyboard/chart/chart_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_videos_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), video=V1EventVideo(path="test", content_type="mp4"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), video=V1EventVideo(height=1, width=1), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), video=V1EventVideo(height=10, width=10, colorspace=2), step=14, ), ] events = V1Events.read( name="foo", kind="video", data=os.path.abspath( "tests/fixtures/polyboard/video/video_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def create_tmp_events(self, run_events: str): metric1 = LoggedEventListSpec( name="metric1", kind=V1ArtifactKind.METRIC, events=[ V1Event.make(step=1, metric=1.1), V1Event.make(step=2, metric=1.2), ], ) self.create_kind_events( run_events=run_events, name="metric1", kind=V1ArtifactKind.METRIC, events=metric1, ) metric2 = LoggedEventListSpec( name="metric2", kind=V1ArtifactKind.METRIC, events=[ V1Event.make(step=1, metric=2.1), V1Event.make(step=2, metric=2.2), ], ) self.create_kind_events( run_events=run_events, name="metric2", kind=V1ArtifactKind.METRIC, events=metric2, )
def test_histograms_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), histogram=V1EventHistogram(values=[10], counts=[1]), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), histogram=V1EventHistogram(values=[10, 1, 1], counts=[1, 1, 1]), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), histogram=V1EventHistogram(values=[10, 112, 12, 1], counts=[12, 1, 1, 1]), step=14, ), ] events = V1Events.read( name="foo", kind="histogram", data=os.path.abspath( "tests/fixtures/polyboard/histogram/histogram_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_video(self): events = LoggedEventListSpec( name="foo", kind="video", events=[ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), video=V1EventVideo(height=1, width=1, colorspace=1, path="path"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 11:24:57"), video=V1EventVideo(height=10, width=1, colorspace=0, path="path"), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 12:24:57"), video=V1EventVideo(height=1, width=10, colorspace=2, path="path"), step=14, ), ], ) events_dict = events.to_dict() assert events_dict == events.from_dict(events_dict).to_dict()
def test_models_read_yaml(self): values = [ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), model=V1EventModel(framework="tensorflow", path="path1"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:25:57"), model=V1EventModel(framework="pytorch", path="path2"), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 10:26:57"), model=V1EventModel(framework="onnx", path="path3"), step=14, ), ] events = V1Events.read( name="foo", kind="model", data=os.path.abspath("tests/fixtures/polyboard/model/model_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_dataframe(self): events = LoggedEventListSpec( name="foo", kind="dataframe", events=[ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), dataframe=V1EventDataframe(path="path", content_type="parquet"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 11:24:57"), dataframe=V1EventDataframe(path="path", content_type="pickle"), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 12:24:57"), dataframe=V1EventDataframe(path="path"), step=14, ), ], ) events_dict = events.to_dict() assert events_dict == events.from_dict(events_dict).to_dict()
def test_audio(self): events = LoggedEventListSpec( name="foo", kind="audio", events=[ V1Event( timestamp=dt_parser.parse("2018-12-11 10:24:57"), audio=V1EventAudio(sample_rate=1.1, num_channels=2, length_frames=2, path="test"), step=12, ), V1Event( timestamp=dt_parser.parse("2018-12-11 11:24:57"), audio=V1EventAudio(sample_rate=1.1, num_channels=2, length_frames=2, path="test"), step=13, ), V1Event( timestamp=dt_parser.parse("2018-12-11 12:24:57"), audio=V1EventAudio(sample_rate=1.12, num_channels=22, length_frames=22, path="test"), step=14, ), ], ) events_dict = events.to_dict() assert events_dict == events.from_dict(events_dict).to_dict()
def test_audios_read_yaml(self): values = [ V1Event( timestamp=parse_datetime("2018-12-11 10:24:57"), audio=V1EventAudio( sample_rate=1.1, num_channels=2, length_frames=2, path="test" ), step=12, ), V1Event( timestamp=parse_datetime("2018-12-11 10:25:57"), audio=V1EventAudio( sample_rate=1.11, num_channels=22, length_frames=22, path="test", content_type="wav", ), step=13, ), V1Event( timestamp=parse_datetime("2018-12-11 10:26:57"), audio=V1EventAudio(path="testwave", content_type="wav"), step=14, ), ] events = V1Events.read( name="foo", kind="audio", data=os.path.abspath("tests/fixtures/polyboard/audio/audio_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_images_read_yaml(self): values = [ V1Event( timestamp=parse_datetime("2018-12-11 10:24:57"), image=V1EventImage(path="test"), step=12, ), V1Event( timestamp=parse_datetime("2018-12-11 10:25:57"), image=V1EventImage(height=1, width=1), step=13, ), V1Event( timestamp=parse_datetime("2018-12-11 10:26:57"), image=V1EventImage(height=10, width=10, colorspace=2), step=14, ), ] events = V1Events.read( name="foo", kind="image", data=os.path.abspath("tests/fixtures/polyboard/image/image_events.plx"), ) assert events.name == "foo" assert len(events.df.values) == 3 for i in range(3): assert events.get_event_at(i).to_dict() == values[i].to_dict()
def log_sklearn_pr_curve(self, name, y_preds, y_targets, step=None, timestamp=None): """Calculates and logs PR curve using sklearn. ```python >>> log_sklearn_pr_curve("pr_value", y_preds, y_targets, step=10) ``` Args: name: str, name of the event y_preds: List[float] or numpy.array y_targets: List[float] or numpy.array step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.sklearn_pr_curve( y_preds=y_preds, y_targets=y_targets, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_pr_curve(self, name, precision, recall, average_precision=None, step=None, timestamp=None): """Logs PR curve. This method expects an already processed values. ```python >>> log_pr_curve("pr_value", precision, recall, step=10) ``` Args: name: str, name of the curve y_preds: List[float] or numpy.array y_targets: List[float] or numpy.array step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.pr_curve( precision=precision, recall=recall, average_precision=average_precision, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_model(self, path, name=None, framework=None, spec=None, step=None, timestamp=None): name = name or os.path.basename(path) ext = None if os.path.isfile(path): ext = get_path_extension(filepath=path) asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.MODEL, name=name, step=step, ext=ext, ) model = events_processors.model_path(from_path=path, asset_path=asset_path, framework=framework, spec=spec) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.MODEL, event=V1Event(timestamp=timestamp, step=step, model=model), ) self._event_logger.add_event(logged_event)
def log_html(self, name, html, step=None, timestamp=None): logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HTML, event=V1Event(timestamp=timestamp, step=step, html=html), ) self._event_logger.add_event(logged_event)
def log_text(self, name, text, step=None, timestamp=None): logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.TEXT, event=V1Event(timestamp=timestamp, step=step, text=text), ) self._event_logger.add_event(logged_event)
def log_image_with_boxes( self, tensor_image, tensor_boxes, name=None, step=None, timestamp=None, rescale=1, dataformats="CHW", ): name = name or "figure" asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.IMAGE, name=name, step=step, ) event_value = events_processors.image_boxes( asset_path=asset_path, tensor_image=tensor_image, tensor_boxes=tensor_boxes, rescale=rescale, dataformats=dataformats, ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.IMAGE, event=V1Event(timestamp=timestamp, step=step, image=event_value), ) self._event_logger.add_event(logged_event)
def log_artifact(self, path, name=None, artifact_kind=None, step=None, timestamp=None): name = name or os.path.basename(name) ext = get_path_extension(filepath=path) artifact_kind = artifact_kind or V1ArtifactKind.FILE asset_path = get_asset_path( run_path=self.artifacts_path, kind=artifact_kind, name=name, step=step, ext=ext, ) artifact = events_processors.artifact_path(from_path=path, asset_path=asset_path, kind=artifact_kind) logged_event = LoggedEventSpec( name=name, kind=artifact_kind, event=V1Event(timestamp=timestamp, step=step, artifact=artifact), ) self._event_logger.add_event(logged_event)
def log_dataframe(self, path, name=None, content_type=None, step=None, timestamp=None): name = name or os.path.basename(path) ext = get_path_extension(filepath=path) asset_path = get_asset_path( run_path=self.artifacts_path, kind=V1ArtifactKind.DATAFRAME, name=name, step=step, ext=ext, ) df = events_processors.dataframe_path(from_path=path, asset_path=asset_path, content_type=content_type) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.DATAFRAME, event=V1Event(timestamp=timestamp, step=step, dataframe=df), ) self._event_logger.add_event(logged_event)
def log_np_histogram(self, name, values, counts, step=None, timestamp=None): """Logs a numpy histogram. ```python >>> values, counts = np.histogram(np.random.randint(255, size=(1000,))) >>> log_np_histogram(name="histo1", values=values, counts=counts, step=1) ``` Args: name: str, name values: np.array counts: np.array step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.np_histogram(values=values, counts=counts) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HISTOGRAM, event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value), ) self._add_event(logged_event)
def log_histogram( self, name, values, bins, max_bins=None, step=None, timestamp=None ): """Logs a histogram. ```python >>> log_histogram(name="histo", values=np.arange(np.prod((1024,)), dtype=float).reshape((1024,)), bins="auto", step=1) # noqa ``` Args: name: str, name values: np.array bins: int or str max_bins: int, optional step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.histogram( values=values, bins=bins, max_bins=max_bins ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.HISTOGRAM, event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value), ) self._add_event(logged_event)
def log_metrics(self, step=None, timestamp=None, **metrics): """Logs multiple metrics. ```python >>> log_metrics(step=123, loss=0.023, accuracy=0.91) ``` > It's very important to log `step` as one of your metrics if you want to compare experiments on the dashboard and use the steps in x-axis instead of timestamps. Args: step: int, optional timestamp: datetime, optional **metrics: **kwargs, key: value """ self._log_has_events() events = [] for metric in metrics: event_value = events_processors.metric(metrics[metric]) if event_value == UNKNOWN: continue events.append( LoggedEventSpec( name=metric, kind=V1ArtifactKind.METRIC, event=V1Event.make(timestamp=timestamp, step=step, metric=event_value), )) if events: self._event_logger.add_events(events)
def log_metric(self, name: str, value: float, step: int = None, timestamp=None): """Logs a metric datapoint. ```python >>> log_metric(name="loss", value=0.01, step=10) ``` > It's very important to log `step` as one of your metrics if you want to compare experiments on the dashboard and use the steps in x-axis instead of timestamps. Args: name: str, metric name value: float, metric value step: int, optional timestamp: datetime, optional """ self._log_has_events() events = [] event_value = events_processors.metric(value) if event_value == UNKNOWN: return events.append( LoggedEventSpec( name=name, kind=V1ArtifactKind.METRIC, event=V1Event.make(timestamp=timestamp, step=step, metric=event_value), ) ) if events: self._add_events(events) self._results[name] = event_value
def log_roc_auc_curve(self, name, fpr, tpr, auc=None, step=None, timestamp=None): """Logs ROC/AUC curve. This method expects an already processed values. ```python >>> log_roc_auc_curve("roc_value", fpr, tpr, auc=0.6, step=1) ``` Args: name: str, name of the curve fpr: List[float] or numpy.array, false positive rate tpr: List[float] or numpy.array, true positive rate auc: float, optional, calculated area under curve step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.roc_auc_curve( fpr=fpr, tpr=tpr, auc=auc, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._event_logger.add_event(logged_event)
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None): """Logs a custom curve. ```python >>> log_curve("pr_value", x, y, annotation="more=info", step=10) ``` Args: name: str, name of the curve x: List[float] or numpy.array y: List[float] or numpy.array annotation: str, optional step: int, optional timestamp: datetime, optional """ self._log_has_events() event_value = events_processors.curve( x=x, y=y, annotation=annotation, ) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CURVE, event=V1Event.make(timestamp=timestamp, step=step, curve=event_value), ) self._add_event(logged_event)
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None): chart = events_processors.mpl_plotly_chart(figure=figure) logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.CHART, event=V1Event(timestamp=timestamp, step=step, chart=chart), ) self._event_logger.add_event(logged_event)
def log_image_with_boxes( self, tensor_image, tensor_boxes, name=None, step=None, timestamp=None, rescale=1, dataformats="CHW", ): """Logs an image with bounding boxes. ```python >>> log_image_with_boxes( >>> name="my_image", >>> tensor_image=np.arange(np.prod((3, 32, 32)), dtype=float).reshape((3, 32, 32)), >>> tensor_boxes=np.array([[10, 10, 40, 40]]), >>> ) ``` Args: tensor_image: numpy.array or str: Image data or file name tensor_boxes: numpy.array or str: Box data (for detected objects) box should be represented as [x1, y1, x2, y2] name: str, name of the image step: int, optional timestamp: datetime, optional rescale: int, optional dataformats: str, optional """ self._log_has_events() name = name or "figure" asset_path = get_asset_path( run_path=self._artifacts_path, kind=V1ArtifactKind.IMAGE, name=name, step=step, ) asset_rel_path = os.path.relpath(asset_path, self._artifacts_path) event_value = events_processors.image_boxes( asset_path=asset_path, tensor_image=tensor_image, tensor_boxes=tensor_boxes, rescale=rescale, dataformats=dataformats, asset_rel_path=asset_rel_path, ) if event_value == UNKNOWN: return logged_event = LoggedEventSpec( name=name, kind=V1ArtifactKind.IMAGE, event=V1Event.make(timestamp=timestamp, step=step, image=event_value), ) self._event_logger.add_event(logged_event)