def test_dataframes_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
dataframe=V1EventDataframe(path="path1",
content_type="parquet"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
dataframe=V1EventDataframe(path="path2",
content_type="pickle"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
dataframe=V1EventDataframe(path="path3"),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="dataframe",
data=os.path.abspath(
"tests/fixtures/polyboard/dataframe/dataframe_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_artifacts_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
artifact=V1EventArtifact(kind="dataframe", path="path1"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
artifact=V1EventArtifact(kind="tsv", path="path2"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
artifact=V1EventArtifact(kind="csv", path="path3"),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="artifact",
data=os.path.abspath(
"tests/fixtures/polyboard/artifact/artifact_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_log_line_has_iso_datetime(self):
parsed = V1Event.make(timestamp="2018-12-11T08:49:07.163495183Z",
step=12)
expected = V1Event(
timestamp=dt_parser.parse("2018-12-11T08:49:07.163495+00:00"),
step=12)
assert parsed == expected
def test_htmls_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
html="<div>1</div>",
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
html="<div>2</div>",
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
html="<div>3</div>",
step=14,
),
]
events = V1Events.read(
name="foo",
kind="html",
data=os.path.abspath(
"tests/fixtures/polyboard/html/html_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_charts_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
chart=V1EventChart(kind="plotly", figure={"foo": "bar"}),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
chart=V1EventChart(kind="vega", figure={"foo2": "bar2"}),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
chart=V1EventChart(kind="bokeh", figure={"foo3": "bar3"}),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="chart",
data=os.path.abspath(
"tests/fixtures/polyboard/chart/chart_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_videos_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
video=V1EventVideo(path="test", content_type="mp4"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
video=V1EventVideo(height=1, width=1),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
video=V1EventVideo(height=10, width=10, colorspace=2),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="video",
data=os.path.abspath(
"tests/fixtures/polyboard/video/video_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def create_tmp_events(self, run_events: str):
metric1 = LoggedEventListSpec(
name="metric1",
kind=V1ArtifactKind.METRIC,
events=[
V1Event.make(step=1, metric=1.1),
V1Event.make(step=2, metric=1.2),
],
)
self.create_kind_events(
run_events=run_events,
name="metric1",
kind=V1ArtifactKind.METRIC,
events=metric1,
)
metric2 = LoggedEventListSpec(
name="metric2",
kind=V1ArtifactKind.METRIC,
events=[
V1Event.make(step=1, metric=2.1),
V1Event.make(step=2, metric=2.2),
],
)
self.create_kind_events(
run_events=run_events,
name="metric2",
kind=V1ArtifactKind.METRIC,
events=metric2,
)
def test_histograms_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
histogram=V1EventHistogram(values=[10], counts=[1]),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
histogram=V1EventHistogram(values=[10, 1, 1], counts=[1, 1,
1]),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
histogram=V1EventHistogram(values=[10, 112, 12, 1],
counts=[12, 1, 1, 1]),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="histogram",
data=os.path.abspath(
"tests/fixtures/polyboard/histogram/histogram_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_video(self):
events = LoggedEventListSpec(
name="foo",
kind="video",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
video=V1EventVideo(height=1,
width=1,
colorspace=1,
path="path"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
video=V1EventVideo(height=10,
width=1,
colorspace=0,
path="path"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
video=V1EventVideo(height=1,
width=10,
colorspace=2,
path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_models_read_yaml(self):
values = [
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
model=V1EventModel(framework="tensorflow", path="path1"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:25:57"),
model=V1EventModel(framework="pytorch", path="path2"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:26:57"),
model=V1EventModel(framework="onnx", path="path3"),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="model",
data=os.path.abspath("tests/fixtures/polyboard/model/model_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_dataframe(self):
events = LoggedEventListSpec(
name="foo",
kind="dataframe",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
dataframe=V1EventDataframe(path="path",
content_type="parquet"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
dataframe=V1EventDataframe(path="path",
content_type="pickle"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
dataframe=V1EventDataframe(path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_audio(self):
events = LoggedEventListSpec(
name="foo",
kind="audio",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
audio=V1EventAudio(sample_rate=1.1,
num_channels=2,
length_frames=2,
path="test"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
audio=V1EventAudio(sample_rate=1.1,
num_channels=2,
length_frames=2,
path="test"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
audio=V1EventAudio(sample_rate=1.12,
num_channels=22,
length_frames=22,
path="test"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_audios_read_yaml(self):
values = [
V1Event(
timestamp=parse_datetime("2018-12-11 10:24:57"),
audio=V1EventAudio(
sample_rate=1.1, num_channels=2, length_frames=2, path="test"
),
step=12,
),
V1Event(
timestamp=parse_datetime("2018-12-11 10:25:57"),
audio=V1EventAudio(
sample_rate=1.11,
num_channels=22,
length_frames=22,
path="test",
content_type="wav",
),
step=13,
),
V1Event(
timestamp=parse_datetime("2018-12-11 10:26:57"),
audio=V1EventAudio(path="testwave", content_type="wav"),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="audio",
data=os.path.abspath("tests/fixtures/polyboard/audio/audio_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def test_images_read_yaml(self):
values = [
V1Event(
timestamp=parse_datetime("2018-12-11 10:24:57"),
image=V1EventImage(path="test"),
step=12,
),
V1Event(
timestamp=parse_datetime("2018-12-11 10:25:57"),
image=V1EventImage(height=1, width=1),
step=13,
),
V1Event(
timestamp=parse_datetime("2018-12-11 10:26:57"),
image=V1EventImage(height=10, width=10, colorspace=2),
step=14,
),
]
events = V1Events.read(
name="foo",
kind="image",
data=os.path.abspath("tests/fixtures/polyboard/image/image_events.plx"),
)
assert events.name == "foo"
assert len(events.df.values) == 3
for i in range(3):
assert events.get_event_at(i).to_dict() == values[i].to_dict()
def log_sklearn_pr_curve(self,
name,
y_preds,
y_targets,
step=None,
timestamp=None):
"""Calculates and logs PR curve using sklearn.
```python
>>> log_sklearn_pr_curve("pr_value", y_preds, y_targets, step=10)
```
Args:
name: str, name of the event
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.sklearn_pr_curve(
y_preds=y_preds,
y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_pr_curve(self,
name,
precision,
recall,
average_precision=None,
step=None,
timestamp=None):
"""Logs PR curve. This method expects an already processed values.
```python
>>> log_pr_curve("pr_value", precision, recall, step=10)
```
Args:
name: str, name of the curve
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.pr_curve(
precision=precision,
recall=recall,
average_precision=average_precision,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_model(self,
path,
name=None,
framework=None,
spec=None,
step=None,
timestamp=None):
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
model = events_processors.model_path(from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
def log_html(self, name, html, step=None, timestamp=None):
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HTML,
event=V1Event(timestamp=timestamp, step=step, html=html),
)
self._event_logger.add_event(logged_event)
def log_text(self, name, text, step=None, timestamp=None):
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.TEXT,
event=V1Event(timestamp=timestamp, step=step, text=text),
)
self._event_logger.add_event(logged_event)
def log_image_with_boxes(
self,
tensor_image,
tensor_boxes,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW",
):
name = name or "figure"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
)
event_value = events_processors.image_boxes(
asset_path=asset_path,
tensor_image=tensor_image,
tensor_boxes=tensor_boxes,
rescale=rescale,
dataformats=dataformats,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event(timestamp=timestamp, step=step, image=event_value),
)
self._event_logger.add_event(logged_event)
def log_artifact(self,
path,
name=None,
artifact_kind=None,
step=None,
timestamp=None):
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
artifact = events_processors.artifact_path(from_path=path,
asset_path=asset_path,
kind=artifact_kind)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event(timestamp=timestamp, step=step, artifact=artifact),
)
self._event_logger.add_event(logged_event)
def log_dataframe(self,
path,
name=None,
content_type=None,
step=None,
timestamp=None):
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
df = events_processors.dataframe_path(from_path=path,
asset_path=asset_path,
content_type=content_type)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
def log_np_histogram(self, name, values, counts, step=None, timestamp=None):
"""Logs a numpy histogram.
```python
>>> values, counts = np.histogram(np.random.randint(255, size=(1000,)))
>>> log_np_histogram(name="histo1", values=values, counts=counts, step=1)
```
Args:
name: str, name
values: np.array
counts: np.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.np_histogram(values=values, counts=counts)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._add_event(logged_event)
def log_histogram(
self, name, values, bins, max_bins=None, step=None, timestamp=None
):
"""Logs a histogram.
```python
>>> log_histogram(name="histo", values=np.arange(np.prod((1024,)), dtype=float).reshape((1024,)), bins="auto", step=1) # noqa
```
Args:
name: str, name
values: np.array
bins: int or str
max_bins: int, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.histogram(
values=values, bins=bins, max_bins=max_bins
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._add_event(logged_event)
def log_metrics(self, step=None, timestamp=None, **metrics):
"""Logs multiple metrics.
```python
>>> log_metrics(step=123, loss=0.023, accuracy=0.91)
```
> It's very important to log `step` as one of your metrics
if you want to compare experiments on the dashboard
and use the steps in x-axis instead of timestamps.
Args:
step: int, optional
timestamp: datetime, optional
**metrics: **kwargs, key: value
"""
self._log_has_events()
events = []
for metric in metrics:
event_value = events_processors.metric(metrics[metric])
if event_value == UNKNOWN:
continue
events.append(
LoggedEventSpec(
name=metric,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp,
step=step,
metric=event_value),
))
if events:
self._event_logger.add_events(events)
def log_metric(self, name: str, value: float, step: int = None, timestamp=None):
"""Logs a metric datapoint.
```python
>>> log_metric(name="loss", value=0.01, step=10)
```
> It's very important to log `step` as one of your metrics
if you want to compare experiments on the dashboard
and use the steps in x-axis instead of timestamps.
Args:
name: str, metric name
value: float, metric value
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
events = []
event_value = events_processors.metric(value)
if event_value == UNKNOWN:
return
events.append(
LoggedEventSpec(
name=name,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp, step=step, metric=event_value),
)
)
if events:
self._add_events(events)
self._results[name] = event_value
def log_roc_auc_curve(self,
name,
fpr,
tpr,
auc=None,
step=None,
timestamp=None):
"""Logs ROC/AUC curve. This method expects an already processed values.
```python
>>> log_roc_auc_curve("roc_value", fpr, tpr, auc=0.6, step=1)
```
Args:
name: str, name of the curve
fpr: List[float] or numpy.array, false positive rate
tpr: List[float] or numpy.array, true positive rate
auc: float, optional, calculated area under curve
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.roc_auc_curve(
fpr=fpr,
tpr=tpr,
auc=auc,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None):
"""Logs a custom curve.
```python
>>> log_curve("pr_value", x, y, annotation="more=info", step=10)
```
Args:
name: str, name of the curve
x: List[float] or numpy.array
y: List[float] or numpy.array
annotation: str, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.curve(
x=x,
y=y,
annotation=annotation,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._add_event(logged_event)
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None):
chart = events_processors.mpl_plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
def log_image_with_boxes(
self,
tensor_image,
tensor_boxes,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW",
):
"""Logs an image with bounding boxes.
```python
>>> log_image_with_boxes(
>>> name="my_image",
>>> tensor_image=np.arange(np.prod((3, 32, 32)), dtype=float).reshape((3, 32, 32)),
>>> tensor_boxes=np.array([[10, 10, 40, 40]]),
>>> )
```
Args:
tensor_image: numpy.array or str: Image data or file name
tensor_boxes: numpy.array or str: Box data (for detected objects)
box should be represented as [x1, y1, x2, y2]
name: str, name of the image
step: int, optional
timestamp: datetime, optional
rescale: int, optional
dataformats: str, optional
"""
self._log_has_events()
name = name or "figure"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
event_value = events_processors.image_boxes(
asset_path=asset_path,
tensor_image=tensor_image,
tensor_boxes=tensor_boxes,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
