def log_histogram(
self, name, values, bins, max_bins=None, step=None, timestamp=None
):
"""Logs a histogram.
```python
>>> log_histogram(name="histo", values=np.arange(np.prod((1024,)), dtype=float).reshape((1024,)), bins="auto", step=1) # noqa
```
Args:
name: str, name
values: np.array
bins: int or str
max_bins: int, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.histogram(
values=values, bins=bins, max_bins=max_bins
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._add_event(logged_event)
def log_artifact(
self, path, name=None, artifact_kind=None, step=None, timestamp=None
):
self._log_has_events()
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
artifact = events_processors.artifact_path(
from_path=path,
asset_path=asset_path,
kind=artifact_kind,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event.make(timestamp=timestamp, step=step, artifact=artifact),
)
self._event_logger.add_event(logged_event)
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None):
"""Logs a custom curve.
```python
>>> log_curve("pr_value", x, y, annotation="more=info", step=10)
```
Args:
name: str, name of the curve
x: List[float] or numpy.array
y: List[float] or numpy.array
annotation: str, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.curve(
x=x,
y=y,
annotation=annotation,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._add_event(logged_event)
def log_dataframe(
self, path, name=None, content_type=None, step=None, timestamp=None
):
self._log_has_events()
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
df = events_processors.dataframe_path(
from_path=path,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event.make(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
def log_metric(self, name: str, value: float, step: int = None, timestamp=None):
"""Logs a metric datapoint.
```python
>>> log_metric(name="loss", value=0.01, step=10)
```
> It's very important to log `step` as one of your metrics
if you want to compare experiments on the dashboard
and use the steps in x-axis instead of timestamps.
Args:
name: str, metric name
value: float, metric value
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
events = []
event_value = events_processors.metric(value)
if event_value == UNKNOWN:
return
events.append(
LoggedEventSpec(
name=name,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp, step=step, metric=event_value),
)
)
if events:
self._add_events(events)
self._results[name] = event_value
def log_model(
self, path, name=None, framework=None, spec=None, step=None, timestamp=None
):
self._log_has_model()
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
model = events_processors.model_path(
from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event.make(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
def test_log_line_has_iso_datetime(self):
parsed = V1Event.make(timestamp="2018-12-11T08:49:07.163495183Z",
step=12)
expected = V1Event(
timestamp=dt_parser.parse("2018-12-11T08:49:07.163495+00:00"),
step=12)
assert parsed == expected
def log_pr_curve(self,
name,
precision,
recall,
average_precision=None,
step=None,
timestamp=None):
"""Logs PR curve. This method expects an already processed values.
```python
>>> log_pr_curve("pr_value", precision, recall, step=10)
```
Args:
name: str, name of the curve
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.pr_curve(
precision=precision,
recall=recall,
average_precision=average_precision,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_sklearn_pr_curve(self,
name,
y_preds,
y_targets,
step=None,
timestamp=None):
"""Calculates and logs PR curve using sklearn.
```python
>>> log_sklearn_pr_curve("pr_value", y_preds, y_targets, step=10)
```
Args:
name: str, name of the event
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.sklearn_pr_curve(
y_preds=y_preds,
y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_metrics(self, step=None, timestamp=None, **metrics):
"""Logs multiple metrics.
```python
>>> log_metrics(step=123, loss=0.023, accuracy=0.91)
```
> It's very important to log `step` as one of your metrics
if you want to compare experiments on the dashboard
and use the steps in x-axis instead of timestamps.
Args:
step: int, optional
timestamp: datetime, optional
**metrics: **kwargs, key: value
"""
self._log_has_events()
events = []
for metric in metrics:
event_value = events_processors.metric(metrics[metric])
if event_value == UNKNOWN:
continue
events.append(
LoggedEventSpec(
name=metric,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp,
step=step,
metric=event_value),
))
if events:
self._event_logger.add_events(events)
def log_np_histogram(self, name, values, counts, step=None, timestamp=None):
"""Logs a numpy histogram.
```python
>>> values, counts = np.histogram(np.random.randint(255, size=(1000,)))
>>> log_np_histogram(name="histo1", values=values, counts=counts, step=1)
```
Args:
name: str, name
values: np.array
counts: np.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.np_histogram(values=values, counts=counts)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._add_event(logged_event)
def log_roc_auc_curve(self,
name,
fpr,
tpr,
auc=None,
step=None,
timestamp=None):
"""Logs ROC/AUC curve. This method expects an already processed values.
```python
>>> log_roc_auc_curve("roc_value", fpr, tpr, auc=0.6, step=1)
```
Args:
name: str, name of the curve
fpr: List[float] or numpy.array, false positive rate
tpr: List[float] or numpy.array, true positive rate
auc: float, optional, calculated area under curve
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.roc_auc_curve(
fpr=fpr,
tpr=tpr,
auc=auc,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_image_with_boxes(
self,
tensor_image,
tensor_boxes,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW",
):
"""Logs an image with bounding boxes.
```python
>>> log_image_with_boxes(
>>> name="my_image",
>>> tensor_image=np.arange(np.prod((3, 32, 32)), dtype=float).reshape((3, 32, 32)),
>>> tensor_boxes=np.array([[10, 10, 40, 40]]),
>>> )
```
Args:
tensor_image: numpy.array or str: Image data or file name
tensor_boxes: numpy.array or str: Box data (for detected objects)
box should be represented as [x1, y1, x2, y2]
name: str, name of the image
step: int, optional
timestamp: datetime, optional
rescale: int, optional
dataformats: str, optional
"""
self._log_has_events()
name = name or "figure"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
event_value = events_processors.image_boxes(
asset_path=asset_path,
tensor_image=tensor_image,
tensor_boxes=tensor_boxes,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
def log_text(self, name, text, step=None, timestamp=None):
self._log_has_events()
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.TEXT,
event=V1Event.make(timestamp=timestamp, step=step, text=text),
)
self._event_logger.add_event(logged_event)
def metrics_dict_to_list(metrics: Dict) -> List:
results = []
for k, v in metrics.items():
results.append(
LoggedEventSpec(
name=k, kind=V1ArtifactKind.METRIC, event=V1Event.make(metric=v),
)
)
return results
def log_html(self, name, html, step=None, timestamp=None):
self._log_has_events()
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HTML,
event=V1Event.make(timestamp=timestamp, step=step, html=html),
)
self._event_logger.add_event(logged_event)
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None):
self._log_has_events()
chart = events_processors.mpl_plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
def log_roc_auc_curve(self, name, fpr, tpr, auc=None, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.roc_auc_curve(fpr=fpr, tpr=tpr, auc=auc,)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.curve(x=x, y=y, annotation=annotation,)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_audio(
self,
data,
name=None,
sample_rate=44100,
step=None,
timestamp=None,
content_type=None,
):
self._log_dashboard()
is_file = isinstance(data, str) and os.path.exists(data)
ext = content_type or "wav"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "audio"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.AUDIO,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.audio_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.audio(
asset_path=asset_path,
tensor=data,
sample_rate=sample_rate,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.AUDIO,
event=V1Event.make(timestamp=timestamp,
step=step,
audio=event_value),
)
self._event_logger.add_event(logged_event)
def log_image(self,
data,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW"):
self._log_dashboard()
is_file = isinstance(data, str) and os.path.exists(data)
ext = "png"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "image"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.image_path(from_path=data,
asset_path=asset_path)
elif hasattr(data, "encoded_image_string"):
event_value = events_processors.encoded_image(
asset_path=asset_path,
data=data,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.image(
asset_path=asset_path,
data=data,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
def log_sklearn_pr_curve(self, name, y_preds, y_targets, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.sklearn_pr_curve(
y_preds=y_preds, y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_video(self,
data,
name=None,
fps=4,
step=None,
timestamp=None,
content_type=None):
self._log_dashboard()
is_file = isinstance(data, str) and os.path.exists(data)
content_type = content_type or "gif"
if is_file:
name = name or os.path.basename(data)
content_type = get_path_extension(filepath=data) or content_type
else:
name = name or "video"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.VIDEO,
name=name,
step=step,
ext=content_type,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.video_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.video(
asset_path=asset_path,
tensor=data,
fps=fps,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.VIDEO,
event=V1Event.make(timestamp=timestamp,
step=step,
video=event_value),
)
self._event_logger.add_event(logged_event)
def log_pr_curve(
self, name, precision, recall, average_precision=None, step=None, timestamp=None
):
self._log_has_events()
event_value = events_processors.pr_curve(
precision=precision, recall=recall, average_precision=average_precision,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
def log_np_histogram(self, name, values, counts, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.np_histogram(values=values, counts=counts)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._event_logger.add_event(logged_event)
def log_metric(self, name, value, step=None, timestamp=None):
events = []
event_value = events_processors.metric(value)
if event_value == UNKNOWN:
return
events.append(
LoggedEventSpec(
name=name,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp, step=step, metric=event_value),
)
)
if events:
self._event_logger.add_events(events)
self._results[name] = event_value
def create_tmp_events(self):
text1 = LoggedEventListSpec(
name="text1",
kind=V1ArtifactKind.TEXT,
events=[
V1Event.make(step=1, text="foo1"),
V1Event.make(step=2, text="boo2"),
],
)
self.create_kind_events(name="text1",
kind=V1ArtifactKind.TEXT,
events=text1)
text2 = LoggedEventListSpec(
name="text2",
kind=V1ArtifactKind.TEXT,
events=[
V1Event.make(step=1, text="foo2"),
V1Event.make(step=2, text="boo2"),
],
)
self.create_kind_events(name="text2",
kind=V1ArtifactKind.TEXT,
events=text2)
html1 = LoggedEventListSpec(
name="html1",
kind=V1ArtifactKind.HTML,
events=[
V1Event.make(step=1, html="foo1"),
V1Event.make(step=2, html="boo2"),
],
)
self.create_kind_events(name="html1",
kind=V1ArtifactKind.HTML,
events=html1)
html2 = LoggedEventListSpec(
name="htm2",
kind=V1ArtifactKind.HTML,
events=[
V1Event.make(step=1, html="foo2"),
V1Event.make(step=2, html="boo2"),
],
)
self.create_kind_events(name="html2",
kind=V1ArtifactKind.HTML,
events=html2)
def log_metrics(self, step=None, timestamp=None, **metrics):
events = []
for metric in metrics:
event_value = events_processors.metric(metrics[metric])
if event_value == UNKNOWN:
continue
events.append(
LoggedEventSpec(
name=metric,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp,
step=step,
metric=event_value),
))
if events:
self._event_logger.add_events(events)
def log_model(self,
path,
name=None,
framework=None,
spec=None,
step=None,
timestamp=None):
"""Logs a model.
Args:
path: str, path to the model to log
name: str, name
framework: str, optional ,name of the framework
spec: Dict, optional, key, value information about the model
step: int, optional
timestamp: datetime, optional
"""
self._log_has_model()
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
model = events_processors.model_path(
from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event.make(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None):
"""Logs a matplotlib figure to plotly figure.
Args:
name: str, name of the figure
figure: figure
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
chart = events_processors.mpl_plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)