def test_audio(self):
events = LoggedEventListSpec(
name="foo",
kind="audio",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
audio=V1EventAudio(sample_rate=1.1,
num_channels=2,
length_frames=2,
path="test"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
audio=V1EventAudio(sample_rate=1.1,
num_channels=2,
length_frames=2,
path="test"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
audio=V1EventAudio(sample_rate=1.12,
num_channels=22,
length_frames=22,
path="test"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_dataframe(self):
events = LoggedEventListSpec(
name="foo",
kind="dataframe",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
dataframe=V1EventDataframe(path="path",
content_type="parquet"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
dataframe=V1EventDataframe(path="path",
content_type="pickle"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
dataframe=V1EventDataframe(path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_init_events(self):
events = LoggedEventListSpec(name="test", kind="metric", events=[])
run_path = tempfile.mkdtemp()
ew = EventWriter(run_path=run_path, backend=EventWriter.EVENTS_BACKEND)
event_file = ew._get_event_path(name=events.name, kind=events.kind)
assert os.path.exists(event_file) is False
ew._init_events(events)
assert os.path.exists(event_file) is True
expected_events = V1Events.read(kind="metric", name="test", data=event_file)
assert expected_events.name == events.name
assert expected_events.kind == events.kind
# Init same file
ew._init_events(events)
assert os.path.exists(event_file) is True
# New file
events = LoggedEventListSpec(name="new", kind="text", events=[])
new_event_file = ew._get_event_path(name=events.name, kind=events.kind)
assert os.path.exists(new_event_file) is False
ew._init_events(events)
assert os.path.exists(new_event_file) is True
expected_events = V1Events.read(kind="text", name="new", data=new_event_file)
assert expected_events.name == events.name
assert expected_events.kind == events.kind
# Previous file should still be there
assert os.path.exists(event_file) is True
def create_tmp_events(self, run_events: str):
metric1 = LoggedEventListSpec(
name="metric1",
kind=V1ArtifactKind.METRIC,
events=[
V1Event.make(step=1, metric=1.1),
V1Event.make(step=2, metric=1.2),
],
)
self.create_kind_events(
run_events=run_events,
name="metric1",
kind=V1ArtifactKind.METRIC,
events=metric1,
)
metric2 = LoggedEventListSpec(
name="metric2",
kind=V1ArtifactKind.METRIC,
events=[
V1Event.make(step=1, metric=2.1),
V1Event.make(step=2, metric=2.2),
],
)
self.create_kind_events(
run_events=run_events,
name="metric2",
kind=V1ArtifactKind.METRIC,
events=metric2,
)
def test_video(self):
events = LoggedEventListSpec(
name="foo",
kind="video",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
video=V1EventVideo(height=1,
width=1,
colorspace=1,
path="path"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
video=V1EventVideo(height=10,
width=1,
colorspace=0,
path="path"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
video=V1EventVideo(height=1,
width=10,
colorspace=2,
path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def _init_events(self, events_spec: LoggedEventListSpec):
event_path = self._get_event_path(kind=events_spec.kind, name=events_spec.name)
# Check if the file exists otherwise initialize
if not os.path.exists(event_path):
check_or_create_path(event_path, is_dir=False)
with open(event_path, "w") as event_file:
event_file.write(events_spec.get_csv_header())
def create_tmp_events(self):
text1 = LoggedEventListSpec(
name="text1",
kind=V1ArtifactKind.TEXT,
events=[
V1Event.make(step=1, text="foo1"),
V1Event.make(step=2, text="boo2"),
],
)
self.create_kind_events(name="text1",
kind=V1ArtifactKind.TEXT,
events=text1)
text2 = LoggedEventListSpec(
name="text2",
kind=V1ArtifactKind.TEXT,
events=[
V1Event.make(step=1, text="foo2"),
V1Event.make(step=2, text="boo2"),
],
)
self.create_kind_events(name="text2",
kind=V1ArtifactKind.TEXT,
events=text2)
html1 = LoggedEventListSpec(
name="html1",
kind=V1ArtifactKind.HTML,
events=[
V1Event.make(step=1, html="foo1"),
V1Event.make(step=2, html="boo2"),
],
)
self.create_kind_events(name="html1",
kind=V1ArtifactKind.HTML,
events=html1)
html2 = LoggedEventListSpec(
name="htm2",
kind=V1ArtifactKind.HTML,
events=[
V1Event.make(step=1, html="foo2"),
V1Event.make(step=2, html="boo2"),
],
)
self.create_kind_events(name="html2",
kind=V1ArtifactKind.HTML,
events=html2)
def _events_to_files(self, events: List[LoggedEventSpec]):
for event in events:
file_name = "{}.{}".format(event.kind, event.name)
if file_name in self._files:
self._files[file_name].events.append(event.event)
else:
self._files[file_name] = LoggedEventListSpec(
kind=event.kind, name=event.name, events=[event.event])
self._init_events(self._files[file_name])
def test_curve(self):
events = LoggedEventListSpec(
name="foo",
kind="curve",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
curve=V1EventCurve(
kind="roc",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.1",
),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
curve=V1EventCurve(
kind="pr",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.21",
),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
curve=V1EventCurve(
kind="custom",
x=[1.1, 3.1, 5.1],
y=[0.1, 0.3, 0.4],
annotation="0.1",
),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_html(self):
events = LoggedEventListSpec(
name="foo",
kind="html",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
html="<div>1</div>",
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
html="<div>2</div>",
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
html="<div>3</div>",
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_metrics(self):
events = LoggedEventListSpec(
name="foo",
kind="metric",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
metric=0.1,
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
metric=0.112,
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
metric=0.1,
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_images(self):
events = LoggedEventListSpec(
name="foo",
kind="metric",
events=[
V1Event(
timestamp=parse_datetime("2018-12-11 10:24:57"),
image=V1EventImage(height=1, width=1, colorspace=1, path="path"),
step=12,
),
V1Event(
timestamp=parse_datetime("2018-12-11 11:24:57"),
image=V1EventImage(height=10, width=1, colorspace=0, path="path"),
step=13,
),
V1Event(
timestamp=parse_datetime("2018-12-11 12:24:57"),
image=V1EventImage(height=1, width=10, colorspace=2, path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_model(self):
events = LoggedEventListSpec(
name="foo",
kind="model",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
model=V1EventModel(framework="tensorflow", path="path"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
model=V1EventModel(framework="pytorch", path="path"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
model=V1EventModel(framework="onnx", path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_artifact(self):
events = LoggedEventListSpec(
name="foo",
kind="artifact",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
artifact=V1EventArtifact(kind="dataframe", path="path"),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
artifact=V1EventArtifact(kind="tsv", path="path"),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
artifact=V1EventArtifact(kind="csv", path="path"),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_chart(self):
events = LoggedEventListSpec(
name="foo",
kind="chart",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
chart=V1EventChart(kind="plotly", figure={"foo": "bar"}),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
chart=V1EventChart(kind="vega", figure={"foo": "bar"}),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
chart=V1EventChart(kind="bokeh", figure={"foo": "bar"}),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def test_histogram(self):
events = LoggedEventListSpec(
name="foo",
kind="histogram",
events=[
V1Event(
timestamp=dt_parser.parse("2018-12-11 10:24:57"),
histogram=V1EventHistogram(values=[10], counts=[1]),
step=12,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 11:24:57"),
histogram=V1EventHistogram(values=[10], counts=[1]),
step=13,
),
V1Event(
timestamp=dt_parser.parse("2018-12-11 12:24:57"),
histogram=V1EventHistogram(values=[10], counts=[1]),
step=14,
),
],
)
events_dict = events.to_dict()
assert events_dict == events.from_dict(events_dict).to_dict()
def _append_events(self, events_spec: LoggedEventListSpec):
event_path = self._get_event_path(kind=events_spec.kind,
name=events_spec.name)
with open(event_path, "a") as event_file:
event_file.write(events_spec.get_csv_events())
