def logged_metrics():
return [
ScalarMetricLogEntry("training.loss", 10, datetime.datetime.utcnow(), 1),
ScalarMetricLogEntry("training.loss", 20, datetime.datetime.utcnow(), 2),
ScalarMetricLogEntry("training.loss", 30, datetime.datetime.utcnow(), 3),
ScalarMetricLogEntry("training.accuracy", 10, datetime.datetime.utcnow(), 100),
ScalarMetricLogEntry("training.accuracy", 20, datetime.datetime.utcnow(), 200),
ScalarMetricLogEntry("training.accuracy", 30, datetime.datetime.utcnow(), 300),
ScalarMetricLogEntry("training.loss", 40, datetime.datetime.utcnow(), 10),
ScalarMetricLogEntry("training.loss", 50, datetime.datetime.utcnow(), 20),
ScalarMetricLogEntry("training.loss", 60, datetime.datetime.utcnow(), 30),
]
def test_linearize_metrics():
entries = [
ScalarMetricLogEntry("training.loss", 10, datetime.datetime.utcnow(),
100),
ScalarMetricLogEntry("training.accuracy", 5,
datetime.datetime.utcnow(), 50),
ScalarMetricLogEntry("training.loss", 20, datetime.datetime.utcnow(),
200),
ScalarMetricLogEntry("training.accuracy", 10,
datetime.datetime.utcnow(), 100),
ScalarMetricLogEntry("training.accuracy", 15,
datetime.datetime.utcnow(), 150),
ScalarMetricLogEntry("training.accuracy", 30,
datetime.datetime.utcnow(), 300),
]
linearized = linearize_metrics(entries)
assert type(linearized) == dict
assert len(linearized.keys()) == 2
assert "training.loss" in linearized
assert "training.accuracy" in linearized
assert len(linearized["training.loss"]["steps"]) == 2
assert len(linearized["training.loss"]["values"]) == 2
assert len(linearized["training.loss"]["timestamps"]) == 2
assert len(linearized["training.accuracy"]["steps"]) == 4
assert len(linearized["training.accuracy"]["values"]) == 4
assert len(linearized["training.accuracy"]["timestamps"]) == 4
assert linearized["training.accuracy"]["steps"] == [5, 10, 15, 30]
assert linearized["training.accuracy"]["values"] == [50, 100, 150, 300]
assert linearized["training.loss"]["steps"] == [10, 20]
assert linearized["training.loss"]["values"] == [100, 200]
def export(self, observer, base_dir, remove_sources=False,
overwrite=None):
"""
Exports the file log into another observer.
Requires sacred to be installed.
Args:
observer: Observer to export to
base_dir: root path to sources
remove_sources: if sources are too complicated to match
overwrite: whether to overwrite an experiment
"""
from sacred.metrics_logger import ScalarMetricLogEntry, linearize_metrics
# Start simulation of run
experiment = self.experiment.copy()
experiment['base_dir'] = base_dir
# FIXME
experiment['sources'] = [] if remove_sources else update_source_path_prefix(base_dir, experiment['sources'])
try:
observer.started_event(
experiment,
self.command,
self.host,
datetime.datetime.fromisoformat(self.start_time),
self.config,
self.meta,
_id=overwrite
)
except FileNotFoundError as e:
raise FileNotFoundError("The sources are incorrect. Try fixing paths or use `remove_sources=True`."
f" Original error: {e}")
# Add artifacts
for artifact_name in self.artifacts:
observer.artifact_event(
name=artifact_name,
filename=(self.path / artifact_name)
)
# Add resources
for resource in self.resources:
observer.resource_event(resource[0])
# Add metrics
size_metrics = {}
# If overwrite, get the already added metrics.
# FIXME: issue if steps are not increasing
if overwrite is not None:
metrics = observer.metrics.find({"run_id": overwrite})
for metric in metrics:
size_metrics[metric['name']] = len(metric['steps'])
log_metrics = []
for metric_name, metric in self.metrics.items():
steps = metric['steps'] if metric_name not in size_metrics else metric['steps'][size_metrics[metric_name]:]
timestamps = metric['timestamps'] if metric_name not in size_metrics else metric['timestamps'][
size_metrics[metric_name]:]
values = metric['values'] if metric_name not in size_metrics else metric['values'][
size_metrics[metric_name]:]
for step, timestamp, value in zip(steps, timestamps, values):
metric_log_entry = ScalarMetricLogEntry(metric_name, step,
datetime.datetime.fromisoformat(timestamp), value)
log_metrics.append(metric_log_entry)
observer.log_metrics(linearize_metrics(log_metrics), {})
observer.heartbeat_event(
info=self.info if 'info' in self.run else None,
captured_out=self.cout,
beat_time=datetime.datetime.fromisoformat(self.heartbeat),
result=self.result
)
# End simulation
if self.status != "RUNNING":
stop_time = datetime.datetime.fromisoformat(self.stop_time)
if self.status in ["COMPLETED", "RUNNING"]: # If still running we force it as a finished experiment
observer.completed_event(stop_time, self.result)
elif self.status == "INTERRUPTED":
observer.interrupted_event(stop_time, 'INTERRUPTED')
elif self.status == "FAILED":
observer.failed_event(stop_time, self.fail_trace)
