def flush(self) -> None:
"""Flush the buffered steps and observations to database."""
n_steps, n_observations = len(self.step_buffer), len(self.observations_buffer)
# Nothing to flush.
if not n_steps:
return
with Timer() as flush_time:
# House keeping notice: Keep these statements in sync with record().
self.cursor.executemany(
"INSERT OR IGNORE INTO States VALUES (?, ?, ?, ?, ?)",
self.step_buffer,
)
self.cursor.executemany(
"INSERT OR IGNORE INTO Observations VALUES (?, ?, ?, ?, ?, ?)",
((k, *v) for k, v in self.observations_buffer.items()),
)
self.step_buffer = []
self.observations_buffer = {}
self.connection.commit()
logging.info(
"Wrote %d state records and %d observations in %s. Last flush %s ago",
n_steps,
n_observations,
flush_time,
humanize_duration(time() - self.last_commit),
)
self.last_commit = time()
def run_sensitivity_analysis_eval(rewards_path: Path,
runtimes_path: Path) -> None:
"""Print a summary of sensitivity analysis logs."""
with open(str(rewards_path)) as f:
rewards_in = f.read().rstrip().split("\n")
with open(str(runtimes_path)) as f:
runtimes_in = f.read().rstrip().split("\n")
rows = []
for rewards_row, runtimes_row in zip(rewards_in, runtimes_in):
name, *rewards = rewards_row.split(",")
_, *runtimes = runtimes_row.split(",")
if rewards == [""]:
rows.append((name, "-", "-", "-", "-", "-"))
continue
rewards = np.array([float(v) for v in rewards])
runtimes = np.array([float(v) for v in runtimes])
rows.append((
name,
humanize_duration(runtimes.mean()),
f"{rewards.mean():.5%}",
f"{np.median(rewards):.5%}",
f"{rewards.max():.5%}",
f"{rewards.std():.5%}",
))
print(
tabulate(
sorted(rows),
headers=(
"Name",
"Time (avg)",
"Δ (avg)",
"Δ (median)",
"Δ (max)",
"Δ (std.)",
),
))
def test_humanize_duration_negative_seconds():
assert timer.humanize_duration(-1.5) == "-1.500s"
def test_humanize_duration_ns():
assert timer.humanize_duration(0.0000005) == "500.0ns"
assert timer.humanize_duration(0.0000000019) == "1.9ns"
def test_humanize_duration_us():
assert timer.humanize_duration(0.0005) == "500.0us"
assert timer.humanize_duration(0.0000119) == "11.9us"
def test_humanize_duration_ms():
assert timer.humanize_duration(0.0055) == "5.5ms"
assert timer.humanize_duration(0.5) == "500.0ms"
assert timer.humanize_duration(0.51) == "510.0ms"
assert timer.humanize_duration(0.9999) == "999.9ms"
def test_humanize_duration_seconds():
assert timer.humanize_duration(5) == "5.000s"
assert timer.humanize_duration(500.111111) == "500.1s"