def plot_longs():
for n in [500, 5000, 20000]:
long_metric = Metric('x', 'y')
for i in range(n):
long_metric.add_record(random.random(), random.random())
plot(long_metric, 'temp', name=f'longer_metric_{n}')
def test_all_empty_no_plot(tmpdir):
files_before = len(os.listdir(tmpdir))
m1 = Metric('empty')
m2 = Metric('empty2')
plot_group([m1, m2], folder=tmpdir)
assert len(os.listdir(tmpdir)) == files_before
def test_some_empty_plots(tmpdir):
files_before = len(os.listdir(tmpdir))
m1 = Metric('empty')
m2 = Metric('empty2')
m2.add_record(1, 1)
plot_group([m1, m2], folder=tmpdir)
assert len(os.listdir(tmpdir)) > files_before
def test_ezplot(tmpdir):
m = Metric('x', 'y')
m.add_ys(1, [1, 2, 3])
m.add_record(2, 4)
m.add_record(2, 4)
m.add_record(2, 4)
m.add_ys(4, [5, 6, 12])
files_before = len(os.listdir(tmpdir))
plot(m, folder=tmpdir, name="bla3")
files_after = len(os.listdir(tmpdir))
assert files_after > files_before
def reinforce(lr: float):
assert 0 < lr < 1
m = Metric(f"lr={lr:.5f}")
env = ShortCoridorEnv()
agent = CoridorAgent()
for episode in range(3_000):
history, total_reward = rollout(agent, env)
m.add_record(episode, total_reward)
G = sum(r for o, a, r in history)
for o, a, r in history:
apply_grad(G, a, agent, lr)
G -= r
def test_empty_no_plot(tmpdir):
files_before = len(os.listdir(tmpdir))
m = Metric('empty')
plot(m, folder=tmpdir)
assert len(os.listdir(tmpdir)) == files_before
def one_series(shift):
metrics = [Metric('x', 'y') for x in range(10)]
for m in metrics:
for i in range(40000):
m.add_record(i,
random.random() + shift + 0.4 * math.sin(i / 200))
m._sort()
return sum(metrics)
def test_samples():
m1 = Metric("x", "y")
m1.add_ys(x=2, ys=[3])
m1.add_ys(x=3, ys=[3])
m2 = Metric("x", "y")
m2.add_ys(x=2, ys=[15, 14])
m2.add_ys(x=3, ys=[10, 12])
assert m1.samples == 1
assert m2.samples == 2
def test_summ_valid():
m1 = Metric("x", "y")
m1.add_ys(x=2, ys=[3])
m1.add_ys(x=3, ys=[3])
m2 = Metric("x", "y")
m2.add_ys(x=2, ys=[15])
m2.add_ys(x=3, ys=[10])
m3 = m1 + m2
assert len(m3.data.keys()) == 2
def test_summ_many():
import random
metrics = [Metric('x', 'y') for x in range(10)]
for m in metrics:
for i in range(50):
m.add_record(x=random.random(), y=random.random())
m11 = sum(metrics)
assert 50 <= len(m11.data) <= 500
assert len(list(m11.data.values())[0]) == 10
def test_summ_many_originals_intact():
import random
metrics = [Metric('x', 'y') for x in range(10)]
for m in metrics:
for i in range(50):
m.add_record(x=random.random(), y=random.random())
m11 = sum(metrics)
m1 = metrics[0]
assert m1.samples == 1
def test_original_metrics_intact():
m1 = Metric("x", "y")
m1.add_ys(x=2, ys=[3])
m1.add_ys(x=3, ys=[3])
m2 = Metric("x", "y")
m2.add_ys(x=2, ys=[15])
m2.add_ys(x=3, ys=[10])
m3 = m1 + m2
assert m1.samples == 1
assert m2.samples == 1
assert m3.samples == 2
def test_cycle_save(tmpdir):
m1 = Metric("x", "y")
m1.add_ys(x=2, ys=[3])
m1.add_ys(x=3, ys=[3])
m1.save(tmpdir)
ms = Metric.load_all(tmpdir)
assert len(ms) == 1
assert m1.__dict__ == ms[0].__dict__
def test_discard_warmup():
m = Metric()
for i in range(10):
m.add_record(i, i % 2)
m.discard_warmup(0.5)
assert 4 < len(m.data) < 6
assert 0 not in m.data
assert 8 in m.data
from pennpaper import Metric, plot m1 = Metric() m1.add_record(1,1) m1.add_record(5,5) m2 = Metric() m2.add_record(2, 3) m2.add_record(4, 3) m3 = m1 + m2 print(m3.data, m3.samples) plot(m3) plot(m3, smoothen=False, name='true')
def test_plot_group(tmpdir):
m = Metric('x', 'y')
m.add_ys(1, [1, 2, 3])
m.add_record(2, 4)
m.add_record(2, 4)
m.add_record(2, 4)
m.add_ys(4, [5, 6, 12])
m2 = Metric('x', 'y')
m2.add_ys(1, [4, 3, 5])
m2.add_record(2, 5)
m2.add_record(2, 9)
m2.add_record(2, 9)
m2.add_ys(4, [5, 6, -1])
files_before = len(os.listdir(tmpdir))
plot_group([m, m2], folder=tmpdir, name="bla3")
files_after = len(os.listdir(tmpdir))
assert files_after > files_before
def test_load_many(tmpdir):
m1 = Metric("m1", "x", "y")
m1.add_ys(x=2, ys=[3])
m1.add_ys(x=3, ys=[3])
m2 = Metric("m2", "x", "y")
m2.add_ys(x=2, ys=[15])
m2.add_ys(x=3, ys=[10])
m1.save(tmpdir)
m2.save(tmpdir)
ms = Metric.load_all(tmpdir)
assert len(ms) == 2
def test_summ_3():
m1 = Metric("x", "y")
m1.add_ys(x=2, ys=[3, 4, 5])
m1.add_ys(x=3, ys=[20, 4, 5])
m2 = Metric("x", "y")
m2.add_ys(x=2, ys=[15])
m2.add_ys(x=3, ys=[20])
m3 = Metric("x", "y")
m3.add_ys(x=2, ys=[15])
m3.add_ys(x=3, ys=[20])
m4 = sum([m1, m2, m3])
assert 2 in m4.data.keys()
from pennpaper import Metric, plot
import random
metrics = [Metric('x', 'y') for x in range(10)]
for m in metrics:
for i in range(50):
m.add_record(random.random(), random.random())
m._sort()
m11 = sum(metrics)
plot(metrics[0], 'temp', name="single_random")
plot(m11, 'temp', name="summ_10_random")
straight = Metric('x', 'y')
for i in range(50):
straight.add_record(i, i + 0.1 * random.random())
plot(straight, 'temp', name='straight')
y += random.gauss(0, 1)
return y
return _
pow2 = noisy_mapping(lambda x: x ** 2)
"""
lets record the pairs (x, f(x)) in a metric and make a plot:
"""
from pennpaper import Metric, plot_group, plot
m1 = Metric("pow2")
for x in X:
m1.add_record(x, pow2(x))
plot(m1)
m2 = Metric("pow2_2")
for x in X:
m2.add_record(x, pow2(x))
plot_group([m1, m2])
# Actually, m1 and m2 are metrics of the same process.
# What if we create a new metric tracking the mean and stddev of this process?
def test_unique():
m1 = Metric("x", "y")
m2 = Metric("x", "y")
assert m1 is not m2
assert not m1 == m2
import numpy as np
from pennpaper import Metric, plot_group
xs = np.arange(0.1, 5, step=0.01)
uni_noise = lambda x: np.random.uniform(size=x.shape) + x
funcs = {}
funcs['uniform'] = lambda x: np.random.uniform(size=x.shape) + x
funcs['weibull'] = lambda x: np.random.weibull(a=1, size=x.shape) + x
funcs['beta'] = lambda x: np.random.beta(a=1, b=1, size=x.shape) + x
metrics = []
for name, f in funcs.items():
m = Metric(name=name)
for i in range(100):
m.add_arrays(uni_noise(xs), f(xs), new_sample=True)
metrics.append(m)
plot_group(metrics)
plot_group(metrics, name='true', smoothen=False)