def test_BestDistancePlotter():
with testing.postgresql.Postgresql() as postgresql:
engine = create_engine(postgresql.url())
distance_table, model_groups = create_sample_distance_table(engine)
plotter = BestDistancePlotter(distance_table)
df_dist = plotter.generate_plot_data(
metric="precision@",
parameter="100_abs",
model_group_ids=[1, 2],
train_end_times=["2014-01-01", "2015-01-01"],
)
# assert that we have the right # of columns and a row for each % diff value
# 202 row because 101 percentiles (0-100 inclusive), 2 model groups
assert df_dist.shape == (101 * 2, 5)
# all of the model groups are within .34 of the best, so pick
# a number higher than that and all should qualify
for value in df_dist[df_dist["distance"] ==
0.35]["pct_of_time"].values:
assert numpy.isclose(value, 1.0)
# model group 1 (stable) should be within 0.11 1/2 of the time
# if we included 2016 in the train_end_times, this would be 1/3!
for value in df_dist[(df_dist["distance"] == 0.11) & (
df_dist["model_group_id"] == 1)]["pct_of_time"].values:
assert numpy.isclose(value, 0.5)
def test_BestDistancePlotter_plot_bounds():
class FakeDistanceTable(object):
@property
def observed_bounds(self):
return {
("precision@", "100_abs"): (0.02, 0.87),
("recall@", "100_abs"): (0.0, 1.0),
("false positives@", "300_abs"): (2, 162),
}
plotter = BestDistancePlotter(FakeDistanceTable())
assert plotter.plot_bounds("precision@", "100_abs") == (0.0, 1.0)
assert plotter.plot_bounds("recall@", "100_abs") == (0.0, 1.0)
assert plotter.plot_bounds("false positives@", "300_abs") == (2, 178)
def test_BestDistancePlotter_plot_bounds():
class FakeDistanceTable(object):
@property
def observed_bounds(self):
return {
('precision@', '100_abs'): (0.02, 0.87),
('recall@', '100_abs'): (0.0, 1.0),
('false positives@', '300_abs'): (2, 162)
}
plotter = BestDistancePlotter(FakeDistanceTable())
assert plotter.plot_bounds('precision@', '100_abs') == (0.0, 1.0)
assert plotter.plot_bounds('recall@', '100_abs') == (0.0, 1.0)
assert plotter.plot_bounds('false positives@', '300_abs') == (2, 178)
def test_BestDistancePlotter_plot():
with patch("triage.component.audition.distance_from_best.plot_cats") as plot_patch:
with testing.postgresql.Postgresql() as postgresql:
engine = create_engine(postgresql.url())
distance_table, model_groups = create_sample_distance_table(engine)
plotter = BestDistancePlotter(distance_table)
plotter.plot_all_best_dist(
[{"metric": "precision@", "parameter": "100_abs"}],
model_group_ids=[1, 2],
train_end_times=["2014-01-01", "2015-01-01"],
)
assert plot_patch.called
args, kwargs = plot_patch.call_args
assert "distance" in kwargs["frame"]
assert "pct_of_time" in kwargs["frame"]
assert kwargs["x_col"] == "distance"
assert kwargs["y_col"] == "pct_of_time"
def test_BestDistancePlotter_plot():
with patch('triage.component.audition.distance_from_best.plot_cats'
) as plot_patch:
with testing.postgresql.Postgresql() as postgresql:
engine = create_engine(postgresql.url())
distance_table, model_groups = create_sample_distance_table(engine)
plotter = BestDistancePlotter(distance_table)
plotter.plot_all_best_dist(
[{
'metric': 'precision@',
'parameter': '100_abs'
}],
model_group_ids=[1, 2],
train_end_times=['2014-01-01', '2015-01-01'],
)
assert plot_patch.called
args, kwargs = plot_patch.call_args
assert 'distance' in kwargs['frame']
assert 'pct_of_time' in kwargs['frame']
assert kwargs['x_col'] == 'distance'
assert kwargs['y_col'] == 'pct_of_time'