def test_BestDistancePlotter(): with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = BestDistancePlotter(distance_table) df_dist = plotter.generate_plot_data( metric="precision@", parameter="100_abs", model_group_ids=[1, 2], train_end_times=["2014-01-01", "2015-01-01"], ) # assert that we have the right # of columns and a row for each % diff value # 202 row because 101 percentiles (0-100 inclusive), 2 model groups assert df_dist.shape == (101 * 2, 5) # all of the model groups are within .34 of the best, so pick # a number higher than that and all should qualify for value in df_dist[df_dist["distance"] == 0.35]["pct_of_time"].values: assert numpy.isclose(value, 1.0) # model group 1 (stable) should be within 0.11 1/2 of the time # if we included 2016 in the train_end_times, this would be 1/3! for value in df_dist[(df_dist["distance"] == 0.11) & ( df_dist["model_group_id"] == 1)]["pct_of_time"].values: assert numpy.isclose(value, 0.5)
def test_BestDistancePlotter_plot_bounds(): class FakeDistanceTable(object): @property def observed_bounds(self): return { ("precision@", "100_abs"): (0.02, 0.87), ("recall@", "100_abs"): (0.0, 1.0), ("false positives@", "300_abs"): (2, 162), } plotter = BestDistancePlotter(FakeDistanceTable()) assert plotter.plot_bounds("precision@", "100_abs") == (0.0, 1.0) assert plotter.plot_bounds("recall@", "100_abs") == (0.0, 1.0) assert plotter.plot_bounds("false positives@", "300_abs") == (2, 178)
def test_BestDistancePlotter_plot_bounds(): class FakeDistanceTable(object): @property def observed_bounds(self): return { ('precision@', '100_abs'): (0.02, 0.87), ('recall@', '100_abs'): (0.0, 1.0), ('false positives@', '300_abs'): (2, 162) } plotter = BestDistancePlotter(FakeDistanceTable()) assert plotter.plot_bounds('precision@', '100_abs') == (0.0, 1.0) assert plotter.plot_bounds('recall@', '100_abs') == (0.0, 1.0) assert plotter.plot_bounds('false positives@', '300_abs') == (2, 178)
def test_BestDistancePlotter_plot(): with patch("triage.component.audition.distance_from_best.plot_cats") as plot_patch: with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = BestDistancePlotter(distance_table) plotter.plot_all_best_dist( [{"metric": "precision@", "parameter": "100_abs"}], model_group_ids=[1, 2], train_end_times=["2014-01-01", "2015-01-01"], ) assert plot_patch.called args, kwargs = plot_patch.call_args assert "distance" in kwargs["frame"] assert "pct_of_time" in kwargs["frame"] assert kwargs["x_col"] == "distance" assert kwargs["y_col"] == "pct_of_time"
def test_BestDistancePlotter_plot(): with patch('triage.component.audition.distance_from_best.plot_cats' ) as plot_patch: with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = BestDistancePlotter(distance_table) plotter.plot_all_best_dist( [{ 'metric': 'precision@', 'parameter': '100_abs' }], model_group_ids=[1, 2], train_end_times=['2014-01-01', '2015-01-01'], ) assert plot_patch.called args, kwargs = plot_patch.call_args assert 'distance' in kwargs['frame'] assert 'pct_of_time' in kwargs['frame'] assert kwargs['x_col'] == 'distance' assert kwargs['y_col'] == 'pct_of_time'