def test_BestDistancePlotter(): with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = BestDistancePlotter(distance_table) df_dist = plotter.generate_plot_data( metric='precision@', parameter='100_abs', model_group_ids=[1, 2], train_end_times=['2014-01-01', '2015-01-01']) # assert that we have the right # of columns and a row for each % diff value # 202 row because 101 percentiles (0-100 inclusive), 2 model groups assert df_dist.shape == (101 * 2, 5) # all of the model groups are within .34 of the best, so pick # a number higher than that and all should qualify for value in df_dist[df_dist['distance'] == 0.35]['pct_of_time'].values: assert numpy.isclose(value, 1.0) # model group 1 (stable) should be within 0.11 1/2 of the time # if we included 2016 in the train_end_times, this would be 1/3! for value in df_dist[(df_dist['distance'] == 0.11) & ( df_dist['model_group_id'] == 1)]['pct_of_time'].values: assert numpy.isclose(value, 0.5)
def test_selection_rule_picker(): with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) def pick_spiky(df, train_end_time): return model_groups['spiky'].model_group_id selection_rule_picker = SelectionRulePicker( distance_from_best_table=distance_table) results = selection_rule_picker.results_for_rule( bound_selection_rule=BoundSelectionRule(descriptive_name='spiky', function=pick_spiky, args={}), model_group_ids=[ mg.model_group_id for mg in model_groups.values() ], train_end_times=['2014-01-01', '2015-01-01', '2016-01-01'], regret_metric='precision@', regret_parameter='100_abs', ) assert [result['dist_from_best_case_next_time'] for result in results] == [0.19, 0.3, 0.12] assert [result['raw_value'] for result in results] == [0.45, 0.84, 0.45]
def test_selection_rule_picker_with_args(): with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) def pick_highest_avg(df, train_end_time, metric, parameter): assert len(df['train_end_time'].unique()) == 2 subsetted = df[(df['metric'] == metric) & (df['parameter'] == parameter)] mean = subsetted.groupby(['model_group_id'])['raw_value'].mean() return mean.nlargest(1).index[0] selection_rule_picker = SelectionRulePicker( distance_from_best_table=distance_table) regrets = [ result['dist_from_best_case_next_time'] for result in selection_rule_picker.results_for_rule( bound_selection_rule=BoundSelectionRule( descriptive_name='pick_highest_avg', function=pick_highest_avg, args={ 'metric': 'recall@', 'parameter': '100_abs' }, ), model_group_ids=[ mg.model_group_id for mg in model_groups.values() ], train_end_times=['2015-01-01'], regret_metric='precision@', regret_parameter='100_abs', ) ] # picking the highest avg recall will pick 'spiky' for this time assert regrets == [0.3]
def test_SelectionRulePerformancePlotter_plot_metrics(): with patch('audition.selection_rule_performance.plot_cats') as plot_patch: with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = SelectionRulePerformancePlotter( SelectionRulePicker(distance_table)) plotter.plot(bound_selection_rules=[ BoundSelectionRule( function_name='best_current_value', args={ 'metric': 'precision@', 'parameter': '100_abs' }, ), BoundSelectionRule( function_name='best_average_value', args={ 'metric': 'precision@', 'parameter': '100_abs' }, ) ], regret_metric='precision@', regret_parameter='100_abs', model_group_ids=[1, 2], train_end_times=['2014-01-01', '2015-01-01'], plot_type='metric') assert plot_patch.called args, kwargs = plot_patch.call_args assert 'raw_value_next_time' in kwargs['frame'] assert 'train_end_time' in kwargs['frame'] assert kwargs['x_col'] == 'train_end_time' assert kwargs['y_col'] == 'raw_value_next_time'
def test_ModelGroupPerformancePlotter_generate_plot_data(): with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = ModelGroupPerformancePlotter(distance_table) df = plotter.generate_plot_data( metric='precision@', parameter='100_abs', model_group_ids=[1, 2], train_end_times=['2014-01-01', '2015-01-01'], ) assert sorted(df['model_type'].unique()) == [ 'best case', 'mySpikeClassifier', 'myStableClassifier' ] for value in df[df['model_group_id'] == 1]['raw_value'].values: assert numpy.isclose(value, 0.5)
def test_ModelGroupPerformancePlotter_plot_all(): with patch('audition.model_group_performance.plot_cats') as plot_patch: with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = ModelGroupPerformancePlotter(distance_table) plotter.plot_all( [{ 'metric': 'precision@', 'parameter': '100_abs' }], model_group_ids=[1, 2], train_end_times=['2014-01-01', '2015-01-01'], ) assert plot_patch.called args, kwargs = plot_patch.call_args assert 'raw_value' in kwargs['frame'] assert 'train_end_time' in kwargs['frame'] assert kwargs['x_col'] == 'train_end_time' assert kwargs['y_col'] == 'raw_value'
def test_BestDistancePlotter_plot(): with patch('audition.distance_from_best.plot_cats') as plot_patch: with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = BestDistancePlotter(distance_table) plotter.plot_all_best_dist( [{ 'metric': 'precision@', 'parameter': '100_abs' }], model_group_ids=[1, 2], train_end_times=['2014-01-01', '2015-01-01'], ) assert plot_patch.called args, kwargs = plot_patch.call_args assert 'distance' in kwargs['frame'] assert 'pct_of_time' in kwargs['frame'] assert kwargs['x_col'] == 'distance' assert kwargs['y_col'] == 'pct_of_time'
def test_SelectionPlotter_create_plot_dataframe(): with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = SelectionRulePlotter( selection_rule_picker=SelectionRulePicker(distance_table)) plot_df = plotter.create_plot_dataframe( bound_selection_rules=[ BoundSelectionRule(descriptive_name='best_current_precision', function=best_current_value, args={ 'metric': 'precision@', 'parameter': '100_abs' }), BoundSelectionRule(descriptive_name='best_avg_precision', function=best_average_value, args={ 'metric': 'precision@', 'parameter': '100_abs' }), ], model_group_ids=[ mg.model_group_id for mg in model_groups.values() ], train_end_times=['2014-01-01', '2015-01-01', '2016-01-01'], regret_metric='precision@', regret_parameter='100_abs', ) # assert that we have the right # of columns and a row for each % diff value assert plot_df.shape == (100 * 2, 3) # both selection rules have a regret lower than 70 for value in plot_df[plot_df['regret'] == 0.70]['pct_of_time'].values: assert numpy.isclose(value, 1.0) # best avg precision rule should be within 0.14 1/3 of the time for value in plot_df[(plot_df['regret'] == 0.14) & (plot_df['selection_rule'] == 'best_avg_precision')]['pct_of_time'].values: assert numpy.isclose(value, 1.0 / 3)
def test_SelectionPlotter_plot(): with patch('audition.regrets.plot_cats') as plot_patch: with testing.postgresql.Postgresql() as postgresql: engine = create_engine(postgresql.url()) distance_table, model_groups = create_sample_distance_table(engine) plotter = SelectionRulePlotter( selection_rule_picker=SelectionRulePicker(distance_table)) plotter.plot_all_selection_rules( bound_selection_rules=[ BoundSelectionRule( descriptive_name='best_current_precision', function=best_current_value, args={ 'metric': 'precision@', 'parameter': '100_abs' }), BoundSelectionRule(descriptive_name='best_avg_precision', function=best_average_value, args={ 'metric': 'precision@', 'parameter': '100_abs' }), ], model_group_ids=[ mg.model_group_id for mg in model_groups.values() ], train_end_times=['2014-01-01', '2015-01-01', '2016-01-01'], regret_metric='precision@', regret_parameter='100_abs', ) assert plot_patch.called args, kwargs = plot_patch.call_args assert 'regret' in kwargs['frame'] assert 'pct_of_time' in kwargs['frame'] assert kwargs['x_col'] == 'regret' assert kwargs['y_col'] == 'pct_of_time'