def rank_by_stat_test_wins_and_average_rank(self): """Rank fuzzers using statistical test wins per benchmark and average rank across benchmarks.""" return data_utils.experiment_level_ranking( self._experiment_snapshots_df, data_utils.benchmark_rank_by_stat_test_wins, data_utils.experiment_rank_by_average_rank)
def rank_by_average_rank_and_average_rank(self): """Rank fuzzers using average rank per benchmark and average rank across benchmarks.""" return data_utils.experiment_level_ranking( self._experiment_snapshots_df, data_utils.benchmark_rank_by_average_rank, data_utils.experiment_rank_by_average_rank)
def _ranking(self, benchmark_level_ranking_function, experiment_level_ranking_function): return data_utils.experiment_level_ranking( self._experiment_snapshots_df, functools.partial(benchmark_level_ranking_function, key=self._relevant_column), experiment_level_ranking_function)
def test_experiment_rank_by_num_firsts(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) ranking = data_utils.experiment_level_ranking( snapshots_df, data_utils.benchmark_rank_by_median, data_utils.experiment_rank_by_num_firsts) expected_ranking = pd.Series(index=['libfuzzer', 'afl'], data=[1.0, 1.0]) assert ranking.equals(expected_ranking)
def test_experiment_rank_by_average_normalized_score(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) ranking = data_utils.experiment_level_ranking( snapshots_df, data_utils.benchmark_rank_by_median, data_utils.experiment_rank_by_average_normalized_score) expected_ranking = pd.Series(index=['libfuzzer', 'afl'], data=[81.81, 80.00]) pd_test.assert_series_equal(ranking, expected_ranking, check_names=False, check_less_precise=True)
def rank_by_median_and_number_of_firsts(self): """Rank fuzzers using median coverage per benchmark and number of first places across benchmarks.""" return data_utils.experiment_level_ranking( self._experiment_snapshots_df, data_utils.benchmark_rank_by_median, data_utils.experiment_rank_by_num_firsts)
def rank_by_median_and_average_normalized_score(self): """Rank fuzzers using median coverage per benchmark and average normalized score across benchmarks.""" return data_utils.experiment_level_ranking( self._experiment_snapshots_df, data_utils.benchmark_rank_by_median, data_utils.experiment_rank_by_average_normalized_score)