def rank_by_stat_test_wins_and_average_rank(self):
"""Rank fuzzers using statistical test wins per benchmark and average
rank across benchmarks."""
return data_utils.experiment_level_ranking(
self._experiment_snapshots_df,
data_utils.benchmark_rank_by_stat_test_wins,
data_utils.experiment_rank_by_average_rank)
def rank_by_average_rank_and_average_rank(self):
"""Rank fuzzers using average rank per benchmark and average rank
across benchmarks."""
return data_utils.experiment_level_ranking(
self._experiment_snapshots_df,
data_utils.benchmark_rank_by_average_rank,
data_utils.experiment_rank_by_average_rank)
def _ranking(self, benchmark_level_ranking_function,
experiment_level_ranking_function):
return data_utils.experiment_level_ranking(
self._experiment_snapshots_df,
functools.partial(benchmark_level_ranking_function,
key=self._relevant_column),
experiment_level_ranking_function)
def test_experiment_rank_by_num_firsts():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
ranking = data_utils.experiment_level_ranking(
snapshots_df, data_utils.benchmark_rank_by_median,
data_utils.experiment_rank_by_num_firsts)
expected_ranking = pd.Series(index=['libfuzzer', 'afl'], data=[1.0, 1.0])
assert ranking.equals(expected_ranking)
def test_experiment_rank_by_average_normalized_score():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
ranking = data_utils.experiment_level_ranking(
snapshots_df, data_utils.benchmark_rank_by_median,
data_utils.experiment_rank_by_average_normalized_score)
expected_ranking = pd.Series(index=['libfuzzer', 'afl'],
data=[81.81, 80.00])
pd_test.assert_series_equal(ranking,
expected_ranking,
check_names=False,
check_less_precise=True)
def rank_by_median_and_number_of_firsts(self):
"""Rank fuzzers using median coverage per benchmark and number of first
places across benchmarks."""
return data_utils.experiment_level_ranking(
self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,
data_utils.experiment_rank_by_num_firsts)
def rank_by_median_and_average_normalized_score(self):
"""Rank fuzzers using median coverage per benchmark and average
normalized score across benchmarks."""
return data_utils.experiment_level_ranking(
self._experiment_snapshots_df, data_utils.benchmark_rank_by_median,
data_utils.experiment_rank_by_average_normalized_score)
