def _relative_summary_table(self, key_column='edges_covered'): """A pivot table of medians ( % of experiment max per benchmark ) for each fuzzer on each benchmark.""" pivot = data_utils.experiment_pivot_table( self._experiment_snapshots_df, functools.partial(data_utils.benchmark_rank_by_percent, key=key_column)) # Remove names pivot = pivot.rename_axis(index=None, columns=None) # Add rows for Median and Mean values nrows, _ = pivot.shape pivot.loc['FuzzerMedian'] = pivot.iloc[0:nrows].median() pivot.loc['FuzzerMean'] = pivot.iloc[0:nrows].mean() # Sort fuzzers left to right by FuzzerMean pivot = pivot.sort_values(by='FuzzerMean', axis=1, ascending=False) # Move Median and Mean to top rows row_index = pivot.index.to_list() pivot = pivot.reindex(row_index[-2:] + row_index[:-2]) # Mean row slicer idx = pd.IndexSlice['FuzzerMean', :] whbl = sns.light_palette('lightblue', n_colors=30, as_cmap=True) pivot = pivot.style\ .background_gradient(axis=1, cmap=whbl, vmin=95, vmax=100)\ .highlight_max(axis=1, color='lightgreen')\ .format("{:.2f}")\ .apply(data_utils.underline_row, axis=1, subset=idx)\ .set_table_styles(self._SUMMARY_TABLE_STYLE) return pivot
def rank_by_stat_test_wins(self): """Ranking across benchmarks (using statistical test wins based per- benchmark ranking.)""" pivot_table_of_stat_test_wins = data_utils.experiment_pivot_table( self._experiment_snapshots_df, data_utils.rank_by_average_rank) return data_utils.experiment_rank_by_average_rank( pivot_table_of_stat_test_wins)
def rank_by_average_rank(self): """Ranking across benchmarks (using rank average based per-benchmark ranking.)""" pivot_table_of_average_ranks = data_utils.experiment_pivot_table( self._experiment_snapshots_df, data_utils.rank_by_average_rank) return data_utils.experiment_rank_by_average_rank( pivot_table_of_average_ranks)
def test_experiment_pivot_table(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) pivot_table = data_utils.experiment_pivot_table( snapshots_df, data_utils.benchmark_rank_by_median) # yapf: disable expected_data = pd.DataFrame([ {'benchmark': 'libpng', 'fuzzer': 'afl', 'median': 150}, {'benchmark': 'libpng', 'fuzzer': 'libfuzzer', 'median': 250}, {'benchmark': 'libxml', 'fuzzer': 'afl', 'median': 1100}, {'benchmark': 'libxml', 'fuzzer': 'libfuzzer', 'median': 700}, ]) # yapf: enable expected_pivot_table = pd.pivot_table(expected_data, index=['benchmark'], columns=['fuzzer'], values='median') assert pivot_table.equals(expected_pivot_table)
def summary_table(self): """A pivot table of medians for each fuzzer on each benchmark.""" return data_utils.experiment_pivot_table( self._experiment_snapshots_df, data_utils.benchmark_rank_by_median)
def summary_table(self): """A pivot table of medians for each fuzzer on each benchmark.""" return data_utils.experiment_pivot_table( self._experiment_snapshots_df, functools.partial(data_utils.benchmark_rank_by_median, key=self._relevant_column))