def _relative_summary_table(self, key_column='edges_covered'):
"""A pivot table of medians ( % of experiment max per benchmark )
for each fuzzer on each benchmark."""
pivot = data_utils.experiment_pivot_table(
self._experiment_snapshots_df,
functools.partial(data_utils.benchmark_rank_by_percent,
key=key_column))
# Remove names
pivot = pivot.rename_axis(index=None, columns=None)
# Add rows for Median and Mean values
nrows, _ = pivot.shape
pivot.loc['FuzzerMedian'] = pivot.iloc[0:nrows].median()
pivot.loc['FuzzerMean'] = pivot.iloc[0:nrows].mean()
# Sort fuzzers left to right by FuzzerMean
pivot = pivot.sort_values(by='FuzzerMean', axis=1, ascending=False)
# Move Median and Mean to top rows
row_index = pivot.index.to_list()
pivot = pivot.reindex(row_index[-2:] + row_index[:-2])
# Mean row slicer
idx = pd.IndexSlice['FuzzerMean', :]
whbl = sns.light_palette('lightblue', n_colors=30, as_cmap=True)
pivot = pivot.style\
.background_gradient(axis=1, cmap=whbl, vmin=95, vmax=100)\
.highlight_max(axis=1, color='lightgreen')\
.format("{:.2f}")\
.apply(data_utils.underline_row, axis=1, subset=idx)\
.set_table_styles(self._SUMMARY_TABLE_STYLE)
return pivot
def rank_by_stat_test_wins(self):
"""Ranking across benchmarks (using statistical test wins based per-
benchmark ranking.)"""
pivot_table_of_stat_test_wins = data_utils.experiment_pivot_table(
self._experiment_snapshots_df, data_utils.rank_by_average_rank)
return data_utils.experiment_rank_by_average_rank(
pivot_table_of_stat_test_wins)
def rank_by_average_rank(self):
"""Ranking across benchmarks (using rank average based per-benchmark
ranking.)"""
pivot_table_of_average_ranks = data_utils.experiment_pivot_table(
self._experiment_snapshots_df, data_utils.rank_by_average_rank)
return data_utils.experiment_rank_by_average_rank(
pivot_table_of_average_ranks)
def test_experiment_pivot_table():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
pivot_table = data_utils.experiment_pivot_table(
snapshots_df, data_utils.benchmark_rank_by_median)
# yapf: disable
expected_data = pd.DataFrame([
{'benchmark': 'libpng', 'fuzzer': 'afl', 'median': 150},
{'benchmark': 'libpng', 'fuzzer': 'libfuzzer', 'median': 250},
{'benchmark': 'libxml', 'fuzzer': 'afl', 'median': 1100},
{'benchmark': 'libxml', 'fuzzer': 'libfuzzer', 'median': 700},
])
# yapf: enable
expected_pivot_table = pd.pivot_table(expected_data,
index=['benchmark'],
columns=['fuzzer'],
values='median')
assert pivot_table.equals(expected_pivot_table)
def summary_table(self):
"""A pivot table of medians for each fuzzer on each benchmark."""
return data_utils.experiment_pivot_table(
self._experiment_snapshots_df, data_utils.benchmark_rank_by_median)
def summary_table(self):
"""A pivot table of medians for each fuzzer on each benchmark."""
return data_utils.experiment_pivot_table(
self._experiment_snapshots_df,
functools.partial(data_utils.benchmark_rank_by_median,
key=self._relevant_column))
