def ranking_plot(self, benchmark_snapshot_df, axes=None):
"""Draws ranking plot.
The fuzzer labels will be in the order of their median coverage.
"""
benchmark_names = benchmark_snapshot_df.benchmark.unique()
assert len(benchmark_names) == 1, 'Not a single benchmark data!'
assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
fuzzer_order = data_utils.benchmark_rank_by_median(
benchmark_snapshot_df).index
axes = sns.barplot(y='edges_covered',
x='fuzzer',
data=benchmark_snapshot_df,
order=fuzzer_order,
estimator=np.median,
palette=self._fuzzer_colors,
ax=axes)
sns.despine(offset=_DEFAULT_SPINE_OFFSET, trim=True)
axes.set_title(_formatted_title(benchmark_snapshot_df))
axes.set(ylabel='Reached edge coverage')
axes.set(xlabel='Fuzzer (highest median coverage on the left)')
plt.xticks(rotation=_DEFAULT_LABEL_ROTATION,
horizontalalignment='right')
def distribution_plot(self, benchmark_snapshot_df, axes=None):
"""Draws distribution plot.
The fuzzer labels will be in the order of their median coverage.
"""
benchmark_names = benchmark_snapshot_df.benchmark.unique()
assert len(benchmark_names) == 1, 'Not a single benchmark data!'
assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
fuzzers_in_order = data_utils.benchmark_rank_by_median(
benchmark_snapshot_df).index
for fuzzer in fuzzers_in_order:
measurements_for_fuzzer = benchmark_snapshot_df[
benchmark_snapshot_df.fuzzer == fuzzer]
sns.distplot(measurements_for_fuzzer['edges_covered'],
hist=False,
label=fuzzer,
color=self._fuzzer_colors[fuzzer],
ax=axes)
axes.set_title(_formatted_title(benchmark_snapshot_df))
axes.legend(loc='upper right', frameon=False)
axes.set(xlabel='Edge coverage')
axes.set(ylabel='Density')
plt.xticks(rotation=_DEFAULT_LABEL_ROTATION,
horizontalalignment='right')
def violin_plot(self, benchmark_snapshot_df, axes=None):
"""Draws violin plot.
The fuzzer labels will be in the order of their median coverage.
"""
benchmark_names = benchmark_snapshot_df.benchmark.unique()
assert len(benchmark_names) == 1, 'Not a single benchmark data!'
assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
fuzzer_order = data_utils.benchmark_rank_by_median(
benchmark_snapshot_df).index
# Another options is to use |boxplot| instead of |violinplot|. With
# boxplot the median/min/max/etc is more visible than on the violin,
# especially with distributions with high variance. It does not have
# however violinplot's kernel density estimation.
sns.violinplot(y='edges_covered',
x='fuzzer',
data=benchmark_snapshot_df,
order=fuzzer_order,
palette=self._fuzzer_colors,
ax=axes)
sns.despine(offset=_DEFAULT_SPINE_OFFSET, trim=True)
axes.set_title(_formatted_title(benchmark_snapshot_df))
axes.set(ylabel='Reached edge coverage')
axes.set(xlabel='Fuzzer (highest median coverage on the left)')
plt.xticks(rotation=_DEFAULT_LABEL_ROTATION,
horizontalalignment='right')
def ranking_plot(self, benchmark_snapshot_df, axes=None, bugs=False):
"""Draws ranking plot.
The fuzzer labels will be in the order of their median coverage.
"""
self._common_datafame_checks(benchmark_snapshot_df, snapshot=True)
column_of_interest = 'bugs_covered' if bugs else 'edges_covered'
fuzzer_order = data_utils.benchmark_rank_by_median(
benchmark_snapshot_df, key=column_of_interest).index
axes = sns.barplot(y=column_of_interest,
x='fuzzer',
data=benchmark_snapshot_df,
order=fuzzer_order,
estimator=np.median,
palette=self._fuzzer_colors,
ax=axes)
axes.set_title(_formatted_title(benchmark_snapshot_df))
ylabel = 'Reached {} coverage'.format('bug' if bugs else 'region')
axes.set(ylabel=ylabel)
axes.set(xlabel='Fuzzer (highest median coverage on the left)')
axes.set_xticklabels(axes.get_xticklabels(),
rotation=_DEFAULT_LABEL_ROTATION,
horizontalalignment='right')
sns.despine(ax=axes, trim=True)
def distribution_plot(self, benchmark_snapshot_df, axes=None, bugs=False):
"""Draws distribution plot.
The fuzzer labels will be in the order of their median coverage.
"""
self._common_datafame_checks(benchmark_snapshot_df, snapshot=True)
column_of_interest = 'bugs_covered' if bugs else 'edges_covered'
fuzzers_in_order = data_utils.benchmark_rank_by_median(
benchmark_snapshot_df, key=column_of_interest).index
for fuzzer in fuzzers_in_order:
measurements_for_fuzzer = benchmark_snapshot_df[
benchmark_snapshot_df.fuzzer == fuzzer]
sns.distplot(measurements_for_fuzzer[column_of_interest],
hist=False,
label=fuzzer,
color=self._fuzzer_colors[fuzzer],
ax=axes)
axes.set_title(_formatted_title(benchmark_snapshot_df))
axes.legend(loc='upper right', frameon=False)
axes.set(xlabel='Bug coverage' if bugs else 'Code region coverage')
axes.set(ylabel='Density')
axes.set_xticklabels(axes.get_xticklabels(),
rotation=_DEFAULT_LABEL_ROTATION,
horizontalalignment='right')
def test_benchmark_rank_by_median():
experiment_df = create_experiment_data()
benchmark_df = experiment_df[experiment_df.benchmark == 'libxml']
snapshot_df = data_utils.get_benchmark_snapshot(benchmark_df)
ranking = data_utils.benchmark_rank_by_median(snapshot_df)
expected_ranking = pd.Series(index=['afl', 'libfuzzer'], data=[1100, 700])
assert ranking.equals(expected_ranking)
def box_or_violin_plot(self,
benchmark_snapshot_df,
axes=None,
bugs=False,
violin=False):
"""Draws a box or violin plot based on parameter.
The fuzzer labels will be in the order of their median coverage.
With boxplot the median/min/max/etc is more visible than on the violin,
especially with distributions with high variance. It does not have
however violinplot's kernel density estimation.
"""
self._common_datafame_checks(benchmark_snapshot_df, snapshot=True)
column_of_interest = 'bugs_covered' if bugs else 'edges_covered'
fuzzer_order = data_utils.benchmark_rank_by_median(
benchmark_snapshot_df, key=column_of_interest).index
mean_props = {
'markersize': '10',
'markeredgecolor': 'black',
'markerfacecolor': 'white'
}
common_args = dict(y=column_of_interest,
x='fuzzer',
data=benchmark_snapshot_df,
order=fuzzer_order,
ax=axes)
if violin:
sns.violinplot(**common_args, palette=self._fuzzer_colors)
else:
sns.boxplot(**common_args,
palette=self._fuzzer_colors,
showmeans=True,
meanprops=mean_props)
sns.stripplot(**common_args, size=3, color="black", alpha=0.6)
axes.set_title(_formatted_title(benchmark_snapshot_df))
ylabel = 'Reached {} coverage'.format('bug' if bugs else 'region')
axes.set(ylabel=ylabel)
axes.set(xlabel='Fuzzer (highest median coverage on the left)')
axes.set_xticklabels(axes.get_xticklabels(),
rotation=_DEFAULT_LABEL_ROTATION,
horizontalalignment='right')
sns.despine(ax=axes, trim=True)
def rank_by_median(self):
"""Fuzzer ranking by median coverage."""
return data_utils.benchmark_rank_by_median(self._benchmark_snapshot_df)
