def test_benchmark_rank_by_mean():
experiment_df = create_experiment_data()
benchmark_df = experiment_df[experiment_df.benchmark == 'libxml']
snapshot_df = data_utils.get_benchmark_snapshot(benchmark_df)
ranking = data_utils.benchmark_rank_by_mean(snapshot_df)
expected_ranking = pd.Series(index=['afl', 'libfuzzer'], data=[1100, 700])
assert ranking.equals(expected_ranking)
def coverage_growth_plot(self, benchmark_df, axes=None):
"""Draws coverage growth plot on given |axes|.
The fuzzer labels will be in the order of their mean coverage at the
snapshot time (typically, the end of experiment).
"""
benchmark_names = benchmark_df.benchmark.unique()
assert len(benchmark_names) == 1, 'Not a single benchmark data!'
benchmark_snapshot_df = data_utils.get_benchmark_snapshot(benchmark_df)
snapshot_time = benchmark_snapshot_df.time.unique()[0]
fuzzer_order = data_utils.benchmark_rank_by_mean(
benchmark_snapshot_df).index
axes = sns.lineplot(
y='edges_covered',
x='time',
hue='fuzzer',
hue_order=fuzzer_order,
data=benchmark_df[benchmark_df.time <= snapshot_time],
ci=None if self._quick else 95,
palette=self._fuzzer_colors,
ax=axes)
axes.set_title(_formatted_title(benchmark_snapshot_df))
# Indicate the snapshot time with a big red vertical line.
axes.axvline(x=snapshot_time, color='r')
# Move legend outside of the plot.
axes.legend(bbox_to_anchor=(1.00, 1),
borderaxespad=0,
loc='upper left',
frameon=False)
axes.set(ylabel='Edge coverage')
axes.set(xlabel='Time (hour:minute)')
if self._logscale:
axes.set_xscale('log')
ticks = np.logspace(
# Start from the time of the first measurement.
np.log10(experiment_utils.DEFAULT_SNAPSHOT_SECONDS),
np.log10(snapshot_time + 1), # Include tick at end time.
_DEFAULT_TICKS_COUNT)
else:
ticks = np.arange(
experiment_utils.DEFAULT_SNAPSHOT_SECONDS,
snapshot_time + 1, # Include tick at end time.
snapshot_time / _DEFAULT_TICKS_COUNT)
axes.set_xticks(ticks)
axes.set_xticklabels([_formatted_hour_min(t) for t in ticks])
sns.despine(ax=axes, trim=True)
def rank_by_mean(self):
"""Fuzzer ranking by mean coverage."""
return data_utils.benchmark_rank_by_mean(self._benchmark_snapshot_df)
def coverage_growth_plot(self,
benchmark_df,
axes=None,
logscale=False,
bugs=False):
"""Draws edge (or bug) coverage growth plot on given |axes|.
The fuzzer labels will be in the order of their mean coverage at the
snapshot time (typically, the end of experiment).
"""
self._common_datafame_checks(benchmark_df)
column_of_interest = 'bugs_covered' if bugs else 'edges_covered'
benchmark_snapshot_df = data_utils.get_benchmark_snapshot(benchmark_df)
snapshot_time = benchmark_snapshot_df.time.unique()[0]
fuzzer_order = data_utils.benchmark_rank_by_mean(
benchmark_snapshot_df, key=column_of_interest).index
axes = sns.lineplot(
y=column_of_interest,
x='time',
hue='fuzzer',
hue_order=fuzzer_order,
data=benchmark_df[benchmark_df.time <= snapshot_time],
ci=None if bugs or self._quick else 95,
estimator=np.median,
palette=self._fuzzer_colors,
style='fuzzer',
dashes=False,
markers=self._fuzzer_markers,
ax=axes)
axes.set_title(_formatted_title(benchmark_snapshot_df))
# Indicate the snapshot time with a big red vertical line.
axes.axvline(x=snapshot_time, color='r')
# Move legend outside of the plot.
axes.legend(bbox_to_anchor=(1.00, 1),
borderaxespad=0,
loc='upper left',
frameon=False)
axes.set(ylabel='Bug coverage' if bugs else 'Code region coverage')
axes.set(xlabel='Time (hour:minute)')
if self._logscale or logscale:
axes.set_xscale('log')
ticks = np.logspace(
# Start from the time of the first measurement.
np.log10(experiment_utils.DEFAULT_SNAPSHOT_SECONDS),
np.log10(snapshot_time + 1), # Include tick at end time.
_DEFAULT_TICKS_COUNT)
else:
ticks = np.arange(
experiment_utils.DEFAULT_SNAPSHOT_SECONDS,
snapshot_time + 1, # Include tick at end time.
snapshot_time / _DEFAULT_TICKS_COUNT)
axes.set_xticks(ticks)
axes.set_xticklabels([_formatted_hour_min(t) for t in ticks])
sns.despine(ax=axes, trim=True)
