def test_experiment_rank_by_num_firsts():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
ranking = data_utils.experiment_level_ranking(
snapshots_df, data_utils.benchmark_rank_by_median,
data_utils.experiment_rank_by_num_firsts)
expected_ranking = pd.Series(index=['libfuzzer', 'afl'], data=[1.0, 1.0])
assert ranking.equals(expected_ranking)
def test_get_experiment_snapshots():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
timestamps_per_trial = snapshots_df[['trial_id', 'time']]
expected_timestamps_per_trial = pd.DataFrame([{
'trial_id': trial,
'time': 9
} for trial in range(8)])
assert timestamps_per_trial.equals(expected_timestamps_per_trial)
def test_experiment_rank_by_average_normalized_score():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
ranking = data_utils.experiment_level_ranking(
snapshots_df, data_utils.benchmark_rank_by_median,
data_utils.experiment_rank_by_average_normalized_score)
expected_ranking = pd.Series(index=['libfuzzer', 'afl'],
data=[81.81, 80.00])
pd_test.assert_series_equal(ranking,
expected_ranking,
check_names=False,
check_less_precise=True)
def test_experiment_summary():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
summary = data_utils.experiment_summary(snapshots_df)
expected_summary = pd.DataFrame({
'benchmark': ['libpng', 'libpng', 'libxml', 'libxml'],
'fuzzer': ['libfuzzer', 'afl', 'afl', 'libfuzzer'],
'time': [9, 9, 9, 9],
'count': [2, 2, 2, 2],
'min': [200, 100, 1000, 600],
'median': [250, 150, 1100, 700],
'max': [300, 200, 1200, 800]
}).set_index(['benchmark', 'fuzzer', 'time']).astype(float)
assert summary[['count', 'min', 'median', 'max']].equals(expected_summary)
def test_experiment_pivot_table():
experiment_df = create_experiment_data()
snapshots_df = data_utils.get_experiment_snapshots(experiment_df)
pivot_table = data_utils.experiment_pivot_table(
snapshots_df, data_utils.benchmark_rank_by_median)
# yapf: disable
expected_data = pd.DataFrame([
{'benchmark': 'libpng', 'fuzzer': 'afl', 'median': 150},
{'benchmark': 'libpng', 'fuzzer': 'libfuzzer', 'median': 250},
{'benchmark': 'libxml', 'fuzzer': 'afl', 'median': 1100},
{'benchmark': 'libxml', 'fuzzer': 'libfuzzer', 'median': 700},
])
# yapf: enable
expected_pivot_table = pd.pivot_table(expected_data,
index=['benchmark'],
columns=['fuzzer'],
values='median')
assert pivot_table.equals(expected_pivot_table)
def _experiment_snapshots_df(self):
"""Data frame containing only the time snapshots, for each benchmark,
based on which we do further analysis, i.e., statistical tests and
ranking."""
return data_utils.get_experiment_snapshots(self._experiment_df)
