def test_experiment_rank_by_num_firsts(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) ranking = data_utils.experiment_level_ranking( snapshots_df, data_utils.benchmark_rank_by_median, data_utils.experiment_rank_by_num_firsts) expected_ranking = pd.Series(index=['libfuzzer', 'afl'], data=[1.0, 1.0]) assert ranking.equals(expected_ranking)
def test_get_experiment_snapshots(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) timestamps_per_trial = snapshots_df[['trial_id', 'time']] expected_timestamps_per_trial = pd.DataFrame([{ 'trial_id': trial, 'time': 9 } for trial in range(8)]) assert timestamps_per_trial.equals(expected_timestamps_per_trial)
def test_experiment_rank_by_average_normalized_score(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) ranking = data_utils.experiment_level_ranking( snapshots_df, data_utils.benchmark_rank_by_median, data_utils.experiment_rank_by_average_normalized_score) expected_ranking = pd.Series(index=['libfuzzer', 'afl'], data=[81.81, 80.00]) pd_test.assert_series_equal(ranking, expected_ranking, check_names=False, check_less_precise=True)
def test_experiment_summary(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) summary = data_utils.experiment_summary(snapshots_df) expected_summary = pd.DataFrame({ 'benchmark': ['libpng', 'libpng', 'libxml', 'libxml'], 'fuzzer': ['libfuzzer', 'afl', 'afl', 'libfuzzer'], 'time': [9, 9, 9, 9], 'count': [2, 2, 2, 2], 'min': [200, 100, 1000, 600], 'median': [250, 150, 1100, 700], 'max': [300, 200, 1200, 800] }).set_index(['benchmark', 'fuzzer', 'time']).astype(float) assert summary[['count', 'min', 'median', 'max']].equals(expected_summary)
def test_experiment_pivot_table(): experiment_df = create_experiment_data() snapshots_df = data_utils.get_experiment_snapshots(experiment_df) pivot_table = data_utils.experiment_pivot_table( snapshots_df, data_utils.benchmark_rank_by_median) # yapf: disable expected_data = pd.DataFrame([ {'benchmark': 'libpng', 'fuzzer': 'afl', 'median': 150}, {'benchmark': 'libpng', 'fuzzer': 'libfuzzer', 'median': 250}, {'benchmark': 'libxml', 'fuzzer': 'afl', 'median': 1100}, {'benchmark': 'libxml', 'fuzzer': 'libfuzzer', 'median': 700}, ]) # yapf: enable expected_pivot_table = pd.pivot_table(expected_data, index=['benchmark'], columns=['fuzzer'], values='median') assert pivot_table.equals(expected_pivot_table)
def _experiment_snapshots_df(self): """Data frame containing only the time snapshots, for each benchmark, based on which we do further analysis, i.e., statistical tests and ranking.""" return data_utils.get_experiment_snapshots(self._experiment_df)