def pending_trials(db, experiment_config): """Adds trials to the database and returns pending trials.""" other_experiment_name = experiment_config['experiment'] + 'other' db_utils.add_all([ models.Experiment(name=experiment_config['experiment']), models.Experiment(name=other_experiment_name) ]) def create_trial(experiment, time_started=None, time_ended=None): """Creates a database trial.""" return models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER, time_started=time_started, time_ended=time_ended) our_pending_trials = [ create_trial(experiment_config['experiment']), create_trial(experiment_config['experiment']) ] other_trials = [ create_trial(other_experiment_name), create_trial(experiment_config['experiment'], datetime.datetime.now()), create_trial(experiment_config['experiment'], datetime.datetime.now()) ] db_utils.add_all(other_trials + our_pending_trials) our_trial_ids = [trial.id for trial in our_pending_trials] return db_utils.query(models.Trial).filter( models.Trial.id.in_(our_trial_ids))
def test_add_nonprivate_experiments_for_merge_with_clobber(db): """Tests that add_nonprivate_experiments_for_merge_with_clobber doesn't include private experiments and returns the expected results in the correct order.""" experiment_names = ['1', '2', '3'] arbitrary_datetime = datetime.datetime(2020, 1, 1) db_utils.add_all([ models.Experiment(name=name, time_created=arbitrary_datetime, private=False) for name in experiment_names ]) db_utils.add_all([ models.Experiment(name='private', time_created=arbitrary_datetime, private=True), models.Experiment(name='later-nonprivate', time_created=arbitrary_datetime + datetime.timedelta(days=1), private=False), models.Experiment(name='nonprivate', time_created=arbitrary_datetime, private=False), ]) expected_results = ['nonprivate', 'later-nonprivate', '1', '2', '3'] results = queries.add_nonprivate_experiments_for_merge_with_clobber( experiment_names) assert results == expected_results
def test_add_nonprivate_experiments_for_merge_with_clobber(db): """Tests that add_nonprivate_experiments_for_merge_with_clobber doesn't include private experiments and returns the expected results in the correct order.""" experiment_names = ['1', '2', '3'] db_utils.add_all([ models.Experiment(name=name, time_created=ARBITRARY_DATETIME, time_ended=ARBITRARY_DATETIME + datetime.timedelta(days=1), private=False) for name in experiment_names ]) db_utils.add_all([ models.Experiment(name='private', time_created=ARBITRARY_DATETIME, private=True), models.Experiment(name='earlier-nonprivate', time_created=ARBITRARY_DATETIME - datetime.timedelta(days=1), time_ended=ARBITRARY_DATETIME, private=False), models.Experiment(name='nonprivate', time_created=ARBITRARY_DATETIME, time_ended=ARBITRARY_DATETIME + datetime.timedelta(days=1), private=False), models.Experiment(name='nonprivate-in-progress', time_created=ARBITRARY_DATETIME, time_ended=None, private=False), ]) expected_results = ['earlier-nonprivate', 'nonprivate', '1', '2', '3'] results = queries.add_nonprivate_experiments_for_merge_with_clobber( experiment_names) assert results == expected_results
def create_experiments(experiment_config): """Create the experiment experiment entity for the experiment in |experiment_config| and create another one and save the results to the db.""" other_experiment_name = get_other_experiment_name(experiment_config) db_utils.add_all([ models.Experiment(name=experiment_config['experiment']), models.Experiment(name=other_experiment_name) ])
def db_experiment(experiment_config, db): """A fixture that populates the database with an experiment entity with the name specified in the experiment_config fixture.""" experiment = models.Experiment(name=experiment_config['experiment']) db_utils.add_all([experiment]) # yield so that the experiment exists until the using function exits. yield
def test_get_fuzzers_changed_since_last_non_master_experiment( mocked_info, mocked_get_changed_files, db_experiment): """Tests that get_fuzzers_changed_since_last returns the correct result when the first experiment's git hash is not in branch""" # Set up a newer, out-of-branch experiment. out_of_branch_experiment = models.Experiment() out_of_branch_experiment.name = 'out-of-branch-experiment' out_of_branch_hash = 'out-of-branch-experiment-hash' out_of_branch_experiment.git_hash = out_of_branch_hash db_utils.add_all([out_of_branch_experiment]) # Update the time of out_of_branch_experiment to come after db_experiment. out_of_branch_experiment.time_created = (db_experiment.time_created + datetime.timedelta(days=1)) db_utils.add_all([out_of_branch_experiment]) def get_changed_files(commit_hash): if commit_hash == 'out-of-branch-experiment-hash': raise diff_utils.DiffError(commit_hash) return AFL_FUZZER_PY mocked_get_changed_files.side_effect = get_changed_files assert not experiment_changes.get_fuzzers_changed_since_last() mocked_info.assert_called_with('Skipping %s. Commit is not in branch.', out_of_branch_hash) mocked_get_changed_files.assert_has_calls( [mock.call(out_of_branch_hash), mock.call('hash')])
def db_experiment(db): """Fixture that creates a database populated the databse with an experiment.""" experiment = models.Experiment() experiment.name = 'experiment' experiment.git_hash = 'hash' db_utils.add_all([experiment]) return experiment
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals self, mocked_is_cycle_unchanged, db, experiment, tmp_path): """Integration test for measure_snapshot_coverage.""" # WORK is set by experiment to a directory that only makes sense in a # fakefs. A directory containing necessary llvm tools is also added to # PATH. llvm_tools_path = get_test_data_path('llvm_tools') os.environ["PATH"] += os.pathsep + llvm_tools_path os.environ['WORK'] = str(tmp_path) mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'ftfuzzer') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer, trial.benchmark, trial.id, SNAPSHOT_LOGGER) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.filestore_utils.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measurer.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 13178
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals self, mocked_is_cycle_unchanged, create_measurer, db, experiment): """Integration test for measure_snapshot_coverage.""" mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'fuzz-target') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = create_measurer(trial.fuzzer, trial.benchmark, trial.id) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.gsutil.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measurer.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 3798
def test_get_last_trial_time_started_called_early(db, experiment_config): """Tests that get_last_trial_time_started raises an exception if called while there are still pending trials.""" experiment = experiment_config['experiment'] db_utils.add_all([ models.Experiment(name=experiment), ]) trial1 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) trial2 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) first_time = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0))) trial1.time_started = first_time trials = [trial1, trial2] db_utils.add_all(trials) with pytest.raises(AssertionError): scheduler.get_last_trial_time_started(experiment)
def test_get_experiment_data_fuzzer_stats(db): """Tests that get_experiment_data handles fuzzer_stats correctly.""" experiment_name = 'experiment-1' db_utils.add_all([ models.Experiment(name=experiment_name, time_created=ARBITRARY_DATETIME, private=False) ]) trial = models.Trial(fuzzer='afl', experiment=experiment_name, benchmark='libpng') db_utils.add_all([trial]) fuzzer_stats = {'execs_per_sec': 100.0} snapshot = models.Snapshot(time=900, trial_id=trial.id, edges_covered=100, fuzzer_stats=fuzzer_stats) db_utils.add_all([snapshot]) experiment_df = queries.get_experiment_data([experiment_name]) # pylint: disable=unused-variable
def test_get_last_trial_time_started(db, experiment_config): """Tests that get_last_trial_time_started returns the time_started of the last trial to be started.""" experiment = experiment_config['experiment'] db_utils.add_all([ models.Experiment(name=experiment), ]) trial1 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) trial2 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) first_time = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0))) trial1.time_started = first_time last_time_started = first_time + datetime.timedelta(days=1) trial2.time_started = last_time_started trials = [trial1, trial2] db_utils.add_all(trials) assert scheduler.get_last_trial_time_started( experiment) == last_time_started