def test_get_preempted_trials_new_preempted(mocked_get_preempted_operations, preempt_exp_conf): """Tests that TrialInstanceManager.get_preempted_trials returns trials that new preempted trials we don't know about until we query for them and not trials that we already knew were preempted.""" trial_instance_manager = get_trial_instance_manager(preempt_exp_conf) # Create trials. experiment = preempt_exp_conf['experiment'] time_started = ARBITRARY_DATETIME known_preempted = models.Trial(experiment=experiment, fuzzer=FUZZER, benchmark=BENCHMARK, time_started=time_started) unknown_preempted = models.Trial(experiment=experiment, fuzzer=FUZZER, benchmark=BENCHMARK, time_started=time_started) trials = [known_preempted, unknown_preempted] db_utils.add_all(trials) mocked_get_preempted_operations.return_value = [ _get_preemption_operation(trial.id, preempt_exp_conf) for trial in trials ] trial_instance_manager.preempted_trials = { known_preempted.id: known_preempted } result = trial_instance_manager.get_preempted_trials() expected_result = [unknown_preempted] assert result == expected_result
def test_initialize_experiment_in_db(dispatcher_experiment): """Tests that _initialize_experiment_in_db adds the right things to the database.""" trials_args = itertools.product(dispatcher_experiment.benchmarks, range(dispatcher_experiment.num_trials), dispatcher_experiment.fuzzers) trials = [ models.Trial(fuzzer=fuzzer, experiment=dispatcher_experiment.experiment_name, benchmark=benchmark) for benchmark, _, fuzzer in trials_args ] dispatcher._initialize_experiment_in_db(dispatcher_experiment.config, trials) db_experiments = db_utils.query(models.Experiment).all() assert len(db_experiments) == 1 db_experiment = db_experiments[0] assert db_experiment.name == os.environ['EXPERIMENT'] trials = db_utils.query(models.Trial).all() fuzzer_and_benchmarks = [(trial.benchmark, trial.fuzzer) for trial in trials] assert fuzzer_and_benchmarks == ([('benchmark-1', 'fuzzer-a'), ('benchmark-1', 'fuzzer-b')] * 4) + [('benchmark-2', 'fuzzer-a'), ('benchmark-2', 'fuzzer-b')] * 4
def build_images_for_trials(fuzzers: List[str], benchmarks: List[str], num_trials: int, preemptible: bool, concurrent_builds=None) -> List[models.Trial]: """Builds the images needed to run |experiment| and returns a list of trials that can be run for experiment. This is the number of trials specified in experiment times each pair of fuzzer+benchmark that builds successfully.""" # This call will raise an exception if the images can't be built which will # halt the experiment. builder.build_base_images() # Only build fuzzers for benchmarks whose measurers built successfully. if concurrent_builds is None: benchmarks = builder.build_all_measurers(benchmarks) build_successes = builder.build_all_fuzzer_benchmarks( fuzzers, benchmarks) else: benchmarks = builder.build_all_measurers(benchmarks, concurrent_builds) build_successes = builder.build_all_fuzzer_benchmarks( fuzzers, benchmarks, concurrent_builds) experiment_name = experiment_utils.get_experiment_name() trials = [] for fuzzer, benchmark in build_successes: fuzzer_benchmark_trials = [ models.Trial(fuzzer=fuzzer, experiment=experiment_name, benchmark=benchmark, preemptible=preemptible) for _ in range(num_trials) ] trials.extend(fuzzer_benchmark_trials) return trials
def create_trial(experiment, time_started=None, time_ended=None): """Creates a database trial.""" return models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER, time_started=time_started, time_ended=time_ended)
def replace_trial(trial, preemptible): """Returns a new trial to replace |trial|. The trial is preemptible if |preemptible|. Sets trial.replacement to the replacement trial.""" replacement = models.Trial(fuzzer=trial.fuzzer, benchmark=trial.benchmark, experiment=trial.experiment, preemptible=preemptible) trial.replacement = replacement.id return replacement
def test_get_last_trial_time_started_called_early(db, experiment_config): """Tests that get_last_trial_time_started raises an exception if called while there are still pending trials.""" experiment = experiment_config['experiment'] db_utils.add_all([ models.Experiment(name=experiment), ]) trial1 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) trial2 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) first_time = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0))) trial1.time_started = first_time trials = [trial1, trial2] db_utils.add_all(trials) with pytest.raises(AssertionError): scheduler.get_last_trial_time_started(experiment)
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals self, mocked_is_cycle_unchanged, db, experiment, tmp_path): """Integration test for measure_snapshot_coverage.""" # WORK is set by experiment to a directory that only makes sense in a # fakefs. A directory containing necessary llvm tools is also added to # PATH. llvm_tools_path = get_test_data_path('llvm_tools') os.environ["PATH"] += os.pathsep + llvm_tools_path os.environ['WORK'] = str(tmp_path) mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'ftfuzzer') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer, trial.benchmark, trial.id, SNAPSHOT_LOGGER) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.filestore_utils.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measurer.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 13178
def test_get_last_trial_time_started(db, experiment_config): """Tests that get_last_trial_time_started returns the time_started of the last trial to be started.""" experiment = experiment_config['experiment'] db_utils.add_all([ models.Experiment(name=experiment), ]) trial1 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) trial2 = models.Trial(experiment=experiment, benchmark=BENCHMARK, fuzzer=FUZZER) first_time = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0))) trial1.time_started = first_time last_time_started = first_time + datetime.timedelta(days=1) trial2.time_started = last_time_started trials = [trial1, trial2] db_utils.add_all(trials) assert scheduler.get_last_trial_time_started( experiment) == last_time_started
def _initialize_experiment_in_db(experiment: str, benchmarks: List[str], fuzzers: List[str], num_trials: int): """Initializes |experiment| in the database by creating the experiment entity and entities for each trial in the experiment.""" db_utils.add_all( [db_utils.get_or_create(models.Experiment, name=experiment)]) trials_args = itertools.product(sorted(benchmarks), range(num_trials), sorted(fuzzers)) trials = [ models.Trial(fuzzer=fuzzer, experiment=experiment, benchmark=benchmark) for benchmark, _, fuzzer in trials_args ] # TODO(metzman): Consider doing this without sqlalchemy. This can get # slow with SQLalchemy (it's much worse with add_all). db_utils.bulk_save(trials)
def test_get_preempted_trials_stale_preempted(_, preempt_exp_conf): """Tests that TrialInstanceManager.get_preempted_trials doesn't return trials that we already know were preempted.""" trial_instance_manager = get_trial_instance_manager(preempt_exp_conf) trial = models.Trial(experiment=preempt_exp_conf['experiment'], fuzzer=FUZZER, benchmark=BENCHMARK) db_utils.add_all([trial]) instance_name = experiment_utils.get_trial_instance_name( preempt_exp_conf['experiment'], trial.id) trial_instance_manager.preempted_trials = {instance_name: trial} with mock.patch( 'experiment.scheduler.TrialInstanceManager.' '_get_started_unfinished_instances', return_value=[instance_name]): assert trial_instance_manager.get_preempted_trials() == []
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals self, mocked_is_cycle_unchanged, create_measurer, db, experiment): """Integration test for measure_snapshot_coverage.""" mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'fuzz-target') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = create_measurer(trial.fuzzer, trial.benchmark, trial.id) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.gsutil.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measurer.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 3798
def test_get_experiment_data_fuzzer_stats(db): """Tests that get_experiment_data handles fuzzer_stats correctly.""" experiment_name = 'experiment-1' db_utils.add_all([ models.Experiment(name=experiment_name, time_created=ARBITRARY_DATETIME, private=False) ]) trial = models.Trial(fuzzer='afl', experiment=experiment_name, benchmark='libpng') db_utils.add_all([trial]) fuzzer_stats = {'execs_per_sec': 100.0} snapshot = models.Snapshot(time=900, trial_id=trial.id, edges_covered=100, fuzzer_stats=fuzzer_stats) db_utils.add_all([snapshot]) experiment_df = queries.get_experiment_data([experiment_name]) # pylint: disable=unused-variable