def build_images_for_trials(fuzzers: List[str], benchmarks: List[str], num_trials: int, preemptible: bool, concurrent_builds=None) -> List[models.Trial]: """Builds the images needed to run |experiment| and returns a list of trials that can be run for experiment. This is the number of trials specified in experiment times each pair of fuzzer+benchmark that builds successfully.""" # This call will raise an exception if the images can't be built which will # halt the experiment. builder.build_base_images() # Only build fuzzers for benchmarks whose measurers built successfully. if concurrent_builds is None: benchmarks = builder.build_all_measurers(benchmarks) build_successes = builder.build_all_fuzzer_benchmarks( fuzzers, benchmarks) else: benchmarks = builder.build_all_measurers(benchmarks, concurrent_builds) build_successes = builder.build_all_fuzzer_benchmarks( fuzzers, benchmarks, concurrent_builds) experiment_name = experiment_utils.get_experiment_name() trials = [] for fuzzer, benchmark in build_successes: fuzzer_benchmark_trials = [ models.Trial(fuzzer=fuzzer, experiment=experiment_name, benchmark=benchmark, preemptible=preemptible) for _ in range(num_trials) ] trials.extend(fuzzer_benchmark_trials) return trials
def test_build_all_measurers(_, mocked_build_measurer, build_measurer_return_value, experiment): """Tests that build_all_measurers works as intendend when build_measurer calls fail.""" mocked_build_measurer.return_value = build_measurer_return_value benchmarks = get_regular_benchmarks() result = builder.build_all_measurers(benchmarks) if build_measurer_return_value: assert result == benchmarks else: assert not result
def test_build_all_measurers(_, mocked_build_measurer, build_measurer_return_value, experiment, fs): """Tests that build_all_measurers works as intendend when build_measurer calls fail.""" fs.add_real_directory(utils.ROOT_DIR) mocked_build_measurer.return_value = build_measurer_return_value benchmarks = get_regular_benchmarks() result = builder.build_all_measurers(benchmarks) if build_measurer_return_value: assert result == benchmarks else: assert not result
def __init__(self, experiment_config_filepath: str): self.config = yaml_utils.read(experiment_config_filepath) benchmarks = self.config['benchmarks'].split(',') self.benchmarks = builder.build_all_measurers(benchmarks) self.fuzzers = [ fuzzer_config_utils.get_fuzzer_name(filename) for filename in os.listdir(fuzzer_config_utils.get_fuzzer_configs_dir()) ] _initialize_experiment_in_db(self.config['experiment'], self.config['git_hash'], self.benchmarks, self.fuzzers, self.config['trials']) self.web_bucket = posixpath.join( self.config['cloud_web_bucket'], experiment_utils.get_experiment_name())
def _test_build_measurers_benchmarks(benchmarks): """Asserts that measurers for each benchmark in |benchmarks| can build.""" assert benchmarks == builder.build_all_measurers(benchmarks)