def build_images_for_trials(fuzzers: List[str],
benchmarks: List[str],
num_trials: int,
preemptible: bool,
concurrent_builds=None) -> List[models.Trial]:
"""Builds the images needed to run |experiment| and returns a list of trials
that can be run for experiment. This is the number of trials specified in
experiment times each pair of fuzzer+benchmark that builds successfully."""
# This call will raise an exception if the images can't be built which will
# halt the experiment.
builder.build_base_images()
# Only build fuzzers for benchmarks whose measurers built successfully.
if concurrent_builds is None:
benchmarks = builder.build_all_measurers(benchmarks)
build_successes = builder.build_all_fuzzer_benchmarks(
fuzzers, benchmarks)
else:
benchmarks = builder.build_all_measurers(benchmarks, concurrent_builds)
build_successes = builder.build_all_fuzzer_benchmarks(
fuzzers, benchmarks, concurrent_builds)
experiment_name = experiment_utils.get_experiment_name()
trials = []
for fuzzer, benchmark in build_successes:
fuzzer_benchmark_trials = [
models.Trial(fuzzer=fuzzer,
experiment=experiment_name,
benchmark=benchmark,
preemptible=preemptible) for _ in range(num_trials)
]
trials.extend(fuzzer_benchmark_trials)
return trials
def test_build_all_measurers(_, mocked_build_measurer,
build_measurer_return_value, experiment):
"""Tests that build_all_measurers works as intendend when build_measurer
calls fail."""
mocked_build_measurer.return_value = build_measurer_return_value
benchmarks = get_regular_benchmarks()
result = builder.build_all_measurers(benchmarks)
if build_measurer_return_value:
assert result == benchmarks
else:
assert not result
def test_build_all_measurers(_, mocked_build_measurer,
build_measurer_return_value, experiment, fs):
"""Tests that build_all_measurers works as intendend when build_measurer
calls fail."""
fs.add_real_directory(utils.ROOT_DIR)
mocked_build_measurer.return_value = build_measurer_return_value
benchmarks = get_regular_benchmarks()
result = builder.build_all_measurers(benchmarks)
if build_measurer_return_value:
assert result == benchmarks
else:
assert not result
def __init__(self, experiment_config_filepath: str):
self.config = yaml_utils.read(experiment_config_filepath)
benchmarks = self.config['benchmarks'].split(',')
self.benchmarks = builder.build_all_measurers(benchmarks)
self.fuzzers = [
fuzzer_config_utils.get_fuzzer_name(filename) for filename in
os.listdir(fuzzer_config_utils.get_fuzzer_configs_dir())
]
_initialize_experiment_in_db(self.config['experiment'],
self.config['git_hash'], self.benchmarks,
self.fuzzers, self.config['trials'])
self.web_bucket = posixpath.join(
self.config['cloud_web_bucket'],
experiment_utils.get_experiment_name())
def _test_build_measurers_benchmarks(benchmarks):
"""Asserts that measurers for each benchmark in |benchmarks| can build."""
assert benchmarks == builder.build_all_measurers(benchmarks)
