def _validate_individual_experiment_requests(experiment_requests): """Returns True if all requests in |experiment_request| are valid in isolation. Does not account for PAUSE_SERVICE_KEYWORD or duplicates.""" all_fuzzers = set(fuzzer_utils.get_fuzzer_names()) valid = True # Validate format. for request in experiment_requests: if not isinstance(request, dict): logger.error('Request: %s is not a dict.', request) experiment_requests.remove(request) valid = False continue if 'experiment' not in request: logger.error('Request: %s does not have field "experiment".', request) valid = False continue experiment = _get_experiment_name(request) if not validate_experiment_name(experiment): valid = False logger.error('Experiment name: %s is not valid.', experiment) # Request isn't so malformed that we can finding other issues. # if present. Don't continue. experiment = request['experiment'] if not request.get('fuzzers'): logger.error('Request: %s does not specify any fuzzers.', request) valid = False continue experiment_fuzzers = request['fuzzers'] for fuzzer in experiment_fuzzers: if fuzzer in all_fuzzers: continue # Fuzzer isn't valid. logger.error('Fuzzer: %s in experiment %s is not valid.', fuzzer, experiment) valid = False description = request.get('description') if description is not None and not isinstance(description, str): logger.error( 'Request: %s "description" attribute is not a valid string.', request) valid = False continue oss_fuzz_corpus = request.get('oss_fuzz_corpus') if oss_fuzz_corpus is not None and not isinstance( oss_fuzz_corpus, bool): logger.error( 'Request: %s "oss_fuzz_corpus" attribute is not a valid bool.', request) valid = False continue return valid
def main(): """Run an experiment in the cloud.""" logs.initialize() parser = argparse.ArgumentParser( description='Begin an experiment that evaluates fuzzers on one or ' 'more benchmarks.') all_benchmarks = benchmark_utils.get_all_benchmarks() all_fuzzers = fuzzer_utils.get_fuzzer_names() parser.add_argument('-b', '--benchmarks', help='Benchmark names. All of them by default.', nargs='+', required=False, default=all_benchmarks, choices=all_benchmarks) parser.add_argument('-c', '--experiment-config', help='Path to the experiment configuration yaml file.', required=True) parser.add_argument('-e', '--experiment-name', help='Experiment name.', required=True) parser.add_argument('-f', '--fuzzers', help='Fuzzers to use.', nargs='+', required=False, default=None, choices=all_fuzzers) parser.add_argument( '-ns', '--no-seeds', help='Should trials be conducted without seed corpora.', required=False, default=False, action='store_true') parser.add_argument( '-nd', '--no-dictionaries', help='Should trials be conducted without dictionaries.', required=False, default=False, action='store_true') args = parser.parse_args() fuzzers = args.fuzzers or all_fuzzers start_experiment(args.experiment_name, args.experiment_config, args.benchmarks, fuzzers, no_seeds=args.no_seeds, no_dictionaries=args.no_dictionaries) return 0
def main(): """Run an experiment in the cloud.""" logs.initialize() parser = argparse.ArgumentParser( description='Begin an experiment that evaluates fuzzers on one or ' 'more benchmarks.') all_benchmarks = benchmark_utils.get_all_benchmarks() all_fuzzers = fuzzer_utils.get_fuzzer_names() parser.add_argument('-b', '--benchmarks', help='Benchmark names. All of them by default.', nargs='+', required=False, default=all_benchmarks, choices=all_benchmarks) parser.add_argument('-c', '--experiment-config', help='Path to the experiment configuration yaml file.', required=True) parser.add_argument('-e', '--experiment-name', help='Experiment name.', required=True) fuzzers_group = parser.add_mutually_exclusive_group() fuzzers_group.add_argument('-f', '--fuzzers', help='Fuzzers to use.', nargs='+', required=False, default=None, choices=all_fuzzers) fuzzers_group.add_argument('-cf', '--changed-fuzzers', help=('Use fuzzers that have changed since the ' 'last experiment. The last experiment is ' 'determined by the database your ' 'experiment uses, not necessarily the ' 'fuzzbench service'), action='store_true', required=False) args = parser.parse_args() if args.changed_fuzzers: fuzzers = experiment_changes.get_fuzzers_changed_since_last() if not fuzzers: logs.error('No fuzzers changed since last experiment. Exiting.') return 1 else: fuzzers = args.fuzzers or all_fuzzers start_experiment(args.experiment_name, args.experiment_config, args.benchmarks, fuzzers) return 0
def get_changed_fuzzers_for_ci(changed_files: List[str] = None) -> List[str]: """Returns a list of fuzzers that have changed functionality based on the files that have changed in |changed_files|. Unlike get_changed_fuzzers this function considers changes that affect building or running fuzzers in CI.""" changed_files = get_absolute_paths(changed_files) if any(changed_file in CI_FILES for changed_file in changed_files): # If any of changed files are in CI_FILES. return fuzzer_utils.get_fuzzer_names() return get_changed_fuzzers(changed_files)
def get_files_dependent_fuzzers(dependency_files: List[str]) -> List[str]: """Returns a list of fuzzer names dependent on |dependency_files|.""" dependency_files = set(dependency_files) dependent_fuzzers = [] for fuzzer in fuzzer_utils.get_fuzzer_names(): fuzzer_dependencies = get_fuzzer_dependencies(fuzzer) if fuzzer_dependencies.intersection(dependency_files): dependent_fuzzers.append(fuzzer) return dependent_fuzzers
def main(): """Generates Makefile with docker image build rules.""" fuzzers = fuzzer_utils.get_fuzzer_names() benchmarks = benchmark_utils.get_all_benchmarks() buildable_images = docker_images.get_images_to_build(fuzzers, benchmarks) print('export DOCKER_BUILDKIT := 1') # Print oss-fuzz benchmarks property variables. _print_benchmark_fuzz_target(benchmarks) for name, image in buildable_images.items(): print_rules_for_image(name, image) # Print build targets for all fuzzer-benchmark pairs (including coverage). fuzzers.append('coverage') for fuzzer in fuzzers: image_type = "runner" if 'coverage' in fuzzer: image_type = "builder" for benchmark in benchmarks: print(('build-{fuzzer}-{benchmark}: ' + '.{fuzzer}-{benchmark}-{image_type}\n').format( fuzzer=fuzzer, benchmark=benchmark, image_type=image_type)) print() # Print fuzzer-all benchmarks build targets. for fuzzer in fuzzers: all_build_targets = ' '.join([ 'build-{0}-{1}'.format(fuzzer, benchmark) for benchmark in benchmarks ]) print('build-{fuzzer}-all: {all_targets}'.format( fuzzer=fuzzer, all_targets=all_build_targets)) all_test_run_targets = ' '.join([ 'test-run-{0}-{1}'.format(fuzzer, benchmark) for benchmark in benchmarks ]) print('test-run-{fuzzer}-all: {all_targets}'.format( fuzzer=fuzzer, all_targets=all_test_run_targets)) # Print all targets build target. all_build_targets = ' '.join( ['build-{0}-all'.format(name) for name in fuzzers]) print('build-all: {all_targets}'.format(all_targets=all_build_targets))
def generate_makefile(): """Generates the contents of the makefile and returns it.""" fuzzers = fuzzer_utils.get_fuzzer_names() benchmarks = benchmark_utils.get_all_benchmarks() buildable_images = docker_images.get_images_to_build(fuzzers, benchmarks) makefile = 'export DOCKER_BUILDKIT := 1\n\n' # Print oss-fuzz benchmarks property variables. makefile += _get_benchmark_fuzz_target(benchmarks) for name, image in buildable_images.items(): makefile += get_rules_for_image(name, image) # Print build targets for all fuzzer-benchmark pairs (including coverage). fuzzers.append('coverage') for fuzzer in fuzzers: image_type = 'runner' if 'coverage' in fuzzer: image_type = 'builder' for benchmark in benchmarks: makefile += (f'build-{fuzzer}-{benchmark}: ' + f'.{fuzzer}-{benchmark}-{image_type}\n') makefile += '\n' # Print fuzzer-all benchmarks build targets. for fuzzer in fuzzers: all_build_targets = ' '.join( [f'build-{fuzzer}-{benchmark}' for benchmark in benchmarks]) makefile += f'build-{fuzzer}-all: {all_build_targets}\n' all_test_run_targets = ' '.join( [f'test-run-{fuzzer}-{benchmark}' for benchmark in benchmarks]) makefile += f'test-run-{fuzzer}-all: {all_test_run_targets}\n' # Print all targets build target. all_build_targets = ' '.join([f'build-{fuzzer}-all' for fuzzer in fuzzers]) makefile += f'build-all: {all_build_targets}' return makefile
def main(): """Run an experiment in the cloud.""" logs.initialize() parser = argparse.ArgumentParser( description='Begin an experiment that evaluates fuzzers on one or ' 'more benchmarks.') all_benchmarks = benchmark_utils.get_all_benchmarks() coverage_benchmarks = benchmark_utils.get_coverage_benchmarks() parser.add_argument('-b', '--benchmarks', help=('Benchmark names. ' 'All code coverage benchmarks of them by ' 'default.'), nargs='+', required=False, default=coverage_benchmarks, choices=all_benchmarks) parser.add_argument('-c', '--experiment-config', help='Path to the experiment configuration yaml file.', required=True) parser.add_argument('-e', '--experiment-name', help='Experiment name.', required=True) parser.add_argument('-d', '--description', help='Description of the experiment.', required=False) parser.add_argument('-cb', '--concurrent-builds', help='Max concurrent builds allowed.', required=False) all_fuzzers = fuzzer_utils.get_fuzzer_names() parser.add_argument('-f', '--fuzzers', help='Fuzzers to use.', nargs='+', required=False, default=None, choices=all_fuzzers) parser.add_argument('-ns', '--no-seeds', help='Should trials be conducted without seed corpora.', required=False, default=False, action='store_true') parser.add_argument('-nd', '--no-dictionaries', help='Should trials be conducted without dictionaries.', required=False, default=False, action='store_true') parser.add_argument('-a', '--allow-uncommitted-changes', help='Skip check that no uncommited changes made.', required=False, default=False, action='store_true') parser.add_argument( '-o', '--oss-fuzz-corpus', help='Should trials be conducted with OSS-Fuzz corpus (if available).', required=False, default=False, action='store_true') args = parser.parse_args() fuzzers = args.fuzzers or all_fuzzers concurrent_builds = args.concurrent_builds if concurrent_builds is not None: if not concurrent_builds.isdigit(): parser.error( "The concurrent build argument must be a positive number") concurrent_builds = int(concurrent_builds) start_experiment(args.experiment_name, args.experiment_config, args.benchmarks, fuzzers, description=args.description, no_seeds=args.no_seeds, no_dictionaries=args.no_dictionaries, oss_fuzz_corpus=args.oss_fuzz_corpus, allow_uncommitted_changes=args.allow_uncommitted_changes, concurrent_builds=concurrent_builds) return 0
def test_get_changed_fuzzers_for_ci(): """Tests that get_changed_fuzzers_for_ci returns all fuzzers when a file that affects all fuzzer build was changed.""" changed_fuzzers = change_utils.get_changed_fuzzers_for_ci( [os.path.join(utils.ROOT_DIR, 'docker', 'build.mk')]) assert changed_fuzzers == fuzzer_utils.get_fuzzer_names()