def _validate_individual_experiment_requests(experiment_requests):
"""Returns True if all requests in |experiment_request| are valid in
isolation. Does not account for PAUSE_SERVICE_KEYWORD or duplicates."""
all_fuzzers = set(fuzzer_utils.get_fuzzer_names())
valid = True
# Validate format.
for request in experiment_requests:
if not isinstance(request, dict):
logger.error('Request: %s is not a dict.', request)
experiment_requests.remove(request)
valid = False
continue
if 'experiment' not in request:
logger.error('Request: %s does not have field "experiment".',
request)
valid = False
continue
experiment = _get_experiment_name(request)
if not validate_experiment_name(experiment):
valid = False
logger.error('Experiment name: %s is not valid.', experiment)
# Request isn't so malformed that we can finding other issues.
# if present. Don't continue.
experiment = request['experiment']
if not request.get('fuzzers'):
logger.error('Request: %s does not specify any fuzzers.', request)
valid = False
continue
experiment_fuzzers = request['fuzzers']
for fuzzer in experiment_fuzzers:
if fuzzer in all_fuzzers:
continue
# Fuzzer isn't valid.
logger.error('Fuzzer: %s in experiment %s is not valid.', fuzzer,
experiment)
valid = False
description = request.get('description')
if description is not None and not isinstance(description, str):
logger.error(
'Request: %s "description" attribute is not a valid string.',
request)
valid = False
continue
oss_fuzz_corpus = request.get('oss_fuzz_corpus')
if oss_fuzz_corpus is not None and not isinstance(
oss_fuzz_corpus, bool):
logger.error(
'Request: %s "oss_fuzz_corpus" attribute is not a valid bool.',
request)
valid = False
continue
return valid
def main():
"""Run an experiment in the cloud."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Begin an experiment that evaluates fuzzers on one or '
'more benchmarks.')
all_benchmarks = benchmark_utils.get_all_benchmarks()
all_fuzzers = fuzzer_utils.get_fuzzer_names()
parser.add_argument('-b',
'--benchmarks',
help='Benchmark names. All of them by default.',
nargs='+',
required=False,
default=all_benchmarks,
choices=all_benchmarks)
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
parser.add_argument('-f',
'--fuzzers',
help='Fuzzers to use.',
nargs='+',
required=False,
default=None,
choices=all_fuzzers)
parser.add_argument(
'-ns',
'--no-seeds',
help='Should trials be conducted without seed corpora.',
required=False,
default=False,
action='store_true')
parser.add_argument(
'-nd',
'--no-dictionaries',
help='Should trials be conducted without dictionaries.',
required=False,
default=False,
action='store_true')
args = parser.parse_args()
fuzzers = args.fuzzers or all_fuzzers
start_experiment(args.experiment_name,
args.experiment_config,
args.benchmarks,
fuzzers,
no_seeds=args.no_seeds,
no_dictionaries=args.no_dictionaries)
return 0
def main():
"""Run an experiment in the cloud."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Begin an experiment that evaluates fuzzers on one or '
'more benchmarks.')
all_benchmarks = benchmark_utils.get_all_benchmarks()
all_fuzzers = fuzzer_utils.get_fuzzer_names()
parser.add_argument('-b',
'--benchmarks',
help='Benchmark names. All of them by default.',
nargs='+',
required=False,
default=all_benchmarks,
choices=all_benchmarks)
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
fuzzers_group = parser.add_mutually_exclusive_group()
fuzzers_group.add_argument('-f',
'--fuzzers',
help='Fuzzers to use.',
nargs='+',
required=False,
default=None,
choices=all_fuzzers)
fuzzers_group.add_argument('-cf',
'--changed-fuzzers',
help=('Use fuzzers that have changed since the '
'last experiment. The last experiment is '
'determined by the database your '
'experiment uses, not necessarily the '
'fuzzbench service'),
action='store_true',
required=False)
args = parser.parse_args()
if args.changed_fuzzers:
fuzzers = experiment_changes.get_fuzzers_changed_since_last()
if not fuzzers:
logs.error('No fuzzers changed since last experiment. Exiting.')
return 1
else:
fuzzers = args.fuzzers or all_fuzzers
start_experiment(args.experiment_name, args.experiment_config,
args.benchmarks, fuzzers)
return 0
def get_changed_fuzzers_for_ci(changed_files: List[str] = None) -> List[str]:
"""Returns a list of fuzzers that have changed functionality based
on the files that have changed in |changed_files|.
Unlike get_changed_fuzzers this function considers changes that affect
building or running fuzzers in CI."""
changed_files = get_absolute_paths(changed_files)
if any(changed_file in CI_FILES for changed_file in changed_files):
# If any of changed files are in CI_FILES.
return fuzzer_utils.get_fuzzer_names()
return get_changed_fuzzers(changed_files)
def get_files_dependent_fuzzers(dependency_files: List[str]) -> List[str]:
"""Returns a list of fuzzer names dependent on |dependency_files|."""
dependency_files = set(dependency_files)
dependent_fuzzers = []
for fuzzer in fuzzer_utils.get_fuzzer_names():
fuzzer_dependencies = get_fuzzer_dependencies(fuzzer)
if fuzzer_dependencies.intersection(dependency_files):
dependent_fuzzers.append(fuzzer)
return dependent_fuzzers
def main():
"""Generates Makefile with docker image build rules."""
fuzzers = fuzzer_utils.get_fuzzer_names()
benchmarks = benchmark_utils.get_all_benchmarks()
buildable_images = docker_images.get_images_to_build(fuzzers, benchmarks)
print('export DOCKER_BUILDKIT := 1')
# Print oss-fuzz benchmarks property variables.
_print_benchmark_fuzz_target(benchmarks)
for name, image in buildable_images.items():
print_rules_for_image(name, image)
# Print build targets for all fuzzer-benchmark pairs (including coverage).
fuzzers.append('coverage')
for fuzzer in fuzzers:
image_type = "runner"
if 'coverage' in fuzzer:
image_type = "builder"
for benchmark in benchmarks:
print(('build-{fuzzer}-{benchmark}: ' +
'.{fuzzer}-{benchmark}-{image_type}\n').format(
fuzzer=fuzzer,
benchmark=benchmark,
image_type=image_type))
print()
# Print fuzzer-all benchmarks build targets.
for fuzzer in fuzzers:
all_build_targets = ' '.join([
'build-{0}-{1}'.format(fuzzer, benchmark)
for benchmark in benchmarks
])
print('build-{fuzzer}-all: {all_targets}'.format(
fuzzer=fuzzer, all_targets=all_build_targets))
all_test_run_targets = ' '.join([
'test-run-{0}-{1}'.format(fuzzer, benchmark)
for benchmark in benchmarks
])
print('test-run-{fuzzer}-all: {all_targets}'.format(
fuzzer=fuzzer, all_targets=all_test_run_targets))
# Print all targets build target.
all_build_targets = ' '.join(
['build-{0}-all'.format(name) for name in fuzzers])
print('build-all: {all_targets}'.format(all_targets=all_build_targets))
def generate_makefile():
"""Generates the contents of the makefile and returns it."""
fuzzers = fuzzer_utils.get_fuzzer_names()
benchmarks = benchmark_utils.get_all_benchmarks()
buildable_images = docker_images.get_images_to_build(fuzzers, benchmarks)
makefile = 'export DOCKER_BUILDKIT := 1\n\n'
# Print oss-fuzz benchmarks property variables.
makefile += _get_benchmark_fuzz_target(benchmarks)
for name, image in buildable_images.items():
makefile += get_rules_for_image(name, image)
# Print build targets for all fuzzer-benchmark pairs (including coverage).
fuzzers.append('coverage')
for fuzzer in fuzzers:
image_type = 'runner'
if 'coverage' in fuzzer:
image_type = 'builder'
for benchmark in benchmarks:
makefile += (f'build-{fuzzer}-{benchmark}: ' +
f'.{fuzzer}-{benchmark}-{image_type}\n')
makefile += '\n'
# Print fuzzer-all benchmarks build targets.
for fuzzer in fuzzers:
all_build_targets = ' '.join(
[f'build-{fuzzer}-{benchmark}' for benchmark in benchmarks])
makefile += f'build-{fuzzer}-all: {all_build_targets}\n'
all_test_run_targets = ' '.join(
[f'test-run-{fuzzer}-{benchmark}' for benchmark in benchmarks])
makefile += f'test-run-{fuzzer}-all: {all_test_run_targets}\n'
# Print all targets build target.
all_build_targets = ' '.join([f'build-{fuzzer}-all' for fuzzer in fuzzers])
makefile += f'build-all: {all_build_targets}'
return makefile
def main():
"""Run an experiment in the cloud."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Begin an experiment that evaluates fuzzers on one or '
'more benchmarks.')
all_benchmarks = benchmark_utils.get_all_benchmarks()
coverage_benchmarks = benchmark_utils.get_coverage_benchmarks()
parser.add_argument('-b',
'--benchmarks',
help=('Benchmark names. '
'All code coverage benchmarks of them by '
'default.'),
nargs='+',
required=False,
default=coverage_benchmarks,
choices=all_benchmarks)
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
parser.add_argument('-d',
'--description',
help='Description of the experiment.',
required=False)
parser.add_argument('-cb',
'--concurrent-builds',
help='Max concurrent builds allowed.',
required=False)
all_fuzzers = fuzzer_utils.get_fuzzer_names()
parser.add_argument('-f',
'--fuzzers',
help='Fuzzers to use.',
nargs='+',
required=False,
default=None,
choices=all_fuzzers)
parser.add_argument('-ns',
'--no-seeds',
help='Should trials be conducted without seed corpora.',
required=False,
default=False,
action='store_true')
parser.add_argument('-nd',
'--no-dictionaries',
help='Should trials be conducted without dictionaries.',
required=False,
default=False,
action='store_true')
parser.add_argument('-a',
'--allow-uncommitted-changes',
help='Skip check that no uncommited changes made.',
required=False,
default=False,
action='store_true')
parser.add_argument(
'-o',
'--oss-fuzz-corpus',
help='Should trials be conducted with OSS-Fuzz corpus (if available).',
required=False,
default=False,
action='store_true')
args = parser.parse_args()
fuzzers = args.fuzzers or all_fuzzers
concurrent_builds = args.concurrent_builds
if concurrent_builds is not None:
if not concurrent_builds.isdigit():
parser.error(
"The concurrent build argument must be a positive number")
concurrent_builds = int(concurrent_builds)
start_experiment(args.experiment_name,
args.experiment_config,
args.benchmarks,
fuzzers,
description=args.description,
no_seeds=args.no_seeds,
no_dictionaries=args.no_dictionaries,
oss_fuzz_corpus=args.oss_fuzz_corpus,
allow_uncommitted_changes=args.allow_uncommitted_changes,
concurrent_builds=concurrent_builds)
return 0
def test_get_changed_fuzzers_for_ci():
"""Tests that get_changed_fuzzers_for_ci returns all fuzzers when a file
that affects all fuzzer build was changed."""
changed_fuzzers = change_utils.get_changed_fuzzers_for_ci(
[os.path.join(utils.ROOT_DIR, 'docker', 'build.mk')])
assert changed_fuzzers == fuzzer_utils.get_fuzzer_names()
