def run_requested_experiment(dry_run):
"""Run the oldest requested experiment that hasn't been run yet in
experiment-requests.yaml."""
requested_experiments = _get_requested_experiments()
# TODO(metzman): Look into supporting benchmarks as an optional parameter so
# that people can add fuzzers that don't support everything.
if PAUSE_SERVICE_KEYWORD in requested_experiments:
# Check if automated experiment service is paused.
logs.warning('Pause service requested, not running experiment.')
return
requested_experiment = None
for experiment_config in reversed(requested_experiments):
experiment_name = _get_experiment_name(experiment_config)
with db_utils.session_scope() as session:
is_new_experiment = session.query(models.Experiment).filter(
models.Experiment.name == experiment_name).first() is None
if is_new_experiment:
requested_experiment = experiment_config
break
if requested_experiment is None:
logs.info('No new experiment to run. Exiting.')
return
experiment_name = _get_experiment_name(requested_experiment)
if not validate_experiment_requests([requested_experiment]):
logs.error('Requested experiment: %s in %s is not valid.',
requested_experiment, REQUESTED_EXPERIMENTS_PATH)
return
fuzzers = requested_experiment['fuzzers']
benchmark_type = requested_experiment.get('type')
if benchmark_type == benchmark_utils.BenchmarkType.BUG.value:
benchmarks = benchmark_utils.get_bug_benchmarks()
else:
benchmarks = benchmark_utils.get_coverage_benchmarks()
logs.info('Running experiment: %s with fuzzers: %s.', experiment_name,
' '.join(fuzzers))
description = _get_description(requested_experiment)
oss_fuzz_corpus = _use_oss_fuzz_corpus(requested_experiment)
_run_experiment(experiment_name, fuzzers, benchmarks, description,
oss_fuzz_corpus, dry_run)
def main():
"""Run an experiment in the cloud."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Begin an experiment that evaluates fuzzers on one or '
'more benchmarks.')
all_benchmarks = benchmark_utils.get_all_benchmarks()
coverage_benchmarks = benchmark_utils.get_coverage_benchmarks()
parser.add_argument('-b',
'--benchmarks',
help=('Benchmark names. '
'All code coverage benchmarks of them by '
'default.'),
nargs='+',
required=False,
default=coverage_benchmarks,
choices=all_benchmarks)
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
parser.add_argument('-d',
'--description',
help='Description of the experiment.',
required=False)
parser.add_argument('-cb',
'--concurrent-builds',
help='Max concurrent builds allowed.',
required=False)
all_fuzzers = fuzzer_utils.get_fuzzer_names()
parser.add_argument('-f',
'--fuzzers',
help='Fuzzers to use.',
nargs='+',
required=False,
default=None,
choices=all_fuzzers)
parser.add_argument('-ns',
'--no-seeds',
help='Should trials be conducted without seed corpora.',
required=False,
default=False,
action='store_true')
parser.add_argument('-nd',
'--no-dictionaries',
help='Should trials be conducted without dictionaries.',
required=False,
default=False,
action='store_true')
parser.add_argument('-a',
'--allow-uncommitted-changes',
help='Skip check that no uncommited changes made.',
required=False,
default=False,
action='store_true')
parser.add_argument(
'-o',
'--oss-fuzz-corpus',
help='Should trials be conducted with OSS-Fuzz corpus (if available).',
required=False,
default=False,
action='store_true')
args = parser.parse_args()
fuzzers = args.fuzzers or all_fuzzers
concurrent_builds = args.concurrent_builds
if concurrent_builds is not None:
if not concurrent_builds.isdigit():
parser.error(
"The concurrent build argument must be a positive number")
concurrent_builds = int(concurrent_builds)
start_experiment(args.experiment_name,
args.experiment_config,
args.benchmarks,
fuzzers,
description=args.description,
no_seeds=args.no_seeds,
no_dictionaries=args.no_dictionaries,
oss_fuzz_corpus=args.oss_fuzz_corpus,
allow_uncommitted_changes=args.allow_uncommitted_changes,
concurrent_builds=concurrent_builds)
return 0