def stop_experiment(experiment_name, experiment_config_filename):
"""Stop the experiment specified by |experiment_config_filename|."""
experiment_config = yaml_utils.read(experiment_config_filename)
if experiment_config.get('local_experiment', False):
raise NotImplementedError(
'Local experiment stop logic is not implemented.')
cloud_project = experiment_config['cloud_project']
cloud_compute_zone = experiment_config['cloud_compute_zone']
gce.initialize()
instances = list(gce.get_instances(cloud_project, cloud_compute_zone))
experiment_instances = []
dispatcher_instance = experiment_utils.get_dispatcher_instance_name(
experiment_name)
if dispatcher_instance not in instances:
logger.warning('Dispatcher instance not running, skip.')
else:
experiment_instances.append(dispatcher_instance)
trial_prefix = 'r-' + experiment_name
experiment_instances.extend([
instance for instance in instances if instance.startswith(trial_prefix)
])
if not experiment_instances:
logger.warning('No experiment instances found, no work to do.')
return True
if not gcloud.delete_instances(experiment_instances, cloud_compute_zone):
logger.error('Failed to stop experiment instances.')
return False
logger.info('Successfully stopped experiment.')
return True
def stop_experiment(experiment_name, experiment_config_filename):
"""Stop the experiment specified by |experiment_config_filename|."""
instances = gcloud.list_instances()
experiment_config = yaml_utils.read(experiment_config_filename)
cloud_compute_zone = experiment_config['cloud_compute_zone']
trial_prefix = 'r-' + experiment_name
experiment_instances = [
instance for instance in instances if instance.startswith(trial_prefix)
]
dispatcher_instance = experiment_utils.get_dispatcher_instance_name(
experiment_name)
if dispatcher_instance not in instances:
logger.warning('Dispatcher instance not running, skip.')
else:
experiment_instances.append(dispatcher_instance)
if not experiment_instances:
logger.warning('No experiment instances found, no work to do.')
return 0
if not gcloud.delete_instances(experiment_instances, cloud_compute_zone):
logger.error('Failed to stop experiment instances.')
return 1
logger.info('Successfully stopped experiment.')
return 0
def read_and_validate_experiment_config(config_filename: str) -> Dict:
"""Reads |config_filename|, validates it, finds as many errors as possible,
and returns it."""
config = yaml_utils.read(config_filename)
filestore_params = {'experiment_filestore', 'report_filestore'}
cloud_config = {'cloud_compute_zone'}
string_params = cloud_config.union(filestore_params)
int_params = {'trials', 'max_total_time'}
required_params = int_params.union(filestore_params)
local_experiment = config.get('local_experiment', False)
if not local_experiment:
required_params = required_params.union(cloud_config)
valid = True
if 'cloud_experiment_bucket' in config or 'cloud_web_bucket' in config:
logs.error('"cloud_experiment_bucket" and "cloud_web_bucket" are now '
'"experiment_filestore" and "report_filestore".')
for param in required_params:
if param not in config:
valid = False
logs.error('Config does not contain "%s".', param)
continue
value = config[param]
if param in int_params and not isinstance(value, int):
valid = False
logs.error('Config parameter "%s" is "%s". It must be an int.',
param, value)
continue
if param in string_params and (not isinstance(value, str) or
value != value.lower()):
valid = False
logs.error(
'Config parameter "%s" is "%s". It must be a lowercase string.',
param, str(value))
continue
if param not in filestore_params:
continue
if local_experiment and not value.startswith('/'):
valid = False
logs.error(
'Config parameter "%s" is "%s". Local experiments only support '
'using Posix file systems as filestores.', param, value)
continue
if not local_experiment and not value.startswith('gs://'):
valid = False
logs.error(
'Config parameter "%s" is "%s". '
'It must start with gs:// when running on Google Cloud.', param,
value)
if not valid:
raise ValidationError('Config: %s is invalid.' % config_filename)
return config
def main():
"""Reproduce a specified experiment."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Reproduce an experiment from a full config file.')
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
parser.add_argument('-d',
'--description',
help='Description of the experiment.',
required=False)
args = parser.parse_args()
config = yaml_utils.read(args.experiment_config)
run_experiment.validate_experiment_name(args.experiment_name)
if args.experiment_name == config['experiment']:
raise Exception('Must use a different experiment name.')
config['experiment'] = args.experiment_name
config['description'] = args.description
validate_config(config)
run_experiment.start_experiment_from_full_config(config)
return 0
def _print_benchmark_fuzz_target(benchmarks):
"""Prints benchmark variables from benchmark.yaml files."""
for benchmark in benchmarks:
benchmark_vars = yaml_utils.read(
os.path.join(BENCHMARK_DIR, benchmark, 'benchmark.yaml'))
print(benchmark + '-fuzz-target=' + benchmark_vars['fuzz_target'])
print()
def get_by_variant_name(fuzzer_variant_name):
"""Get a fuzzer config based on a fuzzer's display name."""
config_directory = get_fuzzer_configs_dir()
for config_filename in os.listdir(config_directory):
config_absolute_filename = config_directory / config_filename
if fuzzer_variant_name == get_fuzzer_name(config_absolute_filename):
return yaml_utils.read(config_absolute_filename)
return None
def __init__(self, experiment_config_filepath: str):
self.config = yaml_utils.read(experiment_config_filepath)
self.benchmarks = self.config['benchmarks']
self.fuzzers = self.config['fuzzers']
self.num_trials = self.config['trials']
self.experiment_name = self.config['experiment']
self.git_hash = self.config['git_hash']
self.preemptible = self.config.get('preemptible_runners')
def main():
"""Write base-images build spec when run from command line."""
image_templates = yaml_utils.read(
os.path.join(ROOT_DIR, 'docker', 'image_types.yaml'))
base_images_spec = create_cloudbuild_spec(
{'base-image': image_templates['base-image']}, build_base_images=True)
base_images_spec_file = os.path.join(ROOT_DIR, 'docker', 'gcb',
'base-images.yaml')
yaml_utils.write(base_images_spec_file, base_images_spec)
def is_fuzzer_tested_in_ci(fuzzer: str) -> bool:
"""Returns True if |fuzzer| is in the list of fuzzers tested in
fuzzers.yml."""
yaml_filepath = _SRC_ROOT / '.github' / 'workflows' / 'fuzzers.yml'
yaml_contents = yaml_utils.read(yaml_filepath)
fuzzer_list = yaml_contents['jobs']['build']['strategy']['matrix']['fuzzer']
is_tested = fuzzer in fuzzer_list
if not is_tested:
print(f'{fuzzer} is not included in fuzzer list in {yaml_filepath}.')
return is_tested
def _get_benchmark_fuzz_target(benchmarks):
"""Returns benchmark variables from benchmark.yaml files."""
variables = ''
for benchmark in benchmarks:
benchmark_vars = yaml_utils.read(
os.path.join(BENCHMARK_DIR, benchmark, 'benchmark.yaml'))
variables += (benchmark + '-fuzz-target=' +
benchmark_vars['fuzz_target'] + '\n')
variables += '\n'
return variables
def get_fuzzer_name(fuzzer_config_filename: str) -> str:
"""Get the fuzzer specified in fuzzer_config_filename"""
fuzzer_config = yaml_utils.read(get_fuzzer_configs_dir() /
fuzzer_config_filename)
# Multiple configurations of the same fuzzer are differentiated by their
# assigned display names, but we default to the in the simple case.
if 'variant_name' in fuzzer_config:
return fuzzer_config['variant_name']
return fuzzer_config['fuzzer']
def main():
"""Set up Redis connection and start the experiment."""
redis_connection = redis.Redis(host="queue-server")
config_path = environment.get('EXPERIMENT_CONFIG',
'fuzzbench/local-experiment-config.yaml')
config = yaml_utils.read(config_path)
config = config_utils.validate_and_expand(config)
with rq.Connection(redis_connection):
return run_experiment(config)
def _setup_experiment_files(fs):
"""Set up experiment config and core-fuzzers files and return experiment
config yaml."""
fs.add_real_file(reporter.CORE_FUZZERS_YAML)
config_filepath = os.path.join(os.path.dirname(__file__), 'test_data',
'experiment-config.yaml')
fs.add_real_file(config_filepath)
experiment_config = yaml_utils.read(config_filepath)
return experiment_config
def main():
"""Run an experiment in the cloud."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Begin an experiment that evaluates fuzzers on one or '
'more benchmarks.')
all_benchmarks = benchmark_utils.get_all_benchmarks()
parser.add_argument('-b',
'--benchmarks',
help='Benchmark names. All of them by default.',
nargs='+',
required=False,
default=all_benchmarks,
choices=all_benchmarks)
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
parser.add_argument('-f',
'--fuzzers',
help='Fuzzers to use.',
nargs='+',
required=False,
default=[])
parser.add_argument('-fc',
'--fuzzer-configs',
help='Fuzzer configurations to use.',
nargs='+',
required=False,
default=[])
args = parser.parse_args()
if not args.fuzzer_configs:
fuzzer_configs = fuzzer_utils.get_fuzzer_configs(fuzzers=args.fuzzers)
else:
fuzzer_configs = [
yaml_utils.read(fuzzer_config)
for fuzzer_config in args.fuzzer_configs
]
start_experiment(args.experiment_name, args.experiment_config,
args.benchmarks, fuzzer_configs)
if not os.getenv('MANUAL_EXPERIMENT'):
stop_experiment.stop_experiment(args.experiment_name,
args.experiment_config)
return 0
def main():
"""Main function for running scheduler independently."""
logs.initialize(default_extras={'component': 'dispatcher'})
if len(sys.argv) != 2:
print('Usage: {} <experiment_config.yaml>'.format(sys.argv[0]))
return 1
experiment_config = yaml_utils.read(sys.argv[1])
schedule_loop(experiment_config)
return 0
def __init__(self, experiment_config_filepath: str):
self.config = yaml_utils.read(experiment_config_filepath)
benchmarks = self.config['benchmarks'].split(',')
self.benchmarks = builder.build_all_measurers(benchmarks)
self.fuzzers = [
fuzzer_config_utils.get_fuzzer_name(filename) for filename in
os.listdir(fuzzer_config_utils.get_fuzzer_configs_dir())
]
_initialize_experiment_in_db(self.config['experiment'], self.benchmarks,
self.fuzzers, self.config['trials'])
self.web_bucket = posixpath.join(self.config['cloud_web_bucket'],
experiment_utils.get_experiment_name())
def __init__(self, experiment_config_filepath: str):
self.config = yaml_utils.read(experiment_config_filepath)
self.benchmarks = self.config['benchmarks'].split(',')
self.fuzzers = [
fuzzer_config_utils.get_fuzzer_name(filename) for filename in
os.listdir(fuzzer_config_utils.get_fuzzer_configs_dir())
]
self.num_trials = self.config['trials']
self.experiment_name = self.config['experiment']
self.git_hash = self.config['git_hash']
self.web_bucket = posixpath.join(
self.config['cloud_web_bucket'],
experiment_utils.get_experiment_name())
def main():
"""Run schedule_measure_workers as a standalone script by calling schedule
in a loop. Useful for debugging."""
logs.initialize(
default_extras={
'experiment': os.environ['EXPERIMENT'],
'component': 'dispatcher',
'subcomponent': 'scheduler'
})
gce.initialize()
config_path = sys.argv[1]
config = yaml_utils.read(config_path)
queue = initialize(config)
while True:
schedule(config, queue)
time.sleep(30)
def read_and_validate_experiment_config(config_filename: str) -> Dict:
"""Reads |config_filename|, validates it, and returns it."""
# TODO(metzman) Consider exceptioning early instead of logging error. It
# will be less useful for users but will simplify this code quite a bit. And
# it isn't like anything expensive happens before this validation is done so
# rerunning it is cheap.
config = yaml_utils.read(config_filename)
bucket_params = {'cloud_experiment_bucket', 'cloud_web_bucket'}
string_params = {
'cloud_compute_zone', 'cloud_experiment_bucket', 'cloud_web_bucket'
}
int_params = {'trials', 'max_total_time'}
required_params = int_params.union(string_params)
valid = True
for param in required_params:
if param not in config:
valid = False
logs.error('Config does not contain "%s".', param)
continue
value = config[param]
if param in int_params and not isinstance(value, int):
valid = False
logs.error('Config parameter "%s" is "%s". It must be an int.',
param, value)
continue
if param in string_params and (not isinstance(value, str) or
value != value.lower()):
valid = False
logs.error(
'Config parameter "%s" is "%s". It must be a lowercase string.',
param, str(value))
continue
if param in bucket_params and not value.startswith('gs://'):
valid = False
logs.error(
'Config parameter "%s" is "%s". It must start with gs://.',
param, value)
if not valid:
raise ValidationError('Config: %s is invalid.' % config_filename)
return config
def _add_build_arguments_to_config(base: str, fuzzer: str) -> str:
"""If there are fuzzer-specific arguments, make a config file with them."""
fuzzer_config = fuzzer_config_utils.get_by_variant_name(fuzzer)
if 'build_arguments' not in fuzzer_config:
return base
# TODO(mbarbella): Rather than rewrite yaml files, use the GCB API.
args = fuzzer_config['build_arguments']
config = yaml_utils.read(base)
for step in config['steps']:
if 'id' in step and step['id'] in BUILDER_STEP_IDS:
# Append additional flags before the final argument.
step['args'] = step['args'][:-1] + args + [step['args'][-1]]
new_config_path = os.path.join(CONFIG_DIR, 'builds', fuzzer + '.yaml')
filesystem.create_directory(os.path.dirname(new_config_path))
yaml_utils.write(new_config_path, config)
return new_config_path
def output_report(experiment_config: dict,
in_progress=False,
coverage_report=False):
"""Generate the HTML report and write it to |web_bucket|."""
experiment_name = experiment_utils.get_experiment_name()
web_filestore_path = posixpath.join(experiment_config['report_filestore'],
experiment_name)
reports_dir = get_reports_dir()
core_fuzzers = yaml_utils.read(CORE_FUZZERS_YAML)['fuzzers']
fuzzers = sorted(set(experiment_config['fuzzers']).union(set(core_fuzzers)))
# Don't merge with nonprivate experiments until the very end as doing it
# while the experiment is in progress will produce unusable realtime
# results.
merge_with_nonprivate = (not in_progress and experiment_config.get(
'merge_with_nonprivate', False))
try:
logger.debug('Generating report.')
filesystem.recreate_directory(reports_dir)
generate_report.generate_report(
[experiment_name],
str(reports_dir),
report_name=experiment_name,
fuzzers=fuzzers,
in_progress=in_progress,
merge_with_clobber_nonprivate=merge_with_nonprivate,
coverage_report=coverage_report)
filestore_utils.rsync(
str(reports_dir),
web_filestore_path,
delete=False, # Don't remove existing coverage jsons.
gsutil_options=[
'-h', 'Cache-Control:public,max-age=0,no-transform'
])
logger.debug('Done generating report.')
except data_utils.EmptyDataError:
logs.warning('No snapshot data.')
except Exception: # pylint: disable=broad-except
logger.error('Error generating HTML report.')
def get_fuzzer_configs(fuzzers=None):
"""Returns the list of all fuzzer and variant configurations."""
# Import it here to avoid yaml dependency in runner.
# pylint: disable=import-outside-toplevel
from common import yaml_utils
fuzzers_dir = os.path.join(utils.ROOT_DIR, 'fuzzers')
fuzzer_configs = []
names = set()
for fuzzer in os.listdir(fuzzers_dir):
if not os.path.isfile(os.path.join(fuzzers_dir, fuzzer, 'fuzzer.py')):
continue
if fuzzer == 'coverage':
continue
if not fuzzers or fuzzer in fuzzers:
# Auto-generate the default configuration for each underlying
# fuzzer.
fuzzer_configs.append({'fuzzer': fuzzer})
variant_config_path = os.path.join(fuzzers_dir, fuzzer,
'variants.yaml')
if not os.path.isfile(variant_config_path):
continue
variant_config = yaml_utils.read(variant_config_path)
assert 'variants' in variant_config, (
'Missing "variants" section of {}'.format(variant_config_path))
for variant in variant_config['variants']:
if not fuzzers or variant['name'] in fuzzers:
assert 'name' in variant, (
'Missing name attribute for fuzzer variant in {}'.format(
variant_config_path))
variant['fuzzer'] = fuzzer
fuzzer_configs.append(variant)
name = variant['name'] if 'name' in variant else variant['fuzzer']
assert name not in names, (
'Multiple fuzzers/variants have the same name: ' + name)
names.add(name)
return fuzzer_configs
def validate_fuzzer_config(fuzzer_config_name: str):
"""Validate |fuzzer_config_name|."""
allowed_fields = ['variant_name', 'env', 'fuzzer']
fuzzer_config = yaml_utils.read(fuzzer_config_name)
if 'fuzzer' not in fuzzer_config:
raise Exception('Fuzzer configuration must include the "fuzzer" field.')
for key in fuzzer_config:
if key not in allowed_fields:
raise Exception('Invalid entry "%s" in fuzzer configuration "%s"' %
(key, fuzzer_config_name))
variant_name = fuzzer_config.get('variant_name')
if variant_name:
if not re.match(FUZZER_NAME_REGEX, variant_name):
raise Exception(
'The "variant_name" option may only contain lowercase letters, '
'numbers, or underscores.')
fuzzer_name = fuzzer_config.get('fuzzer')
if fuzzer_name:
validate_fuzzer(fuzzer_name)
def validate_experiment_requests(paths: List[Path]):
"""Returns False if service/experiment-requests.yaml it is in |paths| and is
not valid."""
if Path(automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH) not in paths:
return True
try:
experiment_requests = yaml_utils.read(
automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH)
except yaml.parser.ParserError:
print('Error parsing %s.' %
automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH)
return False
result = automatic_run_experiment.validate_experiment_requests(
experiment_requests)
if not result:
print('%s is not valid.' %
automatic_run_experiment.REQUESTED_EXPERIMENTS_PATH)
return result
def get_fuzzer_configs(fuzzers=None):
"""Returns the list of all fuzzers."""
# Import it here to avoid yaml dependency in runner.
# pylint: disable=import-outside-toplevel
from common import yaml_utils
fuzzers_dir = os.path.join(utils.ROOT_DIR, 'fuzzers')
fuzzer_configs = []
for fuzzer in os.listdir(fuzzers_dir):
if not os.path.isfile(os.path.join(fuzzers_dir, fuzzer, 'fuzzer.py')):
continue
if fuzzer == 'coverage':
continue
if not fuzzers or fuzzer in fuzzers:
# Auto-generate the default configuration for each base fuzzer.
fuzzer_configs.append({'fuzzer': fuzzer})
variant_config_path = os.path.join(fuzzers_dir, fuzzer, 'variants.yaml')
if not os.path.isfile(variant_config_path):
continue
variant_config = yaml_utils.read(variant_config_path)
assert 'variants' in variant_config, (
'Missing "variants" section of {}'.format(variant_config_path))
for variant in variant_config['variants']:
if not fuzzers or variant['name'] in fuzzers:
# Modify the config from the variants.yaml format to the
# format expected by a fuzzer config.
assert 'name' in variant, (
'Missing name attribute for fuzzer variant in {}'.format(
variant_config_path))
variant['variant_name'] = variant['name']
del variant['name']
variant['fuzzer'] = fuzzer
fuzzer_configs.append(variant)
return fuzzer_configs
def get_core_fuzzers():
"""Return list of core fuzzers to be used for merging experiment data."""
return yaml_utils.read(CORE_FUZZERS_YAML)['fuzzers']
def _get_image_type_templates():
"""Loads the image types config that contains "templates" describing how to
build them and their dependencies."""
return yaml_utils.read('docker/image_types.yaml')
def _get_requested_experiments():
"""Return requested experiments."""
return yaml_utils.read(REQUESTED_EXPERIMENTS_PATH)
def main():
"""Run an experiment in the cloud."""
logs.initialize()
parser = argparse.ArgumentParser(
description='Begin an experiment that evaluates fuzzers on one or '
'more benchmarks.')
all_benchmarks = benchmark_utils.get_all_benchmarks()
all_fuzzers = fuzzer_utils.get_fuzzer_names()
parser.add_argument('-b',
'--benchmarks',
help='Benchmark names. All of them by default.',
nargs='+',
required=False,
default=all_benchmarks,
choices=all_benchmarks)
parser.add_argument('-c',
'--experiment-config',
help='Path to the experiment configuration yaml file.',
required=True)
parser.add_argument('-e',
'--experiment-name',
help='Experiment name.',
required=True)
fuzzers_group = parser.add_mutually_exclusive_group()
fuzzers_group.add_argument('-f',
'--fuzzers',
help='Fuzzers to use.',
nargs='+',
required=False,
default=None,
choices=all_fuzzers)
fuzzers_group.add_argument('-fc',
'--fuzzer-configs',
help='Fuzzer configurations to use.',
nargs='+',
required=False,
default=[])
fuzzers_group.add_argument('-cf',
'--changed-fuzzers',
help=('Use fuzzers that have changed since the '
'last experiment. The last experiment is '
'determined by the database your '
'experiment uses, not necessarily the '
'fuzzbench service'),
action='store_true',
required=False)
args = parser.parse_args()
if args.fuzzer_configs:
fuzzer_configs = [
yaml_utils.read(fuzzer_config)
for fuzzer_config in args.fuzzer_configs
]
else:
if args.changed_fuzzers:
fuzzers = experiment_changes.get_fuzzers_changed_since_last()
if not fuzzers:
logs.error('No fuzzers changed since last experiment. Exiting.')
return 1
else:
fuzzers = args.fuzzers
fuzzer_configs = fuzzer_utils.get_fuzzer_configs(fuzzers)
start_experiment(args.experiment_name, args.experiment_config,
args.benchmarks, fuzzer_configs)
if not os.getenv('MANUAL_EXPERIMENT'):
stop_experiment.stop_experiment(args.experiment_name,
args.experiment_config)
return 0
def experiment_config():
"""Returns the default configuration for end-to-end testing."""
return config_utils.validate_and_expand(
yaml_utils.read('fuzzbench/test_e2e/end-to-end-test-config.yaml'))
