def start_experiment( # pylint: disable=too-many-arguments
experiment_name: str,
config_filename: str,
benchmarks: List[str],
fuzzers: List[str],
no_seeds=False,
no_dictionaries=False,
oss_fuzz_corpus=False):
"""Start a fuzzer benchmarking experiment."""
check_no_local_changes()
validate_experiment_name(experiment_name)
validate_benchmarks(benchmarks)
config = read_and_validate_experiment_config(config_filename)
config['fuzzers'] = fuzzers
config['benchmarks'] = benchmarks
config['experiment'] = experiment_name
config['git_hash'] = get_git_hash()
config['no_seeds'] = no_seeds
config['no_dictionaries'] = no_dictionaries
config['oss_fuzz_corpus'] = oss_fuzz_corpus
set_up_experiment_config_file(config)
# Make sure we can connect to database.
local_experiment = config.get('local_experiment', False)
if not local_experiment:
if 'POSTGRES_PASSWORD' not in os.environ:
raise Exception('Must set POSTGRES_PASSWORD environment variable.')
gcloud.set_default_project(config['cloud_project'])
start_dispatcher(config, CONFIG_DIR)
def start_experiment(experiment_name: str, config_filename: str,
benchmarks: List[str], fuzzers: List[str],
fuzzer_configs: List[str]):
"""Start a fuzzer benchmarking experiment."""
check_no_local_changes()
validate_experiment_name(experiment_name)
validate_benchmarks(benchmarks)
config = read_and_validate_experiment_config(config_filename)
config['benchmarks'] = ','.join(benchmarks)
config['experiment'] = experiment_name
config['git_hash'] = get_git_hash()
set_up_experiment_config_file(config)
set_up_fuzzer_config_files(fuzzers, fuzzer_configs)
# Make sure we can connect to database.
local_experiment = config.get('local_experiment', False)
if not local_experiment:
if 'POSTGRES_PASSWORD' not in os.environ:
raise Exception('Must set POSTGRES_PASSWORD environment variable.')
gcloud.set_default_project(config['cloud_project'])
start_dispatcher(config, CONFIG_DIR)
def start_experiment_from_full_config(config):
"""Start a fuzzer benchmarking experiment from a full (internal) config."""
set_up_experiment_config_file(config)
# Make sure we can connect to database.
local_experiment = config.get('local_experiment', False)
if not local_experiment:
if 'POSTGRES_PASSWORD' not in os.environ:
raise Exception('Must set POSTGRES_PASSWORD environment variable.')
gcloud.set_default_project(config['cloud_project'])
start_dispatcher(config, experiment_utils.CONFIG_DIR)
def start_experiment(experiment_name: str, config_filename: str,
benchmarks: List[str], fuzzers: List[str],
fuzzer_configs: List[str]):
"""Start a fuzzer benchmarking experiment."""
validate_benchmarks(benchmarks)
config = read_and_validate_experiment_config(config_filename)
config['benchmarks'] = ','.join(benchmarks)
validate_experiment_name(experiment_name)
config['experiment'] = experiment_name
config_dir = 'config'
filesystem.recreate_directory(config_dir)
experiment_config_filename = os.path.join(config_dir, 'experiment.yaml')
with open(experiment_config_filename, 'w') as experiment_config_file:
yaml.dump(config, experiment_config_file, default_flow_style=False)
if not fuzzers and not fuzzer_configs:
raise Exception('Need to provide either a list of fuzzers or '
'a list of fuzzer configs.')
fuzzer_config_dir = os.path.join(config_dir, 'fuzzer-configs')
filesystem.recreate_directory(fuzzer_config_dir)
for fuzzer_config in fuzzer_configs:
if fuzzer_configs.count(fuzzer_config) > 1:
raise Exception('Fuzzer config "%s" provided more than once.' %
fuzzer_config)
# Validate the fuzzer yaml attributes e.g. fuzzer, env, etc.
validate_fuzzer_config(fuzzer_config)
shutil.copy(fuzzer_config, fuzzer_config_dir)
for fuzzer in fuzzers:
if fuzzers.count(fuzzer) > 1:
raise Exception('Fuzzer "%s" provided more than once.' % fuzzer)
validate_fuzzer(fuzzer)
fuzzer_config_file_path = os.path.join(fuzzer_config_dir, fuzzer)
# Create a simple yaml with just the fuzzer attribute.
with open(fuzzer_config_file_path, 'w') as file_handle:
file_handle.write('fuzzer: ' + fuzzer)
# Make sure we can connect to database.
if 'POSTGRES_PASSWORD' not in os.environ:
raise Exception('Must set POSTGRES_PASSWORD environment variable.')
gcloud.set_default_project(config['cloud_project'])
dispatcher = Dispatcher(config)
if not os.getenv('MANUAL_EXPERIMENT'):
dispatcher.create_async()
copy_resources_to_bucket(config_dir, config)
if not os.getenv('MANUAL_EXPERIMENT'):
dispatcher.start()
def initialize_global_state(project: str):
"""Set up any global state needed by the dispatcher, including
authenticating to docker and gcloud and setting the project for gcloud."""
# TODO(metzman): Move global state set up into startup script.
gcloud.set_default_project(project)