def initialize_measurement_dirs(self):
"""Initialize directories that will be needed for measuring
coverage."""
for directory in [self.corpus_dir, self.sancov_dir, self.crashes_dir]:
filesystem.recreate_directory(directory)
for directory in [self.report_dir, self.prev_corpus_dir]:
pathlib.Path(directory).mkdir(exist_ok=True)
def output_report(experiment_config: dict, in_progress=False):
"""Generate the HTML report and write it to |web_bucket|."""
experiment_name = experiment_utils.get_experiment_name()
web_filestore_path = posixpath.join(experiment_config['report_filestore'],
experiment_name)
reports_dir = get_reports_dir()
# Don't merge with nonprivate experiments until the very end as doing it
# while the experiment is in progress will produce unusable realtime
# results.
merge_with_nonprivate = (not in_progress and experiment_config.get(
'merge_with_nonprivate', False))
try:
logger.debug('Generating report.')
filesystem.recreate_directory(reports_dir)
generate_report.generate_report(
[experiment_name],
str(reports_dir),
in_progress=in_progress,
merge_with_clobber_nonprivate=merge_with_nonprivate)
filestore_utils.rsync(str(reports_dir),
web_filestore_path,
gsutil_options=[
'-h',
'Cache-Control:public,max-age=0,no-transform'
])
logger.debug('Done generating report.')
except data_utils.EmptyDataError:
logs.warning('No snapshot data.')
except Exception: # pylint: disable=broad-except
logger.error('Error generating HTML report.')
def set_up_experiment_config_file(config):
"""Set up the config file that will actually be used in the
experiment (not the one given to run_experiment.py)."""
filesystem.recreate_directory(CONFIG_DIR)
experiment_config_filename = os.path.join(CONFIG_DIR, 'experiment.yaml')
with open(experiment_config_filename, 'w') as experiment_config_file:
yaml.dump(config, experiment_config_file, default_flow_style=False)
def initialize_measurement_dirs(self):
"""Initialize directories that will be needed for measuring
coverage."""
for directory in [
self.corpus_dir, self.coverage_dir, self.crashes_dir
]:
filesystem.recreate_directory(directory)
filesystem.create_directory(self.report_dir)
def set_up_experiment_config_file(config):
"""Set up the config file that will actually be used in the
experiment (not the one given to run_experiment.py)."""
filesystem.recreate_directory(experiment_utils.CONFIG_DIR)
experiment_config_filename = (
experiment_utils.get_internal_experiment_config_relative_path())
with open(experiment_config_filename, 'w') as experiment_config_file:
yaml.dump(config, experiment_config_file, default_flow_style=False)
def initialize_directories(self):
"""Initialize directories needed for the trial."""
directories = [
self.corpus_dir,
self.corpus_archives_dir,
self.results_dir,
]
for directory in directories:
filesystem.recreate_directory(directory)
def build_all_measurers(benchmarks: List[str]) -> List[str]:
"""Build measurers for benchmarks."""
logger.info('Building measurers.')
filesystem.recreate_directory(build_utils.get_coverage_binaries_dir())
benchmarks = [(benchmark, ) for benchmark in benchmarks]
results = retry_build_loop(build_measurer, benchmarks)
logger.info('Done building measurers.')
# Return list of benchmarks (like the list we were passed as an argument)
# instead of returning a list of tuples each containing a benchmark.
return [result[0] for result in results]
def build_all_measurers(benchmarks: List[str]) -> List[str]:
"""Build measurers for each benchmark in |benchmarks| in parallel
Returns a list of benchmarks built successfully."""
logger.info('Building measurers.')
filesystem.recreate_directory(build_utils.get_coverage_binaries_dir())
build_measurer_args = [(benchmark, ) for benchmark in benchmarks]
successful_calls = retry_build_loop(build_measurer, build_measurer_args)
logger.info('Done building measurers.')
# Return list of benchmarks (like the list we were passed as an argument)
# instead of returning a list of tuples each containing a benchmark.
return [successful_call[0] for successful_call in successful_calls]
def test_recreate_directory_existing(fs):
"""Tests that recreate_directory recreates a directory that already
exists."""
new_directory = 'new-directory'
os.mkdir(new_directory)
new_file = os.path.join(new_directory, 'file')
with open(new_file, 'w') as file_handle:
file_handle.write('hi')
filesystem.recreate_directory(new_directory)
assert os.path.exists(new_directory)
assert not os.path.exists(new_file)
def start_experiment(experiment_name: str, config_filename: str,
benchmarks: List[str], fuzzers: List[str],
fuzzer_configs: List[str]):
"""Start a fuzzer benchmarking experiment."""
validate_benchmarks(benchmarks)
config = read_and_validate_experiment_config(config_filename)
config['benchmarks'] = ','.join(benchmarks)
validate_experiment_name(experiment_name)
config['experiment'] = experiment_name
config_dir = 'config'
filesystem.recreate_directory(config_dir)
experiment_config_filename = os.path.join(config_dir, 'experiment.yaml')
with open(experiment_config_filename, 'w') as experiment_config_file:
yaml.dump(config, experiment_config_file, default_flow_style=False)
if not fuzzers and not fuzzer_configs:
raise Exception('Need to provide either a list of fuzzers or '
'a list of fuzzer configs.')
fuzzer_config_dir = os.path.join(config_dir, 'fuzzer-configs')
filesystem.recreate_directory(fuzzer_config_dir)
for fuzzer_config in fuzzer_configs:
if fuzzer_configs.count(fuzzer_config) > 1:
raise Exception('Fuzzer config "%s" provided more than once.' %
fuzzer_config)
# Validate the fuzzer yaml attributes e.g. fuzzer, env, etc.
validate_fuzzer_config(fuzzer_config)
shutil.copy(fuzzer_config, fuzzer_config_dir)
for fuzzer in fuzzers:
if fuzzers.count(fuzzer) > 1:
raise Exception('Fuzzer "%s" provided more than once.' % fuzzer)
validate_fuzzer(fuzzer)
fuzzer_config_file_path = os.path.join(fuzzer_config_dir, fuzzer)
# Create a simple yaml with just the fuzzer attribute.
with open(fuzzer_config_file_path, 'w') as file_handle:
file_handle.write('fuzzer: ' + fuzzer)
# Make sure we can connect to database.
if 'POSTGRES_PASSWORD' not in os.environ:
raise Exception('Must set POSTGRES_PASSWORD environment variable.')
gcloud.set_default_project(config['cloud_project'])
dispatcher = Dispatcher(config)
if not os.getenv('MANUAL_EXPERIMENT'):
dispatcher.create_async()
copy_resources_to_bucket(config_dir, config)
if not os.getenv('MANUAL_EXPERIMENT'):
dispatcher.start()
def set_up_fuzzer_config_files(fuzzer_configs):
"""Write configurations specified by |fuzzer_configs| to yaml files that
will be used to store configurations."""
if not fuzzer_configs:
raise Exception('Need to provide either a list of fuzzers or '
'a list of fuzzer configs.')
fuzzer_config_dir = os.path.join(CONFIG_DIR, 'fuzzer-configs')
filesystem.recreate_directory(fuzzer_config_dir)
for fuzzer_config in fuzzer_configs:
# Validate the fuzzer yaml attributes e.g. fuzzer, env, etc.
validate_fuzzer_config(fuzzer_config)
config_file_name = os.path.join(fuzzer_config_dir,
get_full_fuzzer_name(fuzzer_config))
yaml_utils.write(config_file_name, fuzzer_config)
def output_report(experiment_config: dict,
in_progress=False,
coverage_report=False):
"""Generate the HTML report and write it to |web_bucket|."""
experiment_name = experiment_utils.get_experiment_name()
reports_dir = get_reports_dir()
core_fuzzers = set(get_core_fuzzers())
experiment_fuzzers = set(experiment_config['fuzzers'])
fuzzers = experiment_fuzzers.union(core_fuzzers)
# Calculate path to store report files in filestore.
web_filestore_path = experiment_config['report_filestore']
if not fuzzers.issubset(core_fuzzers):
# This means that we are running an experimental report with fuzzers
# not in the core list. So, store these in |experimental| sub-directory.
web_filestore_path = os.path.join(web_filestore_path, 'experimental')
web_filestore_path = posixpath.join(web_filestore_path, experiment_name)
# Don't merge with nonprivate experiments until the very end as doing it
# while the experiment is in progress will produce unusable realtime
# results.
merge_with_nonprivate = (not in_progress and experiment_config.get(
'merge_with_nonprivate', False))
try:
logger.debug('Generating report.')
filesystem.recreate_directory(reports_dir)
generate_report.generate_report(
[experiment_name],
str(reports_dir),
report_name=experiment_name,
fuzzers=fuzzers,
in_progress=in_progress,
merge_with_clobber_nonprivate=merge_with_nonprivate,
coverage_report=coverage_report)
filestore_utils.rsync(
str(reports_dir),
web_filestore_path,
delete=False, # Don't remove existing coverage jsons.
gsutil_options=[
'-h', 'Cache-Control:public,max-age=0,no-transform'
])
logger.debug('Done generating report.')
except data_utils.EmptyDataError:
logs.warning('No snapshot data.')
except Exception: # pylint: disable=broad-except
logger.error('Error generating HTML report.')
def output_report(web_bucket):
"""Generate the HTML report and write it to |web_bucket|."""
experiment_name = experiment_utils.get_experiment_name()
reports_dir = get_reports_dir()
try:
logger.debug('Generating report.')
filesystem.recreate_directory(reports_dir)
generate_report.generate_report([experiment_name], str(reports_dir))
gsutil.rsync(str(reports_dir),
web_bucket,
gsutil_options=[
'-h', 'Cache-Control:public,max-age=0,no-transform'
],
parallel=False)
logger.debug('Done generating report.')
except Exception: # pylint: disable=broad-except
logger.error('Error generating HTML report.')
def output_report(web_bucket, in_progress=False):
"""Generate the HTML report and write it to |web_bucket|."""
experiment_name = experiment_utils.get_experiment_name()
reports_dir = get_reports_dir()
try:
logger.debug('Generating report.')
filesystem.recreate_directory(reports_dir)
generate_report.generate_report([experiment_name],
str(reports_dir),
in_progress=in_progress)
filestore_utils.rsync(str(reports_dir),
web_bucket,
gsutil_options=[
'-h',
'Cache-Control:public,max-age=0,no-transform'
])
logger.debug('Done generating report.')
except data_utils.EmptyDataError:
logs.warning('No snapshot data.')
except Exception: # pylint: disable=broad-except
logger.error('Error generating HTML report.')
def set_up_fuzzer_config_files(fuzzers, fuzzer_configs):
"""Set up config files for each fuzzer in |fuzzers| and each
config file provided in |fuzzer_configs|."""
if not fuzzers and not fuzzer_configs:
raise Exception('Need to provide either a list of fuzzers or '
'a list of fuzzer configs.')
fuzzer_config_dir = os.path.join(CONFIG_DIR, 'fuzzer-configs')
filesystem.recreate_directory(fuzzer_config_dir)
for fuzzer_config in fuzzer_configs:
if fuzzer_configs.count(fuzzer_config) > 1:
raise Exception('Fuzzer config "%s" provided more than once.' %
fuzzer_config)
# Validate the fuzzer yaml attributes e.g. fuzzer, env, etc.
validate_fuzzer_config(fuzzer_config)
shutil.copy(fuzzer_config, fuzzer_config_dir)
for fuzzer in fuzzers:
if fuzzers.count(fuzzer) > 1:
raise Exception('Fuzzer "%s" provided more than once.' % fuzzer)
validate_fuzzer(fuzzer)
fuzzer_config_file_path = os.path.join(fuzzer_config_dir, fuzzer)
# Create a simple yaml with just the fuzzer attribute.
with open(fuzzer_config_file_path, 'w') as file_handle:
file_handle.write('fuzzer: ' + fuzzer)
def test_recreate_directory_not_existing(tmp_path):
"""Tests that recreate_directory creates a directory that does not already
exist."""
new_directory = os.path.join(tmp_path, 'new-directory')
filesystem.recreate_directory(new_directory)
assert os.path.exists(new_directory)
def test_recreate_directory_not_existing(fs):
"""Tests that recreate_directory creates a directory that does not already
exist."""
new_directory = 'new-directory'
filesystem.recreate_directory(new_directory)
assert os.path.exists(new_directory)
