def initialize_measurement_dirs(self):
"""Initialize directories that will be needed for measuring
coverage."""
for directory in [
self.corpus_dir, self.coverage_dir, self.crashes_dir
]:
filesystem.recreate_directory(directory)
filesystem.create_directory(self.report_dir)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False):
"""Generate report helper."""
report_name = report_name or experiment_names[0]
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
else:
experiment_df = queries.get_experiment_data(experiment_names)
# Save the raw data along with the report.
experiment_df.to_csv(data_path)
data_utils.validate_data(experiment_df)
if benchmarks is not None:
experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks)
if fuzzers is not None:
experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
if end_time is not None:
experiment_df = data_utils.filter_max_time(experiment_df, end_time)
if merge_with_clobber:
experiment_df = data_utils.clobber_experiments_data(
experiment_df, experiment_names)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df, report_directory, plotter, experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def set_up_coverage_binaries(pool, experiment):
"""Set up coverage binaries for all benchmarks in |experiment|."""
# Use set comprehension to select distinct benchmarks.
benchmarks = [
benchmark_tuple[0]
for benchmark_tuple in db_utils.query(models.Trial.benchmark).distinct(
).filter(models.Trial.experiment == experiment)
]
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
filesystem.create_directory(coverage_binaries_dir)
pool.map(set_up_coverage_binary, benchmarks)
def generate_coverage_regions_json(self):
"""Stores the coverage data in a json file."""
covered_regions = extract_covered_regions_from_summary_json(
self.merged_summary_json_file)
coverage_json_src = os.path.join(self.data_dir, 'covered_regions.json')
coverage_json_dst = exp_path.filestore(coverage_json_src)
filesystem.create_directory(self.data_dir)
with open(coverage_json_src, 'w') as file_handle:
json.dump(covered_regions, file_handle)
filestore_utils.cp(coverage_json_src,
coverage_json_dst,
expect_zero=False)
def cp( # pylint: disable=invalid-name
source,
destination,
recursive=False,
parallel=False): # pylint: disable=unused-argument
"""Executes "cp" command from |source| to |destination|."""
# Create intermediate folders for `cp` command to behave like `gsutil.cp`.
filesystem.create_directory(os.path.dirname(destination))
command = ['cp']
if recursive:
command.append('-r')
command.extend([source, destination])
return new_process.execute(command, expect_zero=True)
def set_up_coverage_binary(benchmark):
"""Set up coverage binaries for |benchmark|."""
initialize_logs()
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark
filesystem.create_directory(benchmark_coverage_binary_dir)
archive_name = 'coverage-build-%s.tar.gz' % benchmark
archive_filestore_path = exp_path.filestore(coverage_binaries_dir /
archive_name)
filestore_utils.cp(archive_filestore_path,
str(benchmark_coverage_binary_dir))
archive_path = benchmark_coverage_binary_dir / archive_name
tar = tarfile.open(archive_path, 'r:gz')
tar.extractall(benchmark_coverage_binary_dir)
os.remove(archive_path)
def _add_build_arguments_to_config(base: str, fuzzer: str) -> str:
"""If there are fuzzer-specific arguments, make a config file with them."""
fuzzer_config = fuzzer_config_utils.get_by_variant_name(fuzzer)
if 'build_arguments' not in fuzzer_config:
return base
# TODO(mbarbella): Rather than rewrite yaml files, use the GCB API.
args = fuzzer_config['build_arguments']
config = yaml_utils.read(base)
for step in config['steps']:
if 'id' in step and step['id'] in BUILDER_STEP_IDS:
# Append additional flags before the final argument.
step['args'] = step['args'][:-1] + args + [step['args'][-1]]
new_config_path = os.path.join(CONFIG_DIR, 'builds', fuzzer + '.yaml')
filesystem.create_directory(os.path.dirname(new_config_path))
yaml_utils.write(new_config_path, config)
return new_config_path
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
fuzzers=None,
report_type='default',
quick=False,
from_cached_data=False):
"""Generate report helper."""
report_name = report_name or experiment_names[0]
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
else:
experiment_df = queries.get_experiment_data(experiment_names)
# Save the raw data along with the report.
experiment_df.to_csv(data_path)
if fuzzers is not None:
experiment_df = filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = label_fuzzers_by_experiment(experiment_df)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df, report_directory, plotter, experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def start(self):
"""Start the experiment on the dispatcher."""
container_name = 'dispatcher-container'
logs.info('Started dispatcher with container name: %s', container_name)
experiment_filestore_path = os.path.abspath(
self.config['experiment_filestore'])
filesystem.create_directory(experiment_filestore_path)
sql_database_arg = (
'SQL_DATABASE_URL=sqlite:///{}?check_same_thread=False'.format(
os.path.join(experiment_filestore_path, 'local.db')))
docker_registry = self.config['docker_registry']
set_instance_name_arg = 'INSTANCE_NAME={instance_name}'.format(
instance_name=self.instance_name)
set_experiment_arg = 'EXPERIMENT={experiment}'.format(
experiment=self.config['experiment'])
shared_experiment_filestore_arg = '{0}:{0}'.format(
self.config['experiment_filestore'])
# TODO: (#484) Use config in function args or set as environment
# variables.
set_docker_registry_arg = 'DOCKER_REGISTRY={}'.format(docker_registry)
set_experiment_filestore_arg = (
'EXPERIMENT_FILESTORE={experiment_filestore}'.format(
experiment_filestore=self.config['experiment_filestore']))
shared_report_filestore_arg = '{0}:{0}'.format(
self.config['report_filestore'])
set_report_filestore_arg = (
'REPORT_FILESTORE={report_filestore}'.format(
report_filestore=self.config['report_filestore']))
docker_image_url = '{docker_registry}/dispatcher-image'.format(
docker_registry=docker_registry)
command = [
'docker',
'run',
'-ti',
'--rm',
'-v',
'/var/run/docker.sock:/var/run/docker.sock',
'-v',
shared_experiment_filestore_arg,
'-v',
shared_report_filestore_arg,
'-e',
set_instance_name_arg,
'-e',
set_experiment_arg,
'-e',
sql_database_arg,
'-e',
set_experiment_filestore_arg,
'-e',
set_report_filestore_arg,
'-e',
set_docker_registry_arg,
'-e',
'LOCAL_EXPERIMENT=True',
'--cap-add=SYS_PTRACE',
'--cap-add=SYS_NICE',
'--name=%s' % container_name,
docker_image_url,
'/bin/bash',
'-c',
'rsync -r '
'"${EXPERIMENT_FILESTORE}/${EXPERIMENT}/input/" ${WORK} && '
'mkdir ${WORK}/src && '
'tar -xvzf ${WORK}/src.tar.gz -C ${WORK}/src && '
'source "${WORK}/.venv/bin/activate" && '
'pip3 install -r "${WORK}/src/requirements.txt" && '
'PYTHONPATH=${WORK}/src python3 '
'${WORK}/src/experiment/dispatcher.py || '
'/bin/bash' # Open shell if experiment fails.
]
return new_process.execute(command, write_to_stdout=True)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False,
merge_with_clobber_nonprivate=False,
coverage_report=False):
"""Generate report helper."""
if merge_with_clobber_nonprivate:
experiment_names = (
queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names))
main_experiment_name = experiment_names[0]
report_name = report_name or main_experiment_name
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
description = "from cached data"
else:
experiment_df = queries.get_experiment_data(experiment_names)
description = queries.get_experiment_description(main_experiment_name)
data_utils.validate_data(experiment_df)
if benchmarks is not None:
experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks)
if fuzzers is not None:
experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
if end_time is not None:
experiment_df = data_utils.filter_max_time(experiment_df, end_time)
if merge_with_clobber or merge_with_clobber_nonprivate:
experiment_df = data_utils.clobber_experiments_data(
experiment_df, experiment_names)
# Save the filtered raw data along with the report if not using cached data
# or if the data does not exist.
if not from_cached_data or not os.path.exists(data_path):
experiment_df.to_csv(data_path)
# Load the coverage json summary file.
coverage_dict = {}
if coverage_report:
coverage_dict = coverage_data_utils.get_covered_regions_dict(
experiment_df)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df,
coverage_dict,
report_directory,
plotter,
experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress, coverage_report,
description)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def start(self):
"""Start the experiment on the dispatcher."""
experiment_filestore_path = os.path.abspath(
self.config['experiment_filestore'])
filesystem.create_directory(experiment_filestore_path)
sql_database_arg = 'SQL_DATABASE_URL=sqlite:///{}'.format(
os.path.join(experiment_filestore_path, 'local.db'))
base_docker_tag = experiment_utils.get_base_docker_tag(
self.config['cloud_project'])
set_instance_name_arg = 'INSTANCE_NAME={instance_name}'.format(
instance_name=self.instance_name)
set_experiment_arg = 'EXPERIMENT={experiment}'.format(
experiment=self.config['experiment'])
set_cloud_project_arg = 'CLOUD_PROJECT={cloud_project}'.format(
cloud_project=self.config['cloud_project'])
shared_experiment_filestore_arg = '{0}:{0}'.format(
self.config['experiment_filestore'])
set_experiment_filestore_arg = (
'EXPERIMENT_FILESTORE={experiment_filestore}'.format(
experiment_filestore=self.config['experiment_filestore']))
shared_report_filestore_arg = '{0}:{0}'.format(
self.config['report_filestore'])
set_report_filestore_arg = (
'REPORT_FILESTORE={report_filestore}'.format(
report_filestore=self.config['report_filestore']))
docker_image_url = '{base_docker_tag}/dispatcher-image'.format(
base_docker_tag=base_docker_tag)
command = [
'docker',
'run',
'-ti',
'--rm',
'-v',
'/var/run/docker.sock:/var/run/docker.sock',
'-v',
shared_experiment_filestore_arg,
'-v',
shared_report_filestore_arg,
'-e',
set_instance_name_arg,
'-e',
set_experiment_arg,
'-e',
set_cloud_project_arg,
'-e',
sql_database_arg,
'-e',
set_experiment_filestore_arg,
'-e',
set_report_filestore_arg,
'-e',
'LOCAL_EXPERIMENT=True',
'--cap-add=SYS_PTRACE',
'--cap-add=SYS_NICE',
'--name=dispatcher-container',
docker_image_url,
'/bin/bash',
'-c',
'rsync -r '
'"${EXPERIMENT_FILESTORE}/${EXPERIMENT}/input/" ${WORK} && '
'mkdir ${WORK}/src && '
'tar -xvzf ${WORK}/src.tar.gz -C ${WORK}/src && '
'source "${WORK}/.venv/bin/activate" && '
'pip3 install -r "${WORK}/src/requirements.txt" && '
'PYTHONPATH=${WORK}/src python3 '
'${WORK}/src/experiment/dispatcher.py || '
'/bin/bash' # Open shell if experiment fails.
]
return new_process.execute(command, write_to_stdout=True)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False,
merge_with_clobber_nonprivate=False,
coverage_report=False):
"""Generate report helper."""
if merge_with_clobber_nonprivate:
experiment_names = (
queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names))
merge_with_clobber = True
main_experiment_name = experiment_names[0]
report_name = report_name or main_experiment_name
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, DATA_FILENAME)
experiment_df, experiment_description = get_experiment_data(
experiment_names, main_experiment_name, from_cached_data, data_path)
# TODO(metzman): Ensure that each experiment is in the df. Otherwise there
# is a good chance user misspelled something.
data_utils.validate_data(experiment_df)
experiment_df = modify_experiment_data_if_requested(
experiment_df, experiment_names, benchmarks, fuzzers,
label_by_experiment, end_time, merge_with_clobber)
# Add |bugs_covered| column prior to export.
experiment_df = data_utils.add_bugs_covered_column(experiment_df)
# Save the filtered raw data along with the report if not using cached data
# or if the data does not exist.
if not from_cached_data or not os.path.exists(data_path):
experiment_df.to_csv(data_path)
# Load the coverage json summary file.
coverage_dict = {}
if coverage_report:
logger.info('Generating coverage report info.')
coverage_dict = coverage_data_utils.get_covered_regions_dict(
experiment_df)
logger.info('Finished generating coverage report info.')
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df,
coverage_dict,
report_directory,
plotter,
experiment_name=report_name)
template = report_type + '.html'
logger.info('Rendering HTML report.')
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress, coverage_report,
experiment_description)
logger.info('Done rendering HTML report.')
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
