def generate_report(experiment_names,
report_directory,
report_name=None,
report_type='default',
quick=False,
from_cached_data=False):
"""Generate report helper."""
report_name = report_name or experiment_names[0]
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
else:
experiment_df = queries.get_experiment_data(experiment_names)
# Save the raw data along with the report.
experiment_df.to_csv(data_path)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df, report_directory, plotter, experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False):
"""Generate report helper."""
report_name = report_name or experiment_names[0]
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
else:
experiment_df = queries.get_experiment_data(experiment_names)
# Save the raw data along with the report.
experiment_df.to_csv(data_path)
data_utils.validate_data(experiment_df)
if benchmarks is not None:
experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks)
if fuzzers is not None:
experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
if end_time is not None:
experiment_df = data_utils.filter_max_time(experiment_df, end_time)
if merge_with_clobber:
experiment_df = data_utils.clobber_experiments_data(
experiment_df, experiment_names)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df, report_directory, plotter, experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def test_pariwise_unique_coverage_heatmap_plot(tmp_path):
"""Tests that pairwise unique coverage heatmap looks as expected (even with
a large number of fuzzers)."""
fuzzer_num = 22
fuzzers = [f'fuzzer-{i}' for i in range(fuzzer_num)]
table_data = [range(1000, 1000 + fuzzer_num)] * fuzzer_num
table_df = pd.DataFrame(table_data, index=fuzzers, columns=fuzzers)
plotter = plotting.Plotter(fuzzers)
image_path = tmp_path / 'out.png'
plotter.write_pairwise_unique_coverage_heatmap_plot(table_df, image_path)
golden_path = 'analysis/test_data/pairwise_unique_coverage_heatmap.png'
plt_cmp.compare_images(image_path, golden_path, tol=0.01)
def test_unique_coverage_ranking_plot(tmp_path):
"""Tests that unique coverage ranking plot looks as expected (even with a
large number of fuzzers)."""
fuzzer_num = 22
fuzzers = [f'fuzzer-{i}' for i in range(fuzzer_num)]
unique_regions = [10 * i for i in range(fuzzer_num)]
total_regions = [1000] * fuzzer_num
df = pd.DataFrame({
'fuzzer': fuzzers,
'unique_regions_covered': unique_regions,
'aggregated_edges_covered': total_regions
})
plotter = plotting.Plotter(fuzzers)
image_path = tmp_path / 'out.png'
plotter.write_unique_coverage_ranking_plot(df, image_path)
golden_path = 'analysis/test_data/unique_coverage_ranking.png'
plt_cmp.compare_images(image_path, golden_path, tol=0.01)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False,
merge_with_clobber_nonprivate=False,
coverage_report=False):
"""Generate report helper."""
if merge_with_clobber_nonprivate:
experiment_names = (
queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names))
main_experiment_name = experiment_names[0]
report_name = report_name or main_experiment_name
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
description = "from cached data"
else:
experiment_df = queries.get_experiment_data(experiment_names)
description = queries.get_experiment_description(main_experiment_name)
data_utils.validate_data(experiment_df)
if benchmarks is not None:
experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks)
if fuzzers is not None:
experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
if end_time is not None:
experiment_df = data_utils.filter_max_time(experiment_df, end_time)
if merge_with_clobber or merge_with_clobber_nonprivate:
experiment_df = data_utils.clobber_experiments_data(
experiment_df, experiment_names)
# Save the filtered raw data along with the report if not using cached data
# or if the data does not exist.
if not from_cached_data or not os.path.exists(data_path):
experiment_df.to_csv(data_path)
# Load the coverage json summary file.
coverage_dict = {}
if coverage_report:
coverage_dict = coverage_data_utils.get_covered_regions_dict(
experiment_df)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df,
coverage_dict,
report_directory,
plotter,
experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress, coverage_report,
description)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False,
merge_with_clobber_nonprivate=False,
coverage_report=False):
"""Generate report helper."""
if merge_with_clobber_nonprivate:
experiment_names = (
queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names))
merge_with_clobber = True
main_experiment_name = experiment_names[0]
report_name = report_name or main_experiment_name
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, DATA_FILENAME)
experiment_df, experiment_description = get_experiment_data(
experiment_names, main_experiment_name, from_cached_data, data_path)
# TODO(metzman): Ensure that each experiment is in the df. Otherwise there
# is a good chance user misspelled something.
data_utils.validate_data(experiment_df)
experiment_df = modify_experiment_data_if_requested(
experiment_df, experiment_names, benchmarks, fuzzers,
label_by_experiment, end_time, merge_with_clobber)
# Add |bugs_covered| column prior to export.
experiment_df = data_utils.add_bugs_covered_column(experiment_df)
# Save the filtered raw data along with the report if not using cached data
# or if the data does not exist.
if not from_cached_data or not os.path.exists(data_path):
experiment_df.to_csv(data_path)
# Load the coverage json summary file.
coverage_dict = {}
if coverage_report:
logger.info('Generating coverage report info.')
coverage_dict = coverage_data_utils.get_covered_regions_dict(
experiment_df)
logger.info('Finished generating coverage report info.')
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df,
coverage_dict,
report_directory,
plotter,
experiment_name=report_name)
template = report_type + '.html'
logger.info('Rendering HTML report.')
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress, coverage_report,
experiment_description)
logger.info('Done rendering HTML report.')
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
