def generate_report(experiment_names,
report_directory,
report_name=None,
report_type='default',
quick=False,
from_cached_data=False):
"""Generate report helper."""
report_name = report_name or experiment_names[0]
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
else:
experiment_df = queries.get_experiment_data(experiment_names)
# Save the raw data along with the report.
experiment_df.to_csv(data_path)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df, report_directory, plotter, experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False):
"""Generate report helper."""
report_name = report_name or experiment_names[0]
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
else:
experiment_df = queries.get_experiment_data(experiment_names)
# Save the raw data along with the report.
experiment_df.to_csv(data_path)
data_utils.validate_data(experiment_df)
if benchmarks is not None:
experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks)
if fuzzers is not None:
experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
if end_time is not None:
experiment_df = data_utils.filter_max_time(experiment_df, end_time)
if merge_with_clobber:
experiment_df = data_utils.clobber_experiments_data(
experiment_df, experiment_names)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df, report_directory, plotter, experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def test_linkify_fuzzer_names_in_ranking():
"""Tests turning fuzzer names into links."""
experiment_df = test_data_utils.create_experiment_data()
results = experiment_results.ExperimentResults(experiment_df,
coverage_dict=None,
output_directory=None,
plotter=None)
ranking = results.rank_by_median_and_average_rank
ranking = results.linkify_names(ranking)
assert ranking.index[0] == (
'<a href="https://github.com/google/fuzzbench/blob/'
'master/fuzzers/afl">afl</a>')
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False,
merge_with_clobber_nonprivate=False,
coverage_report=False):
"""Generate report helper."""
if merge_with_clobber_nonprivate:
experiment_names = (
queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names))
main_experiment_name = experiment_names[0]
report_name = report_name or main_experiment_name
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, 'data.csv.gz')
if from_cached_data and os.path.exists(data_path):
experiment_df = pd.read_csv(data_path)
description = "from cached data"
else:
experiment_df = queries.get_experiment_data(experiment_names)
description = queries.get_experiment_description(main_experiment_name)
data_utils.validate_data(experiment_df)
if benchmarks is not None:
experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks)
if fuzzers is not None:
experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers)
if label_by_experiment:
experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df)
if end_time is not None:
experiment_df = data_utils.filter_max_time(experiment_df, end_time)
if merge_with_clobber or merge_with_clobber_nonprivate:
experiment_df = data_utils.clobber_experiments_data(
experiment_df, experiment_names)
# Save the filtered raw data along with the report if not using cached data
# or if the data does not exist.
if not from_cached_data or not os.path.exists(data_path):
experiment_df.to_csv(data_path)
# Load the coverage json summary file.
coverage_dict = {}
if coverage_report:
coverage_dict = coverage_data_utils.get_covered_regions_dict(
experiment_df)
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df,
coverage_dict,
report_directory,
plotter,
experiment_name=report_name)
template = report_type + '.html'
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress, coverage_report,
description)
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)
def generate_report(experiment_names,
report_directory,
report_name=None,
label_by_experiment=False,
benchmarks=None,
fuzzers=None,
report_type='default',
quick=False,
log_scale=False,
from_cached_data=False,
in_progress=False,
end_time=None,
merge_with_clobber=False,
merge_with_clobber_nonprivate=False,
coverage_report=False):
"""Generate report helper."""
if merge_with_clobber_nonprivate:
experiment_names = (
queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names))
merge_with_clobber = True
main_experiment_name = experiment_names[0]
report_name = report_name or main_experiment_name
filesystem.create_directory(report_directory)
data_path = os.path.join(report_directory, DATA_FILENAME)
experiment_df, experiment_description = get_experiment_data(
experiment_names, main_experiment_name, from_cached_data, data_path)
# TODO(metzman): Ensure that each experiment is in the df. Otherwise there
# is a good chance user misspelled something.
data_utils.validate_data(experiment_df)
experiment_df = modify_experiment_data_if_requested(
experiment_df, experiment_names, benchmarks, fuzzers,
label_by_experiment, end_time, merge_with_clobber)
# Add |bugs_covered| column prior to export.
experiment_df = data_utils.add_bugs_covered_column(experiment_df)
# Save the filtered raw data along with the report if not using cached data
# or if the data does not exist.
if not from_cached_data or not os.path.exists(data_path):
experiment_df.to_csv(data_path)
# Load the coverage json summary file.
coverage_dict = {}
if coverage_report:
logger.info('Generating coverage report info.')
coverage_dict = coverage_data_utils.get_covered_regions_dict(
experiment_df)
logger.info('Finished generating coverage report info.')
fuzzer_names = experiment_df.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
experiment_ctx = experiment_results.ExperimentResults(
experiment_df,
coverage_dict,
report_directory,
plotter,
experiment_name=report_name)
template = report_type + '.html'
logger.info('Rendering HTML report.')
detailed_report = rendering.render_report(experiment_ctx, template,
in_progress, coverage_report,
experiment_description)
logger.info('Done rendering HTML report.')
filesystem.write(os.path.join(report_directory, 'index.html'),
detailed_report)