def main(suite, analyser, verbose, timeout, files, bench):
debug = verbose == 2
exit_code = 0
analyser_instance = None
analysers = list_analysers() if analyser == 'all' else [analyser]
for tool in analysers:
try:
analyser_cls = get_analyser(tool)
analyser_instance = analyser_cls(debug, timeout)
exit_code = run_benchmark_suite(analyser_instance, suite, verbose,
debug, timeout, files, bench)
except AnalyserError as e:
print("Failed to initialize analyzer. "
"Program '{}' failed with {} error code".format(
e.cmd, e.returncode))
exit_code = e.returncode
finally:
if analyser_instance:
print("{} analyser finished. Cleaning up".format(
analyser_instance.get_name()))
analyser_instance.cleanup()
# "click" setting exit code this way...
sys.exit(exit_code)
def generate_benchmark_report(suite):
project_root_dir = code_dir.parent
suite_dir = project_root_dir / 'benchdata' / suite
# Dictionary of analyser reports
reports = {}
# Common benchmark data
data = {'benchmarks': {}, 'analyzers': {}}
for analyser in list_analysers():
yaml_file = suite_dir / (analyser + ".yaml")
# Skip when yaml does not exist
if not yaml_file.exists():
print("Can not find report from '{}' analyser. Skipping...".format(
analyser))
continue
with open(yaml_file, 'r') as fp:
try:
report = yaml.load(fp)
reports[report['analyzer']] = report
# Verify that report is generated by the same benchmark as already loaded reports
if 'suite' in data:
if data['benchmark_url_dir'] != report['benchmark_url_dir']:
raise Exception(
'Report for {} generated by different benchmark {}'
.format(analyser, report['benchmark_url_dir']))
else:
data['suite'] = report['suite']
data['benchmark_url_dir'] = report['benchmark_url_dir']
data['benchmark_link'] = report['benchmark_link']
data['benchmark_subdir'] = report['benchmark_subdir']
# Store general benchmark information
data['analyzers'][analyser] = {
'version': report['version'],
'total_time_str': report['total_time_str'],
'expected': report['expected'],
'error_execution': report['error_execution']
}
# Merge benchmark results with already loaded
for name, result in report['benchmarks'].items():
if name not in data['benchmarks']:
data['benchmarks'][name] = {}
data['benchmarks'][name][analyser] = result
data['benchmarks'][name]['bug_type'] = result.get(
'bug_type', 'Unconfigured')
if 'link' in result.get('expected_data', []):
data['benchmarks'][name]['link'] = result[
'expected_data']['link']
data['benchmark_count'] = len(data['benchmarks'])
except yaml.YAMLError as exc:
print(exc)
print_html_report(data, project_root_dir, suite)
os.makedirs(benchdir, exist_ok=True)
with open(benchdir / (analyser.get_name() + '.yaml'), 'w') as fp:
yaml.dump(out_data, fp)
# TODO add json config lint function?
@click.command()
@click.option('--suite',
'-s',
type=click.Choice(['Suhabe', 'nssc']),
default='Suhabe',
help="Benchmark suite to run; "
"nscc is an abbreviation for not-so-smart-contracts.")
@click.option('--analyser',
'-a',
type=click.Choice(list_analysers() + ['all']),
default='all',
help="Analyser tool to benchmark")
@click.option(
'--verbose',
'-v',
count=True,
help="More verbose output; use twice for the most verbose output.")
@click.option('--timeout',
'-t',
type=float,
default=DEFAULT_TIMEOUT,
help="Maximum time allowed on any single benchmark.")
@click.option('--files/--no-files',
default=False,
help="List files in benchmark and exit.")