def benchmark_size(bench, metrics):
"""Compute the total size of the desired sections in a benchmark. Returns
the size in bytes, which may be zero if the section wasn't found."""
appexe = os.path.join(gp['bd_benchdir'], bench, bench)
sec_sizes = {}
# If the benchmark failed to build, then return a 0 size instead of
# crashing when failing to open the file.
if not os.path.exists(appexe):
return {}
# read format from file and check it is as expected
with open(appexe, 'rb') as fileh:
magic = fileh.read(4)
# lief does not appear to have ability to work from an already opened file
fileh.close()
if (((gp['format'] == 'elf') and (magic != b'\x7fELF'))
or ((gp['format'] == 'macho') and (magic != b'\xcf\xfa\xed\xfe'))):
log.info('ERROR: File format does not match parameter')
sys.exit(1)
binary = lief.parse(appexe)
sections = binary.sections
for metric in metrics:
sec_sizes[metric] = 0
for target_name in gp['secnames'][metric]:
for section in sections:
if ((gp['format'] == 'elf'
and section.name.startswith(target_name))
or (target_name == section.name)):
sec_sizes[metric] += section.size
# Return the section (group) size
return sec_sizes
def main():
"""Main program driving measurement of benchmark size"""
# Establish the root directory of the repository, since we know this file is
# in that directory.
gp['rootdir'] = os.path.abspath(os.path.dirname(__file__))
# Parse arguments using standard technology
parser = build_parser()
args = parser.parse_args()
# Establish logging
setup_logging(args.logdir, 'size')
log_args(args)
# Check args are OK (have to have logging and build directory set up first)
validate_args(args)
# Find the benchmarks
benchmarks = find_benchmarks()
log_benchmarks(benchmarks)
# Collect the size data for the benchmarks
raw_data, rel_data = collect_data(benchmarks)
# We can't compute geometric SD on the fly, so we need to collect all the
# data and then process it in two passes. We could do the first processing
# as we collect the data, but it is clearer to do the three things
# separately. Given the size of datasets with which we are concerned the
# compute overhead is not significant.
if raw_data:
embench_stats(benchmarks, raw_data, rel_data)
log.info('All benchmarks sized successfully')
else:
log.info('ERROR: Failed to compute size benchmarks')
sys.exit(1)
def main():
"""Main program to drive building of benchmarks."""
# Establish the root directory of the repository, since we know this file is
# in that directory.
gp['rootdir'] = os.path.abspath(os.path.dirname(__file__))
# Parse arguments using standard technology
parser = build_parser()
args = parser.parse_args()
# Establish logging, using "build" as the log file prefix.
gp['verbose'] = args.verbose
setup_logging(args.logdir, 'build')
log_args(args)
# Establish build directory
create_builddir(args.builddir, args.clean)
# Check args are OK (have to have logging and build directory set up first)
validate_args(args)
# Find the benchmarks
benchmarks = find_benchmarks()
log_benchmarks(benchmarks)
# Establish other global parameters
set_parameters(args)
log_parameters()
log.debug('General log')
log.debug('===========')
# Set up additional environment variables.
set_environ()
# Track success
successful = compile_support()
if successful:
log.debug('Compilation of support files successful')
for bench in benchmarks:
res = compile_benchmark(bench)
successful &= res
if res:
log.debug('Compilation of benchmark "{bench}" successful'.format(
bench=bench))
res = link_benchmark(bench)
successful &= res
if res:
log.debug('Linking of benchmark "{bench}" successful'.format(
bench=bench))
log.info(bench)
if successful:
log.info('All benchmarks built successfully')
def main():
"""Main program driving measurement of benchmark size"""
# Establish the root directory of the repository, since we know this file is
# in that directory.
gp['rootdir'] = os.path.abspath(os.path.dirname(__file__))
# Parse arguments common to all speed testers, and get list of those
# remaining.
args, remnant = get_common_args()
# Establish logging
setup_logging(args.logdir, 'speed')
log_args(args)
# Check args are OK (have to have logging and build directory set up first)
validate_args(args)
# Find the benchmarks
benchmarks = find_benchmarks()
log_benchmarks(benchmarks)
# Collect the size data for the benchmarks. Pass any remaining args.
raw_data, rel_data = collect_data(benchmarks, remnant)
# We can't compute geometric SD on the fly, so we need to collect all the
# data and then process it in two passes. We could do the first processing
# as we collect the data, but it is clearer to do the three things
# separately. Given the size of datasets with which we are concerned the
# compute overhead is not significant.
if raw_data:
if gp['output_format'] != output_format.BASELINE:
opt_comma = ',' if args.json_comma else ''
embench_stats(benchmarks, raw_data, rel_data, 'speed', opt_comma)
log.info('All benchmarks run successfully')
else:
log.info('ERROR: Failed to compute speed benchmarks')
sys.exit(1)
def collect_data(benchmarks):
"""Collect and log all the raw and optionally relative data associated with
the list of benchmarks supplied in the "benchmarks" argument. Return
the raw data and relative data as a list. The raw data may be empty if
there is a failure. The relative data will be empty if only absolute
results have been requested."""
# Baseline data is held external to the script. Import it here.
gp['baseline_dir'] = os.path.join(gp['rootdir'], 'baseline-data',
'size.json')
with open(gp['baseline_dir']) as fileh:
baseline_all = loads(fileh.read())
# Compute the baseline data we need
baseline = {}
for bench, data in baseline_all.items():
baseline[bench] = 0
for sec in gp['metric']:
if sec in data:
baseline[bench] += data[sec]
# Collect data and output it
successful = True
raw_data = {}
rel_data = {}
log.info('Benchmark size')
log.info('--------- ----')
for bench in benchmarks:
raw_data[bench] = benchmark_size(bench)
rel_data[bench] = {}
output = {}
# Zero is a valid section size, although empty .text should be a
# concern.
if gp['absolute']:
# Want absolute results. Only include non-zero values
output = f'{raw_data[bench]:8,}'
else:
# Want relative results (the default). If baseline is zero, just
# use 0.0 as the value.
if baseline[bench] > 0:
rel_data[bench] = raw_data[bench] / baseline[bench]
else:
rel_data[bench] = 0.0
output = f' {rel_data[bench]:6.2f}'
log.info(f'{bench:15} {output:8}')
if successful:
return raw_data, rel_data
# Otherwise failure return
return [], []
def collect_data(benchmarks, remnant):
"""Collect and log all the raw and optionally relative data associated with
the list of benchmarks supplied in the "benchmarks" argument. "remant"
is left over args from the command line, which may be useful to the
benchmark running procs.
Return the raw data and relative data as a list. The raw data may be
empty if there is a failure. The relative data will be empty if only
absolute results have been requested."""
# Baseline data is held external to the script. Import it here.
gp['baseline_dir'] = os.path.join(
gp['rootdir'], 'baseline-data', 'speed.json'
)
with open(gp['baseline_dir']) as fileh:
baseline = loads(fileh.read())
# Parse target specific args
target_args = get_target_args(remnant)
# Collect data and output it
successful = True
raw_data = {}
rel_data = {}
log.info('Benchmark Speed')
log.info('--------- -----')
for bench in benchmarks:
raw_data[bench] = benchmark_speed(bench, target_args)
rel_data[bench] = 0.0
if raw_data[bench] == 0.0:
del raw_data[bench]
del rel_data[bench]
successful = False
else:
output = ''
if gp['absolute']:
# Want absolute results. Only include non-zero values
output = f'{round(raw_data[bench]):8,}'
else:
# Want relative results (the default). Only use non-zero values.
rel_data[bench] = raw_data[bench] / baseline[bench]
output = f' {rel_data[bench]:6.2f}'
log.info(f'{bench:15} {output:8}')
if successful:
return raw_data, rel_data
# Otherwise failure return
return [], []
def collect_data(benchmarks, remnant):
"""Collect and log all the raw and optionally relative data associated with
the list of benchmarks supplied in the "benchmarks" argument. "remant"
is left over args from the command line, which may be useful to the
benchmark running procs.
Return the raw data and relative data as a list. The raw data may be
empty if there is a failure. The relative data will be empty if only
absolute results have been requested."""
# Baseline data is held external to the script. Import it here.
speed_baseline = os.path.join(gp['baseline_dir'], 'speed.json')
with open(speed_baseline) as fileh:
baseline = loads(fileh.read())
# Parse target specific args
target_args = get_target_args(remnant)
# Collect data
successful = True
raw_data = {}
rel_data = {}
for bench in benchmarks:
raw_data[bench] = benchmark_speed(bench, target_args)
rel_data[bench] = 0.0
if raw_data[bench] == 0.0:
del raw_data[bench]
del rel_data[bench]
successful = False
else:
if (not gp['absolute']
and gp['output_format'] != output_format.BASELINE):
rel_data[bench] = baseline[bench] / raw_data[bench]
# Output it
if gp['output_format'] == output_format.JSON:
log.info(' "speed results" :')
log.info(' { "detailed speed results" :')
for bench in benchmarks:
output = ''
if raw_data[bench] != 0.0:
if gp['absolute']:
output = f'{round(raw_data[bench])}'
else:
output = f'{rel_data[bench]:.2f}'
if bench == benchmarks[0]:
log.info(f' {{ ' + f'"{bench}" : {output},')
elif bench == benchmarks[-1]:
log.info(f' "{bench}" : {output}')
else:
log.info(f' "{bench}" : {output},')
log.info(' },')
elif gp['output_format'] == output_format.TEXT:
log.info('Benchmark Speed')
log.info('--------- -----')
for bench in benchmarks:
output = ''
if raw_data[bench] != 0.0:
if gp['absolute']:
output = f'{round(raw_data[bench]):8,}'
else:
output = f' {rel_data[bench]:6.2f}'
# Want relative results (the default). Only use non-zero values.
log.info(f'{bench:15} {output:8}')
elif gp['output_format'] == output_format.BASELINE:
log.info('{')
for bench in benchmarks:
if bench == benchmarks[-1]:
log.info(f' "{bench}" : {raw_data[bench]}')
else:
log.info(f' "{bench}" : {raw_data[bench]},')
log.info('}')
if successful:
return raw_data, rel_data
# Otherwise failure return
return [], []
def collect_data(benchmarks):
"""Collect and log all the raw and optionally relative data associated with
the list of benchmarks supplied in the "benchmarks" argument. Return
the raw data and relative data as a list. The raw data may be empty if
there is a failure. The relative data will be empty if only absolute
results have been requested.
Note that we manually generate the JSON output, rather than using the
dumps method, because the result will be manually edited, and we want
to guarantee the layout."""
# Baseline data is held external to the script. Import it here.
size_baseline = os.path.join(gp['baseline_dir'], 'size.json')
with open(size_baseline) as fileh:
baseline_all = loads(fileh.read())
# Compute the baseline data we need
baseline = {}
for bench, data in baseline_all.items():
baseline[bench] = 0
for sec in gp['metric']:
baseline[bench] += data[sec]
successful = True
raw_section_data = {}
raw_totals = {}
rel_data = {}
# Collect data
for bench in benchmarks:
if gp['output_format'] == output_format.BASELINE:
raw_section_data[bench] = benchmark_size(bench, ALL_METRICS)
else:
raw_section_data[bench] = benchmark_size(bench, gp['metric'])
raw_totals[bench] = sum(raw_section_data[bench].values())
# Calculate data relative to the baseline if needed
if gp['absolute'] or gp['output_format'] == output_format.BASELINE:
rel_data[bench] = {}
else:
# Want relative results (the default). If baseline is zero, just
# use 0.0 as the value. Note this is inverted compared to the
# speed benchmark, so SMALL is good.
if baseline[bench] > 0:
rel_data[bench] = raw_totals[bench] / baseline[bench]
else:
rel_data[bench] = 0.0
# Output it
if gp['output_format'] == output_format.JSON:
log.info(' "size results" :')
log.info(' { "detailed size results" :')
for bench in benchmarks:
res_output = ''
if gp['absolute']:
res_output = f'{raw_totals[bench]}'
else:
res_output = f'{rel_data[bench]:.2f}'
if bench == benchmarks[0]:
log.info(' { ' + f'"{bench}" : {res_output},')
elif bench == benchmarks[-1]:
log.info(f' "{bench}" : {res_output}')
else:
log.info(f' "{bench}" : {res_output},')
log.info(' },')
elif gp['output_format'] == output_format.TEXT:
log.info('Benchmark size')
log.info('--------- ----')
for bench in benchmarks:
res_output = ''
if gp['absolute']:
res_output = f' {raw_totals[bench]:8,}'
else:
res_output = f' {rel_data[bench]:6.2f}'
log.info(f'{bench:15} {res_output:8}')
elif gp['output_format'] == output_format.BASELINE:
log.info('{')
for bench in benchmarks:
res_output = ''
for metric in ALL_METRICS:
# newline before the first metric
if metric != ALL_METRICS[0]:
res_output += ',\n'
value = raw_section_data[bench][metric]
res_output += f' "{metric}" : {value}'
# comma after all but last benchmark in the log
if bench == benchmarks[-1]:
log.info(f' "{bench}" : {{\n{res_output}\n }}')
else:
log.info(f' "{bench}" : {{\n{res_output}\n }},')
log.info('}')
if successful:
return raw_totals, rel_data
# Otherwise failure return
return [], []
def collect_data(benchmarks):
"""Collect and log all the raw and optionally relative data associated with
the list of benchmarks supplied in the "benchmarks" argument. Return
the raw data and relative data as a list. The raw data may be empty if
there is a failure. The relative data will be empty if only absolute
results have been requested.
Note that we manually generate the JSON output, rather than using the
dumps method, because the result will be manually edited, and we want
to guarantee the layout."""
# Baseline data is held external to the script. Import it here.
gp['baseline_dir'] = os.path.join(gp['rootdir'], 'baseline-data',
'size.json')
with open(gp['baseline_dir']) as fileh:
baseline_all = loads(fileh.read())
# Compute the baseline data we need
baseline = {}
for bench, data in baseline_all.items():
baseline[bench] = 0
for sec in gp['metric']:
baseline[bench] += data[sec]
# Collect data and output it
successful = True
raw_data = {}
rel_data = {}
if gp['json']:
log.info(' "size results" :')
log.info(' { "detailed size results" :')
else:
log.info('Benchmark size')
log.info('--------- ----')
for bench in benchmarks:
raw_data[bench] = benchmark_size(bench)
rel_data[bench] = {}
output = {}
# Zero is a valid section size, although empty .text should be a
# concern.
if gp['absolute']:
# Want absolute results. Only include non-zero values
if gp['json']:
output = f'{raw_data[bench]}'
else:
output = f' {raw_data[bench]:8,}'
else:
# Want relative results (the default). If baseline is zero, just
# use 0.0 as the value. Note this is inverted compared to the
# speed benchmark, so SMALL is good.
if baseline[bench] > 0:
rel_data[bench] = raw_data[bench] / baseline[bench]
else:
rel_data[bench] = 0.0
if gp['json']:
output = f'{rel_data[bench]:.2f}'
else:
output = f' {rel_data[bench]:6.2f}'
if gp['json']:
if bench == benchmarks[0]:
log.info(' { ' + f'"{bench}" : {output},')
elif bench == benchmarks[-1]:
log.info(f' "{bench}" : {output}')
else:
log.info(f' "{bench}" : {output},')
else:
log.info(f'{bench:15} {output:8}')
if gp['json']:
log.info(' },')
if successful:
return raw_data, rel_data
# Otherwise failure return
return [], []
