def benchmark_size(bench, metrics): """Compute the total size of the desired sections in a benchmark. Returns the size in bytes, which may be zero if the section wasn't found.""" appexe = os.path.join(gp['bd_benchdir'], bench, bench) sec_sizes = {} # If the benchmark failed to build, then return a 0 size instead of # crashing when failing to open the file. if not os.path.exists(appexe): return {} # read format from file and check it is as expected with open(appexe, 'rb') as fileh: magic = fileh.read(4) # lief does not appear to have ability to work from an already opened file fileh.close() if (((gp['format'] == 'elf') and (magic != b'\x7fELF')) or ((gp['format'] == 'macho') and (magic != b'\xcf\xfa\xed\xfe'))): log.info('ERROR: File format does not match parameter') sys.exit(1) binary = lief.parse(appexe) sections = binary.sections for metric in metrics: sec_sizes[metric] = 0 for target_name in gp['secnames'][metric]: for section in sections: if ((gp['format'] == 'elf' and section.name.startswith(target_name)) or (target_name == section.name)): sec_sizes[metric] += section.size # Return the section (group) size return sec_sizes
def main(): """Main program driving measurement of benchmark size""" # Establish the root directory of the repository, since we know this file is # in that directory. gp['rootdir'] = os.path.abspath(os.path.dirname(__file__)) # Parse arguments using standard technology parser = build_parser() args = parser.parse_args() # Establish logging setup_logging(args.logdir, 'size') log_args(args) # Check args are OK (have to have logging and build directory set up first) validate_args(args) # Find the benchmarks benchmarks = find_benchmarks() log_benchmarks(benchmarks) # Collect the size data for the benchmarks raw_data, rel_data = collect_data(benchmarks) # We can't compute geometric SD on the fly, so we need to collect all the # data and then process it in two passes. We could do the first processing # as we collect the data, but it is clearer to do the three things # separately. Given the size of datasets with which we are concerned the # compute overhead is not significant. if raw_data: embench_stats(benchmarks, raw_data, rel_data) log.info('All benchmarks sized successfully') else: log.info('ERROR: Failed to compute size benchmarks') sys.exit(1)
def main(): """Main program to drive building of benchmarks.""" # Establish the root directory of the repository, since we know this file is # in that directory. gp['rootdir'] = os.path.abspath(os.path.dirname(__file__)) # Parse arguments using standard technology parser = build_parser() args = parser.parse_args() # Establish logging, using "build" as the log file prefix. gp['verbose'] = args.verbose setup_logging(args.logdir, 'build') log_args(args) # Establish build directory create_builddir(args.builddir, args.clean) # Check args are OK (have to have logging and build directory set up first) validate_args(args) # Find the benchmarks benchmarks = find_benchmarks() log_benchmarks(benchmarks) # Establish other global parameters set_parameters(args) log_parameters() log.debug('General log') log.debug('===========') # Set up additional environment variables. set_environ() # Track success successful = compile_support() if successful: log.debug('Compilation of support files successful') for bench in benchmarks: res = compile_benchmark(bench) successful &= res if res: log.debug('Compilation of benchmark "{bench}" successful'.format( bench=bench)) res = link_benchmark(bench) successful &= res if res: log.debug('Linking of benchmark "{bench}" successful'.format( bench=bench)) log.info(bench) if successful: log.info('All benchmarks built successfully')
def main(): """Main program driving measurement of benchmark size""" # Establish the root directory of the repository, since we know this file is # in that directory. gp['rootdir'] = os.path.abspath(os.path.dirname(__file__)) # Parse arguments common to all speed testers, and get list of those # remaining. args, remnant = get_common_args() # Establish logging setup_logging(args.logdir, 'speed') log_args(args) # Check args are OK (have to have logging and build directory set up first) validate_args(args) # Find the benchmarks benchmarks = find_benchmarks() log_benchmarks(benchmarks) # Collect the size data for the benchmarks. Pass any remaining args. raw_data, rel_data = collect_data(benchmarks, remnant) # We can't compute geometric SD on the fly, so we need to collect all the # data and then process it in two passes. We could do the first processing # as we collect the data, but it is clearer to do the three things # separately. Given the size of datasets with which we are concerned the # compute overhead is not significant. if raw_data: if gp['output_format'] != output_format.BASELINE: opt_comma = ',' if args.json_comma else '' embench_stats(benchmarks, raw_data, rel_data, 'speed', opt_comma) log.info('All benchmarks run successfully') else: log.info('ERROR: Failed to compute speed benchmarks') sys.exit(1)
def collect_data(benchmarks): """Collect and log all the raw and optionally relative data associated with the list of benchmarks supplied in the "benchmarks" argument. Return the raw data and relative data as a list. The raw data may be empty if there is a failure. The relative data will be empty if only absolute results have been requested.""" # Baseline data is held external to the script. Import it here. gp['baseline_dir'] = os.path.join(gp['rootdir'], 'baseline-data', 'size.json') with open(gp['baseline_dir']) as fileh: baseline_all = loads(fileh.read()) # Compute the baseline data we need baseline = {} for bench, data in baseline_all.items(): baseline[bench] = 0 for sec in gp['metric']: if sec in data: baseline[bench] += data[sec] # Collect data and output it successful = True raw_data = {} rel_data = {} log.info('Benchmark size') log.info('--------- ----') for bench in benchmarks: raw_data[bench] = benchmark_size(bench) rel_data[bench] = {} output = {} # Zero is a valid section size, although empty .text should be a # concern. if gp['absolute']: # Want absolute results. Only include non-zero values output = f'{raw_data[bench]:8,}' else: # Want relative results (the default). If baseline is zero, just # use 0.0 as the value. if baseline[bench] > 0: rel_data[bench] = raw_data[bench] / baseline[bench] else: rel_data[bench] = 0.0 output = f' {rel_data[bench]:6.2f}' log.info(f'{bench:15} {output:8}') if successful: return raw_data, rel_data # Otherwise failure return return [], []
def collect_data(benchmarks, remnant): """Collect and log all the raw and optionally relative data associated with the list of benchmarks supplied in the "benchmarks" argument. "remant" is left over args from the command line, which may be useful to the benchmark running procs. Return the raw data and relative data as a list. The raw data may be empty if there is a failure. The relative data will be empty if only absolute results have been requested.""" # Baseline data is held external to the script. Import it here. gp['baseline_dir'] = os.path.join( gp['rootdir'], 'baseline-data', 'speed.json' ) with open(gp['baseline_dir']) as fileh: baseline = loads(fileh.read()) # Parse target specific args target_args = get_target_args(remnant) # Collect data and output it successful = True raw_data = {} rel_data = {} log.info('Benchmark Speed') log.info('--------- -----') for bench in benchmarks: raw_data[bench] = benchmark_speed(bench, target_args) rel_data[bench] = 0.0 if raw_data[bench] == 0.0: del raw_data[bench] del rel_data[bench] successful = False else: output = '' if gp['absolute']: # Want absolute results. Only include non-zero values output = f'{round(raw_data[bench]):8,}' else: # Want relative results (the default). Only use non-zero values. rel_data[bench] = raw_data[bench] / baseline[bench] output = f' {rel_data[bench]:6.2f}' log.info(f'{bench:15} {output:8}') if successful: return raw_data, rel_data # Otherwise failure return return [], []
def collect_data(benchmarks, remnant): """Collect and log all the raw and optionally relative data associated with the list of benchmarks supplied in the "benchmarks" argument. "remant" is left over args from the command line, which may be useful to the benchmark running procs. Return the raw data and relative data as a list. The raw data may be empty if there is a failure. The relative data will be empty if only absolute results have been requested.""" # Baseline data is held external to the script. Import it here. speed_baseline = os.path.join(gp['baseline_dir'], 'speed.json') with open(speed_baseline) as fileh: baseline = loads(fileh.read()) # Parse target specific args target_args = get_target_args(remnant) # Collect data successful = True raw_data = {} rel_data = {} for bench in benchmarks: raw_data[bench] = benchmark_speed(bench, target_args) rel_data[bench] = 0.0 if raw_data[bench] == 0.0: del raw_data[bench] del rel_data[bench] successful = False else: if (not gp['absolute'] and gp['output_format'] != output_format.BASELINE): rel_data[bench] = baseline[bench] / raw_data[bench] # Output it if gp['output_format'] == output_format.JSON: log.info(' "speed results" :') log.info(' { "detailed speed results" :') for bench in benchmarks: output = '' if raw_data[bench] != 0.0: if gp['absolute']: output = f'{round(raw_data[bench])}' else: output = f'{rel_data[bench]:.2f}' if bench == benchmarks[0]: log.info(f' {{ ' + f'"{bench}" : {output},') elif bench == benchmarks[-1]: log.info(f' "{bench}" : {output}') else: log.info(f' "{bench}" : {output},') log.info(' },') elif gp['output_format'] == output_format.TEXT: log.info('Benchmark Speed') log.info('--------- -----') for bench in benchmarks: output = '' if raw_data[bench] != 0.0: if gp['absolute']: output = f'{round(raw_data[bench]):8,}' else: output = f' {rel_data[bench]:6.2f}' # Want relative results (the default). Only use non-zero values. log.info(f'{bench:15} {output:8}') elif gp['output_format'] == output_format.BASELINE: log.info('{') for bench in benchmarks: if bench == benchmarks[-1]: log.info(f' "{bench}" : {raw_data[bench]}') else: log.info(f' "{bench}" : {raw_data[bench]},') log.info('}') if successful: return raw_data, rel_data # Otherwise failure return return [], []
def collect_data(benchmarks): """Collect and log all the raw and optionally relative data associated with the list of benchmarks supplied in the "benchmarks" argument. Return the raw data and relative data as a list. The raw data may be empty if there is a failure. The relative data will be empty if only absolute results have been requested. Note that we manually generate the JSON output, rather than using the dumps method, because the result will be manually edited, and we want to guarantee the layout.""" # Baseline data is held external to the script. Import it here. size_baseline = os.path.join(gp['baseline_dir'], 'size.json') with open(size_baseline) as fileh: baseline_all = loads(fileh.read()) # Compute the baseline data we need baseline = {} for bench, data in baseline_all.items(): baseline[bench] = 0 for sec in gp['metric']: baseline[bench] += data[sec] successful = True raw_section_data = {} raw_totals = {} rel_data = {} # Collect data for bench in benchmarks: if gp['output_format'] == output_format.BASELINE: raw_section_data[bench] = benchmark_size(bench, ALL_METRICS) else: raw_section_data[bench] = benchmark_size(bench, gp['metric']) raw_totals[bench] = sum(raw_section_data[bench].values()) # Calculate data relative to the baseline if needed if gp['absolute'] or gp['output_format'] == output_format.BASELINE: rel_data[bench] = {} else: # Want relative results (the default). If baseline is zero, just # use 0.0 as the value. Note this is inverted compared to the # speed benchmark, so SMALL is good. if baseline[bench] > 0: rel_data[bench] = raw_totals[bench] / baseline[bench] else: rel_data[bench] = 0.0 # Output it if gp['output_format'] == output_format.JSON: log.info(' "size results" :') log.info(' { "detailed size results" :') for bench in benchmarks: res_output = '' if gp['absolute']: res_output = f'{raw_totals[bench]}' else: res_output = f'{rel_data[bench]:.2f}' if bench == benchmarks[0]: log.info(' { ' + f'"{bench}" : {res_output},') elif bench == benchmarks[-1]: log.info(f' "{bench}" : {res_output}') else: log.info(f' "{bench}" : {res_output},') log.info(' },') elif gp['output_format'] == output_format.TEXT: log.info('Benchmark size') log.info('--------- ----') for bench in benchmarks: res_output = '' if gp['absolute']: res_output = f' {raw_totals[bench]:8,}' else: res_output = f' {rel_data[bench]:6.2f}' log.info(f'{bench:15} {res_output:8}') elif gp['output_format'] == output_format.BASELINE: log.info('{') for bench in benchmarks: res_output = '' for metric in ALL_METRICS: # newline before the first metric if metric != ALL_METRICS[0]: res_output += ',\n' value = raw_section_data[bench][metric] res_output += f' "{metric}" : {value}' # comma after all but last benchmark in the log if bench == benchmarks[-1]: log.info(f' "{bench}" : {{\n{res_output}\n }}') else: log.info(f' "{bench}" : {{\n{res_output}\n }},') log.info('}') if successful: return raw_totals, rel_data # Otherwise failure return return [], []
def collect_data(benchmarks): """Collect and log all the raw and optionally relative data associated with the list of benchmarks supplied in the "benchmarks" argument. Return the raw data and relative data as a list. The raw data may be empty if there is a failure. The relative data will be empty if only absolute results have been requested. Note that we manually generate the JSON output, rather than using the dumps method, because the result will be manually edited, and we want to guarantee the layout.""" # Baseline data is held external to the script. Import it here. gp['baseline_dir'] = os.path.join(gp['rootdir'], 'baseline-data', 'size.json') with open(gp['baseline_dir']) as fileh: baseline_all = loads(fileh.read()) # Compute the baseline data we need baseline = {} for bench, data in baseline_all.items(): baseline[bench] = 0 for sec in gp['metric']: baseline[bench] += data[sec] # Collect data and output it successful = True raw_data = {} rel_data = {} if gp['json']: log.info(' "size results" :') log.info(' { "detailed size results" :') else: log.info('Benchmark size') log.info('--------- ----') for bench in benchmarks: raw_data[bench] = benchmark_size(bench) rel_data[bench] = {} output = {} # Zero is a valid section size, although empty .text should be a # concern. if gp['absolute']: # Want absolute results. Only include non-zero values if gp['json']: output = f'{raw_data[bench]}' else: output = f' {raw_data[bench]:8,}' else: # Want relative results (the default). If baseline is zero, just # use 0.0 as the value. Note this is inverted compared to the # speed benchmark, so SMALL is good. if baseline[bench] > 0: rel_data[bench] = raw_data[bench] / baseline[bench] else: rel_data[bench] = 0.0 if gp['json']: output = f'{rel_data[bench]:.2f}' else: output = f' {rel_data[bench]:6.2f}' if gp['json']: if bench == benchmarks[0]: log.info(' { ' + f'"{bench}" : {output},') elif bench == benchmarks[-1]: log.info(f' "{bench}" : {output}') else: log.info(f' "{bench}" : {output},') else: log.info(f'{bench:15} {output:8}') if gp['json']: log.info(' },') if successful: return raw_data, rel_data # Otherwise failure return return [], []