def _Run(vm): """See base method. Args: vm: The vm to run the benchmark on. Returns: A list of sample.Sample objects. """ # swap only if necessary; free local node memory and avoid remote memory; # reset caches; set stack size to unlimited # Also consider setting enable_transparent_hugepages flag to true cmd = ('echo 1 | sudo tee /proc/sys/vm/swappiness && ' 'echo 1 | sudo tee /proc/sys/vm/zone_reclaim_mode && ' 'sync ; echo 3 | sudo tee /proc/sys/vm/drop_caches && ' 'ulimit -s unlimited && ') cmd += 'runcpu ' if FLAGS.spec17_build_only: cmd += '--action build ' if FLAGS.spec17_rebuild: cmd += '--rebuild ' version_specific_parameters = [] # rate runs require 2 GB minimum system main memory per copy, # not including os overhead. Refer to: # https://www.spec.org/cpu2017/Docs/system-requirements.html#memory copies = min(vm.NumCpusForBenchmark(), vm.total_free_memory_kb // (2 * KB_TO_GB_MULTIPLIER)) version_specific_parameters.append(' --copies=%s ' % (FLAGS.spec17_copies or copies)) version_specific_parameters.append( ' --threads=%s ' % (FLAGS.spec17_threads or vm.NumCpusForBenchmark())) if FLAGS.spec17_fdo: version_specific_parameters.append('--feedback ') vm.RemoteCommand('cd /scratch/cpu2017; mkdir fdo_profiles') start_time = time.time() stdout, _ = speccpu.Run(vm, cmd, ' '.join(FLAGS.spec17_subset), version_specific_parameters) if FLAGS.spec17_build_only: if 'Error' in stdout and 'Please review this file' in stdout: raise errors.Benchmarks.RunError('Error during SPEC compilation.') return [ sample.Sample( 'compilation_time', time.time() - start_time, 's', { 'spec17_subset': FLAGS.spec17_subset, 'gcc_version': build_tools.GetVersion(vm, 'gcc') }) ] partial_results = True # Do not allow partial results if any benchmark subset is a full suite. for benchmark_subset in FLAGS.benchmark_subset: if benchmark_subset in ['intspeed', 'fpspeed', 'intrate', 'fprate']: partial_results = False log_files = set() for test in FLAGS.spec17_subset: if test in LOG_FILENAME: log_files.add(LOG_FILENAME[test]) else: if test in INTSPEED_SUITE: log_files.add(LOG_FILENAME['intspeed']) elif test in INTRATE_SUITE: log_files.add(LOG_FILENAME['intrate']) elif test in FPSPEED_SUITE: log_files.add(LOG_FILENAME['fpspeed']) elif test in FPRATE_SUITE: log_files.add(LOG_FILENAME['fprate']) samples = speccpu.ParseOutput(vm, log_files, partial_results, None) for item in samples: item.metadata['vm_name'] = vm.name return samples
def _ExtractScore(stdout, vm, keep_partial_results, runspec_metric): """Extracts the SPEC(int|fp) score from stdout. Args: stdout: String. stdout from running RemoteCommand. vm: The vm instance where SPEC CPU was run. keep_partial_results: Boolean. True if partial results should be extracted in the event that not all benchmarks were successfully run. See the "runspec_keep_partial_results" flag for more info. runspec_metric: String. Indicates whether this is spec speed or rate run. Sample input for SPECint (Refer to unit test for more examples): ... ...Base Peak ============================================= ========================== 400.perlbench 9770 417 23.4 * 9770 417 23.4 * 401.bzip2 9650 565 17.1 * 9650 565 17.1 * 403.gcc 8050 364 22.1 * 429.mcf 9120 364 25.1 * 445.gobmk 10490 499 21.0 * 456.hmmer 9330 491 19.0 * 458.sjeng 12100 588 20.6 * 462.libquantum 20720 468 44.2 * 464.h264ref 22130 700 31.6 * 471.omnetpp 6250 349 17.9 * 473.astar 7020 482 14.6 * 483.xalancbmk 6900 248 27.8 * Est. SPECint(R)_base2006 22.7 Est. SPECint(R)_peak2006 20 Sample input for SPECfp: ... ...Base Peak ============================================= ============================ 410.bwaves 13590 717 19.0 * 13550 710 19.0 * 416.gamess 19580 923 21.2 * 433.milc 9180 480 19.1 * 434.zeusmp 9100 600 15.2 * 435.gromacs 7140 605 11.8 * 436.cactusADM 11950 1289 9.27 * 437.leslie3d 9400 859 10.9 * 444.namd 8020 504 15.9 * 447.dealII 11440 409 28.0 * 450.soplex 8340 272 30.6 * 453.povray 5320 231 23.0 * 454.calculix 8250 993 8.31 * 459.GemsFDTD 10610 775 13.7 * 465.tonto 9840 565 17.4 * 470.lbm 13740 365 37.7 * 481.wrf 11170 788 14.2 * 482.sphinx3 19490 668 29.2 * Est. SPECfp(R)_base2006 17.5 Est. SPECfp(R)_peak2006 20 Returns: A list of sample.Sample objects. """ results = [] speccpu_vm_state = getattr(vm, VM_STATE_ATTR, None) re_begin_section = re.compile('^={1,}') re_end_section = re.compile(speccpu_vm_state.log_format) result_section = [] in_result_section = False at_peak_results_line, peak_name, peak_score = False, None, None # Extract the summary section for line in stdout.splitlines(): if in_result_section: result_section.append(line) # search for begin of result section match = re.search(re_begin_section, line) if match: assert not in_result_section in_result_section = True continue # search for end of result section match = re.search(re_end_section, line) if at_peak_results_line: _, peak_name, peak_score = line.split() at_peak_results_line = False if match: assert in_result_section spec_name = str(match.group(1)) if runspec_metric == 'speed': spec_name += ':speed' try: spec_score = float(match.group(2)) except ValueError: # Partial results may get reported as '--' instead of a number. spec_score = None if FLAGS.spec_runmode != BASE_MODE: at_peak_results_line = True in_result_section = False # remove the final SPEC(int|fp) score, which has only 2 columns. result_section.pop() metadata = { 'runspec_config': speccpu_vm_state.runspec_config, 'runspec_config_md5sum': _GenerateMd5sum(speccpu_vm_state.runspec_config), 'runspec_iterations': str(FLAGS.runspec_iterations), 'runspec_enable_32bit': str(FLAGS.runspec_enable_32bit), 'runspec_define': FLAGS.runspec_define, 'runspec_metric': runspec_metric, 'spec_runmode': FLAGS.spec_runmode, 'spec17_copies': FLAGS.spec17_copies, 'spec17_threads': FLAGS.spec17_threads, 'spec17_fdo': FLAGS.spec17_fdo, 'spec17_subset': FLAGS.spec17_subset, 'gcc_version': build_tools.GetVersion(vm, 'gcc') } missing_results = [] scores = [] for benchmark in result_section: # Skip over failed runs, but count them since they make the overall # result invalid. not_reported = benchmark.count('NR') if not_reported > 1 or (not_reported == 1 and FLAGS.spec_runmode != PEAK_MODE): logging.warning('SPEC CPU missing result: %s', benchmark) missing_results.append(str(benchmark.split()[0])) continue base_score_str, peak_score_str = None, None if FLAGS.spec_runmode == BASE_MODE: # name, copies/threads, time, score, misc name, _, _, base_score_str, _ = benchmark.split() elif FLAGS.spec_runmode == PEAK_MODE: # name, base_not_reported(NR), copies/threads, time, score, misc name, _, _, _, peak_score_str, _ = benchmark.split() else: # name, copies/threads, base time, base score, base misc, # copies/threads, peak time, peak score, peak misc name, _, _, base_score_str, _, _, _, peak_score_str, _ = benchmark.split( ) if runspec_metric == 'speed': name += ':speed' if base_score_str: base_score_float = float(base_score_str) scores.append(base_score_float) results.append( sample.Sample(str(name), base_score_float, '', metadata)) if peak_score_str: peak_score_float = float(peak_score_str) results.append( sample.Sample( str(name) + ':peak', peak_score_float, '', metadata)) if spec_score is None and FLAGS.spec_runmode != PEAK_MODE: missing_results.append(spec_name) if missing_results: if keep_partial_results: metadata['partial'] = 'true' metadata['missing_results'] = ','.join(missing_results) else: raise errors.Benchmarks.RunError( 'speccpu: results missing, see log: ' + ','.join(missing_results)) if spec_score: results.append(sample.Sample(spec_name, spec_score, '', metadata)) elif FLAGS.runspec_estimate_spec: estimated_spec_score = _GeometricMean(scores) results.append( sample.Sample('estimated_' + spec_name, estimated_spec_score, '', metadata)) if peak_score: results.append( sample.Sample(peak_name, float(peak_score), '', metadata)) return results