def get_cpi(dir):
try:
r = sniper_lib.get_results(0, dir)
except ValueError:
msg.PrintMsg('\nERROR: Can\'t get sniper results for:\n ' +
dir)
return 0.0
stats = r['results']
if stats['ncores'] != 1:
msg.PrintMsgPlus(
'Warning: Sniper only supports single-threaded SimPoints')
# This code works with both Sniper/SniperLite
#
instrs = stats['core.instructions']
fs_to_cycles = stats['fs_to_cycles_cores']
fs = stats['barrier.global_time']
for i, d in enumerate(instrs):
cpi = (float(fs[0]) * fs_to_cycles[i]) / instrs[i]
if sum(instrs) <= 0:
msg.PrintMsgPlus(
'\nERROR: No cycles found in Sniper output for pinball:\n '
+ dir)
cpi = 0.0
return cpi
def VerifyLITFiles(self, sim_run_cmd, options):
"""
Use a simulator to verify all the LIT files for the tracing instance are valid.
"""
# Verify the LIT files for each directory.
#
# import pdb; pdb.set_trace()
result = 0
for lit_dir in util.GetLitDir():
if not options.list:
msg.PrintMsgPlus('Verifying files in dir: ' + lit_dir)
# import pdb; pdb.set_trace()
cmd = sim_run_cmd + ' --replay_dir ' + lit_dir
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
cmd += ' --verify'
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
result = util.RunCmd(cmd, options, '',
concurrent=True) # Run jobs concurrently
if result != 0:
msg.PrintMsg('Error found during LIT file verification')
return result
# Wait for all the verification jobs to finish.
#
result = util.WaitJobs(options)
return result
def GetLastMetric(self, sim_file, tid, options):
"""
Get the last metric in a simulator output file. This is the value
for running the entire pinball.
@param sim_file File with simulator results to process
@param options TID of results to be processed
@param options Options given on cmd line
@return metric
"""
# Get a file pointer to the simulator data file.
#
f = util.OpenCompressFile(sim_file)
if f == None:
# Error opening file, return an error.
#
return -1.0
# This is the code which needs to be modified in order to use a
# different metric of interest for a new simulator. The existing code
# uses the metric MPI (misses per thousand instruction).
#
# Current code assume the default Branch Predictor simulator is used. Get the
# number of instructions and misses for this thread in the last line of
# the output.
#
# import pdb ; pdb.set_trace()
instr = misses = 0
for line in f.readlines():
pos = line.find('Icount: ')
if pos != -1:
last = line
# import pdb ; pdb.set_trace()
lst = last.split()
instr = int(lst[1])
misses = int(lst[3])
# Check to make sure there really is valid data. If not, the print a
# warning. No need to exit with an error, because it's possible for
# MPI_MT_MODE applications to have a different number of threads in
# each process. This means some processes may have a thread 'tid',
# while this process may not.
#
if instr > 1:
metric = misses / (float(instr) / 1000)
else:
msg.PrintMsgPlus('WARNING: There were no instructions in simulator output for thread ' + \
str(tid) + ' in file:\n ' + sim_file)
msg.PrintMsg('Prediction error for this process may be suspect.')
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
metric = -1.0 # Error indication
return metric
def CombineFreqVectFiles(self, options):
"""
Combine the BBV and LDV files, applying a scaling factor to allow
different contributions from each file.
@param options Options given on cmd line
@return result of command to generate file
"""
# Format the command and run it.
#
# import pdb; pdb.set_trace()
# Use options for the Python script to generate the CSV file.
#
output_file = 'scaled_combined.out'
try:
fp_error = open(output_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open combined scale error file:\n'
' ' + output_file)
return -1
try:
fp_out = open(self.freq_vect_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open combined scale output file:\n'
' ' + self.freq_vect_file)
fp_error.close()
return -1
cmd = self.csv_bin
cmd += ' --combine ' + str(options.combine)
string = 'Combining BBV and LDV files with scaling factors: BBV: %.3f, LDV: %.3f\n' % \
(options.combine, 1 - options.combine)
msg.PrintMsgPlus(string)
fp_error.write(string)
cmd += ' --normal_bbv ' + self.proj_bbv_file
cmd += ' --normal_ldv ' + self.weight_ldv_file
result = util.RunCmd(cmd, options, '', concurrent=False, f_stdout=fp_out, \
f_stderr=fp_error)
msg.PrintMsg(' Output file: %s' % (self.freq_vect_file))
msg.PrintMsg(' Stderr file: %s\n' % (output_file))
fp_out.close()
fp_error.close()
return result
def RunMPDelete(self, kit, log_key, options):
"""
Delete mp_pool when logging MPI apps.
@param kit Kit instance for these scripts
@param log_key Key to mp_pool to delete
@param options Options given on cmd line
@return No return value
"""
# Start command with pin and pintool.
#
# import pdb; pdb.set_trace()
cmd = os.path.join(kit.path, kit.pin)
cmd += kit.GetPinToolKnob()
# Remove the string '-log:mode_attach' because it's only
# used for logging.
#
pos = log_key.find('-log:mode_attach')
if pos != -1:
log_key = log_key[0:pos]
# Logging mp_pool deletion knobs.
#
cmd += ' -log '
cmd += log_key
cmd += ' -log:mp_delete_pool '
cmd += ' -- echo'
# Execute the cmd
#
if (not hasattr(options, 'no_print_cmd') or not options.no_print_cmd) or\
hasattr(options, 'verbose') and options.verbose:
msg.PrintMsgPlus('Deleting mp_pools for MPI tracing')
msg.PrintMsg(cmd)
if not config.debug:
p = subprocess.Popen(cmd, shell=True)
p.communicate()
del p
return
def PrintHome(self, options):
"""
Print the directories where the Pin kit and Sniper are located.
@return No return value
"""
super(SniperPinPoints,
self).PrintHome(options) # Call base class method PrintHome()
if options.sniper_root:
msg.PrintMsg('Sniper_root: ' +
os.path.realpath(options.sniper_root))
else:
msg.PrintMsgPlus('WARNING: Sniper root not defined.\n')
if options.sniper_options:
msg.PrintMsg('Sniper options: ' +
options.sniper_options)
if options.no_sniperlite or options.sniper_options:
msg.PrintMsg('Sniper type: Full Sniper simulator')
else:
msg.PrintMsg('Sniper type: Sniperlite')
def RunMPCreate(self, kit, options):
"""
Create mp_pool when logging MPI apps.
@param kit Kit instance for these scripts
@param options Options given on cmd line
@return String with MP key and a knob required for mp_pools, or null string
"""
# Start command with pin and pintool.
#
# import pdb; pdb.set_trace()
cmd = os.path.join(kit.path, kit.pin)
cmd += kit.GetPinToolKnob()
# Logging mp_pool creation knobs.
#
log_key = ' -log:mp_key ' + str(random.randint(0, 32728))
cmd += ' -log '
cmd += log_key
cmd += ' -log:mp_create_pool '
cmd += ' -- echo'
# Execute the cmd
#
if (not hasattr(options, 'no_print_cmd') or not options.no_print_cmd) or\
hasattr(options, 'verbose') and options.verbose:
msg.PrintMsgPlus('Creating mp_pools for MPI tracing')
msg.PrintMsg(cmd)
if not config.debug:
p = subprocess.Popen(cmd, shell=True)
p.communicate()
del p
# Return the key to the memory pool just created.
#
return log_key
def ParseCommandLine(self):
"""
Get the options from the command line and check for errors.
@return tuple with parsed options and unparsed args
"""
# Define and get command line options.
#
version = '$Revision: 1.33 $'
version = version.replace('$Revision: ', '')
ver = version.replace(' $', '')
us = '%prog --bbv_file FILE --data_dir DIR FILE --simpoint_file FILE [options]'
desc = 'Runs Simpoint and then generates the region CSV file. ' \
'Input to Simpoint can be just an BBV file or a combination of BBV/LDV files. \n\n' \
'Required options: --bbv_file, --data_dir, --simpoint_file'
util.CheckNonPrintChar(sys.argv)
parser = optparse.OptionParser(
usage=us,
version=ver,
description=desc,
formatter=cmd_options.BlankLinesIndentedHelpFormatter())
cmd_options.debug(parser)
cmd_options.global_file(parser)
cmd_options.list(parser, '')
cmd_options.bbv_file(parser, '')
cmd_options.data_dir(parser)
cmd_options.simpoint_file(parser)
cmd_options.ldv(parser, '')
cmd_options.combine(parser, '')
cmd_options.cutoff(parser, '')
cmd_options.focus_thread(parser, '')
cmd_options.maxk(parser, '')
cmd_options.num_cores(parser, '')
cmd_options.simpoint_options(parser, '')
(options, args) = parser.parse_args()
# Added method cbsp() to 'options' to check if running CBSP.
#
util.AddMethodcbsp(options)
# Must have option '--ldv', even if using option '--combine', in order to
# process BBV/LDV both. Let user know if '--combine' used w/o '--ldv'.
#
if not options.ldv and options.combine != -1.0:
msg.PrintMsgPlus('WARNING: Option \'--combine\' detected without \'--ldv\'. Only using BBV for ' \
'Simpoint. \n Must explicitly specify \'--ldv\' in order to use both BBV/LDV.\n')
if options.ldv:
msg.PrintMsgPlus(
'Using both BBV/LDV files when running Simpoint\n')
# If option combine is not set, then set it to the default value.
# Check to make sure combine an acceptable value.
#
util.SetCombineDefault(options)
util.CheckCombine(options)
# Read in an optional configuration files and set global variables.
#
config_obj = config.ConfigClass()
config_obj.GetCfgGlobals(options,
False) # Don't need to require 4 variables
# Error check input to make sure all required options are on the command line.
#
if options.bbv_file == '':
msg.PrintAndExit(
'Basic block vector file must be defined with option: --bbv_file FILE'
)
if options.data_dir == '':
msg.PrintAndExit(
'Simpoint data directory must be defined with option: --data_dir DIR'
)
if options.simpoint_file == '':
msg.PrintAndExit(
'Simpoint output must be defined with option: --simpoint_file FILE'
)
# The data_dir should exist and contain the BBV file.
#
if not os.path.isdir(options.data_dir):
msg.PrintAndExit('Data directory does not exist: ' +
options.data_dir)
if not os.path.isfile(os.path.join(options.data_dir,
options.bbv_file)):
msg.PrintAndExit('Basic block vector file does not exist: ' +
options.bbv_file)
# Do some 'special' things on native Windows.
#
util.WindowsNativeCheck(options)
return (options, args)
def RunSniper(pp_dir, sim_replay_cmd, phase_length, options):
def round(x, roundby=1000000):
return int(int((x / float(roundby))) * float(roundby))
# List of pinballs used to print Sniper output when all runs are complete.
#
pb_list = []
ret = 0
if not os.path.isdir(pp_dir):
# If running in MPI_MT_MODE, then it's possible for one process to not
# have a thread corresponding to the the current focus thread.
# However, another process might have this thread. Thus, only return
# an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg('WARNING: Directory containing pinballs to run with simulator does not exist:\n ' + \
pp_dir)
msg.PrintMsg('Since tracing mode is \'mpi_mt\', this may be OK.')
return 0
else:
msg.PrintMsg('ERROR: Directory containing pinballs to run with simulator does not exist:\n ' + \
pp_dir)
return -1
# List of output sniper directories.
#
output_dir_list = []
for fn in os.listdir(pp_dir):
if fn.endswith('.address'):
pinball_path = os.path.join(pp_dir, os.path.splitext(fn)[0])
fn = os.path.splitext(fn)[0]
sniper_outputdir = os.path.join(config.sniper_result_dir,
pinball_path)
output_dir_list += [sniper_outputdir]
sniper_outputfile = pinball_path + config.sniper_out_ext
if options.debug:
# If debugging, check to see if the Sniper result files already exist for the pinball. If so,
# then print it out but don't run Sniper again.
#
if os.path.isdir(sniper_outputdir):
msg.PrintMsgPlus(
'WARNING: Skipping Sniper execution because output file already exists.\n'
' %s' % sniper_outputdir)
pb_list.append(pinball_path)
continue
# Select the proper config/options to run the desired version of
# Sniper/SniperLite.
#
# import pdb; pdb.set_trace()
if options.sniper_options:
common_sniper_opts = options.sniper_options
else:
if options.no_sniperlite:
common_sniper_opts = ''
else:
# use_orig = True # Use older SniperLite options
use_orig = False # New SniperLite options
if use_orig:
common_sniper_opts = ' -c dunnington -c cacheonly -c nehalem_cmpsim.cfg ' \
'-g --general/enable_icache_modeling=false ' \
'-g --perf_model/dram/direct_access=true ' \
'-g --perf_model/dram/queue_model/type=contention ' \
'-g --perf_model/dtlb/size=0'
else:
# Older patched Sniper 5.3
#
# common_sniper_opts = ' -c dunnington -c nehalem_cmpsim.cfg -c ccpp1c --pinball-non-sift \
# -g -replay:addr_trans -g --general/enable_icache_modeling=false'
# Newer patched Sniper 5.3
#
# common_sniper_opts = ' -c dunnington -c nehalem_cmpsim.cfg -c cc-fast --pinball-non-sift \
# -g -replay:addr_trans -g --general/enable_icache_modeling=false'
# Production SniperLite 6.0 options
#
common_sniper_opts = ' -c nehalem-lite --pinball-non-sift '
partial_run_cmd = (common_sniper_opts + ' --no-cache-warming')
try:
# If re.search() fails, code falls though to the exception.
#
warmup_region = re.search('warmup(\d+)_prolog(\d+)_region(\d+)_epilog(\d+)_(\d+)_(\d-\d+)', \
pinball_path)
# Get info on the length of regions in the pinball.
#
warmup = int(warmup_region.group(1))
prolog = int(warmup_region.group(2))
file_region = int(warmup_region.group(3))
epilog = int(warmup_region.group(4))
region_num = int(warmup_region.group(5))
weight = warmup_region.group(6).replace('-', '.')
icount = util.GetMaxIcount('', pinball_path)
sim_region_len = icount - warmup
calc_region = sim_region_len - prolog - epilog
if warmup != 0:
# If there are warmups, then need to use options to first do cache warmups in Sniper,
# then simulate the region.
#
partial_run_cmd = (common_sniper_opts + ' -s stop-by-icount:%d:%d ' \
% (sim_region_len, round(warmup)) + ' --roi-script ')
if not options.list:
# Print details about the various sections in the pinball.
#
msg.PrintMsgPlus('Running Sniper on: ' + fn)
msg.PrintMsg(' Warmup count: ' +
locale.format('%14d', warmup, True))
msg.PrintMsg(' Prolog count: ' +
locale.format('%14d', prolog, True))
msg.PrintMsg(' Actual region count: ' + locale.format('%14d', calc_region, True) + \
' (from file name: ' + locale.format('%d', file_region, True) + ')')
msg.PrintMsg(' Epilog count: ' +
locale.format('%14d', epilog, True))
msg.PrintMsg(' Total Instr count: ' +
locale.format('%14d', icount, True))
except AttributeError:
if 'whole_program' not in pinball_path:
# Whole program pinballs cannot be processed using the options
# given above (using -s stop-by-icount:) because they don't have
# warmup instructions to be skipped.
#
# Let the user know the results may be suspect because the dir
# appears to contain whole program pinballs, but the name doesn't
# contain the string 'whole_program'. However, don't flag this as
# an erorr. It's possible for the user to give the name of a WP
# pinball directory which does not contain this string.
#
msg.PrintMsgPlus(
'WARNING: This pinball may not really be a whole program pinball.\n'
' If this is true, the results may not be valid.\n'
' ' + pinball_path)
msg.PrintMsgPlus('Running Sniper on whole program pinball: ' +
pinball_path)
# Format the command and execute it asynchronously.
#
cmd = os.path.join(options.sniper_root, sim_replay_cmd) + partial_run_cmd + \
(' -d "%s" ' % sniper_outputdir) + ' --pinballs ' + pinball_path + \
' > ' + sniper_outputfile + ' 2>&1 '
pb_list.append(pinball_path)
end_str = fn
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=True)
ret = result or ret
result = util.WaitJobs(options)
ret = result or ret
# Make sure some of the important Sniper output files exist for each pinball.
#
file_list = ['sim.stats.sqlite3', 'sim.info', 'sim.cfg', 'sim.out']
for sim_dir in output_dir_list:
for f in file_list:
if not os.path.isfile(os.path.join(sim_dir, f)):
msg.PrintMsg('\nERROR: Sniper output file does not exist:\n' \
' ' + os.path.join(sim_dir, f))
ret = -1
# Define a set of strings which can be ignored as errors if they occur in
# the output. These are special cases which must be added by hand when a
# new application is found that contains a string which might be detected
# as an erorr, but is a normal component of the output from running the
# application.
# Errors generated by Sniper itself which are acceptable.
#
ign_str = [
'void Sift::Writer::Sync(): Assertion',
'Sift::Reader::decodeInstruction'
]
pin_app_term = 'Pin app terminated abnormally'
ign_str += [pin_app_term]
# Strings generated by SPEC CPU2006 benchmarks: dealII-ref-1
#
ign_str += ['Estimated error=']
# Strings generated by MILC
#
ign_str += ['error_per_site', 'No O(a^2) errors', 'error_for_propagator']
# Print the output from running Sniper and look for errors in the output.
#
error = False
for pinball_path in pb_list:
# Get just the pinball name & print it
#
fn = os.path.split(pinball_path)[1]
sniper_outputfile = pinball_path + config.sniper_out_ext
if os.path.isfile(sniper_outputfile):
try:
f_stdout = open(sniper_outputfile, 'r')
except IOError:
msg.PrintMsg(
'ERROR: Can\'t open Sniper output file to look for errors.\n'
' ' + sniper_outputfile)
ret = -1
# Print the output file and look for errors.
#
if not options.list:
if 'whole_program' in pinball_path:
msg.PrintMsg('\nSniper output for: ' + pinball_path)
else:
msg.PrintMsg('\nSniper output for: ' + fn)
for line in f_stdout.readlines():
msg.PrintMsgNoCR(line)
if not options.ignore_sniper_error and \
('ERROR' in line or \
'error' in line or \
'Traceback' in line):
# Check to see if we can accept this line because
# it contains one of the strings we can ignore.
#
ok_ignore = False
for st in ign_str:
if st in line:
ok_ignore = True
break
if not ok_ignore:
# It's truly an error
#
error = True
# Need to print a msg indicating this error is OK to ignore.
#
if pin_app_term in line:
msg.PrintMsg(
'The \'Pin app terminated\' msg can be ignored. '
'It is not an error, just a warning.')
if error:
msg.PrintMsg('\nERROR: Sniper failed with an error.')
ret = -1
error = False # Reset in order to look for error in next pb
return ret
def RunSimPoint(self, kit_path, script_path, options, bin_options):
"""
Run Simpoint in the CBSP Data directory and generate weight files for
each binary in the respective binary Data directory.
@param kit_path Path to kit
@param script_path Explicit path to location in kit where scripts are located
@param options Options given on cmd line
@param bin_options List of options for each CBSP binary
@return exit code from Simpoint
"""
# Get CBSP Data directory and WP pinball basenames. For multi-process
# need to expand list returned by util.GetWPPinballs() to include all
# pinballs in each WP dir.
#
result = 0
cbsp_data_dir = util.GetCBSPDataDir(options)
wp_pinballs = [util.GetWPPinballs(bopt)[0] for bopt in bin_options]
wp_basenames = [
os.path.join(re.sub('whole_program.*/', '', pb))
for pb in wp_pinballs
]
# Go to CBSP Data directory to run Simpoint
#
orig_dir = os.getcwd()
if os.path.isdir(cbsp_data_dir):
os.chdir(cbsp_data_dir)
else:
msg.PrintMsg('ERROR: Unable to change to CBSP Data directory: ' +
cbsp_data_dir)
return -1
# Format the command to run simpoints.
# Use 'bin_options[0]' because parameter should be the same for all
# binaries.
#
sim_out_file = 'run_simpoint_out.txt'
if not options.list:
msg.PrintMsgDate('Running Simpoints for: %s' % options.cbsp_name)
msg.PrintMsgPlus('Simpoint output file (including errors): %s\n' %
os.path.join(cbsp_data_dir, sim_out_file))
if bin_options[0].simpoint_options:
msg.PrintMsgPlus(
'NOTE: Default options for Simpoint not used, only user defined options.'
)
cmd = 'simpoint ' + bin_options[0].simpoint_options
else:
if bin_options[0].maxk:
cmd = 'simpoint -k 2:%d -dim 100 -numInitSeeds 25 -fixedLength off -iters 500' % bin_options[
0].maxk
else:
cmd = 'simpoint -k 2:25 -dim 100 -numInitSeeds 25 -fixedLength off -iters 500'
cmd += ' -saveLabels labels.txt -saveSimpoints simpoints.txt -inputVectorsGzipped -loadFVFile matching-vec-profile.gz'
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
fp_out = open(sim_out_file, 'w')
result = util.RunCmd(cmd,
options,
'',
f_stdout=fp_out,
f_stderr=fp_out)
fp_out.close()
if result != 0:
msg.PrintMsg('\nError found while running Simpoint in dir:\n'
' %s' % os.getcwd())
return result
# Generate the binary Data directories
#
bin_data_dir = []
for basename in wp_basenames:
name = os.path.join('..', '%s.Data' % basename)
bin_data_dir.append(name)
if not os.path.isdir(name):
try:
os.mkdir(name)
except OSError:
msg.PrintAndExit(
'method RunSimPoint(), Unable to make directory: ' +
name)
# Run command to generate weight files for the binaries
#
weight_out_file = 'generate_weights_out.txt'
cmd = os.path.join('make_simpoint_weights.py --weight_file_list ')
for data_dir in bin_data_dir:
cmd += ' %s/weights.txt' % data_dir
if not options.list:
msg.PrintMsgPlus(
'make_simpoint_weights.py output file (including errors): %s\n'
% os.path.join(cbsp_data_dir, weight_out_file))
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
fp_out = open(weight_out_file, 'w')
result = util.RunCmd(cmd,
options,
'',
f_stdout=fp_out,
f_stderr=fp_out)
fp_out.close()
if result != 0:
msg.PrintMsg(
'\nError found while running make_simpoint_weights.py in dir:\n'
' %s' % os.getcwd())
return result
# Copy the simpoints and labels files to binary Data directories
#
#
def copy_file(f, d):
try:
shutil.copy(f, d)
return 0
except IOError:
msg.PrintMsg('\nError found in dir:\n'
' %s\nUnable to copy file:\n %s to %s' %
(os.getcwd(), f, d))
return -1
for data_dir in bin_data_dir:
result = copy_file('simpoints.txt', data_dir)
result = result | copy_file('labels.txt', data_dir)
if result != 0:
return result
# Generate the CSV files in each binary Data directory
#
for data_dir, basename in zip(bin_data_dir, wp_basenames):
# Go to the binary Data directory
#
old_dir = os.getcwd()
os.chdir(data_dir)
# Run the script to generate CSV files for this binary
#
bb_prof = os.path.join('..', cbsp_data_dir,
'%s.bb-profile.bz2' % basename)
csv_file = '%s.pinpoints.csv' % basename
if not options.list:
data_dir_rel_path = os.getcwd().replace(
os.path.join(orig_dir, ''), '')
msg.PrintMsgPlus(
'Any errors from running \'regions.py\' are in: %s' %
os.path.join(data_dir_rel_path, csv_file))
cmd = os.path.join('regions.py --csv_region --bbv_file %s' %
bb_prof)
cmd += ' --region_file=simpoints.txt --weight_file=weights.txt'
cmd += ' > %s 2>&1' % csv_file
msg.PrintMsg('')
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
result = util.RunCmd(cmd, options, '')
if result != 0:
msg.PrintMsg(
'\nError found while generating CSV files in:\n %s' %
os.getcwd())
msg.PrintMsg('Error msgs in file: %s ' % csv_file)
return result
# Return to the CBSP Data directory
#
os.chdir(old_dir)
# Return to original directory
#
os.chdir(orig_dir)
if not options.list:
msg.PrintMsgDate('Finished running Simpoint for: ' +
options.cbsp_name)
return result
def CrossBinaryMatcher(self, kit_path, script_path, options, bin_options):
"""
Run the cross binary matcher on all the binaries in the CBSP experiment.
NOTE: Only works if there is only one WP pinball in each WP pinball
directory (i.e. will NOT work with multipe process applications).
@param options Options given on cmd line
@param kit_path Path to kit
@param script_path Explicit path to location in kit where scripts are located
@param bin_options List of options for each CBSP binary
@return error_code
"""
# Get CBSP Data directory and WP pinball basenames. NOTE: For expanding
# this code to work with multi-process WP pinballs, need to expand list
# returned by util.GetWPPinballs() to include all pinballs in each WP
# dir.
#
result = 0
cbsp_data_dir = util.GetCBSPDataDir(options)
wp_pinballs = []
for bopt in bin_options:
pb = util.GetWPPinballs(bopt)
if not pb:
return -1
wp_pinballs.append(pb[0])
wp_basenames = [
os.path.join(re.sub('whole_program.*/', '', pb))
for pb in wp_pinballs
]
# Go to CBSP Data directory to run matcher
#
orig_dir = os.getcwd()
if os.path.isdir(cbsp_data_dir):
os.chdir(cbsp_data_dir)
else:
msg.PrintMsg('ERROR: Unable to change to CBSP Data directory: ' +
cbsp_data_dir)
return -1
# Matcher output file
#
output_file = 'crossBinaryMatcher_out.txt'
try:
fp_out = open(output_file, 'w')
except IOError:
msg.PrintMsg(
'ERROR: Failed to open cross binary matcher output file:\n'
' ' + output_file)
return -1
# Format cross binary matcher command & execute.
# Use 'bin_options[0]' because parameter should be the same for all
# binaries.
#
if not options.list:
msg.PrintMsgDate('Running cross binary matcher for: ' +
options.cbsp_name)
msg.PrintMsgPlus(
'Cross binary matcher output file (including errors): %s\n' %
os.path.join(cbsp_data_dir, output_file))
cmd = 'crossBinMatcher -graphm_config %s' % os.path.join(
kit_path, script_path, 'graphm.config.txt')
cmd += ' -slice_size %s' % str(bin_options[0].slice_size)
for pb, basename in zip(wp_pinballs, wp_basenames):
cmd += ' %s.{dcfg.json,trace.json}.bz2 ' % os.path.join('..', pb)
cmd += ' %s.bb-profile.bz2 ' % (basename)
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
result = util.RunCmd(cmd,
options,
'',
f_stdout=fp_out,
f_stderr=fp_out)
fp_out.close()
if not options.list:
msg.PrintMsgDate('Finished running cross binary matcher for: ' +
options.cbsp_name)
# Return to original directory
#
os.chdir(orig_dir)
return result
def RunAdditionalPhases(self, sim_replay_cmd, options, bin_options):
"""
Run the CBSP additional phases for SDE which are not run for PinPlay.
@param sim_replay_cmd Script which run the simulator on pinballs
@param options Options given on cmd line
@param options Options for each CBSP binary
@return Exit code from last phase executed
"""
# The SDE phases object needs an SDE kit object.
#
s_phases = sde_phases.SDEPhases()
kit_obj = self.GetKit()
s_phases.SetKit(kit_obj)
# Need the appropriate simulator kit too.
#
sim_kit = self.GetSimKit()
# Generate LMAT/LIT files
#
result = 0
if options.lit_gen or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating traces for region pinballs %s' % \
config.PhaseStr(config.LIT))
util.PhaseBegin(options)
for bopts in bin_options:
wp_dir = util.GetDefaultWPDir(bopts)
log_file_name = util.GetLogFile(bopts)
self.Config.SetPerBinParams(bopts)
result = s_phases.GenAllLitFiles(self.replayer_cmd, bopts)
if result == 0:
if not options.list:
msg.PrintMsgPlus('Waiting on final trace generation')
result = util.WaitJobs(options)
util.CheckResult(result, options, 'Traces for: %s %s' % \
(log_file_name, config.PhaseStr(config.LIT)), intermediate_phase=True)
if not options.list:
msg.PrintMsgDate('Finished generating traces for region pinballs %s' % \
config.PhaseStr(config.LIT))
util.CheckResult(result, options, 'Trace file generation %s' %\
config.PhaseStr(config.LIT))
# Run CMPSim simulator on region pinballs.
#
result = 0
if options.region_sim or options.default_phases:
# Print out CMPSim results every warmup_length instructions.
#
phase_length = options.warmup_length // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
for bopts in bin_options:
self.Config.SetPerBinParams(bopts)
for pp_dir in util.GetRegionPinballDir(bopts):
if not options.list:
msg.PrintMsgDate('Running CMPSim on region pinballs in dir: %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
util.PhaseBegin(options)
result = sim_kit.RunSimulator(pp_dir, sim_replay_cmd,
phase_length, bopts)
util.CheckResult(result, options, 'CMPSim for: %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)), intermediate_phase=True)
if not options.list:
msg.PrintMsgDate('Finished running CMPSim on region pinballs in dir %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
util.CheckResult(result, options, 'CMPSim on region pinballs: %s' % \
config.PhaseStr(config.CMPsim_regions))
# Run CMPSim simulator on whole program pinballs.
#
if options.whole_sim or options.default_phases:
# Set phase_length to print out CMPSim results every slice_size instructions.
#
phase_length = options.slice_size // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
for bopts in bin_options:
if not options.list:
msg.PrintMsgDate('Running CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
util.PhaseBegin(options)
wp_dir = util.GetDefaultWPDir(bopts)
self.Config.SetPerBinParams(bopts)
result = sim_kit.RunSimulator(wp_dir, sim_replay_cmd,
phase_length, bopts)
util.CheckResult(result, options, 'CMPSim for: %s %s' % \
(wp_dir, config.PhaseStr(config.CMPsim_whole)), intermediate_phase=True)
if not options.list:
msg.PrintMsgDate('Finished running CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
util.CheckResult(result, options, 'CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
# Calculate prediction error from simulator data files.
#
if options.pred_error or options.default_phases:
if not options.list:
msg.PrintMsgDate('Calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.PhaseBegin(options)
for bopts in bin_options:
wp_dir = util.GetDefaultWPDir(bopts)
result = s_phases.CalcPredError(wp_dir, sim_kit, bopts)
util.CheckResult(result, options, 'Prediction error for: %s %s' % \
(wp_dir, config.PhaseStr(config.pred_error)), intermediate_phase=True)
if not options.list:
msg.PrintMsgDate('Finished calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.CheckResult(result, options, 'Prediction error calculation %s' % \
config.PhaseStr(config.pred_error))
# Assume nothing has gone wrong at this point.
#
return result
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options):
"""Run the additional phases for SDE which are not run for PinPlay."""
if not options.sniper_root or not os.path.exists(
os.path.join(options.sniper_root, 'run-sniper')):
msg.PrintMsgPlus(
'ERROR: Please set SNIPER_ROOT or --sniper_root to a valid Sniper install.'
)
util.CheckResult(
-1, options,
'Checking for Sniper install location.') # Force error with -1
if options.region_sim or options.default_phases:
# Print out Sniper results every warmup_length instructions.
#
for pp_dir in util.GetRegionPinballDir():
phase_length = options.warmup_length // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
if not options.list:
msg.PrintMsgDate('Running Sniper on region pinballs %s' % \
config.PhaseStr(config.sniper_regions))
util.PhaseBegin(options)
result = RunSniper(pp_dir, sim_replay_cmd, phase_length,
options)
if not options.list:
msg.PrintMsgDate('Finished running Sniper on region pinballs %s' % \
config.PhaseStr(config.sniper_regions))
util.CheckResult(result, options, 'Sniper on region pinballs %s' % \
config.PhaseStr(config.sniper_regions))
if options.whole_sim or options.default_phases:
# Set phase_length to print out Sniper results every slice_size instructions.
#
phase_length = options.slice_size // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
if not options.list:
msg.PrintMsgDate('Running Sniper on whole program pinballs %s' % \
config.PhaseStr(config.sniper_whole))
util.PhaseBegin(options)
result = RunSniper(wp_pb_dir, sim_replay_cmd, phase_length,
options)
if not options.list:
msg.PrintMsgDate('Finished running Sniper on whole program pinballs %s' % \
config.PhaseStr(config.sniper_whole))
util.CheckResult(result, options, 'Sniper on whole program pinballs %s' % \
config.PhaseStr(config.sniper_whole))
if options.pred_error or options.default_phases:
if options.pccount_regions:
msg.PrintMsg('\n Prediction with PCregions is NIY')
return 0
if not options.list:
msg.PrintMsgDate('Calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.PhaseBegin(options)
result = CalcPredError(wp_pb_dir, self.sim_replay_cmd, options)
if not options.list:
msg.PrintMsgDate('Finished calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.CheckResult(result, options, 'Prediction error calculation %s' % \
config.PhaseStr(config.pred_error))
# Assume nothing has gone wrong at this point.
#
return 0
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options):
"""
Run the additional phases for the simulator which are not run as
part of the usual PinPlay phases.
"""
phases_obj = phases.Phases()
self.sim_kit = self.GetSimKit()
msg.PrintMsg('sinuca_tracer_output: ' +
config.sinuca_tracer_output)
msg.PrintMsg('sinuca_tracer_threads: ' +
str(config.sinuca_tracer_threads))
msg.PrintMsg('sinuca_tracer_parallel_start: ' +
str(config.sinuca_tracer_parallel_start))
msg.PrintMsg('sinuca_tracer_parallel_end: ' +
str(config.sinuca_tracer_parallel_end))
if not options.sinuca_tracer_output:
msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_output.')
util.CheckResult(
-1, options,
'Could not find the sinuca_tracer_output parameter.'
) # Force error with -1
# ~ # ~
if not options.sinuca_tracer_threads:
msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_threads.')
util.CheckResult(
-1, options,
'Could not find the sinuca_tracer_threads parameter.'
) # Force error with -1
# ~ # ~
if not options.sinuca_tracer_parallel_start:
msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_parallel_start.')
util.CheckResult(
-1, options,
'Could not find the sinuca_tracer_parallel_start parameter.'
) # Force error with -1
# ~ # ~
if not options.sinuca_tracer_parallel_end:
msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_parallel_end.')
util.CheckResult(
-1, options,
'Could not find the sinuca_tracer_parallel_end parameter.'
) # Force error with -1
# Run branch predictor simulator on region pinballs.
#
if options.region_sim or options.default_phases:
# Print out simulator results every warmup_length instructions.
#
for pp_dir in util.GetRegionPinballDir():
phase_length = options.warmup_length
if phase_length == 0:
phase_length = options.slice_size
if not options.list:
msg.PrintMsgDate('Running simulator on region pinballs: %s' % \
config.PhaseStr(config.sim_regions))
util.PhaseBegin(options)
result = self.sim_kit.RunSimulator(pp_dir, sim_replay_cmd,
options)
if not options.list:
msg.PrintMsgDate('Finished running simulator on region pinballs: %s' % \
config.PhaseStr(config.sim_regions))
util.CheckResult(result, options, 'simulator on region pinballs: %s' % \
config.PhaseStr(config.sim_regions))
# Run branch predictor simulator on whole program pinballs.
#
if options.whole_sim or options.default_phases:
# Set phase_length to print out simulator results every slice_size instructions.
#
phase_length = options.slice_size
if not options.list:
msg.PrintMsgDate('Running simulator on whole program pinballs: %s' % \
config.PhaseStr(config.sim_whole))
util.PhaseBegin(options)
result = self.sim_kit.RunSimulator(wp_pb_dir, sim_replay_cmd,
options)
if not options.list:
msg.PrintMsgDate('Finished running simulator on whole program pinballs: %s' % \
config.PhaseStr(config.sim_whole))
util.CheckResult(result, options, 'simulator on whole program pinballs: %s' % \
config.PhaseStr(config.sim_whole))
return 0
def GenAllLitFiles(self, replay_cmd, options):
"""
Generate LIT files and warmup LMAT files for all region pinballs in the tracing instance.
"""
result = 0
# Get the region pinball directories for this tracing instance.
#
# import pdb; pdb.set_trace()
pp_dirs = util.GetRegionPinballDir(options)
# Put the replay_cmd, options and flavor of files to generate into a
# directory because only one argument can be passed thru to the method
# GenLitFiles(), but three arguments need to be given to this method.
#
# Set the flavor to LMAT file generation.
#
param = {
'replay_cmd': replay_cmd,
'options': options,
'flavor': self.LMAT
}
# First generate LMAT files for each directory.
#
# import pdb; pdb.set_trace()
if hasattr(options, 'list') and not options.list:
msg.PrintMsgDate('Generating LMAT files')
for pdir in pp_dirs:
if not options.list:
msg.PrintMsgPlus(
'Generating LMAT files for pinballs in: ' + pdir)
result = util.RunAllDir(pdir, self.GenLitFiles, True, param)
if result != 0:
msg.PrintMsg('Error found during LIT file generation (1)')
return result
# Need to wait until all the LMAT jobs are complete before the
# LIT files are generated.
#
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Waiting on concurrent LMAT file generation')
result = util.WaitJobs(options)
if hasattr(options, 'list') and not options.list:
msg.PrintMsgDate('Finished generating LMAT files')
# Now set the flavor for LIT file generation.
#
param['flavor'] = self.LIT
# Then generate LCAT files for each directory.
#
# import pdb; pdb.set_trace()
if hasattr(options, 'list') and not options.list:
msg.PrintMsgDate('Generating LIT files')
for pdir in pp_dirs:
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LIT files for pinballs in: ' + pdir)
result = util.RunAllDir(pdir, self.GenLitFiles, True, param)
if result != 0:
msg.PrintMsg('Error found during LIT file generation (2)')
return result
# Need to wait until all the LIT jobs are complete before continuing.
#
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Waiting on concurrent LIT file generation')
result = util.WaitJobs(options)
if hasattr(options, 'list') and not options.list:
msg.PrintMsgDate('Finished generating LIT files')
return result
def GenLitFiles(self, param, dirname, file_name):
"""
Generate either the LIT files or warmup LMAT files for one pinball based on the
value 'flavor' passed in the param dictionary.
The region pinball contains optional warmup instructions before the instructions
in the actual representative regions. If there aren't any warmup instructions for a
pinball, then skip the step which generates LMAT files.
"""
# Setup locale so we can print integers with commas.
#
import locale
locale.setlocale(locale.LC_ALL, "")
# Define a lambda function to print an error message.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: method sde_phases.GenLitFiles() failed to '
'get field: ' + string)
# Unpack replay_cmd, options and flavor from the dictionary param.
# Flavor determines the type of file to generate.
#
if param.has_key('flavor'):
flavor = param['flavor']
else:
err_msg('flavor')
return -1
if param.has_key('options'):
options = param['options']
else:
err_msg('options')
return -1
if param.has_key('replay_cmd'):
replay_cmd = param['replay_cmd']
else:
err_msg('replay_cmd')
return -1
result = 0
# Get information from the pinball and the pinball file name.
# Dummy variables: a, b, c, d, e, f
#
(icount, warmup, prolog, calc_region, epilog, tid,
region) = util.GetRegionInfo(os.path.join(dirname, file_name), options)
(a, b, c, file_region, d, e, f) = util.GetRegionInfo(
os.path.join(dirname, file_name), options,
use_file_fields=True)
# If generating LMAT files, yet there aren't any warmup instructions for this pinball, then skip
# this file by returning from the method.
#
if not options.list:
if flavor == self.LMAT and warmup == 0:
msg.PrintMsgPlus(
"NOTE: No warmup instructions. Not generating LMAT files for trace:\n"
" " + file_name)
return 0
# Check to make sure the TID in file name is the same as the tracing parameter 'focus_thread'.
#
if tid != config.focus_thread and config.focus_thread >= 0:
msg.PrintMsg('ERROR: When generating LIT files, the TID in a pinball\n' \
'file name is not the same as the tracing parameter \'focus_thread\'. \n' \
'Focus thread is: ' + str(config.focus_thread) + ', yet the TID ' \
'in the file name is: ' + str(tid) + '.\n' \
'Please double check to make sure the focus thread parameter is \n' \
'correct (given with \'option -f X\' where X is the TID.)\n' \
' ' + file_name + '\n')
return -1
# Calculate The length of all regions except the warmup
#
lit_len = icount - warmup
# Get the number of threads from the result files
#
field = util.FindResultString(
os.path.join(dirname, file_name + '.result'), 'static_threads')
nthreads = field[0]
if not nthreads:
# Perhaps we are dealing with a pinball version > 2.4. Look in the
# *.global.log file for the number of threads.
#
field = util.FindString(
os.path.join(dirname, file_name + '.global.log'), 'static_threads')
nthreads = field[0]
if nthreads:
nthreads = int(nthreads)
else:
nthreads = 0
# Check to make sure there are a significant number of instructions in the pinball.
#
if calc_region < 1000:
msg.PrintMsg(
'=============================================================')
msg.PrintMsg(
'WARNING: The icount for focus thread region pinball is low:')
msg.PrintMsg(' Dir: ' + dirname)
msg.PrintMsg(' File: ' + file_name)
else:
if not options.list:
msg.PrintMsgPlus('File: ' + file_name)
if not options.list:
# Print details about the various sections in the pinball.
#
msg.PrintMsg(' Warmup count: ' +
locale.format('%14d', warmup, True))
msg.PrintMsg(' Prolog count: ' +
locale.format('%14d', prolog, True))
msg.PrintMsg(' Actual region count: ' + locale.format('%14d', calc_region, True) + \
' (from file name: ' + locale.format('%d', file_region, True) + ')')
msg.PrintMsg(' Epilog count: ' +
locale.format('%14d', epilog, True))
msg.PrintMsg(' Total Instr count: ' +
locale.format('%14d', icount, True))
msg.PrintMsg(' Number of threads: ' +
locale.format('%14d', nthreads, True))
if config.focus_thread >= 0:
msg.PrintMsg(' Focus thread: ' +
str(config.focus_thread))
# Get the 'base' file name without the TID.
#
base_name = util.RemoveTID(file_name)
# Get the LIT directory name from the region pinball directory name.
#
lit_dir = util.ChangeExtension(dirname, '.pp', '.lit')
# Several file names, with path, for pinball and LIT files.
#
# Need to remove TID for the file name given to knob '-log:basename'
# when generating LMAT files. This must be done is because pinLIT will
# add the TID to the LIT file names. Hence it must be removed in the
# 'basename'.
#
# import pdb; pdb.set_trace()
pb_path_file = os.path.join(dirname, file_name)
lit_path_file = os.path.join(lit_dir, file_name)
lit_path_file_no_tid = os.path.join(lit_dir, base_name)
# msg.PrintMsg('pb_path_file: ' + pb_path_file)
# msg.PrintMsg('lit_path_file: ' + lit_path_file)
# msg.PrintMsg('lit_path_file_no_tid: ' + lit_path_file_no_tid)
# Path & file name to the PTOV file which is generated by pinLIT. If
# there are no warmups for the file, then there won't be a 'warmup.ptov'
# file for it.
#
#
ptov_path_file = glob.glob(
os.path.join(lit_dir, base_name) + '*.warmup.ptov')
if ptov_path_file:
ptov_path_file = ptov_path_file[0]
# Format the initial part of the command line which is used for both LMAT and LIT files.
#
cmd_0 = replay_cmd
cmd_0 += ' --replay_file ' + pb_path_file
cmd_0 += ' --log_options '
# Determine if all parameters should be dumped to the global file
# (PinPlay), or just a subset (CBSP).
#
if options.cbsp():
pinplay = False
else:
pinplay = True
cmd_1 = util.AddGlobalFile(self.gv.DumpGlobalVars(pinplay), options)
cmd_1 += util.AddCfgFile(options)
# Add user defined 'lit_options' to both the LMAT and LIT command
# lines. May want sepatate options in the future, but right now
# there's only one.
#
knobs = ''
if hasattr(
options,
'replay_options') and options.replay_options: # Add any user defined replay options
knobs += ' ' + options.replay_options
if hasattr(
options,
'lit_options') and options.lit_options: # Add any user defined replay options
knobs += ' ' + options.lit_options
if flavor == self.LMAT:
# Format the knobs required to generate LMAT files.
#
knobs += ' -log -xyzzy -log:LIT -log:LIT_warmup ' + self.kit_obj.knob_length + ' ' + str(
warmup)
knobs += ' -log:early_out'
knobs += ' -log:basename ' + lit_path_file_no_tid
if hasattr(options, 'compressed'):
knobs += ' -log:compressed ' + options.compressed
knobs += util.GetMsgFileOption(lit_path_file + '.lmat')
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LMAT files: ' + file_name)
end_str = file_name
else:
if warmup > 0 and ptov_path_file:
# Need this knob when generating LIT files if LMAT files already generated
# for this trace.
#
knobs += ' -log:LIT_use_ptov ' + ptov_path_file
# Format the knobs required to generate LIT files.
#
#import pdb; pdb.set_trace()
knobs += ' -log -log:LIT ' + self.kit_obj.knob_skip + ' ' + str(warmup)
if lit_len < 0:
msg.PrintMsg(
'=============================================================')
msg.PrintMsg(
'WARNING: The icount for focus thread region pinball is low:')
msg.PrintMsg(' Dir: ' + dirname)
msg.PrintMsg(' File: ' + file_name)
knobs += ' ' + self.kit_obj.knob_length + ' ' + '100'
msg.PrintMsg(' No LIT will be generated')
else:
knobs += ' ' + self.kit_obj.knob_length + ' ' + str(lit_len)
knobs += ' -log:early_out'
knobs += ' -log:basename ' + lit_path_file_no_tid
if hasattr(options, 'compressed'):
knobs += ' -log:compressed ' + options.compressed
knobs += util.GetMsgFileOption(lit_path_file + '.lit')
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LIT files: ' + file_name)
end_str = file_name
# Format & execute the command.
#
cmd = cmd_0 + '"' + knobs + '"' + cmd_1
# Run the command in the background in order to run concurrent jobs.
#
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=True)
return result
def CalcPredError(wp_pb_dir, sim_replay_cmd, options):
import locale
sys.path.append(os.path.join(options.sniper_root, 'tools'))
import sniper_lib
def get_cpi(dir):
try:
r = sniper_lib.get_results(0, dir)
except ValueError:
msg.PrintMsg('\nERROR: Can\'t get sniper results for:\n ' +
dir)
return 0.0
stats = r['results']
if stats['ncores'] != 1:
msg.PrintMsgPlus(
'Warning: Sniper only supports single-threaded SimPoints')
# This code works with both Sniper/SniperLite
#
instrs = stats['core.instructions']
fs_to_cycles = stats['fs_to_cycles_cores']
fs = stats['barrier.global_time']
for i, d in enumerate(instrs):
cpi = (float(fs[0]) * fs_to_cycles[i]) / instrs[i]
if sum(instrs) <= 0:
msg.PrintMsgPlus(
'\nERROR: No cycles found in Sniper output for pinball:\n '
+ dir)
cpi = 0.0
return cpi
result = 0
for wpdir in glob.glob(
os.path.join(config.sniper_result_dir, wp_pb_dir,
util.GetLogFile() + '*')):
percent_cpi = []
cpi = 0.0
zero_cpi = False
for ppdir in glob.glob(
os.path.join(config.sniper_result_dir,
os.path.basename(wpdir) + '.pp',
os.path.basename(wpdir) + '*')):
percent = float(
re.search(r'warmup.*prolog.*region.*epilog.*_(\d-\d+)\.',
ppdir).group(1).replace('-', '.'))
cpi = get_cpi(ppdir)
if cpi == 0.0:
# Skip this pinball if CPI is 0.0 (which probably means an error).
#
zero_cpi = True
continue
percent_cpi.append((percent, cpi))
if zero_cpi:
continue
abs_diff_from_1 = abs(sum([x[0] for x in percent_cpi]) - 1.0)
if abs_diff_from_1 > 0.00005:
msg.PrintMsgPlus(
'Warning: Weights for all regions don\'t sum up to 1.0, [abs(sum - 1.0) = %6.5f]'
% abs_diff_from_1)
# import pdb; pdb.set_trace()
msg.PrintMsg('\n%s' % os.path.split(wpdir)[1])
predict_cpi = sum([x[0] * x[1] for x in percent_cpi])
msg.PrintMsg(' Intermediate result, predicted CPI: ' +
str(locale.format('%7.4f', predict_cpi, True)))
if predict_cpi == 0.0:
if options.mode == config.MPI_MT_MODE:
msg.PrintMsgPlus('WARNING: Unable to get predicted CPI from region pinballs because of a problem with Sniper results:\n' \
' ' + wpdir + '\n Prediction error for this process will not be calcuated.')
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
# Indicate there was an error, skip this process and try the next one.
#
msg.PrintMsgPlus('ERROR: Unable to get predicted CPI from region pinballs because of a problem with Sniper results:\n' \
' ' + wpdir + '\n Prediction error for this process will not be calcuated.')
result = -1
continue
measure_cpi = get_cpi(wpdir)
msg.PrintMsg(' Intermediate result, measured CPI: ' +
str(locale.format('%7.4f', measure_cpi, True)))
if measure_cpi == 0.0:
# Indicate there was an error, skip this process and try the next one.
#
msg.PrintMsg('\nERROR: Unable to get measured CPI from WP pinballs because of a problem with Sniper results:\n' \
' ' + wpdir + '\n Prediction error will not be calcuated')
result = -1
continue
msg.PrintMsg('\n%s' % os.path.split(wpdir)[1])
msg.PrintMsg(' Predicted CPI: ' +
str(locale.format('%7.4f', predict_cpi, True)))
msg.PrintMsg(' Measured CPI: ' +
str(locale.format('%7.4f', measure_cpi, True)))
msg.PrintMsg(
' Prediction error: ' +
str(locale.format('%7.4f', 1 -
(predict_cpi / measure_cpi), True)) + ' 1-(p/m)')
msg.PrintMsg(
' [Functional correlation: ' +
str(locale.format('%7.4f', predict_cpi / measure_cpi, True)) +
' (p/m)]')
return result
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options):
"""
Run the additional phases for SDE which are not run for PinPlay.
@param wp_pb_dir Directory containing whole program log files (pinballs)
@param sim_replay_cmd Python script used to replay a pinball with a simulator
@param options Options given on cmd line
@return Exit code from last phase executed
"""
# The SDE phases object needs an SDE kit object.
#
s_phases = sde_phases.SDEPhases()
kit_obj = self.GetKit()
s_phases.SetKit(kit_obj)
# Need the appropriate simulator kit too.
#
sim_kit = self.GetSimKit()
# Generate LMAT/LIT files.
#
result = 0
if options.lit_gen or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating traces for region pinballs %s' % \
config.PhaseStr(config.LIT))
util.PhaseBegin(options)
result = s_phases.GenAllLitFiles(self.replayer_cmd, options)
#if result == 0:
# if not options.list:
# msg.PrintMsgPlus('Waiting on final concurrent region pinball generation')
# result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating traces for region pinballs %s' % \
config.PhaseStr(config.LIT))
util.CheckResult(result, options, 'Trace file generation %s' % \
config.PhaseStr(config.LIT))
# Generate traceinfo files.
#
if options.traceinfo or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating traceinfo files %s' % \
config.PhaseStr(config.traceinfo))
util.PhaseBegin(options)
result = s_phases.GenTraceinfoFiles(options)
if not options.list:
msg.PrintMsgDate('Finished generating traceinfo files %s' % \
config.PhaseStr(config.traceinfo))
util.CheckResult(result, options, 'Traceinfo file generation %s' % \
config.PhaseStr(config.traceinfo))
# Run CMPSim simulator on region pinballs.
#
if options.region_sim or options.default_phases:
# Print out CMPSim results every warmup_length instructions.
#
phase_length = options.warmup_length // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
for pp_dir in util.GetRegionPinballDir():
if not options.list:
msg.PrintMsgDate('Running CMPSim on region pinballs in dir: %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
util.PhaseBegin(options)
result = sim_kit.RunSimulator(pp_dir, sim_replay_cmd,
phase_length, options)
if not options.list:
msg.PrintMsgDate('Finished running CMPSim on region pinballs in dir %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
util.CheckResult(result, options, 'CMPSim on pinballs, dir: %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
# Run CMPSim simulator on whole program pinballs.
#
if options.whole_sim or options.default_phases:
# Set phase_length to print out CMPSim results every slice_size instructions.
#
phase_length = options.slice_size // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
if not options.list:
msg.PrintMsgDate('Running CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
util.PhaseBegin(options)
result = sim_kit.RunSimulator(wp_pb_dir, sim_replay_cmd,
phase_length, options)
if not options.list:
msg.PrintMsgDate('Finished running CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
util.CheckResult(result, options, 'CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
# Calculate prediction error from simulator data files.
#
if options.pred_error or options.default_phases:
if not options.list:
msg.PrintMsgDate('Calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.PhaseBegin(options)
result = s_phases.CalcPredError(wp_pb_dir, sim_kit, options)
if not options.list:
msg.PrintMsgDate('Finished calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.CheckResult(result, options, 'Prediction error calculation %s' % \
config.PhaseStr(config.pred_error))
# Verify the LIT files with the simulator.
#
if options.verify:
if not options.list:
msg.PrintMsgDate('Verifying LIT files %s' % \
config.PhaseStr(config.verify_LIT))
util.PhaseBegin(options)
result = s_phases.VerifyLITFiles(self.sim_run_cmd, options)
if not options.list:
msg.PrintMsgDate('Finished verifying LIT files %s' % \
config.PhaseStr(config.verify_LIT))
util.CheckResult(result, options, 'LIT file verification %s' % \
config.PhaseStr(config.verify_LIT))
# Generate instruction mix for the whole program pinballs.
#
if options.imix_whole:
if not options.list:
msg.PrintMsgDate('Generating imix on whole program pinballs %s' % \
config.PhaseStr(config.imix_whole))
# Setup dictionary of parameters for method RunAllDir()
#
param = {'options': options, 'replayer_cmd': self.replayer_cmd}
util.PhaseBegin(options)
result = util.RunAllDir(wp_pb_dir, s_phases.GenImix, True, param)
if result == 0:
if not options.list:
msg.PrintMsgPlus(
'Waiting on final whole program pinball imix generation'
)
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating imix on whole program pinballs %s' % \
config.PhaseStr(config.imix_whole))
util.CheckResult(result, options, 'Imix on whole program pinballs %s' % \
config.PhaseStr(config.imix_whole))
# Generate instruction mix for the LIT files.
#
if options.imix_lit:
if not options.list:
msg.PrintMsgDate('Generating imix on LIT files %s' % \
config.PhaseStr(config.imix_lit))
# Setup dictionary of parameters for method RunAllDir()
#
param = {'options': options, 'replayer_cmd': self.replayer_cmd}
util.PhaseBegin(options)
for lit_dir in util.GetLitDir():
result = util.RunAllDir(lit_dir, s_phases.GenImix, True, param)
if result == 0:
if not options.list:
msg.PrintMsgPlus('Waiting on final LIT files generation')
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating imix on LIT files %s' % \
config.PhaseStr(config.imix_lit))
util.CheckResult(result, options, 'Imix on LIT files %s' % \
config.PhaseStr(config.imix_lit))
# Generate instruction mix for the region pinballs.
#
if options.imix_region:
if not options.list:
msg.PrintMsgDate('Generating imix on region pinballs %s' % \
config.PhaseStr(config.imix_regions))
# Setup dictionary of parameters for method RunAllDir()
#
param = {'options': options, 'replayer_cmd': self.replayer_cmd}
util.PhaseBegin(options)
for pp_dir in util.GetRegionPinballDir():
result = util.RunAllDir(pp_dir, s_phases.GenImix, True, param)
if result == 0:
if not options.list:
msg.PrintMsgPlus(
'Waiting on final region pinballs generation')
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating imix on region pinballs %s' % \
config.PhaseStr(config.imix_regions))
util.CheckResult(result, options, 'Imix on region pinballs %s' % \
config.PhaseStr(config.imix_regions))
# Assume nothing has gone wrong at this point.
#
return result
def Run(self):
""" Get all the user options and run the simulator."""
# import pdb; pdb.set_trace()
options = self.ParseCommandLine()
# Print out the version number
#
if config.debug:
print os.path.basename(sys.argv[0]) + " $Revision: 1.14 $"
# Print out all the parsed options
#
if config.debug:
msg.PrintMsg('Trace basename: ' + options.trace_basename)
msg.PrintMsg('Options: ' + options.sim_options)
# Get the number of instructions in the trace. Set 'add_tid' to
# True to include TID in the file name for pinballs with a TID.
#
# import pdb; pdb.set_trace()
field = util.FindResultString(options.trace_basename, 'inscount:', \
add_tid=True)
icount = field[0]
if icount:
icount = int(icount)
else:
icount = 0
# Explicit path to the binary.
#
# import pdb; pdb.set_trace()
cmd = os.path.join(self.kit.sim_path, self.kit.binary)
# Add some options which are not parameters (and thus not in the global
# data file) to the command line.
#
# import pdb; pdb.set_trace()
if options.sim_options:
cmd += ' ' + options.sim_options
elif self.kit.default_knobs != '':
cmd += ' ' + self.kit.default_knobs
cmd += ' -cfg ' + self.kit.GetSimCfgFile(options)
cmd += ' -max_litcount ' + str(icount)
# Add any unique options required by the simulator.
#
cmd += self.AddUniqueOptions(options)
# Add the name of the trace to simulate.
#
cmd += ' ' + options.trace_basename
# Print out command line before removing any double quotes.
#
msg.PrintMsg(cmd)
# Remove any double quotes (") which still exist in the cmd.
#
cmd = cmd.replace('"', '')
if options.verify:
# If not just listing cmd, print msg letting the user know we are
# verifying a trace.
#
if not options.list:
msg.PrintMsgPlus('Verifying trace: ' + options.trace_basename)
# Setup to write stdout/stderr to a log file.
#
sim_out_filename = options.trace_basename + config.sim_out_ext
sim_out_file = open(sim_out_filename, 'w')
sout = sim_out_file
serr = sim_out_file
# Execute the command line and wait for it to execute. Check for
# errors if validating a trace.
#
# import pdb; pdb.set_trace()
result = 0
if not options.list and not config.debug:
p = subprocess.Popen(cmd, stdout=sout, stderr=serr, \
shell=True)
p.communicate()
result = p.returncode
if options.verify:
# Look for strings in the output file indicating there was an
# error.
#
# import pdb; pdb.set_trace()
sim_out_file = open(sim_out_filename, 'r')
all_lines = sim_out_file.readlines()
sim_out_file.close()
error = False
for line in all_lines:
if 'fault' in line or \
'Unknown error after executing' in line or \
'aborted by microcode after executing' in line or \
'knob library:' in line or \
'Can\'t open config file' in line:
error = True
break
if error:
# If got an error, then print a msg and the output.
#
msg.PrintMsg('ERROR: Verify failed for trace')
msg.PrintMsg(' %s\n' % options.trace_basename)
for line in all_lines:
msg.PrintMsgNoCR(line)
else:
# Else, just print a msg stating it's OK.
#
msg.PrintMsg('\n%s: OK' % options.trace_basename)
if not options.verbose:
# Just print a few select lines of the output.
#
for line in all_lines:
if 'halted after executing' in line or \
' T0: inst' in line:
msg.PrintMsgNoCR(line)
else:
# If verbose, print everything.
#
for line in all_lines:
msg.PrintMsgNoCR(line)
# Exit with the return code from executing the simulator.
#
return result
def GetLastMetric(self, sim_file, tid, options):
"""
Get the last metric in a CMPSim output file. This is the value
for running the entire pinball.
Seek until we are close to the end of the file before start looking
for data. This saves lots of time when processing very large files.
@param sim_file File with simulator results to process
@param options TID of results to be processed
@param options Options given on cmd line
@return metric (-1 if an error occurs)
"""
import struct
# Get the size of the uncompressed simulator data file from the last 4 bytes
# of the compressed file. This value is the file size modulo 2^32.
#
# import pdb ; pdb.set_trace()
try:
fo = open(sim_file, 'rb')
except IOError:
msg.PrintMsg('ERROR: Unable to open CMPSim file for whole program pinball:\n ' + \
sim_file)
return -1.0
try:
fo.seek(-4, 2)
except:
msg.PrintMsg('ERROR: There was a problem accessing data for the WP CMPSim file:\n ' + \
sim_file)
return -1.0
r = fo.read()
fo.close()
size = struct.unpack('<I', r)[0]
# Get a file pointer to the simulator file.
#
f = util.OpenCompressFile(sim_file)
if f == None:
return -1.0
four_GB = 4294967296
seek_past = 100
num_chunk = 0
# First seek to the point in the file given by the 'size'.
#
msg.PrintMsgPlus('Determining size of file: ' + sim_file)
f.seek(size, 1)
current = f.tell()
# For files > 4GB, the value for 'size' is the true file size modulo
# 2^32. If this is the case, seek in 4GB chunks until the true file
# size is found.
#
# import pdb ; pdb.set_trace()
while current - (num_chunk * four_GB) >= size:
# First see if we can seek a few bytes past the current file
# pointer location. If we don't advance the FP, then it's at the
# end of the file. Otherwise, there is a 4GB chunk of the file to
# be bypassed.
#
# import pdb ; pdb.set_trace()
last = current
f.seek(seek_past, 1)
current = f.tell()
if current == last:
break
else:
msg.PrintMsg('Skipping 4GB in CMPSim file')
f.seek(four_GB - seek_past, 1)
num_chunk += 1
current = f.tell()
# Check to see if the last seek reached 'size' modulo 2^32
# bytes. If so, then we are at the end of the file.
#
if current - (num_chunk * four_GB) < size:
break
# import pdb ; pdb.set_trace()
size = num_chunk * four_GB + size
# Skip to 100k bytes before the end of the file. Then start looking for the last set of
# data in the file. This saves a large amount of time, especially for huge files.
#
msg.PrintMsgPlus('Skipping ' + locale.format('%d', size, True) +
' bytes in file: ' + sim_file)
f.seek(0)
f.seek(size - 100000)
# This is the code which needs to be modified in order to use a
# different metric of interest for a new CMPSim. The existing code
# uses the metric CPI.
#
# Current code assume the is used. Get the
# number of instructions and cycles for this thread in the last line of
# the output.
#
instr = cycles = 0
for line in f.readlines():
pos = line.find('Thread: ' + str(tid) + ' Instructions:')
if pos != -1:
last = line
# import pdb ; pdb.set_trace()
lst = last.split()
instr = int(lst[3])
cycles = int(lst[5])
# Check to make sure there really is valid data. If not, the print a
# warning. No need to exit with an error, because it's possible for
# MPI_MT_MODE applications to have a different number of threads in
# each process. This means some processes may have a thread 'tid',
# while this process may not.
#
if instr > 1:
metric = cycles / float(instr)
else:
msg.PrintMsgPlus('WARNING: There were no instructions in WP CMPSim output for thread ' + \
str(tid) + ' in file:\n ' + sim_file)
msg.PrintMsg(
'Prediction error will not be calculated for this process.')
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg('Since tracing mode is \'mpi_mt\', this may be OK.')
metric = -1.0
return metric
def GetRegionMetric(self, sim_file, warmup, tid, options):
"""
Get the metric of interest for just the representative region, not including
any warmup instructions.
It is assumed the first set of CMPSim output data is for the warmup
instructions, if they exist. This is true because when the CMPSim was run
it should have printed out data at 'warmup_len' intervals.
The last set of data will be for both the representative region and
warmup instructions, if any.
Of course, if there's only one set of data, then it is for the region only,
because there aren't any warmup instruction.
@param sim_file File with simulator results to process
@param warmup Number of instructions in warmup section
@param options TID of results to be processed
@param options Options given on cmd line
@return metric
"""
# Get a file pointer to the simulator data file.
#
f = util.OpenCompressFile(sim_file)
if f == None:
return -1.0
# This is the code which needs to be modified in order to use a
# different metric of interest for a new CMPSim. The existing code
# uses the metric CPI.
#
# Get the first and last lines in the output that have the
# cycle/instruction counts. Assume the 1st is always the info for the
# warmup because the CMPSim data is dumped ever 'warmup_length'
# instructions. Assume last data point is for warmup + region. If
# there is only one line, then assume it's only for the region.
#
# Current code assume the default Branch Predictor CMPSim is used.
#
# Always use the data for thread 0 because we don't generate prediction
# error for cooperative region pinballs. Need to fix this when
# this capability is added.
#
# import pdb ; pdb.set_trace()
first = ''
last = ''
for line in f.readlines():
pos = line.find('Thread: ' + str(0) + ' Instructions:')
if pos != -1:
# If the first time, save it.
#
if first == '':
first = line
last = line
# import pdb ; pdb.set_trace()
l_list = last.split()
l_instr = int(l_list[3])
l_cycles = int(l_list[5])
if warmup == 0:
# No warmup. Calc metric from the last set of data.
#
if l_instr > 0:
metric = l_cycles / float(l_instr)
else:
msg.PrintAndExit('(1) Unable to calculate CPI because number of instructions is 0:\n' \
' ' + sim_file)
else:
# Get number of instructions & cycles for first set of data. (from warmup)
#
f_list = first.split()
f_instr = int(f_list[3])
f_cycles = int(f_list[5])
# Calculate region data by subtracting the last values from the
# first values. This gives number of cycles and instructions for
# just the region.
#
# Check to make sure there really is valid data. If not, the print a
# warning. No need to exit with an error, because it's possible for
# MPI_MT_MODE applications to have a different number of threads in
# each process. This means some processes may have a thread 'tid',
# while this process may not.
#
if l_instr - f_instr > 0:
metric = (l_cycles - f_cycles) / float(l_instr - f_instr)
else:
# import pdb ; pdb.set_trace()
msg.PrintMsgPlus('WARNING: It looks like there were no warmup instructions in region CMPSim output for thread ' + \
str(tid) + ' in file:\n ' + sim_file)
msg.PrintMsg('First icount: %s Last icount: %s' % (locale.format('%d', f_instr, True), \
locale.format('%d', l_instr, True)))
if l_instr < config.instr_cmpsim_phase:
msg.PrintMsg(
'Slice size may be too small to calculate prediction error.')
msg.PrintMsg(
'It needs to be at least 1,000,000 for CMPSim to generate valid data.')
msg.PrintMsg('Prediction error for this process may be suspect.')
if hasattr(options,
'mode') and options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
metric = -1.0
return metric
def Run(self):
"""
Get the user's options, read config file and execute the phases desired by the user.
@return Exit code from last phase executed
"""
# Catch signals in order to do an orderly shutdown of the script.
#
self.InstallSigHandler()
# import pdb; pdb.set_trace()
options, bin_options = self.ParseCommandLine()
# Print out the version numbers for all python modules in this directory.
#
if config.debug:
util.PrintModuleVersions()
# If logging, ensure the user has entered a mode.
#
if options.log and config.mode == '':
msg.PrintHelpAndExit(
'Must enter mode of application using: --mode "st|mt|mpi|mpi_mt"')
# Set the defaults for a set of variables which are needed, but not in the list
# of required options in the tracing configuration file.
#
self.SetDefaults(options)
# Add 1500 to the warmup length in order to deal with problems that
# occur when the start of a warmup region and the end of the previous
# region are in the same basic block.
#
config.warmup_length += 1500
# Get variables used in many phases.
#
# import pdb; pdb.set_trace()
wp_pb_dir = util.GetWPDir()
status_file = util.GetCBSPLogFile()
# Print out the tracing configuration parameters.
#
# TODO: Fix print so it prints CBSP info, not PP info
if not options.list:
self.phases.PrintTracingInfo(bin_options, wp_pb_dir,
status_file, self.PrintHome)
#########################################################################
#
# Several phases which clean up tracing instances.
#
#########################################################################
result = 0
if options.delete:
self.phases.DeleteTracingFileDir(options)
util.CheckResult(result, options, 'Deleting tracing instance files')
# Since just deleted all files/dirs for the tracing instance, need
# to set the whole program directory to the default value. Do NOT
# use util.GetWPDir() here because this may return the name of a
# relogged WP directory (which we, of course, just deleted.)
#
wp_pb_dir = util.GetBaseWPDir()
if options.delete_all:
self.phases.DeleteAllTracingFileDir(options)
util.CheckResult(result, options, 'Deleting all tracing files')
# Again, since just deleted all files/dirs, need to set the whole
# program directory to the default value.
#
wp_pb_dir = util.GetBaseWPDir()
#########################################################################
#
# Run the phases of the script. They are in a specific order to ensure
# all prerequisite phases are completed before a given phase is run.
#
#########################################################################
# Logging phase to generate initial whole program pinballs.
#
# import pdb; pdb.set_trace()
if options.log or options.default_phases:
util.PhaseBegin(options)
result = 0
# First check to make sure there aren't any old WP pinball directories.
#
for bopts in bin_options:
wp_pb_dir = util.GetDefaultWPDir(bopts)
if os.path.isdir(wp_pb_dir):
msg.PrintMsg(
"\nERROR: Whole program pinball directory already exists: "
+ wp_pb_dir +
"\nRemove this directory to proceed with generating "
"whole program pinballs.")
util.CheckResult(-1, options, 'Whole program pinball generation %s' % \
config.PhaseStr(config.log_whole))
for bopts in bin_options:
# Add CSBP logging specific knobs to any user defined
# log_options. They are required by this phase in order to
# generate DCFG files.
#
if not hasattr(options, 'log_options'):
setattr(options, 'log_options', '')
dcfg_log_opts = ' -omix %s.%s.mix' % (bopts.program_name, bopts.
input_name)
dcfg_log_opts += config.cbsp_base_log_options
if bopts.log_options:
bopts.log_options += dcfg_log_opts
else:
bopts.log_options = dcfg_log_opts
# Run the logger for this binary
#
wp_pb_dir = util.GetDefaultWPDir(bopts)
log_file_name = util.GetLogFile(bopts)
self.Config.SetPerBinParams(bopts)
result = self.phases.Logger(self.logger_cmd, wp_pb_dir,
log_file_name, bopts)
util.CheckResult(result, options, 'WP pinballs for: %s %s' % \
(log_file_name, config.PhaseStr(config.log_whole)), intermediate_phase=True)
# Now remove the DCFG log options as they are only used in this phase. This
# ensures only the user defined options are used for subsequent phases.
#
bopts.log_options = bopts.log_options.replace(dcfg_log_opts, '')
util.CheckResult(result, options, 'Whole program pinball generation %s' % \
config.PhaseStr(config.log_whole))
# All phases after this require the whole program pinball directories. Exit with an
# error if they don't not exist.
#
# import pdb; pdb.set_trace()
if not options.list and not options.debug and\
not options.delete and not options.delete_all:
err_msg = lambda string: msg.PrintMsg('\nERROR: Can\'t proceed because whole program pinball directory does not exist:\n' + \
' ' + string + \
'\nMust select at least one phase to run. Try using option \'--default_phases\'.' + \
'\nUse \'-h\' for help on selecting phases to run.')
for bopts in bin_options:
wp_dir = util.GetDefaultWPDir(bopts)
if not os.path.isdir(wp_dir):
err_msg(wp_dir)
# Use -1 to force check to fail
util.CheckResult(
-1, options,
'Initial check to see if WP pinballs exist')
# Print out the number of instructions in the whole program pinballs.
#
if not options.list:
for bopts in bin_options:
wp_dir = util.GetDefaultWPDir(bopts)
msg.PrintMsg('')
msg.PrintMsg(wp_dir)
util.PrintInstrCount(util.GetDefaultWPDir(bopts), options)
#########################################################################
#
# These phases are run with the whole progam pinballs defined/generated
# in the previous phase.
#
#########################################################################
# Run the cross binary matcher.
#
if options.cb_match or options.default_phases:
if not options.list:
msg.PrintMsgDate('Running cross binary matcher %s' % \
config.PhaseStr(config.cb_match))
msg.PrintMsg('')
util.PhaseBegin(options)
result = self.phases.CrossBinaryMatcher(self.path, self.script_path,
options, bin_options)
if not options.list:
msg.PrintMsgDate('Finished running cross binary matcher %s' % \
config.PhaseStr(config.cb_match))
util.CheckResult(result, options, 'Cross binary matcher %s' % \
config.PhaseStr(config.cb_match))
# Run Simpoints to generate clusters
#
if options.simpoint or options.default_phases:
if not options.list:
msg.PrintMsgDate('Running Simpoint %s' % \
config.PhaseStr(config.Simpoint))
param = {'options': options}
util.PhaseBegin(options)
result = self.phases.RunSimPoint(self.path, self.script_path,
options, bin_options)
if not options.list:
msg.PrintMsgDate('Finished running Simpoint %s' % \
config.PhaseStr(config.Simpoint))
util.CheckResult(result, options, 'Simpoints generation %s' % \
config.PhaseStr(config.Simpoint))
# Relog to generate representative region pinballs.
#
if options.region_pinball or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating region pinballs %s' % \
config.PhaseStr(config.relog_regions))
util.PhaseBegin(options)
result = 0
for bopts in bin_options:
wp_dir = util.GetDefaultWPDir(bopts)
log_file_name = util.GetLogFile(bopts)
self.Config.SetPerBinParams(bopts)
result = self.phases.MultiIterGenRegionPinballs(
wp_dir, self.replayer_cmd, bopts)
if result == 0:
if not options.list:
msg.PrintMsgPlus(
'Waiting on final concurrent region pinball generation')
result = util.WaitJobs(options)
util.CheckResult(result, options, 'Region pinballs for: %s %s' % \
(log_file_name, config.PhaseStr(config.relog_regions)), intermediate_phase=True)
if not options.list:
msg.PrintMsgDate('Finished generating region pinballs %s' % \
config.PhaseStr(config.relog_regions))
util.CheckResult(result, options, 'Region pinball generation %s' %\
config.PhaseStr(config.relog_regions))
# If there are any additional phases, then run them.
#
self.RunAdditionalPhases(self.sim_replay_cmd, options, bin_options)
# Cleanup and print out a string to indicate the tracing has completed.
#
# import pdb; pdb.set_trace()
util.CleanupTraceEnd(options)
util.PrintTraceEnd(options)
return result
def GenImix(self, param, dirname, base_name):
"""
Generate the instruction mix for one pinball.
"""
# Define a lambda function to print a standard error msg.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: method sde_phases.GenImix() failed to get '
'field: ' + string)
# Get local copies of items in param.
#
if param.has_key('options'):
options = param['options']
else:
err_msg('options')
return -1
if param.has_key('replayer_cmd'):
replayer_cmd = param['replayer_cmd']
else:
err_msg('replayer_cmd')
return -1
# Instruction mix output file name.
#
file_name = os.path.join(dirname, base_name)
imix_file = file_name + '.imix.txt'
# Format the command.
#
cmd = replayer_cmd + ' --replay_file ' + file_name
if options.replay_options: # Add any user defined replay options
cmd += ' ' + options.replay_options
cmd += ' --log_options '
cmd += '"'
cmd += ' -omix ' + imix_file
# Assume if the user gives any knobs to SDE, they replace the
# default knobs added here.
#
if options.pin_options == '':
cmd += ' -top_blocks 100 '
cmd += '"'
if not options.list:
msg.PrintMsgPlus('Generating imix for: ' + base_name)
# Dump the global vars to a pickle file and add this option
# to the command. Then do it.
#
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
if hasattr(options, 'verbose') and options.verbose and not options.list:
msg.PrintMsg('sde_phases.GenImix, calling RunCmd()')
end_str = base_name
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str,
concurrent=True) # Run concurrent jobs here
return result
def GenTraceinfoFiles(self, options):
"""
Generate traceinfo files from sysinfo files. Also do some simple error
checking.
One traceinfo file is generated for each directory which contains
region pinballs. Currently, the script 'create_traceinfo.sh' is used
to generate the traceinfo file. This will be replaced with a Python
script at sometime in the future.
The following info is printed to allow the user to determine if the
required number of region pinballs and traces were generated:
1) The script 'count_traces.sh' gets a count of the number of
expected traces and the region numbers from the region CSV file.
2) The number of actual traces generated and the trace names.
3) The number of region pinballs generated and the pinball names.
"""
# Check to make sure there is at least one region pinball directory for
# this tracing instance.
#
all_pp_dirs = util.GetRegionPinballDir(options)
if all_pp_dirs == [] and not options.debug:
msg.PrintMsg(
'\nERROR: Could not find any PinPoint \'*.pp\' directories.\n'
'Make sure you have run the phase \'-p\'.')
return -1
# Get the region pinball directories for this tracing instance.
#
# import pdb; pdb.set_trace()
all_lit_dirs = util.GetLitDir()
if all_lit_dirs == [] and not options.debug:
msg.PrintMsg(
'\nERROR: Could not find any LIT \'*.lit\' directories.\n'
'Make sure you have run the phase \'-L\'.')
return -1
result = 0
# import pdb; pdb.set_trace()
for lit_dir in all_lit_dirs:
# Get the name of the corresponding region pinball directory
# and make sure it exists.
#
pp_dir = util.ChangeExtension(lit_dir, '.lit', '.pp')
if not os.path.isdir(pp_dir):
if not options.list:
msg.PrintMsgPlus('WARNING: Generating traceinfo files, but the required \'pp\' ' \
'directory does not exist:\n ' + pp_dir)
# If running in MPI_MT_MODE, then it's possible for one process to
# not have a thread corresponding to the the current focus thread.
# However, another process might have this thread.
# Thus, only return an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
return -1
else:
# Not printing any msgs, just skip to the next directory
#
continue
# Make sure the LIT directory exists, then go there.
#
old_dir = os.getcwd()
if os.path.isdir(lit_dir):
os.chdir(lit_dir)
else:
if not options.list:
msg.PrintMsgPlus('WARNING: Generating traceinfo files, but the LIT ' \
'directory does not exist:\n ' + lit_dir)
# If running in MPI_MT_MODE, then it's possible for one process to
# not have a thread corresponding to the the current focus thread.
# However, another process might have this thread.
# Thus, only return an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
return -1
else:
# Not printing any msgs, just skip to the next directory
#
continue
# Copy the traceinfo 'blank' XML files from the SDE kit.
#
blank_path = self.kit_obj.GetTraceinfoBlank()
for blank in config.traceinfo_blank_files:
blank_file = os.path.join(blank_path, blank)
try:
shutil.copy(blank_file, os.getcwd())
except IOError:
msg.PrintMsg(
'\nERROR: Unable to copy traceinfo \'blank\' file:\n '
+ blank_file)
return -1
# Run the script to generate traceinfo XML file. Stdout from the
# script is the XML file. Function util.RunCMD() needs the output XML
# file object in order to write this file.
#
msg.PrintMsg('')
base_name = util.ChangeExtension(lit_dir, '.lit', '')
tr_file = base_name + '.traceinfo.xml'
try:
fp_out = open(tr_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open traceinfo output file:\n'
' ' + tr_file)
return -1
cmd = self.traceinfo_bin + ' ' + base_name
result = util.RunCmd(cmd, options, '',
concurrent=False,
f_stdout=fp_out)
if result != 0:
msg.PrintMsg('Error found while running script \'%s\'' %
self.traceinfo_bin)
return -1
# Print info from the CSV regions file (located in the *.Data
# directory) about the number of expected traces.
#
msg.PrintMsg('')
param = {'in_lit_dir': True}
cluster_info, not_used, total_instr = util.GetClusterInfo(base_name, param)
if cluster_info == {}:
msg.PrintMsg(
'ERROR: Problems getting cluster info from file: %s.pinpoints.csv'
% (base_name))
return -1
cluster_list = util.ParseClusterInfo(cluster_info)
if len(cluster_info) != len(cluster_list):
msg.PrintMsg('ERROR: Did not parse enough clusters from CSV file: %s.pinpoints.csv\n' ' Num clusters: %d\n' \
' Num parsed clusters: %d' % (base_name, len(cluster_info), len(cluster_list)))
return -1
# Print the number of expected traces from the regions CSV file.
#
base_tid = -1
for cl in cluster_list:
if (cl.has_key('tid')):
tid = cl['tid']
else:
msg.PrintMsg(
'ERROR: Parsing cluster info for cluster:\n %s' %
(cl))
return -1
if base_tid == -1:
base_tid = tid
else:
if tid != base_tid:
msg.PrintAndExit(
'ERROR: Expected TID %d, but found TID %d' %
(base_tid, tid))
return -1
msg.PrintMsg('Expected trace count: %d\n' % (len(cluster_info)))
# Print the number of actual traces in the LIT directory and the names.
#
# import pdb; pdb.set_trace()
if not options.list:
msg.PrintMsg(
'Actual trace count: ' + str(util.CountFiles('ami')))
lit_files = glob.glob('*.ami*')
lit_files.sort()
for f in lit_files:
msg.PrintMsg(' ' + f)
# Clean up tmp files in the LIT directory.
#
tmp_files = glob.glob('*blank*.xml')
for f in tmp_files:
os.remove(f)
# Return to the working directory.
#
os.chdir(old_dir)
# Go to the *.pp directory. Print the number of actual pinballs and the names.
#
os.chdir(pp_dir)
if not options.list:
msg.PrintMsg(
'\nPinball count: ' + str(util.CountFiles('address')))
pp_files = glob.glob('*.address')
pp_files.sort()
for f in pp_files:
msg.PrintMsg(' ' + f)
# Print a warning if the expected number of traces are not found.
#
if len(lit_files) != len(pp_files):
msg.PrintMsgPlus(
'WARNING: Number of traces does not match the number of region pinballs.')
msg.PrintMsg('\nGenerated traceinfo file: ' + tr_file)
# Return to the working directory.
#
os.chdir(old_dir)
return result
def Run(self):
"""
Get the user's options, read config file and execute the phases desired by the user.
@return Exit code from last phase executed
"""
# Catch signals in order to do an orderly shutdown of the script.
#
self.InstallSigHandler()
# import pdb; pdb.set_trace()
options = self.ParseCommandLine()
# Print out the version numbers for all python modules in this directory.
#
if config.debug:
util.PrintModuleVersions()
# If logging, ensure the user has entered a mode.
#
if options.log and config.mode == '':
msg.PrintHelpAndExit(
'Must enter mode of application using: --mode "st|mt|mpi|mpi_mt"')
# Set the defaults for a set of variables which are needed, but not in the list
# of required options in the tracing configuration file.
#
self.SetDefaults(options)
# Create an new status file and zero out the file if it already exists.
# If user sets parameter 'append_status', the old file is not deleted.
#
util.NewStatusFile(options)
# Add 1500 to the warmup length in order to deal with problems that
# occur when the start of a warmup region and the end of the previous
# region are in the same basic block.
#
if not config.global_regions:
config.warmup_length += 1500
# Get variables used in many phases.
#
# import pdb; pdb.set_trace()
wp_pb_dir = util.GetWPDir()
log_file_name = util.GetLogFile()
# Print out the tracing configuration parameters.
#
if not options.list:
self.phases.PrintTracingInfo(options, wp_pb_dir, log_file_name,
self.PrintHome)
#########################################################################
#
# Several phases which clean up tracing instances.
#
#########################################################################
result = 0
if options.delete or options.delete_wp:
self.phases.DeleteTracingFileDir(options)
util.CheckResult(result, options, 'Deleting tracing instance files')
# Since just deleted all files/dirs for the tracing instance, need
# to set the whole program directory to the default value. Do NOT
# use util.GetWPDir() here because this may return the name of a
# relogged WP directory (which we, of course, just deleted.)
#
wp_pb_dir = util.GetBaseWPDir()
if options.delete_all:
self.phases.DeleteAllTracingFileDir(options)
util.CheckResult(result, options, 'Deleting all tracing files')
# Again, since just deleted all files/dirs, need to set the whole
# program directory to the default value.
#
wp_pb_dir = util.GetBaseWPDir()
# If the user has given the name of the whole program pinball
# directory, then save it. This must be done after the two delete
# phases because these phases delete this configuration file.
# Also, need to get default WP directory name in 'wp_pb_dir'.
#
# import pdb ; pdb.set_trace()
if hasattr(options, 'whole_pgm_dir') and options.whole_pgm_dir:
config.whole_pgm_dir = options.whole_pgm_dir
self.Config.SaveCfgParameter('whole_pgm_dir', options.whole_pgm_dir)
wp_pb_dir = util.GetWPDir()
#########################################################################
#
# Run the phases of the script. They are in a specific order to ensure
# all prerequisite phases are completed before a given phase is run.
#
#########################################################################
# Run the application without pin/pintools.
#
if options.native_pure:
util.PhaseBegin(options)
result = self.phases.NativePure(options)
util.CheckResult(
result, options,
'Pure native run %s' % config.PhaseStr(config.native_pure))
# Run the application using pin, but no pintools.
#
if options.native_pin:
util.PhaseBegin(options)
result = self.phases.NativePin(self.logger_cmd, wp_pb_dir, log_file_name, \
options)
util.CheckResult(result, options, 'Native run with pin only %s' % \
config.PhaseStr(config.native_pin))
# Logging phase to generate initial whole program pinballs.
#
# import pdb; pdb.set_trace()
if options.log or options.default_phases:
util.PhaseBegin(options)
result = self.phases.Logger(self.logger_cmd, wp_pb_dir, log_file_name, \
options)
util.CheckResult(result, options, 'Whole program pinball generation %s' % \
config.PhaseStr(config.log_whole))
# All phases after this require the whole program pinball directory. Exit with an
# error if it does not exist.
#
# import pdb; pdb.set_trace()
if not os.path.isdir(wp_pb_dir) and not options.list and not options.debug and\
not options.delete and not options.delete_all and not options.delete_wp and \
not options.native_pure and not options.native_pin:
string = '\nERROR: Can\'t proceed because the whole program pinball directory does not exist:\n' + \
' ' + wp_pb_dir + \
'\nMust select at least one phase to run. Try using option \'--default_phases\'.' + \
'\nUse \'-h\' for help on selecting phases to run.'
msg.PrintMsg(string)
# Use -1 to force check to fail
util.CheckResult(-1, options,
'Initial check to see if WP pinballs exist')
# Print out the number of instructions in the whole program pinballs.
#
if not options.list and os.path.isdir(wp_pb_dir):
msg.PrintMsg('')
msg.PrintMsg('Initial whole program pinball(s)')
util.PrintInstrCount(wp_pb_dir, options)
#########################################################################
#
# Phases which relog whole program pinballs using a filter to remove
# certain type of instructions (such as initialization or MPI spin).
#
# The relogging phases must be executed before the basic block vector
# generation phase. This must be done in case one, or more, relogging
# phases are executed. If so then, the BB vector phase needs to use
# the final relogged WP pinballs generated here.
#
#########################################################################
# If the user gives one, or more, of the use_relog_* options, they want
# to explictly define which filtered WP pinballs to use. As a result,
# unset the parameter 'relog_dir' just read from the per instance
# tracing configuration file and from the config object. This removes
# the previous "sticky" value for the filtered WP pinballs. Then use
# the WP pinball directory before any filters have been applied.
#
# import pdb; pdb.set_trace()
if cmd_options.UseRelogOptionsSet(options):
Config = config.ConfigClass()
Config.ClearCfgParameter('relog_dir')
config.relog_dir = ''
wp_pb_dir = util.GetBaseWPDir()
def FinalizeWPDir(wp_dir, string):
"""
Set the filtered WP pinball directory to be the new default WP dir
and print out the number of instructions in the newly filtered
pinballs.
"""
config.relog_dir = wp_dir
if not options.debug and not options.list:
# Only save this parameter if not just debugging or
# listing the commands to be run.
#
self.Config.SaveCfgParameter('relog_dir', wp_dir)
if not options.list and os.path.isdir(wp_dir):
msg.PrintMsg('')
msg.PrintMsg(string)
util.PrintInstrCount(wp_dir, options)
return
# The format of the code for each relogging (filtering) phase contains
# 3 components:
#
# 1) If going to use relogged WP pinballs for the phase (indicated by
# either 'relog_*' or 'use_relog_*' options), then get the
# appropriate WP directory name for this phase. Need to always do
# this, even when not relogging, because a previous run generated
# relogged WP pinballs.
#
# 2) If going to filter with relogging (indicated by a 'relog_*'
# option), then run the relogging phase.
#
# 3) Set the name of the default whole program pinball directory to
# the directory with the relogged WP pinballs.
# Relog with a user defined name for the relogged directory, and hopefully a set of
# knobs to define what action to take when relogging.
#
# import pdb; pdb.set_trace()
if options.use_relog_name != '' or options.relog_name:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_NAME,
options)
if options.relog_name:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs with user defined name: %s %s' % \
(options.relog_name, config.PhaseStr(config.filter_user_defn)))
util.PhaseBegin(options)
result = self.phases.RelogWholeName(self.replay_cmd, wp_pb_dir,
relog_wp_dir, options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs with user defined name: %s %s' % \
(options.relog_name, config.PhaseStr(config.filter_user_defn)))
util.CheckResult(result, options, 'Filtering WP pinballs with user defined name: %s %s' % \
(options.relog_name, config.PhaseStr(config.filter_user_defn)))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered with user defined name: ' +
options.relog_name)
# Relog with a focus thread.
#
if options.use_relog_focus or options.relog_focus:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_FOCUS,
options)
if options.relog_focus:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs with a focus thread %s' % \
config.PhaseStr(config.filter_focus_thread))
# import pdb; pdb.set_trace()
util.PhaseBegin(options)
result = self.phases.RelogWholeFocus(self.replay_cmd,
wp_pb_dir, relog_wp_dir,
options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs with a focus thread %s' % \
config.PhaseStr(config.filter_focus_thread))
util.CheckResult(result, options, 'Filtering WP pinballs with focus thread %s' % \
config.PhaseStr(config.filter_focus_thread))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered with a focus thread')
# If pinballs were relogged with a focus thread, then in the
# remaining phases the focus thread must be 0. Relogging generates
# per thread whole program pinballs (which only have thread 0). To
# enforce this, we change the focus_thread in the config object to
# be 0.
#
config.focus_thread = 0
# Also need to set options.use_relog_focus = True because we are will be using
# WP pinballs which have been relogged with a focus thread.
#
options.use_relog_focus = True
# Relog to remove initialization instructions. Do this before removing cleanup or
# MPI spin instructions.
#
# import pdb; pdb.set_trace()
if options.use_relog_no_init or options.relog_no_init:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir,
config.RELOG_NO_INIT, options)
if options.relog_no_init:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs to remove initialization instructions %s' % \
config.PhaseStr(config.filter_init))
util.PhaseBegin(options)
result = self.phases.RelogWholeRemoveInit(self.replay_cmd,
wp_pb_dir,
relog_wp_dir, options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs to remove initialization instructions %s' % \
config.PhaseStr(config.filter_init))
util.CheckResult(result, options, 'Filtering WP pinballs to remove init instructions %s' % \
config.PhaseStr(config.filter_init))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered to remove initialization instructions')
# Relog to remove cleanup instructions. Do this before removing MPI spin
# instructions.
#
# import pdb; pdb.set_trace()
if options.use_relog_no_cleanup or options.relog_no_cleanup:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir,
config.RELOG_NO_CLEANUP,
options)
if options.relog_no_cleanup:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs to remove cleanup instructions %s' % \
config.PhaseStr(config.filter_cleanup))
util.PhaseBegin(options)
result = self.phases.RelogWholeRemoveCleanup(
self.replay_cmd, wp_pb_dir, relog_wp_dir, options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs to remove cleanup instructions %s' % \
config.PhaseStr(config.filter_cleanup))
util.CheckResult(result, options, 'Filtering WP pinballs to remove cleanup instructions %s' % \
config.PhaseStr(config.filter_cleanup))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered to remove cleanup instructions')
# Relog to exclude code (instructions) between two addresses. Do this
# before removing MPI spin instructions.
#
# import pdb; pdb.set_trace()
if options.use_relog_code_exclude != '' or options.relog_code_exclude:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir,
config.RELOG_CODE_EXCLUDE,
options)
if options.relog_code_exclude:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs with code exclusion %s' % \
config.PhaseStr(config.filter_code_exclude))
util.PhaseBegin(options)
result = self.phases.RelogWholeCodeExclude(
self.replay_cmd, wp_pb_dir, relog_wp_dir, options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs with code exclusion %s' % \
config.PhaseStr(config.filter_code_exclude))
util.CheckResult(result, options, 'Filtering WP pinballs with code exclusion %s' % \
config.PhaseStr(config.filter_code_exclude))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered with code exclusion')
# Relog to remove OpenMP spin instructions.
#
# import pdb; pdb.set_trace()
if options.use_relog_no_omp_spin or options.relog_no_omp_spin:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir,
config.RELOG_NO_OMP_SPIN,
options)
if options.relog_no_omp_spin:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs to remove OpenMP spin instructions %s' % \
config.PhaseStr(config.filter_OMP_spin))
util.PhaseBegin(options)
result = self.phases.RelogWholeRemoveOMPSpin(
self.replay_cmd, wp_pb_dir, relog_wp_dir, options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs to remove OpenMP spin instructions %s' % \
config.PhaseStr(config.filter_OMP_spin))
util.CheckResult(result, options, 'Filtering WP pinballs to remove OpenMP spin instructions %s' % \
config.PhaseStr(config.filter_OMP_spin))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered to remove OpenMP spin instructions')
# Relog to remove MPI spin instructions.
#
# import pdb; pdb.set_trace()
if options.use_relog_no_mpi_spin or options.relog_no_mpi_spin:
relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir,
config.RELOG_NO_MPI_SPIN,
options)
if options.relog_no_mpi_spin:
if not options.list:
msg.PrintMsgDate('Filtering whole program pinballs to remove MPI spin instructions %s' % \
config.PhaseStr(config.filter_MPI_spin))
util.PhaseBegin(options)
result = self.phases.RelogWholeRemoveMPISpin(
self.replay_cmd, wp_pb_dir, relog_wp_dir, options)
if not options.list:
msg.PrintMsgDate('Finished filtering whole program pinballs to remove MPI spin instructions %s' % \
config.PhaseStr(config.filter_MPI_spin))
util.CheckResult(result, options, 'Filtering WP pinballs to remove MPI spin instructions %s' % \
config.PhaseStr(config.filter_MPI_spin))
# No errors, commit to using the new pinballs.
#
wp_pb_dir = relog_wp_dir
FinalizeWPDir(
relog_wp_dir,
'Whole program pinball(s) filtered to remove MPI spin instructions')
if not options.list:
msg.PrintMsgPlus(
'Using whole program pinballs in dir: ' + wp_pb_dir)
if (cmd_options.UseRelogOptionsSet(options) or \
cmd_options.RelogOptionsSet(options)) and \
os.path.isdir(wp_pb_dir):
msg.PrintMsg('')
util.PrintInstrCount(wp_pb_dir, options)
#########################################################################
#
# These phases are run with the whole progam pinballs defined/generated
# in the previous phases.
#
#########################################################################
# Make sure any relogged whole program pinball directory exist. Exit with an
# error if it does not exist.
#
# import pdb; pdb.set_trace()
if not os.path.isdir(wp_pb_dir) and not options.list and not options.debug and\
not options.delete and not options.delete_all and not options.delete_wp and \
not options.native_pure and not options.native_pin:
string = 'ERROR: Can\'t proceed because the whole program pinball directory does not exist:\n' + \
' ' + wp_pb_dir
msg.PrintMsg(string)
# Use -1 to force check to fail
util.CheckResult(-1, options,
'Second check to see if WP pinballs exist')
# Do not run replay whole program pinballs as one of the default
# phases. The user must explicitly include this option. This is to
# save time during the tracing process.
#
if options.replay:
if not options.list:
msg.PrintMsgDate('Replaying all whole program pinballs %s' % \
config.PhaseStr(config.replay_whole))
util.PhaseBegin(options)
result = self.phases.Replay(self.replay_cmd, wp_pb_dir, options)
if not options.list:
msg.PrintMsgDate('Finished replaying all whole program pinballs %s' % \
config.PhaseStr(config.replay_whole))
util.CheckResult(result, options, 'Replay of whole program pinballs %s' % \
config.PhaseStr(config.replay_whole))
# Generate basic block vectors.
#
if options.basic_block_vector or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating basic block vectors %s' % \
config.PhaseStr(config.gen_BBV))
util.PhaseBegin(options)
result = self.phases.BasicBlockVector(self.replay_cmd, wp_pb_dir,
options)
if result == 0:
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished basic block vector generation %s' % \
config.PhaseStr(config.gen_BBV))
util.CheckResult(result, options, 'Basic block vector generation %s' % \
config.PhaseStr(config.gen_BBV))
# Run Simpoints to genenerate representative regions.
#
if options.simpoint or options.default_phases:
if not options.list:
msg.PrintMsgDate('Running Simpoint on all processes %s' % \
config.PhaseStr(config.Simpoint))
# Setup dictionary of parameters for method RunAllDir()
#
msg.PrintMsg('fbm was in pinpoints')
param = {'options': options}
util.PhaseBegin(options)
result = util.RunAllDir(wp_pb_dir, self.phases.RunSimPoint, True,
param)
if not options.list:
msg.PrintMsgDate('Finished running Simpoint for all processes %s' % \
config.PhaseStr(config.Simpoint))
util.CheckResult(result, options, 'Simpoints generation %s' % \
config.PhaseStr(config.Simpoint))
# Relog to generate representative region pinballs.
#
if options.region_pinball or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating region pinballs %s' % \
config.PhaseStr(config.relog_regions))
util.PhaseBegin(options)
result = self.phases.MultiIterGenRegionPinballs(wp_pb_dir,
self.replayer_cmd,
options)
if result == 0:
if not options.list:
msg.PrintMsgPlus(
'Waiting on final concurrent region pinball generation')
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating region pinballs %s' % \
config.PhaseStr(config.relog_regions))
util.CheckResult(result, options, 'Region pinball generation %s' %\
config.PhaseStr(config.relog_regions))
# Do not run replay region pinballs as one of the default phases. The
# user must explicitly include this option. This is to save time
# during the tracing process.
#
if options.replay_region:
result = 0
if not options.list:
msg.PrintMsgDate('Replaying all region pinballs %s' % \
config.PhaseStr(config.replay_regions))
# import pdb; pdb.set_trace()
util.PhaseBegin(options)
for pp_dir in util.GetRegionPinballDir():
# Accumulate any errors which occur, but don't check for errors
# until all the pinballs have been replayed.
#
r = self.phases.Replay(self.replay_cmd, pp_dir, options)
result = result or r
if not options.list:
msg.PrintMsgDate('Finished replaying all region pinballs %s' % \
config.PhaseStr(config.replay_regions))
util.CheckResult(result, options, 'Replay of region pinballs %s' % \
config.PhaseStr(config.replay_regions))
result = 0 # Remove the return values from replaying region pinballs
# If there are any additional phases, then run them.
#
self.RunAdditionalPhases(wp_pb_dir, self.sim_replay_cmd, options)
# Cleanup and print out a string to indicate the tracing has completed.
#
# import pdb; pdb.set_trace()
util.CleanupTraceEnd(options)
util.PrintTraceEnd(options)
return result
def GetRegionMetric(self, sim_file, warmup, tid, options):
"""
Get the metric of interest for just the representative region, not including
any warmup instructions.
It is assumed the first set of simulator output data is for the warmup
instructions, if they exist. This is true because when the simulator was run
the knob to print out data was called with 'warmup_length' instructions.
The last set of data will be for both the representative region and
warmup instructions, if any exist.
Of course, if there's only one set of data, then it is for the region only,
because there aren't any warmup instruction.
Return: metric
"""
# Get a file pointer to the simulator data file.
#
f = util.OpenCompressFile(sim_file)
if f == None:
# Error opening file, return an error.
#
return -1.0
# This is the code which needs to be modified in order to use a
# different metric of interest for a new simulator. The existing code
# uses the metric MPI (misses per thousand instruction).
#
# Current code assume the default Branch Predictor simulator is used.
#
# Get the first and last lines in the output that have the
# cycle/instruction counts. Assume the 1st is always the info for the
# warmup because the simulator data is dumped ever 'warmup_length'
# instructions. Assume last data point is for warmup + region. If
# there is only one line, then assume it's only for the region.
#
# Always use the data for thread 0 because we don't generate functional
# correlation for cooperative region pinballs. Need to fix this when
# this capability is added.
#
# import pdb ; pdb.set_trace()
first = ''
last = ''
for line in f.readlines():
pos = line.find('Icount:')
if pos != -1:
# If the first time, save it.
#
if first == '':
first = line
last = line
# import pdb ; pdb.set_trace()
l_list = last.split()
l_instr = int(l_list[1])
l_misses = int(l_list[3])
if warmup == 0:
# No warmup. Calc metric from the last set of data.
#
if l_instr > 0:
metric = l_misses / (float(l_instr) / 1000)
else:
msg.PrintAndExit('(1) Unable to calculate metric because number of instructions is 0:\n' \
' ' + sim_file)
else:
# Get number of instructions & misses for first set of data. (from warmup)
#
f_list = first.split()
f_instr = int(f_list[1])
f_misses = int(f_list[3])
# Calculate region data by subtracting the last values from the
# first values. This gives number of misses and instructions for
# just the region.
#
# Check to make sure there really is valid data. If not, the print a
# warning. No need to exit with an error, because it's possible for
# MPI_MT_MODE applications to have a different number of threads in
# each process. This means some processes may have a thread 'tid',
# while this process may not.
#
# import pdb ; pdb.set_trace()
if l_instr - f_instr > 0:
metric = (l_misses - f_misses) / (float(l_instr - f_instr) /
1000)
else:
msg.PrintMsgPlus('WARNING: There were no instructions in simulation output for thread ' + \
str(tid) + ' in file:\n ' + sim_file)
msg.PrintMsg(
'Prediction error for this process may be suspect.')
if hasattr(options,
'mode') and options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
metric = -1.0
return metric