def VerifyLITFiles(self, sim_run_cmd, options):
"""
Use a simulator to verify all the LIT files for the tracing instance are valid.
"""
# Verify the LIT files for each directory.
#
# import pdb; pdb.set_trace()
result = 0
for lit_dir in util.GetLitDir():
if not options.list:
msg.PrintMsgPlus('Verifying files in dir: ' + lit_dir)
# import pdb; pdb.set_trace()
cmd = sim_run_cmd + ' --replay_dir ' + lit_dir
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
cmd += ' --verify'
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
result = util.RunCmd(cmd, options, '',
concurrent=True) # Run jobs concurrently
if result != 0:
msg.PrintMsg('Error found during LIT file verification')
return result
# Wait for all the verification jobs to finish.
#
result = util.WaitJobs(options)
return result
def Replay(self, param, dirname, filename):
"""
Run the Simulator on a single LIT file given the command line options and the name of
the file to run. It formats the appropriate command line options,
saves global variables in a pickle file & calls the Sim run script.
"""
# import pdb ; pdb.set_trace()
if 'options' in param:
options = param['options']
else:
msg.PrintAndExit(
'method replay_dir.Replay() failed to get param \'options\'')
basename_file = os.path.join(dirname, filename)
if config.verbose:
msg.PrintMsg("-> Replaying pinball \"" + basename + "\"")
cmd = self.replayer_cmd + ' ' + basename_file
# Check to see if need to add options passed to the Sim run script.
# These are NOT parameters, so they don't get passed in the global
# variables.
#
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
if options.verify:
cmd += ' --verify'
# Add the configuration file, if one exists and print the cmd if just
# debugging.
#
# import pdb ; pdb.set_trace()
cmd += util.AddCfgFile(options)
if options.debug:
msg.PrintMsg(cmd)
result = 0
if not options.debug:
# Dump the global data to a unique file name. Need to add the
# option --global_file with this unique file name to options when
# calling a script.
#
gv = config.GlobalVar()
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
# import pdb ; pdb.set_trace()
if not options.list:
result = util.RunCmd(
cmd, options, filename,
concurrent=True) # Run concurrent jobs here
else:
msg.PrintMsg(cmd)
return result
def RunSimulator(self, dirname, sim_replay_cmd, phase_length, options):
"""
Run Branch Predictor simulator on all the pinballs in a set of
directories given by 'dirname'.
@param dirname Directory containing pinball to process
@param sim_replay_cmd Script used to replay pinball
@param phase_length Number of instructions executed before CMPSim will print next set of output
@param options Options given on cmd line
@return error code from running the simulator
"""
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --log_options'
cmd += ' "'
# 'phase_length' is given in instructions.
#
cmd += ' ' + self.print_data_knob + ' ' + str(phase_length)
cmd += '" '
cmd += ' --sim_add_filename' # This causes simulator output file name to be added to cmd.
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
end_str = '' # Don't want to print anything when this cmd finishes.
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=False)
return result
def RunSimulator(self, dirname, sim_replay_cmd, options):
"""
Run Branch Predictor simulator on all the pinballs in a set of
directories given by 'dirname'.
Return: error code from running the simulator
"""
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = 'pwd;' + sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --log_options'
# SiNUCA options
cmd += ' "'
cmd += ' -output ' + str(options.sinuca_tracer_output)
cmd += ' -threads ' + str(options.sinuca_tracer_threads)
cmd += ' -parallel_start ' + str(options.sinuca_tracer_parallel_start)
cmd += ' -parallel_end ' + str(options.sinuca_tracer_parallel_end)
cmd += ' "'
cmd += ' --sim_add_filename' # This causes simulator output file name to be added to cmd.
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
msg.PrintMsg('DEBUG>> ' + cmd + '\n')
end_str = '' # Don't want to print anything when this cmd finishes.
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, False)
return result
def Replay(self, param, dirname, filename):
"""
Replay a single pinball given the command line options and the name of
the pinball to replay. It formats the appropriate command line options,
saves global variables in a pickle file & calls the replayer script.
@param param Dictionary containing all parameters that need to be
passed into the method. Need a dictionary because this
method is sometimes called by walk_callback() which
only allows one parameter in the functions it calls.
@param dirname Directory where pinball is located
@param filename Pinball base file name
@return Exit code from the replayer script.
"""
if 'options' in param:
options = param['options']
else:
msg.PrintAndExit(
'method replay_dir.Replay() failed to get param \'options\'')
if options.verbose:
msg.PrintMsg('Start of Replay() in replay_dir.py')
# import pdb ; pdb.set_trace()
basename_file = os.path.join(dirname, filename)
command = self.replayer_cmd + ' --replay_file ' + basename_file
if options.verbose:
msg.PrintMsg("-> Replaying pinball \"" + basename_file + "\"")
if options.replay_options:
command += ' --replay_options "' + options.replay_options + '"'
# Check to see if need to add options for BB vector generation. Set
# 'log_opt' to any options the user may have put on the command line.
#
log_opt = options.log_options
if options.bb_add_filename:
file_name = os.path.basename(basename_file)
# If there is a focus thread, then need to remove the TID from the
# file name.
#
# NOTE: This code may need to be fixed when a method of running
# Simpoints on all threads of cooperative pinballs is implemented.
#
file_name = util.RemoveTID(file_name)
# Write BB vector files to the newly created *.Data directory.
#
data_dir = file_name + '.Data'
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
log_opt += ' -o ' + os.path.join(data_dir, file_name)
# Check to see if need to add options when running a simulator.
#
# import pdb ; pdb.set_trace()
if options.sim_add_filename:
# Need to instantiate a kit of the type simulator being used.
# This is required on order to get some kit specific information.
#
# NOTE: If you are adding a kit for a new simulator, then you need
# to modify this code.
#
if config.sim_kit_type == config.BRPRED:
import brpred_kit
sim_kit = brpred_kit.BrPredKit()
elif config.sim_kit_type == config.SINUCA:
import sinuca_kit
sim_kit = sinuca_kit.Sinuca_TracerKit()
elif config.sim_kit_type == config.CMPSIM:
import sde_cmpsim_kit
sim_kit = sde_cmpsim_kit.CMPsimKit()
else:
msg.PrintAndExit('Undefined kit type in method replay_dir.Replay(): ' + \
str(config.sim_kit_type))
# Add the simulator knob to specify the file for the output from
# the simulator.
#
# ~ log_opt += ' ' + sim_kit.GetSimOutputFile(basename_file)
# When 'log_opt' is added to the command line below, it will
# put double quotes (") around all the options. Therefore, need to
# remove any exising double quotes in the current value for the
# string 'log_opt'.
#
log_opt = log_opt.replace('"', '')
# If relogging WP pinballs, need to add the -log:basename knob with
# the relogged pinball path/name.
#
# import pdb ; pdb.set_trace()
if options.wp_relog_dir:
ft = util.GetFocusThreadPB(basename_file)
if ft > -1 and not options.relog_focus:
# If WP pinballs were relogged with a focus thread, then the
# resulting pinballs were 'per thread', not 'cooperative'. If
# relogging with a different filter (i.e. options.relog_focus ==
# False) then need to remove TID from base file name given to
# the knob -log:basename.
#
file_name = os.path.basename(util.RemoveTID(basename_file))
else:
file_name = os.path.basename(basename_file)
log_opt += ' -log:basename ' + os.path.join(options.wp_relog_dir,
file_name)
if not options.list:
msg.PrintMsgDate('Relog whole program pinball: ' + file_name)
if log_opt:
command += ' --log_options "' + log_opt + '"'
if options.playout:
command += ' --playout '
# if not options.list:
# msg.PrintMsg(command)
# If not just listing the command, then dump the global
# variables so the next Python script can have access to them.
# Then run the script.
#
result = 0
if not options.list:
# Dump the global data to a unique file name. Need to add the
# option --global_file with this unique file name to options when
# calling a script.
#
gv = config.GlobalVar()
command += util.AddGlobalFile(gv.DumpGlobalVars(), options)
command += util.AddCfgFile(options)
result = util.RunCmd(command, options, filename,
concurrent=True) # Run concurrent jobs here
else:
# If the option 'list' is defined, then just list out the
# commands to be exectuted, but don't execute them.
#
msg.PrintMsg(command)
return result
def RunSimulator(self, dirname, sim_replay_cmd, phase_length, options):
"""
Run a simulator on all the pinballs in a directory given by 'dirname'.
@param dirname Directory containing pinball to process
@param sim_replay_cmd Script used to replay pinball
@param phase_length Number of instructions executed before CMPSim will print next set of output
@param options Options given on cmd line
@return Error code from running the simulator
"""
# Check to make sure the directory at least exists.
#
# import pdb; pdb.set_trace()
if not os.path.exists(dirname):
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg('WARNING: Directory containing pinballs to run with simulator does not exist:\n ' + \
dirname)
msg.PrintMsg('Since tracing mode is \'mpi_mt\', this may be OK.')
return 0
else:
msg.PrintMsg('ERROR: Directory containing pinballs to run with simulator does not exist:\n ' + \
dirname)
return -1
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Get the number of threads in the pinball and round it up
# to the next power of 2. This is a requirement of CMPSim.
#
# import pdb; pdb.set_trace()
threads = util.GetNumThreadsWP(options)
threads = util.RoundupPow2(threads)
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --replay_options'
cmd += ' "' + options.replay_options + '"'
cmd += ' --log_options'
cmd += ' "'
cmd += '-threads ' + str(threads)
# If the directory contains the WP directory name for this tracing
# instance, then get the icount for each process/thread in the WP
# pinballs. Use the longest icount as the phase_length for CMPSim.
#
# This is done to reduce the CMPSim output to the very minimum for WP
# pinballs. For very large WP pinballs, using the 'default' value for
# phase_length generates CMPSim output which can be multiple GBs in size.
#
# import pdb; pdb.set_trace()
if util.GetWPDir() in dirname:
max_icount = 0
pinballs = glob.glob(os.path.join(dirname, '*.address'))
for pb in pinballs:
fname = os.path.basename(pb)
icount = util.GetMaxIcount(dirname, fname)
if icount > max_icount:
max_icount = icount
# Check for errors
#
# import pdb; pdb.set_trace()
if max_icount != 0:
phase_length = max_icount / 1000000
if phase_length == 0:
# Must have a length of at least 1
#
phase_length = 1
cmd += ' ' + self.print_data_knob + ' ' + str(phase_length)
cmd += '" '
cmd += ' --sim_add_filename' # This causes CMPSim output file name to be added to cmd
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
# import pdb; pdb.set_trace()
end_str = '' # Don't want to print anything when this cmd finishes.
result = util.RunCmd(cmd, options, end_str, concurrent=False)
return result
def GenImix(self, param, dirname, base_name):
"""
Generate the instruction mix for one pinball.
"""
# Define a lambda function to print a standard error msg.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: method sde_phases.GenImix() failed to get '
'field: ' + string)
# Get local copies of items in param.
#
if param.has_key('options'):
options = param['options']
else:
err_msg('options')
return -1
if param.has_key('replayer_cmd'):
replayer_cmd = param['replayer_cmd']
else:
err_msg('replayer_cmd')
return -1
# Instruction mix output file name.
#
file_name = os.path.join(dirname, base_name)
imix_file = file_name + '.imix.txt'
# Format the command.
#
cmd = replayer_cmd + ' --replay_file ' + file_name
if options.replay_options: # Add any user defined replay options
cmd += ' ' + options.replay_options
cmd += ' --log_options '
cmd += '"'
cmd += ' -omix ' + imix_file
# Assume if the user gives any knobs to SDE, they replace the
# default knobs added here.
#
if options.pin_options == '':
cmd += ' -top_blocks 100 '
cmd += '"'
if not options.list:
msg.PrintMsgPlus('Generating imix for: ' + base_name)
# Dump the global vars to a pickle file and add this option
# to the command. Then do it.
#
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
if hasattr(options, 'verbose') and options.verbose and not options.list:
msg.PrintMsg('sde_phases.GenImix, calling RunCmd()')
end_str = base_name
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str,
concurrent=True) # Run concurrent jobs here
return result
def GenLitFiles(self, param, dirname, file_name):
"""
Generate either the LIT files or warmup LMAT files for one pinball based on the
value 'flavor' passed in the param dictionary.
The region pinball contains optional warmup instructions before the instructions
in the actual representative regions. If there aren't any warmup instructions for a
pinball, then skip the step which generates LMAT files.
"""
# Setup locale so we can print integers with commas.
#
import locale
locale.setlocale(locale.LC_ALL, "")
# Define a lambda function to print an error message.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: method sde_phases.GenLitFiles() failed to '
'get field: ' + string)
# Unpack replay_cmd, options and flavor from the dictionary param.
# Flavor determines the type of file to generate.
#
if param.has_key('flavor'):
flavor = param['flavor']
else:
err_msg('flavor')
return -1
if param.has_key('options'):
options = param['options']
else:
err_msg('options')
return -1
if param.has_key('replay_cmd'):
replay_cmd = param['replay_cmd']
else:
err_msg('replay_cmd')
return -1
result = 0
# Get information from the pinball and the pinball file name.
# Dummy variables: a, b, c, d, e, f
#
(icount, warmup, prolog, calc_region, epilog, tid,
region) = util.GetRegionInfo(os.path.join(dirname, file_name), options)
(a, b, c, file_region, d, e, f) = util.GetRegionInfo(
os.path.join(dirname, file_name), options,
use_file_fields=True)
# If generating LMAT files, yet there aren't any warmup instructions for this pinball, then skip
# this file by returning from the method.
#
if not options.list:
if flavor == self.LMAT and warmup == 0:
msg.PrintMsgPlus(
"NOTE: No warmup instructions. Not generating LMAT files for trace:\n"
" " + file_name)
return 0
# Check to make sure the TID in file name is the same as the tracing parameter 'focus_thread'.
#
if tid != config.focus_thread and config.focus_thread >= 0:
msg.PrintMsg('ERROR: When generating LIT files, the TID in a pinball\n' \
'file name is not the same as the tracing parameter \'focus_thread\'. \n' \
'Focus thread is: ' + str(config.focus_thread) + ', yet the TID ' \
'in the file name is: ' + str(tid) + '.\n' \
'Please double check to make sure the focus thread parameter is \n' \
'correct (given with \'option -f X\' where X is the TID.)\n' \
' ' + file_name + '\n')
return -1
# Calculate The length of all regions except the warmup
#
lit_len = icount - warmup
# Get the number of threads from the result files
#
field = util.FindResultString(
os.path.join(dirname, file_name + '.result'), 'static_threads')
nthreads = field[0]
if not nthreads:
# Perhaps we are dealing with a pinball version > 2.4. Look in the
# *.global.log file for the number of threads.
#
field = util.FindString(
os.path.join(dirname, file_name + '.global.log'), 'static_threads')
nthreads = field[0]
if nthreads:
nthreads = int(nthreads)
else:
nthreads = 0
# Check to make sure there are a significant number of instructions in the pinball.
#
if calc_region < 1000:
msg.PrintMsg(
'=============================================================')
msg.PrintMsg(
'WARNING: The icount for focus thread region pinball is low:')
msg.PrintMsg(' Dir: ' + dirname)
msg.PrintMsg(' File: ' + file_name)
else:
if not options.list:
msg.PrintMsgPlus('File: ' + file_name)
if not options.list:
# Print details about the various sections in the pinball.
#
msg.PrintMsg(' Warmup count: ' +
locale.format('%14d', warmup, True))
msg.PrintMsg(' Prolog count: ' +
locale.format('%14d', prolog, True))
msg.PrintMsg(' Actual region count: ' + locale.format('%14d', calc_region, True) + \
' (from file name: ' + locale.format('%d', file_region, True) + ')')
msg.PrintMsg(' Epilog count: ' +
locale.format('%14d', epilog, True))
msg.PrintMsg(' Total Instr count: ' +
locale.format('%14d', icount, True))
msg.PrintMsg(' Number of threads: ' +
locale.format('%14d', nthreads, True))
if config.focus_thread >= 0:
msg.PrintMsg(' Focus thread: ' +
str(config.focus_thread))
# Get the 'base' file name without the TID.
#
base_name = util.RemoveTID(file_name)
# Get the LIT directory name from the region pinball directory name.
#
lit_dir = util.ChangeExtension(dirname, '.pp', '.lit')
# Several file names, with path, for pinball and LIT files.
#
# Need to remove TID for the file name given to knob '-log:basename'
# when generating LMAT files. This must be done is because pinLIT will
# add the TID to the LIT file names. Hence it must be removed in the
# 'basename'.
#
# import pdb; pdb.set_trace()
pb_path_file = os.path.join(dirname, file_name)
lit_path_file = os.path.join(lit_dir, file_name)
lit_path_file_no_tid = os.path.join(lit_dir, base_name)
# msg.PrintMsg('pb_path_file: ' + pb_path_file)
# msg.PrintMsg('lit_path_file: ' + lit_path_file)
# msg.PrintMsg('lit_path_file_no_tid: ' + lit_path_file_no_tid)
# Path & file name to the PTOV file which is generated by pinLIT. If
# there are no warmups for the file, then there won't be a 'warmup.ptov'
# file for it.
#
#
ptov_path_file = glob.glob(
os.path.join(lit_dir, base_name) + '*.warmup.ptov')
if ptov_path_file:
ptov_path_file = ptov_path_file[0]
# Format the initial part of the command line which is used for both LMAT and LIT files.
#
cmd_0 = replay_cmd
cmd_0 += ' --replay_file ' + pb_path_file
cmd_0 += ' --log_options '
# Determine if all parameters should be dumped to the global file
# (PinPlay), or just a subset (CBSP).
#
if options.cbsp():
pinplay = False
else:
pinplay = True
cmd_1 = util.AddGlobalFile(self.gv.DumpGlobalVars(pinplay), options)
cmd_1 += util.AddCfgFile(options)
# Add user defined 'lit_options' to both the LMAT and LIT command
# lines. May want sepatate options in the future, but right now
# there's only one.
#
knobs = ''
if hasattr(
options,
'replay_options') and options.replay_options: # Add any user defined replay options
knobs += ' ' + options.replay_options
if hasattr(
options,
'lit_options') and options.lit_options: # Add any user defined replay options
knobs += ' ' + options.lit_options
if flavor == self.LMAT:
# Format the knobs required to generate LMAT files.
#
knobs += ' -log -xyzzy -log:LIT -log:LIT_warmup ' + self.kit_obj.knob_length + ' ' + str(
warmup)
knobs += ' -log:early_out'
knobs += ' -log:basename ' + lit_path_file_no_tid
if hasattr(options, 'compressed'):
knobs += ' -log:compressed ' + options.compressed
knobs += util.GetMsgFileOption(lit_path_file + '.lmat')
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LMAT files: ' + file_name)
end_str = file_name
else:
if warmup > 0 and ptov_path_file:
# Need this knob when generating LIT files if LMAT files already generated
# for this trace.
#
knobs += ' -log:LIT_use_ptov ' + ptov_path_file
# Format the knobs required to generate LIT files.
#
#import pdb; pdb.set_trace()
knobs += ' -log -log:LIT ' + self.kit_obj.knob_skip + ' ' + str(warmup)
if lit_len < 0:
msg.PrintMsg(
'=============================================================')
msg.PrintMsg(
'WARNING: The icount for focus thread region pinball is low:')
msg.PrintMsg(' Dir: ' + dirname)
msg.PrintMsg(' File: ' + file_name)
knobs += ' ' + self.kit_obj.knob_length + ' ' + '100'
msg.PrintMsg(' No LIT will be generated')
else:
knobs += ' ' + self.kit_obj.knob_length + ' ' + str(lit_len)
knobs += ' -log:early_out'
knobs += ' -log:basename ' + lit_path_file_no_tid
if hasattr(options, 'compressed'):
knobs += ' -log:compressed ' + options.compressed
knobs += util.GetMsgFileOption(lit_path_file + '.lit')
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LIT files: ' + file_name)
end_str = file_name
# Format & execute the command.
#
cmd = cmd_0 + '"' + knobs + '"' + cmd_1
# Run the command in the background in order to run concurrent jobs.
#
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=True)
return result
def Run(self):
"""
Get all the user options and run the logger.
@return Exit code from the logger pintool
"""
# import pdb; pdb.set_trace()
options = self.ParseCommandLine()
# Script which will actually do the logging
#
cmd = "./" + self.log_cmd
# If the user has given these options, add them
#
if hasattr(options, 'compressed') and options.compressed:
cmd += ' --compressed=%s ' % options.compressed
if hasattr(options, 'log_file') and options.log_file:
cmd += ' --log_file ' + options.log_file
log_opts = ' --log_options "-log:syminfo -log:pid '
if hasattr(options, 'log_options') and options.log_options:
log_opts += '%s" ' % options.log_options
else:
log_opts += '"'
cmd += log_opts
if hasattr(options, 'no_log') and options.no_log:
cmd += ' --no_log '
if hasattr(options, 'pid') and options.pid:
cmd += ' --pid ' + str(options.pid)
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
# Finally add program and arguments
#
# cmd += ' ' + options.command
# If the command is not already in double quotes, then quote it now.
# This takes care of cases where the command may redirect input or output
# or the command has options which contain the char '-'.
#
# Assume if there is one double quote, then a 2nd double quote should also
# already be in the command.
#
if not (hasattr(options, 'pid') and options.pid):
if options.command.find('"') == -1:
cmd += ' -- "' + options.command + '" '
else:
cmd += ' -- ' + options.command
# Print out command line used for pin and pintool
#
string = '\n' + cmd
msg.PrintMsg(string)
# Finally execute the command line and gather stdin and stdout.
# Exit with the return code from executing the logger.
#
result = 0
# import pdb; pdb.set_trace()
if not config.debug and not options.list:
platform = util.Platform()
if platform != config.WIN_NATIVE:
cmd = 'time ' + cmd
p = subprocess.Popen(cmd, shell=True)
p.communicate()
result = p.returncode
return result
