def PintoolHelpCmd(cmd, options):
"""
Print the pintool help msg using the script given in the
string 'cmd'.
@param cmd Command line to run lower level script
@param options Options given on cmd line
@return exit code from running cmd
"""
# Do we want to print out the command executed by util.RunCmd() or
# just run silently?
#
if options.verbose or options.debug:
pcmd = True
else:
pcmd = False
cmd += ' --pintool_help'
result = util.RunCmd(cmd,
options,
'',
concurrent=False,
print_time=False,
print_cmd=pcmd)
return result
def VerifyLITFiles(self, sim_run_cmd, options):
"""
Use a simulator to verify all the LIT files for the tracing instance are valid.
"""
# Verify the LIT files for each directory.
#
# import pdb; pdb.set_trace()
result = 0
for lit_dir in util.GetLitDir():
if not options.list:
msg.PrintMsgPlus('Verifying files in dir: ' + lit_dir)
# import pdb; pdb.set_trace()
cmd = sim_run_cmd + ' --replay_dir ' + lit_dir
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
cmd += ' --verify'
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
result = util.RunCmd(cmd, options, '',
concurrent=True) # Run jobs concurrently
if result != 0:
msg.PrintMsg('Error found during LIT file verification')
return result
# Wait for all the verification jobs to finish.
#
result = util.WaitJobs(options)
return result
def RunScriptBack(cmd, options):
"""
Run the script in the background, not the foreground.
Look for any errors from SDE which indicate there was a problem
setting up the port used to connect SDE with GDB. If detected,
then print error msg and exit script.
@param cmd Command line to run lower level script
@param options Options given on cmd line
@return result from running SDE
"""
# Do we want to print out the command executed by util.RunCmd() or
# just run silently?
#
if options.verbose or options.debug:
pcmd = True
else:
pcmd = False
result = 0
if not options.debug:
result = util.RunCmd(cmd,
options,
'',
concurrent=True,
print_time=False,
print_cmd=pcmd)
return result
def Replay(self, param, dirname, filename):
"""
Run the Simulator on a single LIT file given the command line options and the name of
the file to run. It formats the appropriate command line options,
saves global variables in a pickle file & calls the Sim run script.
"""
# import pdb ; pdb.set_trace()
if 'options' in param:
options = param['options']
else:
msg.PrintAndExit(
'method replay_dir.Replay() failed to get param \'options\'')
basename_file = os.path.join(dirname, filename)
if config.verbose:
msg.PrintMsg("-> Replaying pinball \"" + basename + "\"")
cmd = self.replayer_cmd + ' ' + basename_file
# Check to see if need to add options passed to the Sim run script.
# These are NOT parameters, so they don't get passed in the global
# variables.
#
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
if options.verify:
cmd += ' --verify'
# Add the configuration file, if one exists and print the cmd if just
# debugging.
#
# import pdb ; pdb.set_trace()
cmd += util.AddCfgFile(options)
if options.debug:
msg.PrintMsg(cmd)
result = 0
if not options.debug:
# Dump the global data to a unique file name. Need to add the
# option --global_file with this unique file name to options when
# calling a script.
#
gv = config.GlobalVar()
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
# import pdb ; pdb.set_trace()
if not options.list:
result = util.RunCmd(
cmd, options, filename,
concurrent=True) # Run concurrent jobs here
else:
msg.PrintMsg(cmd)
return result
def NormProjectBBV(self, options):
"""
Normalize and project the basic block vector file instead of doing this in Simpoint.
This is required so we can combine BBV and LDV files as frequency vector file given
to Simpoint.
@param options Options given on cmd line
@return result of command to generate projected file
"""
# Format the command and run it.
#
# import pdb; pdb.set_trace()
# Use options for the Python script to generate the CSV file.
#
output_file = 'normalize_project.out'
try:
fp_error = open(output_file, 'w')
except IOError:
msg.PrintMsg(
'ERROR: Failed to open normalize/project error file:\n'
' ' + output_file)
return -1
try:
fp_out = open(self.proj_bbv_file, 'w')
except IOError:
msg.PrintMsg(
'ERROR: Failed to open normalize/project output file:\n'
' ' + self.proj_bbv_file)
fp_error.close()
return -1
cmd = self.csv_bin
cmd += ' --project_bbv'
cmd += ' --bbv_file'
cmd += ' ' + self.generic_bbv_name
cmd += ' --dimensions 16'
# msg.PrintMsg('')
# print 'cmd: ' + cmd
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, '', concurrent=False, f_stdout=fp_out, \
f_stderr=fp_error)
msg.PrintMsg(' Output file: %s' % (self.proj_bbv_file))
msg.PrintMsg(' Stderr file: %s\n' % (output_file))
fp_out.close()
fp_error.close()
return result
def NormWeightLDV(self, options):
"""
Normalize and apply weights to LRU stack distance vector files.
@param options Options given on cmd line
@return result of command to generate file
"""
# Ensure there's an LDV file to process.
#
if not os.path.isfile(self.generic_ldv_name):
msg.PrintMsg('ERROR: Can\'t open LDV vector file.\n '
' ' + self.generic_ldv_name)
return -1
# Use options for the Python script to process the LDV file.
#
# import pdb; pdb.set_trace()
output_file = 'normalize_weight.out'
try:
fp_error = open(output_file, 'w')
except IOError:
msg.PrintMsg(
'ERROR: Failed to open normalize weights error file:\n'
' ' + output_file)
return -1
try:
fp_out = open(self.weight_ldv_file, 'w')
except IOError:
msg.PrintMsg(
'ERROR: Failed to open normalize weights output file:\n'
' ' + self.weight_ldv_file)
fp_error.close()
return -1
cmd = self.csv_bin
cmd += ' --weight_ldv '
cmd += ' --ldv_file ' + self.generic_ldv_name
cmd += ' --dimensions 16'
# msg.PrintMsg('')
# print 'cmd: ' + cmd
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, '', concurrent=False, f_stdout=fp_out, \
f_stderr=fp_error)
msg.PrintMsg(' Output file: %s' % (self.weight_ldv_file))
msg.PrintMsg(' Stderr file: %s\n' % (output_file))
fp_out.close()
fp_error.close()
return result
def RunSimpoint(self, options):
"""
Format and execute the command to run Simpoint.
@param options Options given on cmd line
@return result of running Simpoint
"""
import subprocess
# Output file for simpoint
#
output_file = 'simpoint.out'
try:
fp_out = open(output_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open simpoint error file:\n'
' ' + output_file)
return -1
# Format the Simpoint command and run it.
#
# import pdb; pdb.set_trace()
cmd = self.simpoint_bin
if options.ldv:
cmd += ' -fixedLength on -loadVectorsTxtFmt ' + self.freq_vect_file
else:
cmd += ' -loadFVFile ' + self.generic_bbv_name
# Add either the default options used to configure Simpoints or the
# Simpoint options from the user.
#
if options.simpoint_options == '':
cmd += ' -coveragePct ' + str(options.cutoff)
cmd += ' -maxK ' + str(options.maxk)
else:
cmd += ' ' + options.simpoint_options
cmd += ' -saveSimpoints ./t.simpoints -saveSimpointWeights ./t.weights -saveLabels t.labels'
result = util.RunCmd(cmd,
options,
'',
concurrent=False,
f_stdout=fp_out,
f_stderr=subprocess.STDOUT)
msg.PrintMsg(' Output file: %s' % (output_file))
msg.PrintMsg(' Stderr file: %s' % (output_file))
fp_out.close()
return result
def CombineFreqVectFiles(self, options):
"""
Combine the BBV and LDV files, applying a scaling factor to allow
different contributions from each file.
@param options Options given on cmd line
@return result of command to generate file
"""
# Format the command and run it.
#
# import pdb; pdb.set_trace()
# Use options for the Python script to generate the CSV file.
#
output_file = 'scaled_combined.out'
try:
fp_error = open(output_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open combined scale error file:\n'
' ' + output_file)
return -1
try:
fp_out = open(self.freq_vect_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open combined scale output file:\n'
' ' + self.freq_vect_file)
fp_error.close()
return -1
cmd = self.csv_bin
cmd += ' --combine ' + str(options.combine)
string = 'Combining BBV and LDV files with scaling factors: BBV: %.3f, LDV: %.3f\n' % \
(options.combine, 1 - options.combine)
msg.PrintMsgPlus(string)
fp_error.write(string)
cmd += ' --normal_bbv ' + self.proj_bbv_file
cmd += ' --normal_ldv ' + self.weight_ldv_file
result = util.RunCmd(cmd, options, '', concurrent=False, f_stdout=fp_out, \
f_stderr=fp_error)
msg.PrintMsg(' Output file: %s' % (self.freq_vect_file))
msg.PrintMsg(' Stderr file: %s\n' % (output_file))
fp_out.close()
fp_error.close()
return result
def RunScriptFore(cmd, options):
"""
Run the script in the foreground.
@param cmd Command line to run lower level script
@param options Options given on cmd line
@return error code from running script
"""
result = 0
if not options.debug:
result = util.RunCmd(cmd, options, '',
concurrent=False,
print_time=False,
print_cmd=False)
return result
def RunSimulator(self, dirname, sim_replay_cmd, phase_length, options):
"""
Run Branch Predictor simulator on all the pinballs in a set of
directories given by 'dirname'.
@param dirname Directory containing pinball to process
@param sim_replay_cmd Script used to replay pinball
@param phase_length Number of instructions executed before CMPSim will print next set of output
@param options Options given on cmd line
@return error code from running the simulator
"""
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --log_options'
cmd += ' "'
# 'phase_length' is given in instructions.
#
cmd += ' ' + self.print_data_knob + ' ' + str(phase_length)
cmd += '" '
cmd += ' --sim_add_filename' # This causes simulator output file name to be added to cmd.
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
end_str = '' # Don't want to print anything when this cmd finishes.
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=False)
return result
def RunSimulator(self, dirname, sim_replay_cmd, options):
"""
Run Branch Predictor simulator on all the pinballs in a set of
directories given by 'dirname'.
Return: error code from running the simulator
"""
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = 'pwd;' + sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --log_options'
# SiNUCA options
cmd += ' "'
cmd += ' -output ' + str(options.sinuca_tracer_output)
cmd += ' -threads ' + str(options.sinuca_tracer_threads)
cmd += ' -parallel_start ' + str(options.sinuca_tracer_parallel_start)
cmd += ' -parallel_end ' + str(options.sinuca_tracer_parallel_end)
cmd += ' "'
cmd += ' --sim_add_filename' # This causes simulator output file name to be added to cmd.
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
msg.PrintMsg('DEBUG>> ' + cmd + '\n')
end_str = '' # Don't want to print anything when this cmd finishes.
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, False)
return result
def RunScriptBack(cmd, options):
"""
When the script in the background, not the foreground.
@return error code from invoking the script (not result of running it)
"""
# Do we want to print out the command executed by util.RunCmd() or
# just run silently?
#
if options.verbose or options.debug:
pcmd = True
else:
pcmd = False
result = 0
if not options.debug:
result = util.RunCmd(cmd,
options,
'',
concurrent=True,
print_time=False,
print_cmd=pcmd)
return result
def RunScriptFore(cmd, options):
"""
Run the script in the foreground.
@return error code from running script
"""
# Do we want to print out the command executed by util.RunCmd() or
# just run silently?
#
if options.verbose or options.debug:
pcmd = True
else:
pcmd = False
result = 0
if not options.debug:
result = util.RunCmd(cmd,
options,
'',
concurrent=False,
print_time=False,
print_cmd=pcmd)
return result
def CrossBinaryMatcher(self, kit_path, script_path, options, bin_options):
"""
Run the cross binary matcher on all the binaries in the CBSP experiment.
NOTE: Only works if there is only one WP pinball in each WP pinball
directory (i.e. will NOT work with multipe process applications).
@param options Options given on cmd line
@param kit_path Path to kit
@param script_path Explicit path to location in kit where scripts are located
@param bin_options List of options for each CBSP binary
@return error_code
"""
# Get CBSP Data directory and WP pinball basenames. NOTE: For expanding
# this code to work with multi-process WP pinballs, need to expand list
# returned by util.GetWPPinballs() to include all pinballs in each WP
# dir.
#
result = 0
cbsp_data_dir = util.GetCBSPDataDir(options)
wp_pinballs = []
for bopt in bin_options:
pb = util.GetWPPinballs(bopt)
if not pb:
return -1
wp_pinballs.append(pb[0])
wp_basenames = [
os.path.join(re.sub('whole_program.*/', '', pb))
for pb in wp_pinballs
]
# Go to CBSP Data directory to run matcher
#
orig_dir = os.getcwd()
if os.path.isdir(cbsp_data_dir):
os.chdir(cbsp_data_dir)
else:
msg.PrintMsg('ERROR: Unable to change to CBSP Data directory: ' +
cbsp_data_dir)
return -1
# Matcher output file
#
output_file = 'crossBinaryMatcher_out.txt'
try:
fp_out = open(output_file, 'w')
except IOError:
msg.PrintMsg(
'ERROR: Failed to open cross binary matcher output file:\n'
' ' + output_file)
return -1
# Format cross binary matcher command & execute.
# Use 'bin_options[0]' because parameter should be the same for all
# binaries.
#
if not options.list:
msg.PrintMsgDate('Running cross binary matcher for: ' +
options.cbsp_name)
msg.PrintMsgPlus(
'Cross binary matcher output file (including errors): %s\n' %
os.path.join(cbsp_data_dir, output_file))
cmd = 'crossBinMatcher -graphm_config %s' % os.path.join(
kit_path, script_path, 'graphm.config.txt')
cmd += ' -slice_size %s' % str(bin_options[0].slice_size)
for pb, basename in zip(wp_pinballs, wp_basenames):
cmd += ' %s.{dcfg.json,trace.json}.bz2 ' % os.path.join('..', pb)
cmd += ' %s.bb-profile.bz2 ' % (basename)
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
result = util.RunCmd(cmd,
options,
'',
f_stdout=fp_out,
f_stderr=fp_out)
fp_out.close()
if not options.list:
msg.PrintMsgDate('Finished running cross binary matcher for: ' +
options.cbsp_name)
# Return to original directory
#
os.chdir(orig_dir)
return result
def FinalizeGDB(kit_script_path, options):
"""
Execute the final section of the program. If the user has not explicitly
given the port to use for communications between Pin and GDB, look in the
GDB command output file for the string 'target remote'. Timeout if not
found in a reasonable amount of time. Then add some more commands to the
GDB command file and run GDB.
@param kit_script_path Explicit path to location in kit where scripts are located
@param options Options given on cmd line
@return exit code from running GDB
"""
# Format the string 'target_str' to use the appropriate port
#
if options.debug_port:
# Use a specific port if the user defines one
#
# import pdb; pdb.set_trace()
target_str = 'target remote :%s' % options.debug_port
gdb_file = open(config.gdb_cmd_file, 'w')
else:
# Use the port chose by Pin. Pin will write it out to the
# GDB command file. Exit with an error if it's not there
# within 30 seconds.
#
target_str = 'target remote'
timeout = 30
count = 0
found = False
while count < timeout:
with open(config.gdb_cmd_file, 'r') as gdb_file:
if target_str in gdb_file.read():
found = True
break
time.sleep(1)
if hasattr(options, 'verbose') and options.verbose:
msg.PrintMsg('Waiting for "target remote"')
count += 1
found = True
if not found:
msg.PrintAndExit('Unable to find GDB string \'%s\' in file %s' %
(target_str, config.gdb_cmd_file))
time.sleep(1)
if hasattr(options, 'verbose') and options.verbose:
with open(config.gdb_cmd_file, 'r') as gdb_file:
msg.PrintMsg('Target cmd: ' + gdb_file.read())
# Get the port selected by Pin
#
# import pdb; pdb.set_trace()
with open(config.gdb_cmd_file, 'r') as gdb_file:
target_str = gdb_file.read()
if hasattr(options, 'verbose') and options.verbose:
msg.PrintMsg('Target cmd: ' + target_str)
gdb_file = open(config.gdb_cmd_file, 'w')
# Write some control info and the command to load Pin Python file to the
# GDB command file. Set PYTHONPATH to the location of the scripts.
#
# import pdb; pdb.set_trace()
time.sleep(1)
gdb_file.write('set remotetimeout 30000\n')
pin_python = os.path.join(kit_script_path, 'pin.py')
gdb_file.write('source %s\n' % (pin_python))
gdb_file.write('%s\n' % (target_str))
gdb_file.close()
time.sleep(1)
os.environ["PYTHONPATH"] = kit_script_path
if options.verbose:
print "\nGDB cmd file:"
os.system('cat ' + config.gdb_cmd_file)
print
# Format command and run GDB
#
cmd = gdb_path
cmd += ' --command=%s' % config.gdb_cmd_file
if hasattr(options, "gdb_options") and options.gdb_options:
cmd += ' %s' % options.gdb_options
cmd += ' %s' % options.command
if hasattr(options, 'verbose') and options.verbose and \
not options.debug:
msg.PrintMsg(cmd)
result = util.RunCmd(cmd,
options,
'',
concurrent=False,
print_time=False,
print_cmd=False)
return result
def FinalizeGDB(kit_script_path, options):
"""
Execute the final section of the program. Look in the GDB command output
file for the string 'target remote'. Timeout if not found in a reasonable
amount of time. Then add some more commands to the GDB command file
and run GDB.
@return exit code from running GDB
"""
# Look for the string 'target remote' in the GDB command file. Time out
# if not found within a limited amount of time.
#
target_str = 'target remote'
timeout = 30
count = 0
found = False
while count < timeout:
with open(config.gdb_cmd_file, 'r') as gdb_file:
if target_str in gdb_file.read():
found = True
break
time.sleep(1)
if hasattr(options, 'verbose') and options.verbose:
msg.PrintMsg('Waiting for "target remote"')
count += 1
found = True
if not found:
msg.PrintAndExit('Unable to find GDB string \'%s\' in file %s' %
(target_str, config.gdb_cmd_file))
time.sleep(1)
if hasattr(options, 'verbose') and options.verbose:
with open(config.gdb_cmd_file, 'r') as gdb_file:
msg.PrintMsg('Target cmd: ' + gdb_file.read())
# Write some control info and command to load Pin Python file to the GDB
# command file. Set PYTHONPATH to the location of the scripts.
#
# import pdb; pdb.set_trace()
gdb_file = open(config.gdb_cmd_file, 'a')
gdb_file.write('set remotetimeout 30000\n')
pin_python = os.path.join(kit_script_path, 'pin.py')
gdb_file.write('source %s\n' % (pin_python))
gdb_file.close()
time.sleep(1)
os.environ["PYTHONPATH"] = kit_script_path
# Run GDB
#
cmd = '%s --command=%s %s' % (gdb_path, config.gdb_cmd_file,
options.command)
if hasattr(options, 'verbose') and options.verbose:
msg.PrintMsg(cmd)
if options.verbose or options.debug:
pcmd = True
else:
pcmd = False
result = util.RunCmd(cmd,
options,
'',
concurrent=False,
print_time=False,
print_cmd=pcmd)
return result
# have parsed data for every cluster.
#
#import pdb; pdb.set_trace()
cluster_info, not_used, total_instr = util.GetClusterInfo(base_name, param)
if cluster_info == {}:
msg.PrintAndExit('Error reading file: ' + base_name)
cluster_list = util.ParseClusterInfo(cluster_info)
if len(cluster_info) != len(cluster_list):
msg.PrintAndExit('traceinfo.py() did not parse enough clusters.\n' ' Num clusters: %d\n' \
' Num parsed clusters: %d' % (len(cluster_info), len(cluster_list)))
# First print some info which is independent of the specific traces in the current directory.
#
count = len(cluster_info)
util.RunCmd(FmtPrintCmd(blank_DTD), options, '',
concurrent=False,
print_time=False,
print_cmd=False)
util.RunCmd(FmtPrintCmd(blank_head), options, '',
concurrent=False,
print_time=False,
print_cmd=False)
msg.PrintMsg('<traces>')
msg.PrintMsg('<trace-details trace-count="%d" total-instruction-count="%d" publication-workweek="CHANGEME" publication-year="2006">' % \
(count, total_instr))
# Print the trace specific information
#
err_msg = lambda string: msg.PrintAndExit('traceinfo.py() encountered '
'an error parsing \'' + string)
for cl in cluster_list:
if (cl.has_key('cluster_num')):
def RunSimulator(self, dirname, sim_replay_cmd, phase_length, options):
"""
Run a simulator on all the pinballs in a directory given by 'dirname'.
@param dirname Directory containing pinball to process
@param sim_replay_cmd Script used to replay pinball
@param phase_length Number of instructions executed before CMPSim will print next set of output
@param options Options given on cmd line
@return Error code from running the simulator
"""
# Check to make sure the directory at least exists.
#
# import pdb; pdb.set_trace()
if not os.path.exists(dirname):
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg('WARNING: Directory containing pinballs to run with simulator does not exist:\n ' + \
dirname)
msg.PrintMsg('Since tracing mode is \'mpi_mt\', this may be OK.')
return 0
else:
msg.PrintMsg('ERROR: Directory containing pinballs to run with simulator does not exist:\n ' + \
dirname)
return -1
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Get the number of threads in the pinball and round it up
# to the next power of 2. This is a requirement of CMPSim.
#
# import pdb; pdb.set_trace()
threads = util.GetNumThreadsWP(options)
threads = util.RoundupPow2(threads)
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --replay_options'
cmd += ' "' + options.replay_options + '"'
cmd += ' --log_options'
cmd += ' "'
cmd += '-threads ' + str(threads)
# If the directory contains the WP directory name for this tracing
# instance, then get the icount for each process/thread in the WP
# pinballs. Use the longest icount as the phase_length for CMPSim.
#
# This is done to reduce the CMPSim output to the very minimum for WP
# pinballs. For very large WP pinballs, using the 'default' value for
# phase_length generates CMPSim output which can be multiple GBs in size.
#
# import pdb; pdb.set_trace()
if util.GetWPDir() in dirname:
max_icount = 0
pinballs = glob.glob(os.path.join(dirname, '*.address'))
for pb in pinballs:
fname = os.path.basename(pb)
icount = util.GetMaxIcount(dirname, fname)
if icount > max_icount:
max_icount = icount
# Check for errors
#
# import pdb; pdb.set_trace()
if max_icount != 0:
phase_length = max_icount / 1000000
if phase_length == 0:
# Must have a length of at least 1
#
phase_length = 1
cmd += ' ' + self.print_data_knob + ' ' + str(phase_length)
cmd += '" '
cmd += ' --sim_add_filename' # This causes CMPSim output file name to be added to cmd
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
# import pdb; pdb.set_trace()
end_str = '' # Don't want to print anything when this cmd finishes.
result = util.RunCmd(cmd, options, end_str, concurrent=False)
return result
def GenRegionCSVFile(self, options):
"""
Format and execute the command to generate the regions CSV file.
@param options Options given on cmd line
@return result of command to generating CSV file
"""
# Setup some stuff for generating regions CSV files.
#
# import pdb; pdb.set_trace()
cutoff_suffix = ''
if options.cutoff < 1.0:
cutoff_suffix = '.lpt' + str(options.cutoff)
pos = options.data_dir.find('.Data')
# Output and error files
#
regions_csv_file = options.data_dir[:pos] + '.pinpoints.csv'
try:
fp_out = open(regions_csv_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open CSV output file:\n'
' ' + regions_csv_file)
return -1
output_file = 'create_region_file.out'
try:
fp_error = open(output_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open CSV error file:\n'
' ' + output_file)
fp_out.close()
return -1
# Format the command to generate the region CSV file and run it.
#
# import pdb; pdb.set_trace()
if options.focus_thread < 0:
tid = 0
else:
tid = options.focus_thread
# use_orig = True # Use Chuck's original Perl script
use_orig = False # Use regions.py script
if use_orig:
# Use Chuck's Perl script to generate the CSV file.
#
cmd = 'create_region_file.pl'
cmd += ' ' + self.generic_bbv_name
cmd += ' -seq_region_ids -tid ' + str(tid)
cmd += ' -region_file t.simpoints' + cutoff_suffix
cmd += ' -weight_file t.weights' + cutoff_suffix
else:
# Use the new Python script to generate the CSV file.
#
cmd = self.csv_bin
cmd += ' -f ' + str(tid)
cmd += ' --csv_region '
cmd += ' --bbv_file ' + self.generic_bbv_name + cutoff_suffix
cmd += ' --region_file t.simpoints' + cutoff_suffix
cmd += ' --weight_file t.weights' + cutoff_suffix
msg.PrintMsg('')
result = util.RunCmd(cmd, options, '', concurrent=False, print_time=False, f_stdout=fp_out, \
f_stderr=fp_error)
msg.PrintMsg(
' NOTE: For this script, problems can be in either the output or stderr files. Check them both!'
)
msg.PrintMsg(' Output file: %s' % (regions_csv_file))
msg.PrintMsg(' Stderr file: %s\n' % (output_file))
fp_out.close()
fp_error.close()
return result, regions_csv_file
def RunSniper(pp_dir, sim_replay_cmd, phase_length, options):
def round(x, roundby=1000000):
return int(int((x / float(roundby))) * float(roundby))
# List of pinballs used to print Sniper output when all runs are complete.
#
pb_list = []
ret = 0
if not os.path.isdir(pp_dir):
# If running in MPI_MT_MODE, then it's possible for one process to not
# have a thread corresponding to the the current focus thread.
# However, another process might have this thread. Thus, only return
# an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg('WARNING: Directory containing pinballs to run with simulator does not exist:\n ' + \
pp_dir)
msg.PrintMsg('Since tracing mode is \'mpi_mt\', this may be OK.')
return 0
else:
msg.PrintMsg('ERROR: Directory containing pinballs to run with simulator does not exist:\n ' + \
pp_dir)
return -1
# List of output sniper directories.
#
output_dir_list = []
for fn in os.listdir(pp_dir):
if fn.endswith('.address'):
pinball_path = os.path.join(pp_dir, os.path.splitext(fn)[0])
fn = os.path.splitext(fn)[0]
sniper_outputdir = os.path.join(config.sniper_result_dir,
pinball_path)
output_dir_list += [sniper_outputdir]
sniper_outputfile = pinball_path + config.sniper_out_ext
if options.debug:
# If debugging, check to see if the Sniper result files already exist for the pinball. If so,
# then print it out but don't run Sniper again.
#
if os.path.isdir(sniper_outputdir):
msg.PrintMsgPlus(
'WARNING: Skipping Sniper execution because output file already exists.\n'
' %s' % sniper_outputdir)
pb_list.append(pinball_path)
continue
# Select the proper config/options to run the desired version of
# Sniper/SniperLite.
#
# import pdb; pdb.set_trace()
if options.sniper_options:
common_sniper_opts = options.sniper_options
else:
if options.no_sniperlite:
common_sniper_opts = ''
else:
# use_orig = True # Use older SniperLite options
use_orig = False # New SniperLite options
if use_orig:
common_sniper_opts = ' -c dunnington -c cacheonly -c nehalem_cmpsim.cfg ' \
'-g --general/enable_icache_modeling=false ' \
'-g --perf_model/dram/direct_access=true ' \
'-g --perf_model/dram/queue_model/type=contention ' \
'-g --perf_model/dtlb/size=0'
else:
# Older patched Sniper 5.3
#
# common_sniper_opts = ' -c dunnington -c nehalem_cmpsim.cfg -c ccpp1c --pinball-non-sift \
# -g -replay:addr_trans -g --general/enable_icache_modeling=false'
# Newer patched Sniper 5.3
#
# common_sniper_opts = ' -c dunnington -c nehalem_cmpsim.cfg -c cc-fast --pinball-non-sift \
# -g -replay:addr_trans -g --general/enable_icache_modeling=false'
# Production SniperLite 6.0 options
#
common_sniper_opts = ' -c nehalem-lite --pinball-non-sift '
partial_run_cmd = (common_sniper_opts + ' --no-cache-warming')
try:
# If re.search() fails, code falls though to the exception.
#
warmup_region = re.search('warmup(\d+)_prolog(\d+)_region(\d+)_epilog(\d+)_(\d+)_(\d-\d+)', \
pinball_path)
# Get info on the length of regions in the pinball.
#
warmup = int(warmup_region.group(1))
prolog = int(warmup_region.group(2))
file_region = int(warmup_region.group(3))
epilog = int(warmup_region.group(4))
region_num = int(warmup_region.group(5))
weight = warmup_region.group(6).replace('-', '.')
icount = util.GetMaxIcount('', pinball_path)
sim_region_len = icount - warmup
calc_region = sim_region_len - prolog - epilog
if warmup != 0:
# If there are warmups, then need to use options to first do cache warmups in Sniper,
# then simulate the region.
#
partial_run_cmd = (common_sniper_opts + ' -s stop-by-icount:%d:%d ' \
% (sim_region_len, round(warmup)) + ' --roi-script ')
if not options.list:
# Print details about the various sections in the pinball.
#
msg.PrintMsgPlus('Running Sniper on: ' + fn)
msg.PrintMsg(' Warmup count: ' +
locale.format('%14d', warmup, True))
msg.PrintMsg(' Prolog count: ' +
locale.format('%14d', prolog, True))
msg.PrintMsg(' Actual region count: ' + locale.format('%14d', calc_region, True) + \
' (from file name: ' + locale.format('%d', file_region, True) + ')')
msg.PrintMsg(' Epilog count: ' +
locale.format('%14d', epilog, True))
msg.PrintMsg(' Total Instr count: ' +
locale.format('%14d', icount, True))
except AttributeError:
if 'whole_program' not in pinball_path:
# Whole program pinballs cannot be processed using the options
# given above (using -s stop-by-icount:) because they don't have
# warmup instructions to be skipped.
#
# Let the user know the results may be suspect because the dir
# appears to contain whole program pinballs, but the name doesn't
# contain the string 'whole_program'. However, don't flag this as
# an erorr. It's possible for the user to give the name of a WP
# pinball directory which does not contain this string.
#
msg.PrintMsgPlus(
'WARNING: This pinball may not really be a whole program pinball.\n'
' If this is true, the results may not be valid.\n'
' ' + pinball_path)
msg.PrintMsgPlus('Running Sniper on whole program pinball: ' +
pinball_path)
# Format the command and execute it asynchronously.
#
cmd = os.path.join(options.sniper_root, sim_replay_cmd) + partial_run_cmd + \
(' -d "%s" ' % sniper_outputdir) + ' --pinballs ' + pinball_path + \
' > ' + sniper_outputfile + ' 2>&1 '
pb_list.append(pinball_path)
end_str = fn
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=True)
ret = result or ret
result = util.WaitJobs(options)
ret = result or ret
# Make sure some of the important Sniper output files exist for each pinball.
#
file_list = ['sim.stats.sqlite3', 'sim.info', 'sim.cfg', 'sim.out']
for sim_dir in output_dir_list:
for f in file_list:
if not os.path.isfile(os.path.join(sim_dir, f)):
msg.PrintMsg('\nERROR: Sniper output file does not exist:\n' \
' ' + os.path.join(sim_dir, f))
ret = -1
# Define a set of strings which can be ignored as errors if they occur in
# the output. These are special cases which must be added by hand when a
# new application is found that contains a string which might be detected
# as an erorr, but is a normal component of the output from running the
# application.
# Errors generated by Sniper itself which are acceptable.
#
ign_str = [
'void Sift::Writer::Sync(): Assertion',
'Sift::Reader::decodeInstruction'
]
pin_app_term = 'Pin app terminated abnormally'
ign_str += [pin_app_term]
# Strings generated by SPEC CPU2006 benchmarks: dealII-ref-1
#
ign_str += ['Estimated error=']
# Strings generated by MILC
#
ign_str += ['error_per_site', 'No O(a^2) errors', 'error_for_propagator']
# Print the output from running Sniper and look for errors in the output.
#
error = False
for pinball_path in pb_list:
# Get just the pinball name & print it
#
fn = os.path.split(pinball_path)[1]
sniper_outputfile = pinball_path + config.sniper_out_ext
if os.path.isfile(sniper_outputfile):
try:
f_stdout = open(sniper_outputfile, 'r')
except IOError:
msg.PrintMsg(
'ERROR: Can\'t open Sniper output file to look for errors.\n'
' ' + sniper_outputfile)
ret = -1
# Print the output file and look for errors.
#
if not options.list:
if 'whole_program' in pinball_path:
msg.PrintMsg('\nSniper output for: ' + pinball_path)
else:
msg.PrintMsg('\nSniper output for: ' + fn)
for line in f_stdout.readlines():
msg.PrintMsgNoCR(line)
if not options.ignore_sniper_error and \
('ERROR' in line or \
'error' in line or \
'Traceback' in line):
# Check to see if we can accept this line because
# it contains one of the strings we can ignore.
#
ok_ignore = False
for st in ign_str:
if st in line:
ok_ignore = True
break
if not ok_ignore:
# It's truly an error
#
error = True
# Need to print a msg indicating this error is OK to ignore.
#
if pin_app_term in line:
msg.PrintMsg(
'The \'Pin app terminated\' msg can be ignored. '
'It is not an error, just a warning.')
if error:
msg.PrintMsg('\nERROR: Sniper failed with an error.')
ret = -1
error = False # Reset in order to look for error in next pb
return ret
def GenLitFiles(self, param, dirname, file_name):
"""
Generate either the LIT files or warmup LMAT files for one pinball based on the
value 'flavor' passed in the param dictionary.
The region pinball contains optional warmup instructions before the instructions
in the actual representative regions. If there aren't any warmup instructions for a
pinball, then skip the step which generates LMAT files.
"""
# Setup locale so we can print integers with commas.
#
import locale
locale.setlocale(locale.LC_ALL, "")
# Define a lambda function to print an error message.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: method sde_phases.GenLitFiles() failed to '
'get field: ' + string)
# Unpack replay_cmd, options and flavor from the dictionary param.
# Flavor determines the type of file to generate.
#
if param.has_key('flavor'):
flavor = param['flavor']
else:
err_msg('flavor')
return -1
if param.has_key('options'):
options = param['options']
else:
err_msg('options')
return -1
if param.has_key('replay_cmd'):
replay_cmd = param['replay_cmd']
else:
err_msg('replay_cmd')
return -1
result = 0
# Get information from the pinball and the pinball file name.
# Dummy variables: a, b, c, d, e, f
#
(icount, warmup, prolog, calc_region, epilog, tid,
region) = util.GetRegionInfo(os.path.join(dirname, file_name), options)
(a, b, c, file_region, d, e, f) = util.GetRegionInfo(
os.path.join(dirname, file_name), options,
use_file_fields=True)
# If generating LMAT files, yet there aren't any warmup instructions for this pinball, then skip
# this file by returning from the method.
#
if not options.list:
if flavor == self.LMAT and warmup == 0:
msg.PrintMsgPlus(
"NOTE: No warmup instructions. Not generating LMAT files for trace:\n"
" " + file_name)
return 0
# Check to make sure the TID in file name is the same as the tracing parameter 'focus_thread'.
#
if tid != config.focus_thread and config.focus_thread >= 0:
msg.PrintMsg('ERROR: When generating LIT files, the TID in a pinball\n' \
'file name is not the same as the tracing parameter \'focus_thread\'. \n' \
'Focus thread is: ' + str(config.focus_thread) + ', yet the TID ' \
'in the file name is: ' + str(tid) + '.\n' \
'Please double check to make sure the focus thread parameter is \n' \
'correct (given with \'option -f X\' where X is the TID.)\n' \
' ' + file_name + '\n')
return -1
# Calculate The length of all regions except the warmup
#
lit_len = icount - warmup
# Get the number of threads from the result files
#
field = util.FindResultString(
os.path.join(dirname, file_name + '.result'), 'static_threads')
nthreads = field[0]
if not nthreads:
# Perhaps we are dealing with a pinball version > 2.4. Look in the
# *.global.log file for the number of threads.
#
field = util.FindString(
os.path.join(dirname, file_name + '.global.log'), 'static_threads')
nthreads = field[0]
if nthreads:
nthreads = int(nthreads)
else:
nthreads = 0
# Check to make sure there are a significant number of instructions in the pinball.
#
if calc_region < 1000:
msg.PrintMsg(
'=============================================================')
msg.PrintMsg(
'WARNING: The icount for focus thread region pinball is low:')
msg.PrintMsg(' Dir: ' + dirname)
msg.PrintMsg(' File: ' + file_name)
else:
if not options.list:
msg.PrintMsgPlus('File: ' + file_name)
if not options.list:
# Print details about the various sections in the pinball.
#
msg.PrintMsg(' Warmup count: ' +
locale.format('%14d', warmup, True))
msg.PrintMsg(' Prolog count: ' +
locale.format('%14d', prolog, True))
msg.PrintMsg(' Actual region count: ' + locale.format('%14d', calc_region, True) + \
' (from file name: ' + locale.format('%d', file_region, True) + ')')
msg.PrintMsg(' Epilog count: ' +
locale.format('%14d', epilog, True))
msg.PrintMsg(' Total Instr count: ' +
locale.format('%14d', icount, True))
msg.PrintMsg(' Number of threads: ' +
locale.format('%14d', nthreads, True))
if config.focus_thread >= 0:
msg.PrintMsg(' Focus thread: ' +
str(config.focus_thread))
# Get the 'base' file name without the TID.
#
base_name = util.RemoveTID(file_name)
# Get the LIT directory name from the region pinball directory name.
#
lit_dir = util.ChangeExtension(dirname, '.pp', '.lit')
# Several file names, with path, for pinball and LIT files.
#
# Need to remove TID for the file name given to knob '-log:basename'
# when generating LMAT files. This must be done is because pinLIT will
# add the TID to the LIT file names. Hence it must be removed in the
# 'basename'.
#
# import pdb; pdb.set_trace()
pb_path_file = os.path.join(dirname, file_name)
lit_path_file = os.path.join(lit_dir, file_name)
lit_path_file_no_tid = os.path.join(lit_dir, base_name)
# msg.PrintMsg('pb_path_file: ' + pb_path_file)
# msg.PrintMsg('lit_path_file: ' + lit_path_file)
# msg.PrintMsg('lit_path_file_no_tid: ' + lit_path_file_no_tid)
# Path & file name to the PTOV file which is generated by pinLIT. If
# there are no warmups for the file, then there won't be a 'warmup.ptov'
# file for it.
#
#
ptov_path_file = glob.glob(
os.path.join(lit_dir, base_name) + '*.warmup.ptov')
if ptov_path_file:
ptov_path_file = ptov_path_file[0]
# Format the initial part of the command line which is used for both LMAT and LIT files.
#
cmd_0 = replay_cmd
cmd_0 += ' --replay_file ' + pb_path_file
cmd_0 += ' --log_options '
# Determine if all parameters should be dumped to the global file
# (PinPlay), or just a subset (CBSP).
#
if options.cbsp():
pinplay = False
else:
pinplay = True
cmd_1 = util.AddGlobalFile(self.gv.DumpGlobalVars(pinplay), options)
cmd_1 += util.AddCfgFile(options)
# Add user defined 'lit_options' to both the LMAT and LIT command
# lines. May want sepatate options in the future, but right now
# there's only one.
#
knobs = ''
if hasattr(
options,
'replay_options') and options.replay_options: # Add any user defined replay options
knobs += ' ' + options.replay_options
if hasattr(
options,
'lit_options') and options.lit_options: # Add any user defined replay options
knobs += ' ' + options.lit_options
if flavor == self.LMAT:
# Format the knobs required to generate LMAT files.
#
knobs += ' -log -xyzzy -log:LIT -log:LIT_warmup ' + self.kit_obj.knob_length + ' ' + str(
warmup)
knobs += ' -log:early_out'
knobs += ' -log:basename ' + lit_path_file_no_tid
if hasattr(options, 'compressed'):
knobs += ' -log:compressed ' + options.compressed
knobs += util.GetMsgFileOption(lit_path_file + '.lmat')
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LMAT files: ' + file_name)
end_str = file_name
else:
if warmup > 0 and ptov_path_file:
# Need this knob when generating LIT files if LMAT files already generated
# for this trace.
#
knobs += ' -log:LIT_use_ptov ' + ptov_path_file
# Format the knobs required to generate LIT files.
#
#import pdb; pdb.set_trace()
knobs += ' -log -log:LIT ' + self.kit_obj.knob_skip + ' ' + str(warmup)
if lit_len < 0:
msg.PrintMsg(
'=============================================================')
msg.PrintMsg(
'WARNING: The icount for focus thread region pinball is low:')
msg.PrintMsg(' Dir: ' + dirname)
msg.PrintMsg(' File: ' + file_name)
knobs += ' ' + self.kit_obj.knob_length + ' ' + '100'
msg.PrintMsg(' No LIT will be generated')
else:
knobs += ' ' + self.kit_obj.knob_length + ' ' + str(lit_len)
knobs += ' -log:early_out'
knobs += ' -log:basename ' + lit_path_file_no_tid
if hasattr(options, 'compressed'):
knobs += ' -log:compressed ' + options.compressed
knobs += util.GetMsgFileOption(lit_path_file + '.lit')
if hasattr(options, 'list') and not options.list:
msg.PrintMsgPlus('Generating LIT files: ' + file_name)
end_str = file_name
# Format & execute the command.
#
cmd = cmd_0 + '"' + knobs + '"' + cmd_1
# Run the command in the background in order to run concurrent jobs.
#
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=True)
return result
def GenImix(self, param, dirname, base_name):
"""
Generate the instruction mix for one pinball.
"""
# Define a lambda function to print a standard error msg.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: method sde_phases.GenImix() failed to get '
'field: ' + string)
# Get local copies of items in param.
#
if param.has_key('options'):
options = param['options']
else:
err_msg('options')
return -1
if param.has_key('replayer_cmd'):
replayer_cmd = param['replayer_cmd']
else:
err_msg('replayer_cmd')
return -1
# Instruction mix output file name.
#
file_name = os.path.join(dirname, base_name)
imix_file = file_name + '.imix.txt'
# Format the command.
#
cmd = replayer_cmd + ' --replay_file ' + file_name
if options.replay_options: # Add any user defined replay options
cmd += ' ' + options.replay_options
cmd += ' --log_options '
cmd += '"'
cmd += ' -omix ' + imix_file
# Assume if the user gives any knobs to SDE, they replace the
# default knobs added here.
#
if options.pin_options == '':
cmd += ' -top_blocks 100 '
cmd += '"'
if not options.list:
msg.PrintMsgPlus('Generating imix for: ' + base_name)
# Dump the global vars to a pickle file and add this option
# to the command. Then do it.
#
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
if hasattr(options, 'verbose') and options.verbose and not options.list:
msg.PrintMsg('sde_phases.GenImix, calling RunCmd()')
end_str = base_name
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str,
concurrent=True) # Run concurrent jobs here
return result
def GenTraceinfoFiles(self, options):
"""
Generate traceinfo files from sysinfo files. Also do some simple error
checking.
One traceinfo file is generated for each directory which contains
region pinballs. Currently, the script 'create_traceinfo.sh' is used
to generate the traceinfo file. This will be replaced with a Python
script at sometime in the future.
The following info is printed to allow the user to determine if the
required number of region pinballs and traces were generated:
1) The script 'count_traces.sh' gets a count of the number of
expected traces and the region numbers from the region CSV file.
2) The number of actual traces generated and the trace names.
3) The number of region pinballs generated and the pinball names.
"""
# Check to make sure there is at least one region pinball directory for
# this tracing instance.
#
all_pp_dirs = util.GetRegionPinballDir(options)
if all_pp_dirs == [] and not options.debug:
msg.PrintMsg(
'\nERROR: Could not find any PinPoint \'*.pp\' directories.\n'
'Make sure you have run the phase \'-p\'.')
return -1
# Get the region pinball directories for this tracing instance.
#
# import pdb; pdb.set_trace()
all_lit_dirs = util.GetLitDir()
if all_lit_dirs == [] and not options.debug:
msg.PrintMsg(
'\nERROR: Could not find any LIT \'*.lit\' directories.\n'
'Make sure you have run the phase \'-L\'.')
return -1
result = 0
# import pdb; pdb.set_trace()
for lit_dir in all_lit_dirs:
# Get the name of the corresponding region pinball directory
# and make sure it exists.
#
pp_dir = util.ChangeExtension(lit_dir, '.lit', '.pp')
if not os.path.isdir(pp_dir):
if not options.list:
msg.PrintMsgPlus('WARNING: Generating traceinfo files, but the required \'pp\' ' \
'directory does not exist:\n ' + pp_dir)
# If running in MPI_MT_MODE, then it's possible for one process to
# not have a thread corresponding to the the current focus thread.
# However, another process might have this thread.
# Thus, only return an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
return -1
else:
# Not printing any msgs, just skip to the next directory
#
continue
# Make sure the LIT directory exists, then go there.
#
old_dir = os.getcwd()
if os.path.isdir(lit_dir):
os.chdir(lit_dir)
else:
if not options.list:
msg.PrintMsgPlus('WARNING: Generating traceinfo files, but the LIT ' \
'directory does not exist:\n ' + lit_dir)
# If running in MPI_MT_MODE, then it's possible for one process to
# not have a thread corresponding to the the current focus thread.
# However, another process might have this thread.
# Thus, only return an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
return -1
else:
# Not printing any msgs, just skip to the next directory
#
continue
# Copy the traceinfo 'blank' XML files from the SDE kit.
#
blank_path = self.kit_obj.GetTraceinfoBlank()
for blank in config.traceinfo_blank_files:
blank_file = os.path.join(blank_path, blank)
try:
shutil.copy(blank_file, os.getcwd())
except IOError:
msg.PrintMsg(
'\nERROR: Unable to copy traceinfo \'blank\' file:\n '
+ blank_file)
return -1
# Run the script to generate traceinfo XML file. Stdout from the
# script is the XML file. Function util.RunCMD() needs the output XML
# file object in order to write this file.
#
msg.PrintMsg('')
base_name = util.ChangeExtension(lit_dir, '.lit', '')
tr_file = base_name + '.traceinfo.xml'
try:
fp_out = open(tr_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open traceinfo output file:\n'
' ' + tr_file)
return -1
cmd = self.traceinfo_bin + ' ' + base_name
result = util.RunCmd(cmd, options, '',
concurrent=False,
f_stdout=fp_out)
if result != 0:
msg.PrintMsg('Error found while running script \'%s\'' %
self.traceinfo_bin)
return -1
# Print info from the CSV regions file (located in the *.Data
# directory) about the number of expected traces.
#
msg.PrintMsg('')
param = {'in_lit_dir': True}
cluster_info, not_used, total_instr = util.GetClusterInfo(base_name, param)
if cluster_info == {}:
msg.PrintMsg(
'ERROR: Problems getting cluster info from file: %s.pinpoints.csv'
% (base_name))
return -1
cluster_list = util.ParseClusterInfo(cluster_info)
if len(cluster_info) != len(cluster_list):
msg.PrintMsg('ERROR: Did not parse enough clusters from CSV file: %s.pinpoints.csv\n' ' Num clusters: %d\n' \
' Num parsed clusters: %d' % (base_name, len(cluster_info), len(cluster_list)))
return -1
# Print the number of expected traces from the regions CSV file.
#
base_tid = -1
for cl in cluster_list:
if (cl.has_key('tid')):
tid = cl['tid']
else:
msg.PrintMsg(
'ERROR: Parsing cluster info for cluster:\n %s' %
(cl))
return -1
if base_tid == -1:
base_tid = tid
else:
if tid != base_tid:
msg.PrintAndExit(
'ERROR: Expected TID %d, but found TID %d' %
(base_tid, tid))
return -1
msg.PrintMsg('Expected trace count: %d\n' % (len(cluster_info)))
# Print the number of actual traces in the LIT directory and the names.
#
# import pdb; pdb.set_trace()
if not options.list:
msg.PrintMsg(
'Actual trace count: ' + str(util.CountFiles('ami')))
lit_files = glob.glob('*.ami*')
lit_files.sort()
for f in lit_files:
msg.PrintMsg(' ' + f)
# Clean up tmp files in the LIT directory.
#
tmp_files = glob.glob('*blank*.xml')
for f in tmp_files:
os.remove(f)
# Return to the working directory.
#
os.chdir(old_dir)
# Go to the *.pp directory. Print the number of actual pinballs and the names.
#
os.chdir(pp_dir)
if not options.list:
msg.PrintMsg(
'\nPinball count: ' + str(util.CountFiles('address')))
pp_files = glob.glob('*.address')
pp_files.sort()
for f in pp_files:
msg.PrintMsg(' ' + f)
# Print a warning if the expected number of traces are not found.
#
if len(lit_files) != len(pp_files):
msg.PrintMsgPlus(
'WARNING: Number of traces does not match the number of region pinballs.')
msg.PrintMsg('\nGenerated traceinfo file: ' + tr_file)
# Return to the working directory.
#
os.chdir(old_dir)
return result
def Replay(self, param, dirname, filename):
"""
Replay a single pinball given the command line options and the name of
the pinball to replay. It formats the appropriate command line options,
saves global variables in a pickle file & calls the replayer script.
@param param Dictionary containing all parameters that need to be
passed into the method. Need a dictionary because this
method is sometimes called by walk_callback() which
only allows one parameter in the functions it calls.
@param dirname Directory where pinball is located
@param filename Pinball base file name
@return Exit code from the replayer script.
"""
if 'options' in param:
options = param['options']
else:
msg.PrintAndExit(
'method replay_dir.Replay() failed to get param \'options\'')
if options.verbose:
msg.PrintMsg('Start of Replay() in replay_dir.py')
# import pdb ; pdb.set_trace()
basename_file = os.path.join(dirname, filename)
command = self.replayer_cmd + ' --replay_file ' + basename_file
if options.verbose:
msg.PrintMsg("-> Replaying pinball \"" + basename_file + "\"")
if options.replay_options:
command += ' --replay_options "' + options.replay_options + '"'
# Check to see if need to add options for BB vector generation. Set
# 'log_opt' to any options the user may have put on the command line.
#
log_opt = options.log_options
if options.bb_add_filename:
file_name = os.path.basename(basename_file)
# If there is a focus thread, then need to remove the TID from the
# file name.
#
# NOTE: This code may need to be fixed when a method of running
# Simpoints on all threads of cooperative pinballs is implemented.
#
file_name = util.RemoveTID(file_name)
# Write BB vector files to the newly created *.Data directory.
#
data_dir = file_name + '.Data'
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
log_opt += ' -o ' + os.path.join(data_dir, file_name)
# Check to see if need to add options when running a simulator.
#
# import pdb ; pdb.set_trace()
if options.sim_add_filename:
# Need to instantiate a kit of the type simulator being used.
# This is required on order to get some kit specific information.
#
# NOTE: If you are adding a kit for a new simulator, then you need
# to modify this code.
#
if config.sim_kit_type == config.BRPRED:
import brpred_kit
sim_kit = brpred_kit.BrPredKit()
elif config.sim_kit_type == config.SINUCA:
import sinuca_kit
sim_kit = sinuca_kit.Sinuca_TracerKit()
elif config.sim_kit_type == config.CMPSIM:
import sde_cmpsim_kit
sim_kit = sde_cmpsim_kit.CMPsimKit()
else:
msg.PrintAndExit('Undefined kit type in method replay_dir.Replay(): ' + \
str(config.sim_kit_type))
# Add the simulator knob to specify the file for the output from
# the simulator.
#
# ~ log_opt += ' ' + sim_kit.GetSimOutputFile(basename_file)
# When 'log_opt' is added to the command line below, it will
# put double quotes (") around all the options. Therefore, need to
# remove any exising double quotes in the current value for the
# string 'log_opt'.
#
log_opt = log_opt.replace('"', '')
# If relogging WP pinballs, need to add the -log:basename knob with
# the relogged pinball path/name.
#
# import pdb ; pdb.set_trace()
if options.wp_relog_dir:
ft = util.GetFocusThreadPB(basename_file)
if ft > -1 and not options.relog_focus:
# If WP pinballs were relogged with a focus thread, then the
# resulting pinballs were 'per thread', not 'cooperative'. If
# relogging with a different filter (i.e. options.relog_focus ==
# False) then need to remove TID from base file name given to
# the knob -log:basename.
#
file_name = os.path.basename(util.RemoveTID(basename_file))
else:
file_name = os.path.basename(basename_file)
log_opt += ' -log:basename ' + os.path.join(options.wp_relog_dir,
file_name)
if not options.list:
msg.PrintMsgDate('Relog whole program pinball: ' + file_name)
if log_opt:
command += ' --log_options "' + log_opt + '"'
if options.playout:
command += ' --playout '
# if not options.list:
# msg.PrintMsg(command)
# If not just listing the command, then dump the global
# variables so the next Python script can have access to them.
# Then run the script.
#
result = 0
if not options.list:
# Dump the global data to a unique file name. Need to add the
# option --global_file with this unique file name to options when
# calling a script.
#
gv = config.GlobalVar()
command += util.AddGlobalFile(gv.DumpGlobalVars(), options)
command += util.AddCfgFile(options)
result = util.RunCmd(command, options, filename,
concurrent=True) # Run concurrent jobs here
else:
# If the option 'list' is defined, then just list out the
# commands to be exectuted, but don't execute them.
#
msg.PrintMsg(command)
return result
def RunSimPoint(self, kit_path, script_path, options, bin_options):
"""
Run Simpoint in the CBSP Data directory and generate weight files for
each binary in the respective binary Data directory.
@param kit_path Path to kit
@param script_path Explicit path to location in kit where scripts are located
@param options Options given on cmd line
@param bin_options List of options for each CBSP binary
@return exit code from Simpoint
"""
# Get CBSP Data directory and WP pinball basenames. For multi-process
# need to expand list returned by util.GetWPPinballs() to include all
# pinballs in each WP dir.
#
result = 0
cbsp_data_dir = util.GetCBSPDataDir(options)
wp_pinballs = [util.GetWPPinballs(bopt)[0] for bopt in bin_options]
wp_basenames = [
os.path.join(re.sub('whole_program.*/', '', pb))
for pb in wp_pinballs
]
# Go to CBSP Data directory to run Simpoint
#
orig_dir = os.getcwd()
if os.path.isdir(cbsp_data_dir):
os.chdir(cbsp_data_dir)
else:
msg.PrintMsg('ERROR: Unable to change to CBSP Data directory: ' +
cbsp_data_dir)
return -1
# Format the command to run simpoints.
# Use 'bin_options[0]' because parameter should be the same for all
# binaries.
#
sim_out_file = 'run_simpoint_out.txt'
if not options.list:
msg.PrintMsgDate('Running Simpoints for: %s' % options.cbsp_name)
msg.PrintMsgPlus('Simpoint output file (including errors): %s\n' %
os.path.join(cbsp_data_dir, sim_out_file))
if bin_options[0].simpoint_options:
msg.PrintMsgPlus(
'NOTE: Default options for Simpoint not used, only user defined options.'
)
cmd = 'simpoint ' + bin_options[0].simpoint_options
else:
if bin_options[0].maxk:
cmd = 'simpoint -k 2:%d -dim 100 -numInitSeeds 25 -fixedLength off -iters 500' % bin_options[
0].maxk
else:
cmd = 'simpoint -k 2:25 -dim 100 -numInitSeeds 25 -fixedLength off -iters 500'
cmd += ' -saveLabels labels.txt -saveSimpoints simpoints.txt -inputVectorsGzipped -loadFVFile matching-vec-profile.gz'
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
fp_out = open(sim_out_file, 'w')
result = util.RunCmd(cmd,
options,
'',
f_stdout=fp_out,
f_stderr=fp_out)
fp_out.close()
if result != 0:
msg.PrintMsg('\nError found while running Simpoint in dir:\n'
' %s' % os.getcwd())
return result
# Generate the binary Data directories
#
bin_data_dir = []
for basename in wp_basenames:
name = os.path.join('..', '%s.Data' % basename)
bin_data_dir.append(name)
if not os.path.isdir(name):
try:
os.mkdir(name)
except OSError:
msg.PrintAndExit(
'method RunSimPoint(), Unable to make directory: ' +
name)
# Run command to generate weight files for the binaries
#
weight_out_file = 'generate_weights_out.txt'
cmd = os.path.join('make_simpoint_weights.py --weight_file_list ')
for data_dir in bin_data_dir:
cmd += ' %s/weights.txt' % data_dir
if not options.list:
msg.PrintMsgPlus(
'make_simpoint_weights.py output file (including errors): %s\n'
% os.path.join(cbsp_data_dir, weight_out_file))
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
fp_out = open(weight_out_file, 'w')
result = util.RunCmd(cmd,
options,
'',
f_stdout=fp_out,
f_stderr=fp_out)
fp_out.close()
if result != 0:
msg.PrintMsg(
'\nError found while running make_simpoint_weights.py in dir:\n'
' %s' % os.getcwd())
return result
# Copy the simpoints and labels files to binary Data directories
#
#
def copy_file(f, d):
try:
shutil.copy(f, d)
return 0
except IOError:
msg.PrintMsg('\nError found in dir:\n'
' %s\nUnable to copy file:\n %s to %s' %
(os.getcwd(), f, d))
return -1
for data_dir in bin_data_dir:
result = copy_file('simpoints.txt', data_dir)
result = result | copy_file('labels.txt', data_dir)
if result != 0:
return result
# Generate the CSV files in each binary Data directory
#
for data_dir, basename in zip(bin_data_dir, wp_basenames):
# Go to the binary Data directory
#
old_dir = os.getcwd()
os.chdir(data_dir)
# Run the script to generate CSV files for this binary
#
bb_prof = os.path.join('..', cbsp_data_dir,
'%s.bb-profile.bz2' % basename)
csv_file = '%s.pinpoints.csv' % basename
if not options.list:
data_dir_rel_path = os.getcwd().replace(
os.path.join(orig_dir, ''), '')
msg.PrintMsgPlus(
'Any errors from running \'regions.py\' are in: %s' %
os.path.join(data_dir_rel_path, csv_file))
cmd = os.path.join('regions.py --csv_region --bbv_file %s' %
bb_prof)
cmd += ' --region_file=simpoints.txt --weight_file=weights.txt'
cmd += ' > %s 2>&1' % csv_file
msg.PrintMsg('')
if options.list or options.debug:
msg.PrintMsg(cmd)
else:
result = util.RunCmd(cmd, options, '')
if result != 0:
msg.PrintMsg(
'\nError found while generating CSV files in:\n %s' %
os.getcwd())
msg.PrintMsg('Error msgs in file: %s ' % csv_file)
return result
# Return to the CBSP Data directory
#
os.chdir(old_dir)
# Return to original directory
#
os.chdir(orig_dir)
if not options.list:
msg.PrintMsgDate('Finished running Simpoint for: ' +
options.cbsp_name)
return result
