def Replay(self, param, dirname, filename):
"""
Run the Simulator on a single LIT file given the command line options and the name of
the file to run. It formats the appropriate command line options,
saves global variables in a pickle file & calls the Sim run script.
"""
# import pdb ; pdb.set_trace()
if 'options' in param:
options = param['options']
else:
msg.PrintAndExit(
'method replay_dir.Replay() failed to get param \'options\'')
basename_file = os.path.join(dirname, filename)
if config.verbose:
msg.PrintMsg("-> Replaying pinball \"" + basename + "\"")
cmd = self.replayer_cmd + ' ' + basename_file
# Check to see if need to add options passed to the Sim run script.
# These are NOT parameters, so they don't get passed in the global
# variables.
#
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
if options.verify:
cmd += ' --verify'
# Add the configuration file, if one exists and print the cmd if just
# debugging.
#
# import pdb ; pdb.set_trace()
cmd += util.AddCfgFile(options)
if options.debug:
msg.PrintMsg(cmd)
result = 0
if not options.debug:
# Dump the global data to a unique file name. Need to add the
# option --global_file with this unique file name to options when
# calling a script.
#
gv = config.GlobalVar()
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
# import pdb ; pdb.set_trace()
if not options.list:
result = util.RunCmd(
cmd, options, filename,
concurrent=True) # Run concurrent jobs here
else:
msg.PrintMsg(cmd)
return result
def RunSimulator(self, dirname, sim_replay_cmd, phase_length, options):
"""
Run Branch Predictor simulator on all the pinballs in a set of
directories given by 'dirname'.
@param dirname Directory containing pinball to process
@param sim_replay_cmd Script used to replay pinball
@param phase_length Number of instructions executed before CMPSim will print next set of output
@param options Options given on cmd line
@return error code from running the simulator
"""
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --log_options'
cmd += ' "'
# 'phase_length' is given in instructions.
#
cmd += ' ' + self.print_data_knob + ' ' + str(phase_length)
cmd += '" '
cmd += ' --sim_add_filename' # This causes simulator output file name to be added to cmd.
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
end_str = '' # Don't want to print anything when this cmd finishes.
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, concurrent=False)
return result
def RunSimulator(self, dirname, sim_replay_cmd, options):
"""
Run Branch Predictor simulator on all the pinballs in a set of
directories given by 'dirname'.
Return: error code from running the simulator
"""
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = 'pwd;' + sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --log_options'
# SiNUCA options
cmd += ' "'
cmd += ' -output ' + str(options.sinuca_tracer_output)
cmd += ' -threads ' + str(options.sinuca_tracer_threads)
cmd += ' -parallel_start ' + str(options.sinuca_tracer_parallel_start)
cmd += ' -parallel_end ' + str(options.sinuca_tracer_parallel_end)
cmd += ' "'
cmd += ' --sim_add_filename' # This causes simulator output file name to be added to cmd.
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
msg.PrintMsg('DEBUG>> ' + cmd + '\n')
end_str = '' # Don't want to print anything when this cmd finishes.
# import pdb; pdb.set_trace()
result = util.RunCmd(cmd, options, end_str, False)
return result
def Run(self):
"""
Get all the user options and replay all the desired pinballs.
@return Exit code from replaying the last pinball
"""
# Debugger will run when this is executed
# import pdb; pdb.set_trace()
#import pdb ; pdb.set_trace()
options = self.ParseCommandLine()
result = self.ProcessCommandLine(options)
# Clean up any global data files which still exist.
#
gv = config.GlobalVar()
gv.RmGlobalFiles(options)
return result
def Replay(self, param, dirname, filename):
"""
Replay a single pinball given the command line options and the name of
the pinball to replay. It formats the appropriate command line options,
saves global variables in a pickle file & calls the replayer script.
@param param Dictionary containing all parameters that need to be
passed into the method. Need a dictionary because this
method is sometimes called by walk_callback() which
only allows one parameter in the functions it calls.
@param dirname Directory where pinball is located
@param filename Pinball base file name
@return Exit code from the replayer script.
"""
if 'options' in param:
options = param['options']
else:
msg.PrintAndExit(
'method replay_dir.Replay() failed to get param \'options\'')
if options.verbose:
msg.PrintMsg('Start of Replay() in replay_dir.py')
# import pdb ; pdb.set_trace()
basename_file = os.path.join(dirname, filename)
command = self.replayer_cmd + ' --replay_file ' + basename_file
if options.verbose:
msg.PrintMsg("-> Replaying pinball \"" + basename_file + "\"")
if options.replay_options:
command += ' --replay_options "' + options.replay_options + '"'
# Check to see if need to add options for BB vector generation. Set
# 'log_opt' to any options the user may have put on the command line.
#
log_opt = options.log_options
if options.bb_add_filename:
file_name = os.path.basename(basename_file)
# If there is a focus thread, then need to remove the TID from the
# file name.
#
# NOTE: This code may need to be fixed when a method of running
# Simpoints on all threads of cooperative pinballs is implemented.
#
file_name = util.RemoveTID(file_name)
# Write BB vector files to the newly created *.Data directory.
#
data_dir = file_name + '.Data'
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
log_opt += ' -o ' + os.path.join(data_dir, file_name)
# Check to see if need to add options when running a simulator.
#
# import pdb ; pdb.set_trace()
if options.sim_add_filename:
# Need to instantiate a kit of the type simulator being used.
# This is required on order to get some kit specific information.
#
# NOTE: If you are adding a kit for a new simulator, then you need
# to modify this code.
#
if config.sim_kit_type == config.BRPRED:
import brpred_kit
sim_kit = brpred_kit.BrPredKit()
elif config.sim_kit_type == config.SINUCA:
import sinuca_kit
sim_kit = sinuca_kit.Sinuca_TracerKit()
elif config.sim_kit_type == config.CMPSIM:
import sde_cmpsim_kit
sim_kit = sde_cmpsim_kit.CMPsimKit()
else:
msg.PrintAndExit('Undefined kit type in method replay_dir.Replay(): ' + \
str(config.sim_kit_type))
# Add the simulator knob to specify the file for the output from
# the simulator.
#
# ~ log_opt += ' ' + sim_kit.GetSimOutputFile(basename_file)
# When 'log_opt' is added to the command line below, it will
# put double quotes (") around all the options. Therefore, need to
# remove any exising double quotes in the current value for the
# string 'log_opt'.
#
log_opt = log_opt.replace('"', '')
# If relogging WP pinballs, need to add the -log:basename knob with
# the relogged pinball path/name.
#
# import pdb ; pdb.set_trace()
if options.wp_relog_dir:
ft = util.GetFocusThreadPB(basename_file)
if ft > -1 and not options.relog_focus:
# If WP pinballs were relogged with a focus thread, then the
# resulting pinballs were 'per thread', not 'cooperative'. If
# relogging with a different filter (i.e. options.relog_focus ==
# False) then need to remove TID from base file name given to
# the knob -log:basename.
#
file_name = os.path.basename(util.RemoveTID(basename_file))
else:
file_name = os.path.basename(basename_file)
log_opt += ' -log:basename ' + os.path.join(options.wp_relog_dir,
file_name)
if not options.list:
msg.PrintMsgDate('Relog whole program pinball: ' + file_name)
if log_opt:
command += ' --log_options "' + log_opt + '"'
if options.playout:
command += ' --playout '
# if not options.list:
# msg.PrintMsg(command)
# If not just listing the command, then dump the global
# variables so the next Python script can have access to them.
# Then run the script.
#
result = 0
if not options.list:
# Dump the global data to a unique file name. Need to add the
# option --global_file with this unique file name to options when
# calling a script.
#
gv = config.GlobalVar()
command += util.AddGlobalFile(gv.DumpGlobalVars(), options)
command += util.AddCfgFile(options)
result = util.RunCmd(command, options, filename,
concurrent=True) # Run concurrent jobs here
else:
# If the option 'list' is defined, then just list out the
# commands to be exectuted, but don't execute them.
#
msg.PrintMsg(command)
return result
def RunSimulator(self, dirname, sim_replay_cmd, phase_length, options):
"""
Run a simulator on all the pinballs in a directory given by 'dirname'.
@param dirname Directory containing pinball to process
@param sim_replay_cmd Script used to replay pinball
@param phase_length Number of instructions executed before CMPSim will print next set of output
@param options Options given on cmd line
@return Error code from running the simulator
"""
# Check to make sure the directory at least exists.
#
# import pdb; pdb.set_trace()
if not os.path.exists(dirname):
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg('WARNING: Directory containing pinballs to run with simulator does not exist:\n ' + \
dirname)
msg.PrintMsg('Since tracing mode is \'mpi_mt\', this may be OK.')
return 0
else:
msg.PrintMsg('ERROR: Directory containing pinballs to run with simulator does not exist:\n ' + \
dirname)
return -1
# Don't do anything if running in debug mode.
#
if options.debug:
return 0
gv = config.GlobalVar()
# Get the number of threads in the pinball and round it up
# to the next power of 2. This is a requirement of CMPSim.
#
# import pdb; pdb.set_trace()
threads = util.GetNumThreadsWP(options)
threads = util.RoundupPow2(threads)
# Need to let script 'sim_replay_cmd' know type of kit is calling it.
#
config.sim_kit_type = self.kit_type
# Run on all the pinballs in 'dirname'.
#
cmd = sim_replay_cmd + ' --replay_dir ' + dirname
cmd += ' --replay_options'
cmd += ' "' + options.replay_options + '"'
cmd += ' --log_options'
cmd += ' "'
cmd += '-threads ' + str(threads)
# If the directory contains the WP directory name for this tracing
# instance, then get the icount for each process/thread in the WP
# pinballs. Use the longest icount as the phase_length for CMPSim.
#
# This is done to reduce the CMPSim output to the very minimum for WP
# pinballs. For very large WP pinballs, using the 'default' value for
# phase_length generates CMPSim output which can be multiple GBs in size.
#
# import pdb; pdb.set_trace()
if util.GetWPDir() in dirname:
max_icount = 0
pinballs = glob.glob(os.path.join(dirname, '*.address'))
for pb in pinballs:
fname = os.path.basename(pb)
icount = util.GetMaxIcount(dirname, fname)
if icount > max_icount:
max_icount = icount
# Check for errors
#
# import pdb; pdb.set_trace()
if max_icount != 0:
phase_length = max_icount / 1000000
if phase_length == 0:
# Must have a length of at least 1
#
phase_length = 1
cmd += ' ' + self.print_data_knob + ' ' + str(phase_length)
cmd += '" '
cmd += ' --sim_add_filename' # This causes CMPSim output file name to be added to cmd
cmd += util.AddCfgFile(options)
cmd += util.AddGlobalFile(gv.DumpGlobalVars(), options)
# import pdb; pdb.set_trace()
end_str = '' # Don't want to print anything when this cmd finishes.
result = util.RunCmd(cmd, options, end_str, concurrent=False)
return result
def ParseCommandLine(self):
"""Process command line arguments and do error checking."""
version = '$Revision: 1.28 $'
version = version.replace(' ', '')
ver = version.replace(' $', '')
us = '%prog sim_data_dir | options \nVersion: ' + ver
desc = 'When run with only the argument \'sim_data_dir\', the script calculates the predicted '\
'metric of interest from the simulator data file contained in this directory. '\
'If any options are given, then only the actions defined by the options will be executed. '\
' '\
' '\
'NOTE: Must specify both options \'--actual_metric\' and \'--predicted_metric\' '\
'when using the option \'--pred_error\'.'
# Command line options to control the tool's behavior.
#
util.CheckNonPrintChar(sys.argv)
parser = optparse.OptionParser(usage=us, description=desc, version=ver)
cmd_options.debug(parser)
cmd_options.pred_error(parser, '')
cmd_options.actual_metric(parser)
cmd_options.pinplayhome(parser, '')
cmd_options.predicted_metric(parser)
cmd_options.verbose(parser)
# Parse the command line options.
#
(options, args) = parser.parse_args()
# Added method cbsp() to 'options' to check if running CBSP.
#
util.AddMethodcbsp(options)
# Error if no arguments or options on command line.
#
if len(sys.argv) == 1:
msg.PrintAndExit(
'Give either the name of a directory containing\n'
'simulator data files or one, or more, options on the command line.\n'
'See \'-h\' for more info.')
# Make sure all required options are given if user wants prediction
# error.
#
if options.pred_error:
if options.predicted_metric == '' or options.actual_metric == '':
msg.PrintAndExit('Must give both \'--predicted_metric\' and '
'\'--actual_metric\' with \'--pred_error\'.')
predicted_dir = ''
actual_dir = ''
if len(args) == 1:
# If there are only 2 arguments, assume just calculating predicted
# metric. Check to see if a valid directory name was given on the
# command line.
#
dir_found = False
for dir_name in args:
if os.path.isdir(os.path.realpath(dir_name)):
dir_found = True
break
if not dir_found:
msg.PrintAndExit(
'\'' + sys.argv[1] + '\' is not a valid directory.\n'
'Give either the name of a directory containing simulator data files \n'
'or one, or more, options on the command line.\n'
'See \'-h\' for more info.')
predicted_dir = os.path.realpath(dir_name)
else:
# Check directories given as arguments to options to make sure they exist.
#
if options.predicted_metric:
predicted_dir = os.path.realpath(options.predicted_metric)
if not os.path.isdir(predicted_dir):
msg.PrintAndExit('Predicted metric directory does not exist: ' + \
options.predicted_metric)
if options.actual_metric:
actual_dir = os.path.realpath(options.actual_metric)
if not os.path.isdir(actual_dir):
msg.PrintAndExit('Actual metric directory does not exist: ' + \
options.actual_metric)
# Set the global variables in config object from the options. Thus, if the
# user sets option 'pinplayhome' on the command line, it will be used when
# determining which kit to use.
#
# import pdb; pdb.set_trace()
gv = config.GlobalVar()
gv.SetGlobalVars(options)
self.kit_obj = self.GetKit()
return (predicted_dir, actual_dir, options)
class Logger(object):
# Class attributes
#
log_cmd = 'log.py'
gv = config.GlobalVar()
"""
Generate log files for an application
This class is the low level primative which runs an application with the
logger to generate pinballs (log files). It works with these classes of
applications:
single-threaded
multi-threaded
MPI single-threaded
MPI multi-threaded
multi-process single-threaded (which use fork/exec to create new proc)
multi-process multi-threaded (which use fork/exec to create new proc)
"""
def AddOptionIfNotPresent(self, options, opt, arg):
"""
This function adds an option to the tool if the option was not already
passed in the --log_options switch
@param options dictionary of user options
@param opt Option to add
@param arg Argument for option
@return String containing option and argument
"""
# import pdb; pdb.set_trace()
raw_options = options.log_options
# option was not passed in the command line
#
if raw_options.find(opt) == -1:
return ' ' + opt + ' ' + arg + ' '
return ''
def ParseCommandLine(self):
"""
Parse command line arguments and returns pin and tools options and app command line.
@return List containing: options
"""
# command line options for the driver
#
version = '$Revision: 1.81 $'
version = version.replace('$Revision: ', '')
ver = version.replace(' $', '')
us = '%prog [options] --mode MODE --log_file FILE binary args'
desc = 'Runs binary with the logger Pintool and generates whole program '\
'pinballs. Required arguments include:\n'\
' 1) Binary to run and it\'s arguments.\n'\
' 2) Name for the log file (pinball) using \'--log_file FILE\'\n'\
' Name can include a dir as well as a file.\n' \
' 3) Mode of the binary using \'--mode MODE\'.'
util.CheckNonPrintChar(sys.argv)
parser = optparse.OptionParser(
usage=us,
description=desc,
version=ver,
formatter=cmd_options.BlankLinesIndentedHelpFormatter())
# Define the command line options which control the behavior of the
# script. Some of these methods take a 2nd argument which is the empty
# string ''. If the script uses option groups, then this parameter is
# the group. However, this script does not use option groups, so the
# argument is empty.
#
cmd_options.compressed(parser, '')
cmd_options.config_file(parser)
cmd_options.debug(parser)
cmd_options.global_file(parser)
cmd_options.list(parser, '')
cmd_options.log_file(parser)
cmd_options.log_options(parser)
cmd_options.mode(parser, '')
cmd_options.mpi_options(parser, '')
cmd_options.msgfile_ext(parser)
cmd_options.no_log(parser)
cmd_options.num_proc(parser, '')
cmd_options.pid(parser)
cmd_options.pinplayhome(parser, '')
cmd_options.pintool(parser)
cmd_options.pin_options(parser)
cmd_options.save_global(parser)
cmd_options.sdehome(parser, '')
cmd_options.verbose(parser)
# import pdb; pdb.set_trace()
(options, args) = parser.parse_args()
# Read in configuration files and set global variables.
# No need to read in a config file.
#
config_obj = config.ConfigClass()
config_obj.GetCfgGlobals(options,
False) # No, don't need required parameters
# Make sure user gave an application mode
#
if options.mode:
options.mode = util.ParseMode(options.mode)
else:
parser.error(
"Application mode was not given.\n"
"Need to use option --mode MODE. Choose MODE from: 'st', 'mt', 'mpi', 'mpi_mt', 'mp', 'mp_mt',"
)
# Log file name must be given.
#
# import pdb; pdb.set_trace()
if options.log_file == '':
parser.error('Log file basename was not given.\n' \
'Must give basename with option: --log_file FILE')
# Get the application command line
#
# import pdb; pdb.set_trace()
cmd_line = " ".join(args)
if not options.pid:
if cmd_line:
setattr(options, 'command', cmd_line)
else:
parser.error('no program command line specified.\n'
'Need to add binary and it\'s arguments.')
return options
def GetKit(self, parser):
"""
Get the PinPlay kit.
@return PinPlay kit
"""
return kit.Kit()
def Run(self):
"""
Get all the user options and run the logger.
@return Exit code from the logger pintool
"""
# import pdb; pdb.set_trace()
options = self.ParseCommandLine()
# Script which will actually do the logging
#
cmd = "./" + self.log_cmd
# If the user has given these options, add them
#
if hasattr(options, 'compressed') and options.compressed:
cmd += ' --compressed=%s ' % options.compressed
if hasattr(options, 'log_file') and options.log_file:
cmd += ' --log_file ' + options.log_file
log_opts = ' --log_options "-log:syminfo -log:pid '
if hasattr(options, 'log_options') and options.log_options:
log_opts += '%s" ' % options.log_options
else:
log_opts += '"'
cmd += log_opts
if hasattr(options, 'no_log') and options.no_log:
cmd += ' --no_log '
if hasattr(options, 'pid') and options.pid:
cmd += ' --pid ' + str(options.pid)
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
# Finally add program and arguments
#
# cmd += ' ' + options.command
# If the command is not already in double quotes, then quote it now.
# This takes care of cases where the command may redirect input or output
# or the command has options which contain the char '-'.
#
# Assume if there is one double quote, then a 2nd double quote should also
# already be in the command.
#
if not (hasattr(options, 'pid') and options.pid):
if options.command.find('"') == -1:
cmd += ' -- "' + options.command + '" '
else:
cmd += ' -- ' + options.command
# Print out command line used for pin and pintool
#
string = '\n' + cmd
msg.PrintMsg(string)
# Finally execute the command line and gather stdin and stdout.
# Exit with the return code from executing the logger.
#
result = 0
# import pdb; pdb.set_trace()
if not config.debug and not options.list:
platform = util.Platform()
if platform != config.WIN_NATIVE:
cmd = 'time ' + cmd
p = subprocess.Popen(cmd, shell=True)
p.communicate()
result = p.returncode
return result