def VerifyLITFiles(self, sim_run_cmd, options):
"""
Use a simulator to verify all the LIT files for the tracing instance are valid.
"""
# Verify the LIT files for each directory.
#
# import pdb; pdb.set_trace()
result = 0
for lit_dir in util.GetLitDir():
if not options.list:
msg.PrintMsgPlus('Verifying files in dir: ' + lit_dir)
# import pdb; pdb.set_trace()
cmd = sim_run_cmd + ' --replay_dir ' + lit_dir
if options.sim_options:
cmd += ' --sim_options ' + options.sim_options
cmd += ' --verify'
cmd += util.AddGlobalFile(self.gv.DumpGlobalVars(), options)
cmd += util.AddCfgFile(options)
result = util.RunCmd(cmd, options, '',
concurrent=True) # Run jobs concurrently
if result != 0:
msg.PrintMsg('Error found during LIT file verification')
return result
# Wait for all the verification jobs to finish.
#
result = util.WaitJobs(options)
return result
host = 'pin'
operating_sys = 'linux64'
options = GetOptions()
# Define a lambda function to print an error message.
#
err_msg = lambda string: msg.PrintMsg(
'ERROR: util.ParseFileName() failed to get field: ' + string)
# import pdb; pdb.set_trace()
# For each LIT directory, create a link to each file which is formatted properly
# to be submitted to the GTR.
#
dirs = util.GetLitDir()
dirs.sort()
for lit_dir in dirs:
# Make sure the LIT directory exists, then go there.
#
# import pdb; pdb.set_trace()
old_dir = os.getcwd()
if os.path.isdir(lit_dir):
os.chdir(lit_dir)
msg.PrintMsg('Dir: ' + os.path.basename(os.getcwd()))
else:
msg.PrintAndExit('ERROR: The LIT directory does not exist:\n ' + \
lit_dir)
# Get a list of the simulator files for each trace. Don't include any of the
def DeleteTracingFileDir(self, options):
"""
Delete files & directories generated for only the current tracing instance.
TODO: Implement this for CBSP.
@param options Options given on cmd line
@return no return
"""
# The "base" WP pinball directory is the directory before any filters have
# been applied.
#
# import pdb; pdb.set_trace()
wp_pb_dir = util.GetBaseWPDir()
# Variables used to define different file types for this tracing instance.
#
relog_wp_dir = wp_pb_dir + config.relog_dir_str
log_file = util.GetLogFile()
pb_list = [p.replace('.Data', '') for p in util.GetDataDir()]
csv_list = [p + '*.csv' for p in pb_list]
sim_list = [p + '_simpoint_out.txt' for p in pb_list]
# Only delete these files/dirs which are specific to the current
# tracing instance defined by tracing parameters.
#
util.Delete(options, ' '.join(util.GetDataDir()))
util.Delete(options, ' '.join(util.GetRegionPinballDir()))
util.Delete(options, ' '.join(util.GetLitDir()))
util.Delete(options, relog_wp_dir + '.*')
util.Delete(options, ' '.join(csv_list))
util.Delete(options, ' '.join(sim_list))
util.Delete(options, log_file + '*.NATIVE.TIME')
util.Delete(options, 'verify_' + log_file + '_out.txt')
util.Delete(options, util.GetStatusFileName())
util.Delete(options, 'PARALLEL.PARAM')
# Delete sniper result directories for this tracing instance
#
sniper_dirs = glob.glob(
os.path.join(config.sniper_result_dir, wp_pb_dir,
util.GetLogFile() + '*'))
sniper_dirs += [
os.path.join(config.sniper_result_dir,
os.path.basename(w) + '.pp') for w in sniper_dirs
]
for dir in sniper_dirs:
util.Delete(options, dir)
util.Delete(options, os.path.join(config.sniper_result_dir, wp_pb_dir))
# Only delete the whole program pinball directory if it was not
# specified by the user in a previous run of the scripts or the user
# gave the option '--delete_wp'.
#
# import pdb; pdb.set_trace()
if (hasattr(config, 'whole_pgm_dir') and config.whole_pgm_dir == '') or \
options.delete_wp:
util.Delete(options,
self.Config.GetInstanceFileName(config.config_ext))
util.Delete(options, wp_pb_dir)
else:
msg.PrintMsg(
'\nNOTE: User defined whole program directory was used. Will NOT\n'
'delete this directory.')
msg.PrintMsg(' ' + config.whole_pgm_dir)
# No error checking for deleting
#
return 0
def GenTraceinfoFiles(self, options):
"""
Generate traceinfo files from sysinfo files. Also do some simple error
checking.
One traceinfo file is generated for each directory which contains
region pinballs. Currently, the script 'create_traceinfo.sh' is used
to generate the traceinfo file. This will be replaced with a Python
script at sometime in the future.
The following info is printed to allow the user to determine if the
required number of region pinballs and traces were generated:
1) The script 'count_traces.sh' gets a count of the number of
expected traces and the region numbers from the region CSV file.
2) The number of actual traces generated and the trace names.
3) The number of region pinballs generated and the pinball names.
"""
# Check to make sure there is at least one region pinball directory for
# this tracing instance.
#
all_pp_dirs = util.GetRegionPinballDir(options)
if all_pp_dirs == [] and not options.debug:
msg.PrintMsg(
'\nERROR: Could not find any PinPoint \'*.pp\' directories.\n'
'Make sure you have run the phase \'-p\'.')
return -1
# Get the region pinball directories for this tracing instance.
#
# import pdb; pdb.set_trace()
all_lit_dirs = util.GetLitDir()
if all_lit_dirs == [] and not options.debug:
msg.PrintMsg(
'\nERROR: Could not find any LIT \'*.lit\' directories.\n'
'Make sure you have run the phase \'-L\'.')
return -1
result = 0
# import pdb; pdb.set_trace()
for lit_dir in all_lit_dirs:
# Get the name of the corresponding region pinball directory
# and make sure it exists.
#
pp_dir = util.ChangeExtension(lit_dir, '.lit', '.pp')
if not os.path.isdir(pp_dir):
if not options.list:
msg.PrintMsgPlus('WARNING: Generating traceinfo files, but the required \'pp\' ' \
'directory does not exist:\n ' + pp_dir)
# If running in MPI_MT_MODE, then it's possible for one process to
# not have a thread corresponding to the the current focus thread.
# However, another process might have this thread.
# Thus, only return an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
return -1
else:
# Not printing any msgs, just skip to the next directory
#
continue
# Make sure the LIT directory exists, then go there.
#
old_dir = os.getcwd()
if os.path.isdir(lit_dir):
os.chdir(lit_dir)
else:
if not options.list:
msg.PrintMsgPlus('WARNING: Generating traceinfo files, but the LIT ' \
'directory does not exist:\n ' + lit_dir)
# If running in MPI_MT_MODE, then it's possible for one process to
# not have a thread corresponding to the the current focus thread.
# However, another process might have this thread.
# Thus, only return an error if not tracing a MPI_MT application.
#
if options.mode == config.MPI_MT_MODE:
msg.PrintMsg(
'Since tracing mode is \'mpi_mt\', this may be OK.')
continue
else:
return -1
else:
# Not printing any msgs, just skip to the next directory
#
continue
# Copy the traceinfo 'blank' XML files from the SDE kit.
#
blank_path = self.kit_obj.GetTraceinfoBlank()
for blank in config.traceinfo_blank_files:
blank_file = os.path.join(blank_path, blank)
try:
shutil.copy(blank_file, os.getcwd())
except IOError:
msg.PrintMsg(
'\nERROR: Unable to copy traceinfo \'blank\' file:\n '
+ blank_file)
return -1
# Run the script to generate traceinfo XML file. Stdout from the
# script is the XML file. Function util.RunCMD() needs the output XML
# file object in order to write this file.
#
msg.PrintMsg('')
base_name = util.ChangeExtension(lit_dir, '.lit', '')
tr_file = base_name + '.traceinfo.xml'
try:
fp_out = open(tr_file, 'w')
except IOError:
msg.PrintMsg('ERROR: Failed to open traceinfo output file:\n'
' ' + tr_file)
return -1
cmd = self.traceinfo_bin + ' ' + base_name
result = util.RunCmd(cmd, options, '',
concurrent=False,
f_stdout=fp_out)
if result != 0:
msg.PrintMsg('Error found while running script \'%s\'' %
self.traceinfo_bin)
return -1
# Print info from the CSV regions file (located in the *.Data
# directory) about the number of expected traces.
#
msg.PrintMsg('')
param = {'in_lit_dir': True}
cluster_info, not_used, total_instr = util.GetClusterInfo(base_name, param)
if cluster_info == {}:
msg.PrintMsg(
'ERROR: Problems getting cluster info from file: %s.pinpoints.csv'
% (base_name))
return -1
cluster_list = util.ParseClusterInfo(cluster_info)
if len(cluster_info) != len(cluster_list):
msg.PrintMsg('ERROR: Did not parse enough clusters from CSV file: %s.pinpoints.csv\n' ' Num clusters: %d\n' \
' Num parsed clusters: %d' % (base_name, len(cluster_info), len(cluster_list)))
return -1
# Print the number of expected traces from the regions CSV file.
#
base_tid = -1
for cl in cluster_list:
if (cl.has_key('tid')):
tid = cl['tid']
else:
msg.PrintMsg(
'ERROR: Parsing cluster info for cluster:\n %s' %
(cl))
return -1
if base_tid == -1:
base_tid = tid
else:
if tid != base_tid:
msg.PrintAndExit(
'ERROR: Expected TID %d, but found TID %d' %
(base_tid, tid))
return -1
msg.PrintMsg('Expected trace count: %d\n' % (len(cluster_info)))
# Print the number of actual traces in the LIT directory and the names.
#
# import pdb; pdb.set_trace()
if not options.list:
msg.PrintMsg(
'Actual trace count: ' + str(util.CountFiles('ami')))
lit_files = glob.glob('*.ami*')
lit_files.sort()
for f in lit_files:
msg.PrintMsg(' ' + f)
# Clean up tmp files in the LIT directory.
#
tmp_files = glob.glob('*blank*.xml')
for f in tmp_files:
os.remove(f)
# Return to the working directory.
#
os.chdir(old_dir)
# Go to the *.pp directory. Print the number of actual pinballs and the names.
#
os.chdir(pp_dir)
if not options.list:
msg.PrintMsg(
'\nPinball count: ' + str(util.CountFiles('address')))
pp_files = glob.glob('*.address')
pp_files.sort()
for f in pp_files:
msg.PrintMsg(' ' + f)
# Print a warning if the expected number of traces are not found.
#
if len(lit_files) != len(pp_files):
msg.PrintMsgPlus(
'WARNING: Number of traces does not match the number of region pinballs.')
msg.PrintMsg('\nGenerated traceinfo file: ' + tr_file)
# Return to the working directory.
#
os.chdir(old_dir)
return result
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options):
"""
Run the additional phases for SDE which are not run for PinPlay.
@param wp_pb_dir Directory containing whole program log files (pinballs)
@param sim_replay_cmd Python script used to replay a pinball with a simulator
@param options Options given on cmd line
@return Exit code from last phase executed
"""
# The SDE phases object needs an SDE kit object.
#
s_phases = sde_phases.SDEPhases()
kit_obj = self.GetKit()
s_phases.SetKit(kit_obj)
# Need the appropriate simulator kit too.
#
sim_kit = self.GetSimKit()
# Generate LMAT/LIT files.
#
result = 0
if options.lit_gen or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating traces for region pinballs %s' % \
config.PhaseStr(config.LIT))
util.PhaseBegin(options)
result = s_phases.GenAllLitFiles(self.replayer_cmd, options)
#if result == 0:
# if not options.list:
# msg.PrintMsgPlus('Waiting on final concurrent region pinball generation')
# result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating traces for region pinballs %s' % \
config.PhaseStr(config.LIT))
util.CheckResult(result, options, 'Trace file generation %s' % \
config.PhaseStr(config.LIT))
# Generate traceinfo files.
#
if options.traceinfo or options.default_phases:
if not options.list:
msg.PrintMsgDate('Generating traceinfo files %s' % \
config.PhaseStr(config.traceinfo))
util.PhaseBegin(options)
result = s_phases.GenTraceinfoFiles(options)
if not options.list:
msg.PrintMsgDate('Finished generating traceinfo files %s' % \
config.PhaseStr(config.traceinfo))
util.CheckResult(result, options, 'Traceinfo file generation %s' % \
config.PhaseStr(config.traceinfo))
# Run CMPSim simulator on region pinballs.
#
if options.region_sim or options.default_phases:
# Print out CMPSim results every warmup_length instructions.
#
phase_length = options.warmup_length // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
for pp_dir in util.GetRegionPinballDir():
if not options.list:
msg.PrintMsgDate('Running CMPSim on region pinballs in dir: %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
util.PhaseBegin(options)
result = sim_kit.RunSimulator(pp_dir, sim_replay_cmd,
phase_length, options)
if not options.list:
msg.PrintMsgDate('Finished running CMPSim on region pinballs in dir %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
util.CheckResult(result, options, 'CMPSim on pinballs, dir: %s %s' % \
(pp_dir, config.PhaseStr(config.CMPsim_regions)))
# Run CMPSim simulator on whole program pinballs.
#
if options.whole_sim or options.default_phases:
# Set phase_length to print out CMPSim results every slice_size instructions.
#
phase_length = options.slice_size // config.instr_cmpsim_phase
if phase_length == 0:
phase_length = 1
if not options.list:
msg.PrintMsgDate('Running CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
util.PhaseBegin(options)
result = sim_kit.RunSimulator(wp_pb_dir, sim_replay_cmd,
phase_length, options)
if not options.list:
msg.PrintMsgDate('Finished running CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
util.CheckResult(result, options, 'CMPSim on whole program pinballs %s' % \
config.PhaseStr(config.CMPsim_whole))
# Calculate prediction error from simulator data files.
#
if options.pred_error or options.default_phases:
if not options.list:
msg.PrintMsgDate('Calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.PhaseBegin(options)
result = s_phases.CalcPredError(wp_pb_dir, sim_kit, options)
if not options.list:
msg.PrintMsgDate('Finished calculating prediction error %s' % \
config.PhaseStr(config.pred_error))
util.CheckResult(result, options, 'Prediction error calculation %s' % \
config.PhaseStr(config.pred_error))
# Verify the LIT files with the simulator.
#
if options.verify:
if not options.list:
msg.PrintMsgDate('Verifying LIT files %s' % \
config.PhaseStr(config.verify_LIT))
util.PhaseBegin(options)
result = s_phases.VerifyLITFiles(self.sim_run_cmd, options)
if not options.list:
msg.PrintMsgDate('Finished verifying LIT files %s' % \
config.PhaseStr(config.verify_LIT))
util.CheckResult(result, options, 'LIT file verification %s' % \
config.PhaseStr(config.verify_LIT))
# Generate instruction mix for the whole program pinballs.
#
if options.imix_whole:
if not options.list:
msg.PrintMsgDate('Generating imix on whole program pinballs %s' % \
config.PhaseStr(config.imix_whole))
# Setup dictionary of parameters for method RunAllDir()
#
param = {'options': options, 'replayer_cmd': self.replayer_cmd}
util.PhaseBegin(options)
result = util.RunAllDir(wp_pb_dir, s_phases.GenImix, True, param)
if result == 0:
if not options.list:
msg.PrintMsgPlus(
'Waiting on final whole program pinball imix generation'
)
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating imix on whole program pinballs %s' % \
config.PhaseStr(config.imix_whole))
util.CheckResult(result, options, 'Imix on whole program pinballs %s' % \
config.PhaseStr(config.imix_whole))
# Generate instruction mix for the LIT files.
#
if options.imix_lit:
if not options.list:
msg.PrintMsgDate('Generating imix on LIT files %s' % \
config.PhaseStr(config.imix_lit))
# Setup dictionary of parameters for method RunAllDir()
#
param = {'options': options, 'replayer_cmd': self.replayer_cmd}
util.PhaseBegin(options)
for lit_dir in util.GetLitDir():
result = util.RunAllDir(lit_dir, s_phases.GenImix, True, param)
if result == 0:
if not options.list:
msg.PrintMsgPlus('Waiting on final LIT files generation')
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating imix on LIT files %s' % \
config.PhaseStr(config.imix_lit))
util.CheckResult(result, options, 'Imix on LIT files %s' % \
config.PhaseStr(config.imix_lit))
# Generate instruction mix for the region pinballs.
#
if options.imix_region:
if not options.list:
msg.PrintMsgDate('Generating imix on region pinballs %s' % \
config.PhaseStr(config.imix_regions))
# Setup dictionary of parameters for method RunAllDir()
#
param = {'options': options, 'replayer_cmd': self.replayer_cmd}
util.PhaseBegin(options)
for pp_dir in util.GetRegionPinballDir():
result = util.RunAllDir(pp_dir, s_phases.GenImix, True, param)
if result == 0:
if not options.list:
msg.PrintMsgPlus(
'Waiting on final region pinballs generation')
result = util.WaitJobs(options)
if not options.list:
msg.PrintMsgDate('Finished generating imix on region pinballs %s' % \
config.PhaseStr(config.imix_regions))
util.CheckResult(result, options, 'Imix on region pinballs %s' % \
config.PhaseStr(config.imix_regions))
# Assume nothing has gone wrong at this point.
#
return result