def Run(self): """ Get the user's options, read config file and execute the phases desired by the user. @return Exit code from last phase executed """ # Catch signals in order to do an orderly shutdown of the script. # self.InstallSigHandler() # import pdb; pdb.set_trace() options = self.ParseCommandLine() # Print out the version numbers for all python modules in this directory. # if config.debug: util.PrintModuleVersions() # If logging, ensure the user has entered a mode. # if options.log and config.mode == '': msg.PrintHelpAndExit( 'Must enter mode of application using: --mode "st|mt|mpi|mpi_mt"') # Set the defaults for a set of variables which are needed, but not in the list # of required options in the tracing configuration file. # self.SetDefaults(options) # Create an new status file and zero out the file if it already exists. # If user sets parameter 'append_status', the old file is not deleted. # util.NewStatusFile(options) # Add 1500 to the warmup length in order to deal with problems that # occur when the start of a warmup region and the end of the previous # region are in the same basic block. # if not config.global_regions: config.warmup_length += 1500 # Get variables used in many phases. # # import pdb; pdb.set_trace() wp_pb_dir = util.GetWPDir() log_file_name = util.GetLogFile() # Print out the tracing configuration parameters. # if not options.list: self.phases.PrintTracingInfo(options, wp_pb_dir, log_file_name, self.PrintHome) ######################################################################### # # Several phases which clean up tracing instances. # ######################################################################### result = 0 if options.delete or options.delete_wp: self.phases.DeleteTracingFileDir(options) util.CheckResult(result, options, 'Deleting tracing instance files') # Since just deleted all files/dirs for the tracing instance, need # to set the whole program directory to the default value. Do NOT # use util.GetWPDir() here because this may return the name of a # relogged WP directory (which we, of course, just deleted.) # wp_pb_dir = util.GetBaseWPDir() if options.delete_all: self.phases.DeleteAllTracingFileDir(options) util.CheckResult(result, options, 'Deleting all tracing files') # Again, since just deleted all files/dirs, need to set the whole # program directory to the default value. # wp_pb_dir = util.GetBaseWPDir() # If the user has given the name of the whole program pinball # directory, then save it. This must be done after the two delete # phases because these phases delete this configuration file. # Also, need to get default WP directory name in 'wp_pb_dir'. # # import pdb ; pdb.set_trace() if hasattr(options, 'whole_pgm_dir') and options.whole_pgm_dir: config.whole_pgm_dir = options.whole_pgm_dir self.Config.SaveCfgParameter('whole_pgm_dir', options.whole_pgm_dir) wp_pb_dir = util.GetWPDir() ######################################################################### # # Run the phases of the script. They are in a specific order to ensure # all prerequisite phases are completed before a given phase is run. # ######################################################################### # Run the application without pin/pintools. # if options.native_pure: util.PhaseBegin(options) result = self.phases.NativePure(options) util.CheckResult( result, options, 'Pure native run %s' % config.PhaseStr(config.native_pure)) # Run the application using pin, but no pintools. # if options.native_pin: util.PhaseBegin(options) result = self.phases.NativePin(self.logger_cmd, wp_pb_dir, log_file_name, \ options) util.CheckResult(result, options, 'Native run with pin only %s' % \ config.PhaseStr(config.native_pin)) # Logging phase to generate initial whole program pinballs. # # import pdb; pdb.set_trace() if options.log or options.default_phases: util.PhaseBegin(options) result = self.phases.Logger(self.logger_cmd, wp_pb_dir, log_file_name, \ options) util.CheckResult(result, options, 'Whole program pinball generation %s' % \ config.PhaseStr(config.log_whole)) # All phases after this require the whole program pinball directory. Exit with an # error if it does not exist. # # import pdb; pdb.set_trace() if not os.path.isdir(wp_pb_dir) and not options.list and not options.debug and\ not options.delete and not options.delete_all and not options.delete_wp and \ not options.native_pure and not options.native_pin: string = '\nERROR: Can\'t proceed because the whole program pinball directory does not exist:\n' + \ ' ' + wp_pb_dir + \ '\nMust select at least one phase to run. Try using option \'--default_phases\'.' + \ '\nUse \'-h\' for help on selecting phases to run.' msg.PrintMsg(string) # Use -1 to force check to fail util.CheckResult(-1, options, 'Initial check to see if WP pinballs exist') # Print out the number of instructions in the whole program pinballs. # if not options.list and os.path.isdir(wp_pb_dir): msg.PrintMsg('') msg.PrintMsg('Initial whole program pinball(s)') util.PrintInstrCount(wp_pb_dir, options) ######################################################################### # # Phases which relog whole program pinballs using a filter to remove # certain type of instructions (such as initialization or MPI spin). # # The relogging phases must be executed before the basic block vector # generation phase. This must be done in case one, or more, relogging # phases are executed. If so then, the BB vector phase needs to use # the final relogged WP pinballs generated here. # ######################################################################### # If the user gives one, or more, of the use_relog_* options, they want # to explictly define which filtered WP pinballs to use. As a result, # unset the parameter 'relog_dir' just read from the per instance # tracing configuration file and from the config object. This removes # the previous "sticky" value for the filtered WP pinballs. Then use # the WP pinball directory before any filters have been applied. # # import pdb; pdb.set_trace() if cmd_options.UseRelogOptionsSet(options): Config = config.ConfigClass() Config.ClearCfgParameter('relog_dir') config.relog_dir = '' wp_pb_dir = util.GetBaseWPDir() def FinalizeWPDir(wp_dir, string): """ Set the filtered WP pinball directory to be the new default WP dir and print out the number of instructions in the newly filtered pinballs. """ config.relog_dir = wp_dir if not options.debug and not options.list: # Only save this parameter if not just debugging or # listing the commands to be run. # self.Config.SaveCfgParameter('relog_dir', wp_dir) if not options.list and os.path.isdir(wp_dir): msg.PrintMsg('') msg.PrintMsg(string) util.PrintInstrCount(wp_dir, options) return # The format of the code for each relogging (filtering) phase contains # 3 components: # # 1) If going to use relogged WP pinballs for the phase (indicated by # either 'relog_*' or 'use_relog_*' options), then get the # appropriate WP directory name for this phase. Need to always do # this, even when not relogging, because a previous run generated # relogged WP pinballs. # # 2) If going to filter with relogging (indicated by a 'relog_*' # option), then run the relogging phase. # # 3) Set the name of the default whole program pinball directory to # the directory with the relogged WP pinballs. # Relog with a user defined name for the relogged directory, and hopefully a set of # knobs to define what action to take when relogging. # # import pdb; pdb.set_trace() if options.use_relog_name != '' or options.relog_name: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_NAME, options) if options.relog_name: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs with user defined name: %s %s' % \ (options.relog_name, config.PhaseStr(config.filter_user_defn))) util.PhaseBegin(options) result = self.phases.RelogWholeName(self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs with user defined name: %s %s' % \ (options.relog_name, config.PhaseStr(config.filter_user_defn))) util.CheckResult(result, options, 'Filtering WP pinballs with user defined name: %s %s' % \ (options.relog_name, config.PhaseStr(config.filter_user_defn))) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered with user defined name: ' + options.relog_name) # Relog with a focus thread. # if options.use_relog_focus or options.relog_focus: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_FOCUS, options) if options.relog_focus: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs with a focus thread %s' % \ config.PhaseStr(config.filter_focus_thread)) # import pdb; pdb.set_trace() util.PhaseBegin(options) result = self.phases.RelogWholeFocus(self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs with a focus thread %s' % \ config.PhaseStr(config.filter_focus_thread)) util.CheckResult(result, options, 'Filtering WP pinballs with focus thread %s' % \ config.PhaseStr(config.filter_focus_thread)) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered with a focus thread') # If pinballs were relogged with a focus thread, then in the # remaining phases the focus thread must be 0. Relogging generates # per thread whole program pinballs (which only have thread 0). To # enforce this, we change the focus_thread in the config object to # be 0. # config.focus_thread = 0 # Also need to set options.use_relog_focus = True because we are will be using # WP pinballs which have been relogged with a focus thread. # options.use_relog_focus = True # Relog to remove initialization instructions. Do this before removing cleanup or # MPI spin instructions. # # import pdb; pdb.set_trace() if options.use_relog_no_init or options.relog_no_init: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_NO_INIT, options) if options.relog_no_init: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs to remove initialization instructions %s' % \ config.PhaseStr(config.filter_init)) util.PhaseBegin(options) result = self.phases.RelogWholeRemoveInit(self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs to remove initialization instructions %s' % \ config.PhaseStr(config.filter_init)) util.CheckResult(result, options, 'Filtering WP pinballs to remove init instructions %s' % \ config.PhaseStr(config.filter_init)) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered to remove initialization instructions') # Relog to remove cleanup instructions. Do this before removing MPI spin # instructions. # # import pdb; pdb.set_trace() if options.use_relog_no_cleanup or options.relog_no_cleanup: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_NO_CLEANUP, options) if options.relog_no_cleanup: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs to remove cleanup instructions %s' % \ config.PhaseStr(config.filter_cleanup)) util.PhaseBegin(options) result = self.phases.RelogWholeRemoveCleanup( self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs to remove cleanup instructions %s' % \ config.PhaseStr(config.filter_cleanup)) util.CheckResult(result, options, 'Filtering WP pinballs to remove cleanup instructions %s' % \ config.PhaseStr(config.filter_cleanup)) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered to remove cleanup instructions') # Relog to exclude code (instructions) between two addresses. Do this # before removing MPI spin instructions. # # import pdb; pdb.set_trace() if options.use_relog_code_exclude != '' or options.relog_code_exclude: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_CODE_EXCLUDE, options) if options.relog_code_exclude: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs with code exclusion %s' % \ config.PhaseStr(config.filter_code_exclude)) util.PhaseBegin(options) result = self.phases.RelogWholeCodeExclude( self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs with code exclusion %s' % \ config.PhaseStr(config.filter_code_exclude)) util.CheckResult(result, options, 'Filtering WP pinballs with code exclusion %s' % \ config.PhaseStr(config.filter_code_exclude)) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered with code exclusion') # Relog to remove OpenMP spin instructions. # # import pdb; pdb.set_trace() if options.use_relog_no_omp_spin or options.relog_no_omp_spin: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_NO_OMP_SPIN, options) if options.relog_no_omp_spin: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs to remove OpenMP spin instructions %s' % \ config.PhaseStr(config.filter_OMP_spin)) util.PhaseBegin(options) result = self.phases.RelogWholeRemoveOMPSpin( self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs to remove OpenMP spin instructions %s' % \ config.PhaseStr(config.filter_OMP_spin)) util.CheckResult(result, options, 'Filtering WP pinballs to remove OpenMP spin instructions %s' % \ config.PhaseStr(config.filter_OMP_spin)) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered to remove OpenMP spin instructions') # Relog to remove MPI spin instructions. # # import pdb; pdb.set_trace() if options.use_relog_no_mpi_spin or options.relog_no_mpi_spin: relog_wp_dir = util.GetRelogPhaseDir(wp_pb_dir, config.RELOG_NO_MPI_SPIN, options) if options.relog_no_mpi_spin: if not options.list: msg.PrintMsgDate('Filtering whole program pinballs to remove MPI spin instructions %s' % \ config.PhaseStr(config.filter_MPI_spin)) util.PhaseBegin(options) result = self.phases.RelogWholeRemoveMPISpin( self.replay_cmd, wp_pb_dir, relog_wp_dir, options) if not options.list: msg.PrintMsgDate('Finished filtering whole program pinballs to remove MPI spin instructions %s' % \ config.PhaseStr(config.filter_MPI_spin)) util.CheckResult(result, options, 'Filtering WP pinballs to remove MPI spin instructions %s' % \ config.PhaseStr(config.filter_MPI_spin)) # No errors, commit to using the new pinballs. # wp_pb_dir = relog_wp_dir FinalizeWPDir( relog_wp_dir, 'Whole program pinball(s) filtered to remove MPI spin instructions') if not options.list: msg.PrintMsgPlus( 'Using whole program pinballs in dir: ' + wp_pb_dir) if (cmd_options.UseRelogOptionsSet(options) or \ cmd_options.RelogOptionsSet(options)) and \ os.path.isdir(wp_pb_dir): msg.PrintMsg('') util.PrintInstrCount(wp_pb_dir, options) ######################################################################### # # These phases are run with the whole progam pinballs defined/generated # in the previous phases. # ######################################################################### # Make sure any relogged whole program pinball directory exist. Exit with an # error if it does not exist. # # import pdb; pdb.set_trace() if not os.path.isdir(wp_pb_dir) and not options.list and not options.debug and\ not options.delete and not options.delete_all and not options.delete_wp and \ not options.native_pure and not options.native_pin: string = 'ERROR: Can\'t proceed because the whole program pinball directory does not exist:\n' + \ ' ' + wp_pb_dir msg.PrintMsg(string) # Use -1 to force check to fail util.CheckResult(-1, options, 'Second check to see if WP pinballs exist') # Do not run replay whole program pinballs as one of the default # phases. The user must explicitly include this option. This is to # save time during the tracing process. # if options.replay: if not options.list: msg.PrintMsgDate('Replaying all whole program pinballs %s' % \ config.PhaseStr(config.replay_whole)) util.PhaseBegin(options) result = self.phases.Replay(self.replay_cmd, wp_pb_dir, options) if not options.list: msg.PrintMsgDate('Finished replaying all whole program pinballs %s' % \ config.PhaseStr(config.replay_whole)) util.CheckResult(result, options, 'Replay of whole program pinballs %s' % \ config.PhaseStr(config.replay_whole)) # Generate basic block vectors. # if options.basic_block_vector or options.default_phases: if not options.list: msg.PrintMsgDate('Generating basic block vectors %s' % \ config.PhaseStr(config.gen_BBV)) util.PhaseBegin(options) result = self.phases.BasicBlockVector(self.replay_cmd, wp_pb_dir, options) if result == 0: result = util.WaitJobs(options) if not options.list: msg.PrintMsgDate('Finished basic block vector generation %s' % \ config.PhaseStr(config.gen_BBV)) util.CheckResult(result, options, 'Basic block vector generation %s' % \ config.PhaseStr(config.gen_BBV)) # Run Simpoints to genenerate representative regions. # if options.simpoint or options.default_phases: if not options.list: msg.PrintMsgDate('Running Simpoint on all processes %s' % \ config.PhaseStr(config.Simpoint)) # Setup dictionary of parameters for method RunAllDir() # msg.PrintMsg('fbm was in pinpoints') param = {'options': options} util.PhaseBegin(options) result = util.RunAllDir(wp_pb_dir, self.phases.RunSimPoint, True, param) if not options.list: msg.PrintMsgDate('Finished running Simpoint for all processes %s' % \ config.PhaseStr(config.Simpoint)) util.CheckResult(result, options, 'Simpoints generation %s' % \ config.PhaseStr(config.Simpoint)) # Relog to generate representative region pinballs. # if options.region_pinball or options.default_phases: if not options.list: msg.PrintMsgDate('Generating region pinballs %s' % \ config.PhaseStr(config.relog_regions)) util.PhaseBegin(options) result = self.phases.MultiIterGenRegionPinballs(wp_pb_dir, self.replayer_cmd, options) if result == 0: if not options.list: msg.PrintMsgPlus( 'Waiting on final concurrent region pinball generation') result = util.WaitJobs(options) if not options.list: msg.PrintMsgDate('Finished generating region pinballs %s' % \ config.PhaseStr(config.relog_regions)) util.CheckResult(result, options, 'Region pinball generation %s' %\ config.PhaseStr(config.relog_regions)) # Do not run replay region pinballs as one of the default phases. The # user must explicitly include this option. This is to save time # during the tracing process. # if options.replay_region: result = 0 if not options.list: msg.PrintMsgDate('Replaying all region pinballs %s' % \ config.PhaseStr(config.replay_regions)) # import pdb; pdb.set_trace() util.PhaseBegin(options) for pp_dir in util.GetRegionPinballDir(): # Accumulate any errors which occur, but don't check for errors # until all the pinballs have been replayed. # r = self.phases.Replay(self.replay_cmd, pp_dir, options) result = result or r if not options.list: msg.PrintMsgDate('Finished replaying all region pinballs %s' % \ config.PhaseStr(config.replay_regions)) util.CheckResult(result, options, 'Replay of region pinballs %s' % \ config.PhaseStr(config.replay_regions)) result = 0 # Remove the return values from replaying region pinballs # If there are any additional phases, then run them. # self.RunAdditionalPhases(wp_pb_dir, self.sim_replay_cmd, options) # Cleanup and print out a string to indicate the tracing has completed. # # import pdb; pdb.set_trace() util.CleanupTraceEnd(options) util.PrintTraceEnd(options) return result
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options): """ Run the additional phases for the simulator which are not run as part of the usual PinPlay phases. """ phases_obj = phases.Phases() self.sim_kit = self.GetSimKit() msg.PrintMsg('sinuca_tracer_output: ' + config.sinuca_tracer_output) msg.PrintMsg('sinuca_tracer_threads: ' + str(config.sinuca_tracer_threads)) msg.PrintMsg('sinuca_tracer_parallel_start: ' + str(config.sinuca_tracer_parallel_start)) msg.PrintMsg('sinuca_tracer_parallel_end: ' + str(config.sinuca_tracer_parallel_end)) if not options.sinuca_tracer_output: msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_output.') util.CheckResult( -1, options, 'Could not find the sinuca_tracer_output parameter.' ) # Force error with -1 # ~ # ~ if not options.sinuca_tracer_threads: msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_threads.') util.CheckResult( -1, options, 'Could not find the sinuca_tracer_threads parameter.' ) # Force error with -1 # ~ # ~ if not options.sinuca_tracer_parallel_start: msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_parallel_start.') util.CheckResult( -1, options, 'Could not find the sinuca_tracer_parallel_start parameter.' ) # Force error with -1 # ~ # ~ if not options.sinuca_tracer_parallel_end: msg.PrintMsgPlus('ERROR: Please set sinuca_tracer_parallel_end.') util.CheckResult( -1, options, 'Could not find the sinuca_tracer_parallel_end parameter.' ) # Force error with -1 # Run branch predictor simulator on region pinballs. # if options.region_sim or options.default_phases: # Print out simulator results every warmup_length instructions. # for pp_dir in util.GetRegionPinballDir(): phase_length = options.warmup_length if phase_length == 0: phase_length = options.slice_size if not options.list: msg.PrintMsgDate('Running simulator on region pinballs: %s' % \ config.PhaseStr(config.sim_regions)) util.PhaseBegin(options) result = self.sim_kit.RunSimulator(pp_dir, sim_replay_cmd, options) if not options.list: msg.PrintMsgDate('Finished running simulator on region pinballs: %s' % \ config.PhaseStr(config.sim_regions)) util.CheckResult(result, options, 'simulator on region pinballs: %s' % \ config.PhaseStr(config.sim_regions)) # Run branch predictor simulator on whole program pinballs. # if options.whole_sim or options.default_phases: # Set phase_length to print out simulator results every slice_size instructions. # phase_length = options.slice_size if not options.list: msg.PrintMsgDate('Running simulator on whole program pinballs: %s' % \ config.PhaseStr(config.sim_whole)) util.PhaseBegin(options) result = self.sim_kit.RunSimulator(wp_pb_dir, sim_replay_cmd, options) if not options.list: msg.PrintMsgDate('Finished running simulator on whole program pinballs: %s' % \ config.PhaseStr(config.sim_whole)) util.CheckResult(result, options, 'simulator on whole program pinballs: %s' % \ config.PhaseStr(config.sim_whole)) return 0
def ParseCommandLine(self): """ Process command line arguments and read in config file/globals. @return List of options parsed from command line """ version = '$Revision: 1.100 $' version = version.replace(' ', '') ver = version.replace(' $', '') us = '%prog phase [options] \nVersion: ' + ver desc = 'The script generates traces for an application using the PinPoints methodology '\ '(based on PinPlay: Pin-based record/replay tools). '\ 'At least one command line option must be given to chose the phases to run. ' \ 'Use option \'--default_phases\' to run all the default phases. ' \ 'See section "Phase options" for a list of phases. \n\n'\ '' \ 'The two parameters \'program_name\' and \'input_name\' must be defined in order '\ 'to run this script. If running the logging phase (-l), then two more parameters, '\ '\'command\' and \'mode\', must also be defined. '\ 'Parameters can be given either in a tracing configuration file or with command '\ 'line options. \n\n'\ '' \ 'The default configuration file is "tracing.cfg". '\ 'If it exists, the script will always read this cfg file first. '\ 'Use the option "--cfg" to select additional files. '\ 'Each cfg file on the command line is processed in order. '\ 'Command line options over-ride values read from configuration files. '\ 'All parameters listed in the three parameter '\ 'sections below can be defined in a cfg file. The long option name, '\ 'not the single letter option, must be used in the cfg files.\n\n'\ '' \ 'Example parameter configuration file:'\ ' '\ '--------------------------------------------'\ ' '\ '[Parameters]'\ ' '\ 'program_name: omnetpp'\ ' '\ 'input_name: p10000-s10'\ ' '\ 'command: ./dtlb5-lin64 -p10000 -s10'\ ' '\ 'mode: st'\ ' '\ '--------------------------------------------' util.CheckNonPrintChar(sys.argv) parser = optparse.OptionParser( usage=us, description=desc, version=ver, formatter=cmd_options.BlankLinesIndentedHelpFormatter()) # Command line options to control the tools behavior. These are not in any # of the option groups. # # import pdb; pdb.set_trace() cmd_options.add_program_wp(parser) cmd_options.append_status(parser) cmd_options.config_file(parser) cmd_options.debug(parser) cmd_options.delete(parser) cmd_options.delete_all(parser) cmd_options.delete_wp(parser) cmd_options.dir_separator(parser) cmd_options.log_options(parser) cmd_options.msgfile_ext(parser) cmd_options.no_glob(parser) cmd_options.pintool(parser) cmd_options.pin_options(parser) cmd_options.replay_options(parser) cmd_options.save_global(parser) cmd_options.verbose(parser) self.AddAdditionalOptions(parser) # Top level command line options in the default group allow the user # to choose the phases to run. # phase_group = cmd_options.PhaseGroup(parser) cmd_options.default_phases(parser, phase_group) cmd_options.native_pure(parser, phase_group) cmd_options.log(parser, phase_group) cmd_options.replay(parser, phase_group) cmd_options.basic_block_vector(parser, phase_group) cmd_options.simpoint(parser, phase_group) cmd_options.region_pinball(parser, phase_group) cmd_options.replay_region(parser, phase_group) self.AddAdditionalPhaseOptions(parser, phase_group) parser.add_option_group(phase_group) # Top level command line options in param group are used to set various # tracing parameters. # param_group = cmd_options.ParameterGroup(parser) cmd_options.command(parser, param_group) cmd_options.compressed(parser, param_group) cmd_options.focus_thread(parser, param_group) cmd_options.input_name(parser, param_group) cmd_options.mpi_options(parser, param_group) cmd_options.mode(parser, param_group) cmd_options.num_cores(parser, param_group) cmd_options.num_proc(parser, param_group) cmd_options.program_name(parser, param_group) self.KitOption(parser, param_group) parser.add_option_group(param_group) # Top level command line options which only apply to the Simpoint phase. # simpoint_phase_group = cmd_options.SimpointPhaseGroup(parser) cmd_options.combine(parser, simpoint_phase_group) cmd_options.cutoff(parser, simpoint_phase_group) cmd_options.ldv(parser, simpoint_phase_group) cmd_options.maxk(parser, simpoint_phase_group) cmd_options.slice_size(parser, simpoint_phase_group) cmd_options.warmup_factor(parser, simpoint_phase_group) cmd_options.pccount_regions(parser, simpoint_phase_group) cmd_options.global_regions(parser, simpoint_phase_group) cmd_options.simpoint_options(parser, simpoint_phase_group) parser.add_option_group(simpoint_phase_group) # Top level command line options which only apply to the region pinball generation phase. # region_pb_phase_group = cmd_options.RegionPBPhaseGroup(parser) cmd_options.epilog_length(parser, region_pb_phase_group) cmd_options.prolog_length(parser, region_pb_phase_group) cmd_options.warmup_length(parser, region_pb_phase_group) parser.add_option_group(region_pb_phase_group) # Add verify options. # self.AddVerifyOptions(parser) # Top level command line options in modify group modify the behavior of phases. # modify_group = cmd_options.ModifyGroup(parser) # These should go before the rest of the modify options. # cmd_options.coop_pinball(parser, modify_group) cmd_options.cross_os(parser, modify_group) cmd_options.list(parser, modify_group) cmd_options.native_pin(parser, modify_group) cmd_options.no_focus_thread(parser, modify_group) cmd_options.whole_pgm_dir(parser, modify_group) self.AddAdditionalModifyOptions(parser, modify_group) parser.add_option_group(modify_group) # Top level command line options in which only apply to the region pinball generation phase. # wp_filter_group = cmd_options.WPFilterGroup(parser) cmd_options.relog_name(parser, wp_filter_group) cmd_options.relog_focus(parser, wp_filter_group) cmd_options.relog_no_init(parser, wp_filter_group) cmd_options.relog_no_cleanup(parser, wp_filter_group) cmd_options.relog_code_exclude(parser, wp_filter_group) cmd_options.relog_no_omp_spin(parser, wp_filter_group) cmd_options.relog_no_mpi_spin(parser, wp_filter_group) cmd_options.use_relog_name(parser, wp_filter_group) cmd_options.use_relog_focus(parser, wp_filter_group) cmd_options.use_relog_no_cleanup(parser, wp_filter_group) cmd_options.use_relog_no_init(parser, wp_filter_group) cmd_options.use_relog_code_exclude(parser, wp_filter_group) cmd_options.use_relog_no_omp_spin(parser, wp_filter_group) cmd_options.use_relog_no_mpi_spin(parser, wp_filter_group) parser.add_option_group(wp_filter_group) # Parse the command line options. # # import pdb; pdb.set_trace() (options, args) = parser.parse_args() # Added method cbsp() to 'options' to check if running CBSP. # util.AddMethodcbsp(options) # Check to make sure there was at least one command line option given. # # import pdb; pdb.set_trace() if len(sys.argv) == 1: msg.PrintMsg( "ERROR: Must use command line options to chose at least one phase to run.\n" "Use the option '--default_phases' to run the default phases. Use '--help' for more info.") util.CheckResult(-1, options, 'Checking command line options') # If user does not just want to delete all the files, then # read in configuration files and set global variables. # # import pdb; pdb.set_trace() if not options.delete_all: self.Config.GetCfgGlobals(options, True) # Yes, need 4 required parameters # Once the tracing configuration parameters are read, get the kit # in case pinplayhome was set on the command line. Also, need to # reset the path in object 'self' and set the kit in self.phases. # # import pdb; pdb.set_trace() kit_obj = self.GetKit() self.path = kit_obj.path self.script_path = kit_obj.path self.phases.SetKit(kit_obj) # If required, check to see if there any 'forbidden' char in some of # the parameters. # # import pdb; pdb.set_trace() if self.kit_type == config.SDE and hasattr(options, 'spec') and not options.spec: self.Config.CheckForbiddenChar() # If doing code exclusion, check to make sure the file exists. # if options.relog_code_exclude: if not os.path.exists(options.relog_code_exclude): msg.PrintMsg( 'ERROR: The code exclusion file used for filtering does not exist:\n' ' ' + options.relog_code_exclude) util.CheckResult(-1, options, 'Checking command line options') # Do some 'special' things on native Windows. # util.WindowsNativeCheck(options) return options
def RunAdditionalPhases(self, sim_replay_cmd, options, bin_options): """ Run the CBSP additional phases for SDE which are not run for PinPlay. @param sim_replay_cmd Script which run the simulator on pinballs @param options Options given on cmd line @param options Options for each CBSP binary @return Exit code from last phase executed """ # The SDE phases object needs an SDE kit object. # s_phases = sde_phases.SDEPhases() kit_obj = self.GetKit() s_phases.SetKit(kit_obj) # Need the appropriate simulator kit too. # sim_kit = self.GetSimKit() # Generate LMAT/LIT files # result = 0 if options.lit_gen or options.default_phases: if not options.list: msg.PrintMsgDate('Generating traces for region pinballs %s' % \ config.PhaseStr(config.LIT)) util.PhaseBegin(options) for bopts in bin_options: wp_dir = util.GetDefaultWPDir(bopts) log_file_name = util.GetLogFile(bopts) self.Config.SetPerBinParams(bopts) result = s_phases.GenAllLitFiles(self.replayer_cmd, bopts) if result == 0: if not options.list: msg.PrintMsgPlus('Waiting on final trace generation') result = util.WaitJobs(options) util.CheckResult(result, options, 'Traces for: %s %s' % \ (log_file_name, config.PhaseStr(config.LIT)), intermediate_phase=True) if not options.list: msg.PrintMsgDate('Finished generating traces for region pinballs %s' % \ config.PhaseStr(config.LIT)) util.CheckResult(result, options, 'Trace file generation %s' %\ config.PhaseStr(config.LIT)) # Run CMPSim simulator on region pinballs. # result = 0 if options.region_sim or options.default_phases: # Print out CMPSim results every warmup_length instructions. # phase_length = options.warmup_length // config.instr_cmpsim_phase if phase_length == 0: phase_length = 1 for bopts in bin_options: self.Config.SetPerBinParams(bopts) for pp_dir in util.GetRegionPinballDir(bopts): if not options.list: msg.PrintMsgDate('Running CMPSim on region pinballs in dir: %s %s' % \ (pp_dir, config.PhaseStr(config.CMPsim_regions))) util.PhaseBegin(options) result = sim_kit.RunSimulator(pp_dir, sim_replay_cmd, phase_length, bopts) util.CheckResult(result, options, 'CMPSim for: %s %s' % \ (pp_dir, config.PhaseStr(config.CMPsim_regions)), intermediate_phase=True) if not options.list: msg.PrintMsgDate('Finished running CMPSim on region pinballs in dir %s %s' % \ (pp_dir, config.PhaseStr(config.CMPsim_regions))) util.CheckResult(result, options, 'CMPSim on region pinballs: %s' % \ config.PhaseStr(config.CMPsim_regions)) # Run CMPSim simulator on whole program pinballs. # if options.whole_sim or options.default_phases: # Set phase_length to print out CMPSim results every slice_size instructions. # phase_length = options.slice_size // config.instr_cmpsim_phase if phase_length == 0: phase_length = 1 for bopts in bin_options: if not options.list: msg.PrintMsgDate('Running CMPSim on whole program pinballs %s' % \ config.PhaseStr(config.CMPsim_whole)) util.PhaseBegin(options) wp_dir = util.GetDefaultWPDir(bopts) self.Config.SetPerBinParams(bopts) result = sim_kit.RunSimulator(wp_dir, sim_replay_cmd, phase_length, bopts) util.CheckResult(result, options, 'CMPSim for: %s %s' % \ (wp_dir, config.PhaseStr(config.CMPsim_whole)), intermediate_phase=True) if not options.list: msg.PrintMsgDate('Finished running CMPSim on whole program pinballs %s' % \ config.PhaseStr(config.CMPsim_whole)) util.CheckResult(result, options, 'CMPSim on whole program pinballs %s' % \ config.PhaseStr(config.CMPsim_whole)) # Calculate prediction error from simulator data files. # if options.pred_error or options.default_phases: if not options.list: msg.PrintMsgDate('Calculating prediction error %s' % \ config.PhaseStr(config.pred_error)) util.PhaseBegin(options) for bopts in bin_options: wp_dir = util.GetDefaultWPDir(bopts) result = s_phases.CalcPredError(wp_dir, sim_kit, bopts) util.CheckResult(result, options, 'Prediction error for: %s %s' % \ (wp_dir, config.PhaseStr(config.pred_error)), intermediate_phase=True) if not options.list: msg.PrintMsgDate('Finished calculating prediction error %s' % \ config.PhaseStr(config.pred_error)) util.CheckResult(result, options, 'Prediction error calculation %s' % \ config.PhaseStr(config.pred_error)) # Assume nothing has gone wrong at this point. # return result
def Run(self): """Run the scripts required to run simpoint and generate a region CSV file with the results.""" msg.PrintMsg('') # Get the command line options # (options, args) = self.ParseCommandLine() # Make sure required utilities exist and are executable. # # import pdb; pdb.set_trace() if util.Which(self.simpoint_bin) == None: msg.PrintAndExit('simpoint binary not in path.\n' 'Add directory where it exists to your path.') if util.Which(self.csv_bin) == None: msg.PrintAndExit( 'script to generate the region CSV file not in path.\n' 'Add directory where it exists to your path.') # Go to the data directory. Both utilities should be run in this directory. # os.chdir(options.data_dir) # Always copy the specific BBV to the generic name used by simpoint. # If LDV file exists, then copy it to generic name. # if os.path.isfile(self.generic_bbv_name): os.remove(self.generic_bbv_name) shutil.copy(options.bbv_file, self.generic_bbv_name) ldv_file = options.bbv_file.replace('.bb', '.ldv') if os.path.isfile(ldv_file): if os.path.isfile(self.generic_ldv_name): os.remove(self.generic_ldv_name) shutil.copy(options.bbv_file.replace('.bb', '.ldv'), self.generic_ldv_name) # Get the instruction count and slice_size from the BBV file. # try: f = open(self.generic_bbv_name) except IOError: msg.PrintAndExit('problem opening BBV file: ' + self.generic_bbv_name) instr_count = 0 slice_size = 0 for line in f.readlines(): if line.find('Dynamic') != -1: tmp = line.split() count = tmp[len(tmp) - 1] if count > instr_count: instr_count = int(count) if line.find('SliceSize') != -1: tmp = line.split() size = tmp[len(tmp) - 1] if size > slice_size: slice_size = int(size) # Check to make sure instruction count > slice_size. # if slice_size > instr_count: import locale locale.setlocale(locale.LC_ALL, "") msg.PrintAndExit('Slice size is greater than the number of instructions. Reduce parameter \'slice_size\'.' + \ '\nInstruction count: ' + locale.format('%14d', int(instr_count), True) + \ '\nSlice size: ' + locale.format('%14d', int(slice_size), True)) if options.ldv: # Run to generate regions CSV file using both BBV and LDV files. # result = self.NormProjectBBV(options) util.CheckResult( result, options, 'normalizing and projecting BBV with: ' + self.csv_bin) result = self.NormWeightLDV(options) util.CheckResult( result, options, 'normalizing and applying weights to LDV with: ' + self.csv_bin) result = self.CombineFreqVectFiles(options) util.CheckResult( result, options, 'scaling and combining BBV and LDV with: ' + self.csv_bin) result = self.RunSimpoint(options) util.CheckResult( result, options, 'generating clusters (regions) with: ' + self.simpoint_bin) result, regions_csv_file = self.GenRegionCSVFile(options) util.CheckResult(result, options, 'creating regions CSV file with: ' + self.csv_bin) msg.PrintMsg('\nRegions CSV file: ' + os.path.join(options.data_dir, regions_csv_file)) else: # Run scripts to generate regions CSV file using the new method with just the BBV file. # result = self.NormProjectBBV(options) util.CheckResult( result, options, 'normalizing and projecting BBV with: ' + self.csv_bin) result = self.RunSimpoint(options) util.CheckResult( result, options, 'generating clusters (regions) with: ' + self.simpoint_bin) result, regions_csv_file = self.GenRegionCSVFile(options) util.CheckResult(result, options, 'creating regions CSF file with: ' + self.csv_bin) msg.PrintMsg('\nRegions CSV file: ' + os.path.join(options.data_dir, regions_csv_file)) return result
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options): """ Run the additional phases for SDE which are not run for PinPlay. @param wp_pb_dir Directory containing whole program log files (pinballs) @param sim_replay_cmd Python script used to replay a pinball with a simulator @param options Options given on cmd line @return Exit code from last phase executed """ # The SDE phases object needs an SDE kit object. # s_phases = sde_phases.SDEPhases() kit_obj = self.GetKit() s_phases.SetKit(kit_obj) # Need the appropriate simulator kit too. # sim_kit = self.GetSimKit() # Generate LMAT/LIT files. # result = 0 if options.lit_gen or options.default_phases: if not options.list: msg.PrintMsgDate('Generating traces for region pinballs %s' % \ config.PhaseStr(config.LIT)) util.PhaseBegin(options) result = s_phases.GenAllLitFiles(self.replayer_cmd, options) #if result == 0: # if not options.list: # msg.PrintMsgPlus('Waiting on final concurrent region pinball generation') # result = util.WaitJobs(options) if not options.list: msg.PrintMsgDate('Finished generating traces for region pinballs %s' % \ config.PhaseStr(config.LIT)) util.CheckResult(result, options, 'Trace file generation %s' % \ config.PhaseStr(config.LIT)) # Generate traceinfo files. # if options.traceinfo or options.default_phases: if not options.list: msg.PrintMsgDate('Generating traceinfo files %s' % \ config.PhaseStr(config.traceinfo)) util.PhaseBegin(options) result = s_phases.GenTraceinfoFiles(options) if not options.list: msg.PrintMsgDate('Finished generating traceinfo files %s' % \ config.PhaseStr(config.traceinfo)) util.CheckResult(result, options, 'Traceinfo file generation %s' % \ config.PhaseStr(config.traceinfo)) # Run CMPSim simulator on region pinballs. # if options.region_sim or options.default_phases: # Print out CMPSim results every warmup_length instructions. # phase_length = options.warmup_length // config.instr_cmpsim_phase if phase_length == 0: phase_length = 1 for pp_dir in util.GetRegionPinballDir(): if not options.list: msg.PrintMsgDate('Running CMPSim on region pinballs in dir: %s %s' % \ (pp_dir, config.PhaseStr(config.CMPsim_regions))) util.PhaseBegin(options) result = sim_kit.RunSimulator(pp_dir, sim_replay_cmd, phase_length, options) if not options.list: msg.PrintMsgDate('Finished running CMPSim on region pinballs in dir %s %s' % \ (pp_dir, config.PhaseStr(config.CMPsim_regions))) util.CheckResult(result, options, 'CMPSim on pinballs, dir: %s %s' % \ (pp_dir, config.PhaseStr(config.CMPsim_regions))) # Run CMPSim simulator on whole program pinballs. # if options.whole_sim or options.default_phases: # Set phase_length to print out CMPSim results every slice_size instructions. # phase_length = options.slice_size // config.instr_cmpsim_phase if phase_length == 0: phase_length = 1 if not options.list: msg.PrintMsgDate('Running CMPSim on whole program pinballs %s' % \ config.PhaseStr(config.CMPsim_whole)) util.PhaseBegin(options) result = sim_kit.RunSimulator(wp_pb_dir, sim_replay_cmd, phase_length, options) if not options.list: msg.PrintMsgDate('Finished running CMPSim on whole program pinballs %s' % \ config.PhaseStr(config.CMPsim_whole)) util.CheckResult(result, options, 'CMPSim on whole program pinballs %s' % \ config.PhaseStr(config.CMPsim_whole)) # Calculate prediction error from simulator data files. # if options.pred_error or options.default_phases: if not options.list: msg.PrintMsgDate('Calculating prediction error %s' % \ config.PhaseStr(config.pred_error)) util.PhaseBegin(options) result = s_phases.CalcPredError(wp_pb_dir, sim_kit, options) if not options.list: msg.PrintMsgDate('Finished calculating prediction error %s' % \ config.PhaseStr(config.pred_error)) util.CheckResult(result, options, 'Prediction error calculation %s' % \ config.PhaseStr(config.pred_error)) # Verify the LIT files with the simulator. # if options.verify: if not options.list: msg.PrintMsgDate('Verifying LIT files %s' % \ config.PhaseStr(config.verify_LIT)) util.PhaseBegin(options) result = s_phases.VerifyLITFiles(self.sim_run_cmd, options) if not options.list: msg.PrintMsgDate('Finished verifying LIT files %s' % \ config.PhaseStr(config.verify_LIT)) util.CheckResult(result, options, 'LIT file verification %s' % \ config.PhaseStr(config.verify_LIT)) # Generate instruction mix for the whole program pinballs. # if options.imix_whole: if not options.list: msg.PrintMsgDate('Generating imix on whole program pinballs %s' % \ config.PhaseStr(config.imix_whole)) # Setup dictionary of parameters for method RunAllDir() # param = {'options': options, 'replayer_cmd': self.replayer_cmd} util.PhaseBegin(options) result = util.RunAllDir(wp_pb_dir, s_phases.GenImix, True, param) if result == 0: if not options.list: msg.PrintMsgPlus( 'Waiting on final whole program pinball imix generation' ) result = util.WaitJobs(options) if not options.list: msg.PrintMsgDate('Finished generating imix on whole program pinballs %s' % \ config.PhaseStr(config.imix_whole)) util.CheckResult(result, options, 'Imix on whole program pinballs %s' % \ config.PhaseStr(config.imix_whole)) # Generate instruction mix for the LIT files. # if options.imix_lit: if not options.list: msg.PrintMsgDate('Generating imix on LIT files %s' % \ config.PhaseStr(config.imix_lit)) # Setup dictionary of parameters for method RunAllDir() # param = {'options': options, 'replayer_cmd': self.replayer_cmd} util.PhaseBegin(options) for lit_dir in util.GetLitDir(): result = util.RunAllDir(lit_dir, s_phases.GenImix, True, param) if result == 0: if not options.list: msg.PrintMsgPlus('Waiting on final LIT files generation') result = util.WaitJobs(options) if not options.list: msg.PrintMsgDate('Finished generating imix on LIT files %s' % \ config.PhaseStr(config.imix_lit)) util.CheckResult(result, options, 'Imix on LIT files %s' % \ config.PhaseStr(config.imix_lit)) # Generate instruction mix for the region pinballs. # if options.imix_region: if not options.list: msg.PrintMsgDate('Generating imix on region pinballs %s' % \ config.PhaseStr(config.imix_regions)) # Setup dictionary of parameters for method RunAllDir() # param = {'options': options, 'replayer_cmd': self.replayer_cmd} util.PhaseBegin(options) for pp_dir in util.GetRegionPinballDir(): result = util.RunAllDir(pp_dir, s_phases.GenImix, True, param) if result == 0: if not options.list: msg.PrintMsgPlus( 'Waiting on final region pinballs generation') result = util.WaitJobs(options) if not options.list: msg.PrintMsgDate('Finished generating imix on region pinballs %s' % \ config.PhaseStr(config.imix_regions)) util.CheckResult(result, options, 'Imix on region pinballs %s' % \ config.PhaseStr(config.imix_regions)) # Assume nothing has gone wrong at this point. # return result
def ParseCommandLine(self): """ Process command line arguments, get Kit, tool options, and their paths. @return List containing: pin_options, pintool_options, options.replay_file, kit_obj """ # import pdb; pdb.set_trace() version = '$Revision: 1.63 $' version = version.replace(' ', '') ver = version.replace(' $', '') us = '%prog [options] pinball_basename \nVersion: ' + ver desc = 'Replays one pinball. Use \'--replay_options\' or ' \ '\'--log_options\' to modify the pintool behavior during replay.' util.CheckNonPrintChar(sys.argv) parser = optparse.OptionParser(usage=us, version=ver, description=desc) # Define the command line options which control the behavior of the # script. # # Some of these methods take a 2nd argument which is the empty string # ''. If this script used option groups, then the 2nd parameter would # be the group. However, this script does not use option groups, so # the argument is empty. # cmd_options.arch(parser, '') cmd_options.config_file(parser) cmd_options.cross_os(parser, '') cmd_options.debug(parser) cmd_options.global_file(parser) cmd_options.log_options(parser) cmd_options.msgfile_ext(parser) cmd_options.no_print_cmd(parser) cmd_options.pintool(parser) cmd_options.pintool_help(parser) cmd_options.pin_options(parser) cmd_options.pinplayhome(parser, '') cmd_options.playout(parser) cmd_options.replay_file(parser) cmd_options.replay_options(parser) cmd_options.save_global(parser) cmd_options.sdehome(parser, '') cmd_options.verbose(parser) # import pdb; pdb.set_trace() (options, args) = parser.parse_args() if options.verbose: msg.PrintMsg('Started replayer.py') # Check to make sure the pinball basename has been given as an argument or # command line option. # # import pdb; pdb.set_trace() if options.replay_file == '' and \ not (hasattr(options, 'pintool_help') and options.pintool_help): if len(sys.argv) == 1 or len(args) == 0: msg.PrintMsg( "ERROR: Must have a trace basename on the command line.\n" "Usage: %s [options] pinball_basename" % os.path.basename(sys.argv[0])) util.CheckResult(-1, options, 'Checking command line options') options.replay_file = args[0] # Read in an optional configuration files and set global variables. # config_obj = config.ConfigClass() config_obj.GetCfgGlobals(options, False) # Don't need to require 4 variables # Once the tracing configuration parameters are read, get the kit in # case pinplayhome was set on the command line. # # import pdb; pdb.set_trace() kit_obj = self.GetKit() # If user just wants 'pintool_help' go ahead and print it, then exit # the script. # if hasattr(options, 'pintool_help') and options.pintool_help: result = util.PintoolHelpKit(kit_obj, options) sys.exit(result) # Translate the 'arch' value given by the user into # the internal arch type used by the scripts. # if hasattr(options, 'arch') and options.arch: if 'intel64' in options.arch: options.arch = config.ARCH_INTEL64 elif 'ia32' in options.arch: options.arch = config.ARCH_IA32 else: options.arch = config.ARCH_INVALID # Now that we know the type of the binary, set the user defined pintool, # if one exists. Need to wait until now to set the tool because the # user may only have the tool in the architecture dependent directory # for this type of application. Thus we need the binary type in order # to find it. # # import pdb; pdb.set_trace() kit_obj.binary_type = options.arch pin_options = '' pintool_options = '' # Check to see if there is a pinball to replay. # if options.replay_file == "": msg.PrintHelpAndExit('Replay file not specified!') # If the user specified a pintool, replace the default pintool in the kit with # it. # if hasattr(options, "pintool") and options.pintool: kit_obj.SetPinTool(options.pintool, options.replay_file) platform = util.Platform() if platform == config.LINUX: pin_options = ' ' + kit_obj.prefix + ' -xyzzy ' # If using NOT using Linux tools to work with whole program pinballs generated on Windows, # then need a set of knobs for the pin binary itself. # if not options.cross_os: pin_options += kit_obj.prefix + ' -reserve_memory ' pin_options += kit_obj.prefix + ' ' + options.replay_file + '.address ' pintool_options += ' -replay:basename ' + options.replay_file if options.playout or '-replay:playout 1' in options.replay_options: # If present, need to remove the knob '-replay:playout 1' from # options.replay_options because it can only be given once on the # command line. # pintool_options += ' -replay:playout 1 ' options.replay_options = options.replay_options.replace( '-replay:playout 1', '') else: pintool_options += ' -replay:playout 0 ' # If running Windows WP pinballs on Linux, then need this knob for the replayer pintool. # if options.cross_os: pintool_options += ' -replay:addr_trans' # Add knobs for Pin and replay/logging user gave on the command line. # pin_options += ' ' + options.pin_options pintool_options += ' ' + options.replay_options + ' ' + options.log_options # If user has log options, then may need to at multi-thread knob. # if options.log_options: pintool_options += util.AddMt(options, options.replay_file) return pin_options, pintool_options, options.replay_file, kit_obj, options
def ParseCommandLine(self): """ Process command line arguments. Generate pin and tool options, and their paths. """ # import pdb; pdb.set_trace() version = '$Revision: 1.14 $' version = version.replace(' ', '') ver = version.replace(' $', '') us = '%prog [options] trace_basename \nVersion: ' + ver desc = 'Run a simulator on a trace.' util.CheckNonPrintChar(sys.argv) parser = optparse.OptionParser(usage=us, version=ver, description=desc) # Define the command line options which control the behavior of the # script. # # Some of these methods take a 2nd argument which is the empty string # ''. If this script used option groups, then the 2nd parameter would # be the group. However, this script does not use option groups, so # the argument is empty. # cmd_options.config_file(parser) cmd_options.debug(parser) cmd_options.global_file(parser) cmd_options.list(parser, '') cmd_options.processor(parser, '') cmd_options.save_global(parser) cmd_options.simhome(parser, '') cmd_options.sim_options(parser, '') cmd_options.trace_basename(parser) cmd_options.verbose(parser) cmd_options.verify(parser, '') self.AddAdditionalCmdOptions(parser) # import pdb; pdb.set_trace() (options, args) = parser.parse_args() # Check to make sure the trace basename has been given as an argument. # # import pdb; pdb.set_trace() if len(sys.argv) == 1 or len(args) == 0: msg.PrintMsg( "ERROR: Must have a trace basename on the command line.\n" "Usage: %s [options] trace_basename" % os.path.basename(sys.argv[0])) util.CheckResult(-1, options, 'Checking command line options') options.trace_basename = args[0] # Read in optional configuration files and set global variables. # config_obj = config.ConfigClass() config_obj.GetCfgGlobals(options, False) # Don't need all 4 required parameters # Just need this parameter. # self.Config.CheckRequiredParameter('processor') # Initialize the kit after reading in parameters, in case the user # specified anything which is used to intialize the kit. # self.kit = self.GetKit() return options
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options): """Run the additional phases for SDE which are not run for PinPlay.""" if not options.sniper_root or not os.path.exists( os.path.join(options.sniper_root, 'run-sniper')): msg.PrintMsgPlus( 'ERROR: Please set SNIPER_ROOT or --sniper_root to a valid Sniper install.' ) util.CheckResult( -1, options, 'Checking for Sniper install location.') # Force error with -1 if options.region_sim or options.default_phases: # Print out Sniper results every warmup_length instructions. # for pp_dir in util.GetRegionPinballDir(): phase_length = options.warmup_length // config.instr_cmpsim_phase if phase_length == 0: phase_length = 1 if not options.list: msg.PrintMsgDate('Running Sniper on region pinballs %s' % \ config.PhaseStr(config.sniper_regions)) util.PhaseBegin(options) result = RunSniper(pp_dir, sim_replay_cmd, phase_length, options) if not options.list: msg.PrintMsgDate('Finished running Sniper on region pinballs %s' % \ config.PhaseStr(config.sniper_regions)) util.CheckResult(result, options, 'Sniper on region pinballs %s' % \ config.PhaseStr(config.sniper_regions)) if options.whole_sim or options.default_phases: # Set phase_length to print out Sniper results every slice_size instructions. # phase_length = options.slice_size // config.instr_cmpsim_phase if phase_length == 0: phase_length = 1 if not options.list: msg.PrintMsgDate('Running Sniper on whole program pinballs %s' % \ config.PhaseStr(config.sniper_whole)) util.PhaseBegin(options) result = RunSniper(wp_pb_dir, sim_replay_cmd, phase_length, options) if not options.list: msg.PrintMsgDate('Finished running Sniper on whole program pinballs %s' % \ config.PhaseStr(config.sniper_whole)) util.CheckResult(result, options, 'Sniper on whole program pinballs %s' % \ config.PhaseStr(config.sniper_whole)) if options.pred_error or options.default_phases: if options.pccount_regions: msg.PrintMsg('\n Prediction with PCregions is NIY') return 0 if not options.list: msg.PrintMsgDate('Calculating prediction error %s' % \ config.PhaseStr(config.pred_error)) util.PhaseBegin(options) result = CalcPredError(wp_pb_dir, self.sim_replay_cmd, options) if not options.list: msg.PrintMsgDate('Finished calculating prediction error %s' % \ config.PhaseStr(config.pred_error)) util.CheckResult(result, options, 'Prediction error calculation %s' % \ config.PhaseStr(config.pred_error)) # Assume nothing has gone wrong at this point. # return 0
def Run(self): """ Get the user's options, read config file and execute the phases desired by the user. @return Exit code from last phase executed """ # Catch signals in order to do an orderly shutdown of the script. # self.InstallSigHandler() # import pdb; pdb.set_trace() options, bin_options = self.ParseCommandLine() # Print out the version numbers for all python modules in this directory. # if config.debug: util.PrintModuleVersions() # If logging, ensure the user has entered a mode. # if options.log and config.mode == '': msg.PrintHelpAndExit( 'Must enter mode of application using: --mode "st|mt|mpi|mpi_mt"') # Set the defaults for a set of variables which are needed, but not in the list # of required options in the tracing configuration file. # self.SetDefaults(options) # Add 1500 to the warmup length in order to deal with problems that # occur when the start of a warmup region and the end of the previous # region are in the same basic block. # config.warmup_length += 1500 # Get variables used in many phases. # # import pdb; pdb.set_trace() wp_pb_dir = util.GetWPDir() status_file = util.GetCBSPLogFile() # Print out the tracing configuration parameters. # # TODO: Fix print so it prints CBSP info, not PP info if not options.list: self.phases.PrintTracingInfo(bin_options, wp_pb_dir, status_file, self.PrintHome) ######################################################################### # # Several phases which clean up tracing instances. # ######################################################################### result = 0 if options.delete: self.phases.DeleteTracingFileDir(options) util.CheckResult(result, options, 'Deleting tracing instance files') # Since just deleted all files/dirs for the tracing instance, need # to set the whole program directory to the default value. Do NOT # use util.GetWPDir() here because this may return the name of a # relogged WP directory (which we, of course, just deleted.) # wp_pb_dir = util.GetBaseWPDir() if options.delete_all: self.phases.DeleteAllTracingFileDir(options) util.CheckResult(result, options, 'Deleting all tracing files') # Again, since just deleted all files/dirs, need to set the whole # program directory to the default value. # wp_pb_dir = util.GetBaseWPDir() ######################################################################### # # Run the phases of the script. They are in a specific order to ensure # all prerequisite phases are completed before a given phase is run. # ######################################################################### # Logging phase to generate initial whole program pinballs. # # import pdb; pdb.set_trace() if options.log or options.default_phases: util.PhaseBegin(options) result = 0 # First check to make sure there aren't any old WP pinball directories. # for bopts in bin_options: wp_pb_dir = util.GetDefaultWPDir(bopts) if os.path.isdir(wp_pb_dir): msg.PrintMsg( "\nERROR: Whole program pinball directory already exists: " + wp_pb_dir + "\nRemove this directory to proceed with generating " "whole program pinballs.") util.CheckResult(-1, options, 'Whole program pinball generation %s' % \ config.PhaseStr(config.log_whole)) for bopts in bin_options: # Add CSBP logging specific knobs to any user defined # log_options. They are required by this phase in order to # generate DCFG files. # if not hasattr(options, 'log_options'): setattr(options, 'log_options', '') dcfg_log_opts = ' -omix %s.%s.mix' % (bopts.program_name, bopts. input_name) dcfg_log_opts += config.cbsp_base_log_options if bopts.log_options: bopts.log_options += dcfg_log_opts else: bopts.log_options = dcfg_log_opts # Run the logger for this binary # wp_pb_dir = util.GetDefaultWPDir(bopts) log_file_name = util.GetLogFile(bopts) self.Config.SetPerBinParams(bopts) result = self.phases.Logger(self.logger_cmd, wp_pb_dir, log_file_name, bopts) util.CheckResult(result, options, 'WP pinballs for: %s %s' % \ (log_file_name, config.PhaseStr(config.log_whole)), intermediate_phase=True) # Now remove the DCFG log options as they are only used in this phase. This # ensures only the user defined options are used for subsequent phases. # bopts.log_options = bopts.log_options.replace(dcfg_log_opts, '') util.CheckResult(result, options, 'Whole program pinball generation %s' % \ config.PhaseStr(config.log_whole)) # All phases after this require the whole program pinball directories. Exit with an # error if they don't not exist. # # import pdb; pdb.set_trace() if not options.list and not options.debug and\ not options.delete and not options.delete_all: err_msg = lambda string: msg.PrintMsg('\nERROR: Can\'t proceed because whole program pinball directory does not exist:\n' + \ ' ' + string + \ '\nMust select at least one phase to run. Try using option \'--default_phases\'.' + \ '\nUse \'-h\' for help on selecting phases to run.') for bopts in bin_options: wp_dir = util.GetDefaultWPDir(bopts) if not os.path.isdir(wp_dir): err_msg(wp_dir) # Use -1 to force check to fail util.CheckResult( -1, options, 'Initial check to see if WP pinballs exist') # Print out the number of instructions in the whole program pinballs. # if not options.list: for bopts in bin_options: wp_dir = util.GetDefaultWPDir(bopts) msg.PrintMsg('') msg.PrintMsg(wp_dir) util.PrintInstrCount(util.GetDefaultWPDir(bopts), options) ######################################################################### # # These phases are run with the whole progam pinballs defined/generated # in the previous phase. # ######################################################################### # Run the cross binary matcher. # if options.cb_match or options.default_phases: if not options.list: msg.PrintMsgDate('Running cross binary matcher %s' % \ config.PhaseStr(config.cb_match)) msg.PrintMsg('') util.PhaseBegin(options) result = self.phases.CrossBinaryMatcher(self.path, self.script_path, options, bin_options) if not options.list: msg.PrintMsgDate('Finished running cross binary matcher %s' % \ config.PhaseStr(config.cb_match)) util.CheckResult(result, options, 'Cross binary matcher %s' % \ config.PhaseStr(config.cb_match)) # Run Simpoints to generate clusters # if options.simpoint or options.default_phases: if not options.list: msg.PrintMsgDate('Running Simpoint %s' % \ config.PhaseStr(config.Simpoint)) param = {'options': options} util.PhaseBegin(options) result = self.phases.RunSimPoint(self.path, self.script_path, options, bin_options) if not options.list: msg.PrintMsgDate('Finished running Simpoint %s' % \ config.PhaseStr(config.Simpoint)) util.CheckResult(result, options, 'Simpoints generation %s' % \ config.PhaseStr(config.Simpoint)) # Relog to generate representative region pinballs. # if options.region_pinball or options.default_phases: if not options.list: msg.PrintMsgDate('Generating region pinballs %s' % \ config.PhaseStr(config.relog_regions)) util.PhaseBegin(options) result = 0 for bopts in bin_options: wp_dir = util.GetDefaultWPDir(bopts) log_file_name = util.GetLogFile(bopts) self.Config.SetPerBinParams(bopts) result = self.phases.MultiIterGenRegionPinballs( wp_dir, self.replayer_cmd, bopts) if result == 0: if not options.list: msg.PrintMsgPlus( 'Waiting on final concurrent region pinball generation') result = util.WaitJobs(options) util.CheckResult(result, options, 'Region pinballs for: %s %s' % \ (log_file_name, config.PhaseStr(config.relog_regions)), intermediate_phase=True) if not options.list: msg.PrintMsgDate('Finished generating region pinballs %s' % \ config.PhaseStr(config.relog_regions)) util.CheckResult(result, options, 'Region pinball generation %s' %\ config.PhaseStr(config.relog_regions)) # If there are any additional phases, then run them. # self.RunAdditionalPhases(self.sim_replay_cmd, options, bin_options) # Cleanup and print out a string to indicate the tracing has completed. # # import pdb; pdb.set_trace() util.CleanupTraceEnd(options) util.PrintTraceEnd(options) return result
def ParseCommandLine(self): """ Process command line arguments and read in config file/globals. @return List of options parsed from command line """ version = '$Revision: 1.11 $' version = version.replace(' ', '') ver = version.replace(' $', '') us = '%prog <phase> --cbsp_cfgs CFG_FILE,CFG_FILE[,CFG_FILE] [options] ' desc = 'This script uses Cross Binary SimPoint (CBSP) and PinPlay tools ' \ 'to create a set of equal-work traces from two, or more, binaries compiled from the same source.\n\n'\ '' \ 'At least one command line option must be given to chose the phase(s) of the script to run. ' \ 'Use option \'--default_phases\' to run all phases. ' \ 'See section "Phase options" for a list of phases. \n\n'\ '' \ 'The option \'--cbsp_cfgs\' is also required to run this scripts. It contains a list of PinPlay configuration '\ 'files. Each file defines one of the binaries to be traced. (See \'sde_pinpoints.py -h\' for more info on PinPlay config files.)\n\n'\ '' \ 'Optional master CBSP config files may be given with the option \'-cfg\'. The parameters '\ 'defined in each master cfg file are used for every binary. Master CBSP config files are not required.\n\n'\ '' \ 'Master CBSP cfg files are processed in order. '\ 'Command line options over-ride values read from master CBSP configuration files. '\ 'The long option name, '\ 'not the single letter option, must be used in the config files.\n\n'\ '' \ 'The parameter \'cbsp_name\' MUST be defined in at least one configuration file in order '\ 'to run this script. It may be defined in master/binary CBSP config files or by command line option.\n\n'\ '' \ 'Example configuration file:'\ ' '\ '--------------------------------------------'\ ' '\ '[Parameters]'\ ' '\ 'cbsp_name: lipack_test'\ ' '\ 'program_name: linpack'\ ' '\ 'input_name: avx'\ ' '\ 'command: AVX_linpack 8'\ ' '\ 'mode: st'\ ' '\ '--------------------------------------------\n\n'\ '' \ 'Advanced Usage: Since CBSP is an extension of PinPlay, it\'s possible to combine the two '\ 'tool chains to generate traces. For example, you can use sde_pinplay.py to generate whole program '\ 'pinballs in parallel (possibly using NetBatch). Then use the CBSP script to run the \'cb_match\' '\ 'and \'simpoint\' phases to select equivalent work clusters in each binary. Finally, you can run '\ 'sde_pinpoints.py with NetBatch to generate region pinballs/LIT files in parallel.\n\n'\ '' \ 'If using this method, you must use the configuration file for just one binary with each invocation of '\ 'the sde_pinpoints.py script. Also, in order to combine these tools, you must add the option '\ '\'--add_program_wp\' to each sde_pinpoints.py run. This is required so the directories names '\ 'generated/used by the two chains are compatible. '\ '' util.CheckNonPrintChar(sys.argv) parser = optparse.OptionParser( usage=us, description=desc, version=ver, formatter=cmd_options.BlankLinesIndentedHelpFormatter()) # Command line options to control the tools behavior. These are not in any # of the option groups. # # import pdb; pdb.set_trace() cmd_options.add_program_wp(parser) cmd_options.append_status(parser) cmd_options.cbsp_cfgs(parser) cmd_options.config_file(parser) cmd_options.debug(parser) cmd_options.delete(parser) cmd_options.delete_all(parser) # cmd_options.dir_separator(parser) # Only works for PinPlay, need to fix for CBSP cmd_options.log_options(parser) cmd_options.pintool(parser) cmd_options.pin_options(parser) cmd_options.replay_options(parser) cmd_options.save_global(parser) cmd_options.verbose(parser) self.AddAdditionalOptions(parser) # Top level command line options in the default group allow the user # to choose the phases to run. # phase_group = cmd_options.PhaseGroup(parser) self.default_phases(parser, phase_group) cmd_options.log(parser, phase_group) cmd_options.cb_match(parser, phase_group) cmd_options.simpoint(parser, phase_group) cmd_options.region_pinball(parser, phase_group) self.AddAdditionalPhaseOptions(parser, phase_group) parser.add_option_group(phase_group) # Top level command line options in param group are used to set various # tracing parameters. # param_group = cmd_options.ParameterGroup(parser) cmd_options.cbsp_name(parser, param_group) cmd_options.input_name(parser, param_group) cmd_options.num_cores(parser, param_group) cmd_options.program_name(parser, param_group) self.KitOption(parser, param_group) parser.add_option_group(param_group) # Top level command line options which only apply to the Simpoint phase. # simpoint_phase_group = cmd_options.SimpointPhaseGroup(parser) cmd_options.maxk(parser, simpoint_phase_group) cmd_options.simpoint_options(parser, simpoint_phase_group) cmd_options.slice_size(parser, simpoint_phase_group) parser.add_option_group(simpoint_phase_group) # Top level command line options which only apply to the region pinball generation phase. # region_pb_phase_group = cmd_options.RegionPBPhaseGroup(parser) cmd_options.epilog_length(parser, region_pb_phase_group) cmd_options.prolog_length(parser, region_pb_phase_group) cmd_options.warmup_length(parser, region_pb_phase_group) parser.add_option_group(region_pb_phase_group) # Top level command line options in modify group modify the behavior of phases. # modify_group = cmd_options.ModifyGroup(parser) # These should go before the rest of the modify options. # # cmd_options.coop_pinball(parser, modify_group) cmd_options.cross_os(parser, modify_group) cmd_options.list(parser, modify_group) parser.add_option_group(modify_group) # Parse the command line options. # # import pdb; pdb.set_trace() (options, args) = parser.parse_args() # Added method cbsp() to 'options' to check if running CBSP. # util.AddMethodcbsp(options) # Check to make sure there was at least one command line option given. # # import pdb; pdb.set_trace() if len(sys.argv) == 1: msg.PrintMsg( "\nERROR: Must use command line options to chose at least one phase to run.\n" "Use the option '--default_phases' to run the default phases. Use '--help' for more info.") util.CheckResult(-1, options, 'Check command line options - no phase given') # User must give option/parameter 'cbsp_cfgs' (unless deleting files). # # import pdb; pdb.set_trace() if not options.delete_all and \ (not hasattr(options, 'cbsp_cfgs') or not options.cbsp_cfgs): msg.PrintMsg( '\nERROR: Must give list of configuration files for the CBSP binaries\n' 'using parameter \'--cbsp_cfgs\'.') util.CheckResult(-1, options, 'Check command line options - ' 'Looking for parameter \'cbsp_cfgs\'.') # Get a list of the parameters for each CBSP binary. # # import pdb; pdb.set_trace() bin_options = self.Config.GetAllCBSPBinParams(options) # Added method cbsp() to 'options' to check if running CBSP. # util.AddMethodcbsp(options) # Now that we have all the parameters, create a new status file or, # zero out the file if it already exists. Need all the parameters # because 'cbsp_name' is used as part of the status file name. If user # sets parameter 'append_status', the old file is not deleted. # util.NewStatusFile(options, cbsp=True) # Ensure each binary has a unique program_name/input_name # all_names = set() for bopts in bin_options: name = '%s %s' % (bopts.program_name, bopts.input_name) if name in all_names: msg.PrintMsg( '\nERROR: Must have unique program_name & input_name for ' 'each binary.\n' 'Found duplicates: %s' % name) util.CheckResult(-1, options, 'Check command line options - ' 'Found duplicate program_name & input_name') all_names.add(name) # For debugging, print out the WP directory names for each binary # if hasattr(options, 'verbose') and options.verbose: for bopts in bin_options: msg.PrintMsg('WP dir: ' + util.GetDefaultWPDir(bopts)) # Once the tracing configuration parameters are read, get the kit in # case sdehome/pinplayhome was set on the command line or in a config # file. Also, need to reset paths to kit/scripts in object 'self' and # set the kit in self.phases. # kit_obj = self.GetKit() self.path = kit_obj.path self.script_path = kit_obj.script_path self.phases.SetKit(kit_obj) # If required, check to see if there any 'forbidden' char in some of # the parameters. # if self.kit_type == config.SDE and hasattr(options, 'spec') and not options.spec: self.Config.CheckForbiddenChar() # Do some 'special' things on native Windows. # util.WindowsNativeCheck(options) return options, bin_options
def RunAdditionalPhases(self, wp_pb_dir, sim_replay_cmd, options): """ Run the additional phases for the simulator which are not run as part of the usual PinPlay phases. """ phases_obj = phases.Phases() self.sim_kit = self.GetSimKit() # Run branch predictor simulator on region pinballs. # if options.region_sim or options.default_phases: # Print out simulator results every warmup_length instructions. # for pp_dir in util.GetRegionPinballDir(): phase_length = options.warmup_length if phase_length == 0: phase_length = options.slice_size if not options.list: msg.PrintMsgDate('Running simulator on region pinballs: %s' % \ config.PhaseStr(config.sim_regions)) util.PhaseBegin(options) result = self.sim_kit.RunSimulator(pp_dir, sim_replay_cmd, phase_length, options) if not options.list: msg.PrintMsgDate('Finished running simulator on region pinballs: %s' % \ config.PhaseStr(config.sim_regions)) util.CheckResult(result, options, 'simulator on region pinballs: %s' % \ config.PhaseStr(config.sim_regions)) # Run branch predictor simulator on whole program pinballs. # if options.whole_sim or options.default_phases: # Set phase_length to print out simulator results every slice_size instructions. # phase_length = options.slice_size if not options.list: msg.PrintMsgDate('Running simulator on whole program pinballs: %s' % \ config.PhaseStr(config.sim_whole)) util.PhaseBegin(options) result = self.sim_kit.RunSimulator(wp_pb_dir, sim_replay_cmd, phase_length, options) if not options.list: msg.PrintMsgDate('Finished running simulator on whole program pinballs: %s' % \ config.PhaseStr(config.sim_whole)) util.CheckResult(result, options, 'simulator on whole program pinballs: %s' % \ config.PhaseStr(config.sim_whole)) # Calculate prediction error. # if options.pred_error or options.default_phases: if hasattr(options, 'pccount_regions') and options.pccount_regions: msg.PrintMsg('\n Prediction with PCregions is NIY') return 0 if not options.list: msg.PrintMsgDate('Calculating prediction error: %s' % \ config.PhaseStr(config.pred_error)) util.PhaseBegin(options) result = phases_obj.CalcPredError(wp_pb_dir, self.sim_kit, options) if not options.list: msg.PrintMsgDate('Finished calculating prediction error: %s' % \ config.PhaseStr(config.pred_error)) util.CheckResult(result, options, 'Prediction error calculation: %s' % \ config.PhaseStr(config.pred_error)) # Assume nothing has gone wrong at this point. # return 0