def __init__(self):
try:
self.mCmdLineParms = CommandLineParameters
self.mConfigArgs = retrieve_config_argument(sys.argv[1:])
except SystemExit as aSysExit:
sys.exit(int(str(aSysExit)))
except BaseException:
print(
"[ERROR] - An Unhandled Error has Occurred during applications"
" setup of " + str(sys.argv[0]))
traceback.print_exc(file=sys.stdout)
sys.exit(44)
super().__init__(CmdLine.Switches[CmdLine.msg_lev], Defaults.msg_level)
self.fctrl_dir = None
self.mode = None
self.summary = None
self.process_queue = None
self.client_lev = False
self.default_root = ""
self.ctrl_item = ControlItem()
# no propogate values
self.num_runs = None
self.sum_level = None
self.item_data = {}
self.options = {}
self.max_fails = 0
self.limit_fails = False
self.crit_sec = HiCriticalSection()
self.processor_name = None
self.fctrl = None
self.iss = None
self.generator = None
self.rtl = None
self.performance = None
self.regression = None
self.master_config = None
self.fctrl_name = None
self.output_root = None
self.launcher_type = None
self.output_dir = None
self.process_cmd = None
self.terminated = False
# this is a proc that will decrement the fails counter until it is 0
# if the proc is None then this will not occur
self.on_fail_proc = None
self.is_term_proc = None
# if an external terminate event is captured then this call back will
# shutdown master run and all associated threads gracefully
global shutdown_proc
shutdown_proc = self.trigger_shutdown
def load(self):
# Msg.dbg( "ForrestRun::load" )
my_frun_path = self.option_def(CmdLine.Switches[CmdLine.control_name],
None)
if my_frun_path is None:
raise Exception(
"F-Run Control File Not Found on the Forrest Run Command Line: Given Path: %s",
str((my_frun_path)))
# good to here
self.locate_frun(my_frun_path)
Msg.user("File Path: %s" % (my_frun_path))
my_content = open(self.frun_name).read()
my_glb, my_loc = SysUtils.exec_content(my_content)
Msg.dbg(str(my_loc))
self.fcontrol = my_loc["control_items"]
my_ctrl_dict = self.fcontrol[0]
my_ctrl_item = ControlItem()
my_ctrl_item.load(self._mAppsInfo, my_ctrl_dict)
# Msg.lout( my_ctrl_dict, "user", "Forrest Parent Data ...." )
self.check_simulator()
self.fctrl = ExecuteController(self._mAppsInfo)
self.fctrl.set_frun(self.frun_name)
self.fctrl.load(my_ctrl_item)
def __init__(self):
try:
self.mCmdLineParms = CommandLineParameters
self.mConfigArgs = retrieveConfigArgument(sys.argv[1:])
self.mAppsSetup = ApplicationsSetup(self.mCmdLineParms, sys.argv, self.mConfigArgs, True, True)
self._mAppsInfo = self.mAppsSetup.getApplicationsInfo()
except SystemExit as aSysExit:
sys.exit(int(str(aSysExit)))
except:
print("[ERROR] - An Unhandled Error has Occurred during applications setup of " + str(sys.argv[0]))
traceback.print_exc(file=sys.stdout)
sys.exit(44)
self.module_dir, self.module_name = PathUtils.split_path(PathUtils.real_path(sys.argv[0]))
# self._mAppsInfo = apps_info
self.load_message_levels(CmdLine.Switches[CmdLine.msg_lev], Defaults.msg_level)
#print( "Got here (2.1)" )
# persistent values
self._mAppsInfo.mMainAppPath = PathUtils.include_trailing_path_delimiter(self.module_dir) + "../.." # TODO still keep main app path for now.
self._mAppsInfo.mProcessMax = self.option_def( CmdLine.Switches[CmdLine.process_max], None, self.to_int )
self._mAppsInfo.mTestBaseDir = None
self._mAppsInfo.mToolPath = self.module_dir
self.fctrl_dir = None
self.mode = None
self.summary = None
self.process_queue = None
self.client_lev = False
self.default_root = ""
self.ctrl_item = ControlItem()
# no propogate values
self.num_runs = None
self.sum_level = None
self.item_data = {}
self.options = {}
self.max_fails = 0
self.limit_fails = False
self.crit_sec = HiCriticalSection()
self.processor_name = None
self.fctrl = None
self.terminated = False
# this is a proc that will decrement the fails counter until it is 0
# if the proc is None then this will not occur
self.on_fail_proc = None
self.is_term_proc = None
# if an external terminate event is captured then this call back will shutdown
# master run and all associated threads gracefully
global shutdown_proc
shutdown_proc = self.trigger_shutdown
def __init__(self, aAppsInfo):
# print( "Got here (2.0)" )
super().__init__(CmdLine.Switches[CmdLine.msg_lev], Defaults.msg_level,
aAppsInfo)
#print( "Got here (2.1)" )
# persistent values
self._mAppsInfo.mMainAppPath = PathUtils.include_trailing_path_delimiter(
self.module_dir
) + "../.." # TODO still keep main app path for now.
self._mAppsInfo.mProcessMax = self.option_def(
CmdLine.Switches[CmdLine.process_max], None, self.to_int)
self._mAppsInfo.mTestBaseDir = None
self._mAppsInfo.mToolPath = self.module_dir
self.fctrl_dir = None
self.mode = None
self.summary = None
self.process_queue = None
self.client_lev = False
self.default_root = ""
self.ctrl_item = ControlItem()
# no propogate values
self.num_runs = None
self.sum_level = None
self.item_data = {}
self.options = {}
self.max_fails = 0
self.limit_fails = False
self.crit_sec = HiCriticalSection()
self.processor_name = None
self.terminated = False
# this is a proc that will decrement the fails counter until it is 0
# if the proc is None then this will not occur
self.on_fail_proc = None
self.is_term_proc = None
# if an external terminate event is captured then this call back will shutdown
# master run and all associated threads gracefully
global shutdown_proc
shutdown_proc = self.trigger_shutdown
def process(self):
# Msg.dbg( "FileController::process()" )
# Msg.dbg( "FileController Contents: \n\"" + str( self.fcontrol ) + "\n" )
try:
# Msg.lout( my_options["work-dir"], MsgLevel.dbg, "Work Director Stack" )
# a control file may iterate through to completion according to the amount set in num_runs
for my_ndx in range(self.ctrl_item.iterations):
# Msg.user( "Executing %d of %d Iterations, Control File: %s" % ( my_ndx + 1, self.ctrl_item.iterations, self.ctrl_item.file_path()) , "FILE-ITERATION")
if self.is_terminated():
break
try:
my_item_ndx = 0
# each item in the control set exists as a dictionary
for my_item_dict in self.fcontrol:
try:
if self.is_terminated():
break
my_item_ndx += 1
# Msg.user( "Processing Line: %s" % (str( my_item_dict )), "CTRL-FILE" ) )
my_ctrl_item = ControlItem()
my_ctrl_item.parent_fctrl = self.parent_fctrl
my_ctrl_item.fctrl_item = str(my_item_dict)
try:
my_ctrl_item.load(self.mAppsInfo, my_item_dict,
self.ctrl_item)
except:
raise
my_item_type = my_ctrl_item.item_type()
my_controller = None
if my_item_type == ControlItemType.TaskItem:
# Msg.dbg( "\nControl Item is a Control Task ..." )
my_controller = TaskController(
self.mProcessQueue, self.mAppsInfo)
elif my_item_type == ControlItemType.FileItem:
# Msg.dbg( "\nControl Item is a Control File ..." )
my_controller = FileController(
self.mProcessQueue, self.mAppsInfo,
self.mControlFileLocals)
else:
raise Exception(
"\"" + my_fctrl_name +
"\": Unknown Item Type ...\nUnable to Process ... "
)
if my_controller.load(my_ctrl_item):
my_controller.set_on_fail_proc(
self.on_fail_proc)
my_controller.set_is_term_proc(
self.is_term_proc)
my_controller.process()
# Msg.dbg( "%s: Controller Creation Complete" % ( my_ctrl_item.fctrl_name ))
except TypeError as arg_ex:
Msg.err(str(arg_ex))
my_err_queue_item = SummaryErrorQueueItem({
"error":
"Item #%s Contains an Invalid Type" %
(str(my_item_ndx)),
"message":
arg_ex,
"path":
self.ctrl_item.file_path(),
"type":
str(type(arg_ex))
})
if self.mProcessQueue.summary is not None:
self.mProcessQueue.summary.queue.enqueue(
my_err_queue_item)
Msg.blank()
except FileNotFoundError as arg_ex:
Msg.err(str(arg_ex))
my_err_queue_item = SummaryErrorQueueItem({
"error":
arg_ex,
"message":
"Control File Not Found ...",
"path":
self.ctrl_item.file_path(),
"type":
str(type(arg_ex))
})
if self.mProcessQueue.summary is not None:
self.mProcessQueue.summary.queue.enqueue(
my_err_queue_item)
Msg.blank()
except Exception as arg_ex:
Msg.error_trace(str(arg_ex))
Msg.err(str(arg_ex))
Msg.blank()
finally:
my_controller = None
my_item_dict = None
except Exception as arg_ex:
Msg.error_trace("[ERROR] - " + str(arg_ex))
Msg.err(str(arg_ex))
finally:
pass
finally:
pass
# Msg.dbg()
return True
class MasterRun(ModuleRun):
cLsfWaitTime = 30 # seconds
def __init__(self):
try:
self.mCmdLineParms = CommandLineParameters
self.mConfigArgs = retrieve_config_argument(sys.argv[1:])
except SystemExit as aSysExit:
sys.exit(int(str(aSysExit)))
except BaseException:
print(
"[ERROR] - An Unhandled Error has Occurred during applications"
" setup of " + str(sys.argv[0]))
traceback.print_exc(file=sys.stdout)
sys.exit(44)
super().__init__(CmdLine.Switches[CmdLine.msg_lev], Defaults.msg_level)
self.fctrl_dir = None
self.mode = None
self.summary = None
self.process_queue = None
self.client_lev = False
self.default_root = ""
self.ctrl_item = ControlItem()
# no propogate values
self.num_runs = None
self.sum_level = None
self.item_data = {}
self.options = {}
self.max_fails = 0
self.limit_fails = False
self.crit_sec = HiCriticalSection()
self.processor_name = None
self.fctrl = None
self.iss = None
self.generator = None
self.rtl = None
self.performance = None
self.regression = None
self.master_config = None
self.fctrl_name = None
self.output_root = None
self.launcher_type = None
self.output_dir = None
self.process_cmd = None
self.terminated = False
# this is a proc that will decrement the fails counter until it is 0
# if the proc is None then this will not occur
self.on_fail_proc = None
self.is_term_proc = None
# if an external terminate event is captured then this call back will
# shutdown master run and all associated threads gracefully
global shutdown_proc
shutdown_proc = self.trigger_shutdown
def init_app_setup(self):
if not self.m_app_setup:
self.m_app_setup = ApplicationsSetup(self.mCmdLineParms, sys.argv,
self.mConfigArgs, True, True)
self.m_app_info = self.m_app_setup.getApplicationsInfo()
self.m_app_info.mMainAppPath = (
PathUtils.include_trailing_path_delimiter(self.module_dir) +
"../..")
self.m_app_info.mProcessMax = self.option_def(
CmdLine.Switches[CmdLine.process_max], None, self.to_int)
self.m_app_info.mTestBaseDir = None
self.m_app_info.mToolPath = self.module_dir
def load(self):
# Msg.user( "MasterRun::load" )
self.init_all()
# create the top level FileController
self.fctrl = FileController(self.process_queue, self.m_app_info)
# Msg.lout( self.options, "user", "Initial Option Values" )
self.item_data[CtrlItmKeys.fname] = self.fctrl_name
self.item_data[CtrlItmKeys.options] = self.options
if self.rtl is not None:
self.item_data["rtl"] = self.rtl
try:
self.ctrl_item.load(self.m_app_info, self.item_data)
except BaseException:
Msg.err("Unable to load initial control item.")
raise
# populate the controller
if not self.fctrl.load(self.ctrl_item):
raise LoadError("Unable to load initial Control File .... ")
# initialize the callbacks, if these are None then these are ignored
self.fctrl.set_on_fail_proc(self.on_fail_proc)
self.fctrl.set_is_term_proc(self.is_term_proc)
def run(self):
Msg.dbg("MasterRun::run()")
# Run single run applications here before anything else is done
for app_cfg in self.m_app_info.mSingleRunApps:
app_executor = app_cfg.createExecutor()
app_executor.load(self.ctrl_item)
if not app_executor.skip():
Msg.info("Currently executing %s app" % app_cfg.name())
app_executor.pre()
app_executor.execute()
app_executor.post()
Msg.info("Finished executing %s app" % app_cfg.name())
for my_ndx in range(self.num_runs):
if self.terminated:
break
Msg.info(Formats.exec_num_runs % (my_ndx + 1, self.num_runs))
Msg.blank()
self.fctrl.process()
# Wait until all the threads are done
self.process_queue.fully_loaded = True
workers_done_event.wait()
summary_done_event.wait()
if self.summary:
Msg.dbg(Formats.summ_level % (self.sum_level))
self.summary.process_summary(self.sum_level)
# make cycle count and instruction count available in the shared
# object
if isinstance(self.summary, RegressionSummary):
instruction_count = self.summary.total_instruction_count
cycle_count = self.summary.total_cycle_count
self.m_app_info.mTagToReportInfo.update({
"master_run": {
"total_cycle_count": cycle_count,
"total_instruction_count": instruction_count,
"initial_control_file": self.fctrl_name,
"output_dir": self.output_dir,
}
})
if self.mode == "count":
Msg.info("Total tasks counted in control file tree: " +
str(self.m_app_info.mNumTestsCount) + "\n")
if self.terminated:
Msg.info("####\n#### Reached max fails limit before test was "
"completed.\n####")
self.writeVersionInfo()
self.modulesReport()
# callback for the various points of failure
def handle_on_fail(self, arg_sender):
Msg.user("Fail from: %s" % (str(arg_sender)))
with self.crit_sec:
if self.terminated:
return
self.max_fails -= 1
if self.max_fails > 0:
return
Msg.info("####\n#### Reached max fails limit, setting terminated "
"status flag and triggering regression shutdown.\n####\n")
self.terminated = True
def query_terminated(self):
with self.crit_sec:
return self.terminated
def trigger_shutdown(self):
with self.crit_sec:
self.terminated = True
self.shutdown()
# add a callback that will execute and shutdown all threads when executed
def shutdown(self):
pass
# initializes everything
def init_all(self):
# load options and initialize
# == Old ==>> self.resolve_initial_dirs()
self.check_config()
self.initialize_directories()
self.initialize_output()
# self.initialize_tools( )
Msg.dbg(
" ----------------------------------------------------------------"
)
self.initialize_processor_cmd()
self.populate_options()
self.process_general_options()
self.initialize_summary()
# finally create the process queue
self.initialize_process_queue()
def check_config(self):
my_config_file = self.option_def(CmdLine.Switches[CmdLine.config],
None)
if my_config_file is None:
self.iss = {}
self.generator = {}
self.rtl = {}
self.performance = CtrlItmDefs.performance
self.regression = CtrlItmDefs.regression
Msg.user("iss : %s" % (str(self.iss)))
Msg.user("generator : %s" % (str(self.generator)))
Msg.user("rtl : %s" % (str(self.rtl)))
Msg.user("performance: %s" % (str(self.performance)))
Msg.user("regression : %s" % (str(self.regression)))
return True
return self.load_config(my_config_file)
def load_config(self, arg_cfg_file):
# ok there is a config file to load, there are two possibilities,
# neither require path manipulation
# 1. the config file is specified as a relative path from the launch
# directory
# 2. the config file is specified as a fully qualified path
try:
# load the config file if it exists
my_content = open(arg_cfg_file).read()
except Exception as arg_ex:
Msg.err("Unable to open Config File: %s" % (str(arg_cfg_file)))
raise
try:
my_glb, my_loc = SysUtils.exec_content(my_content)
self.load_config_data(my_loc)
except Exception as arg_ex:
Msg.err("Unable to Process Config File, Message: %s" %
(str(arg_ex)))
raise
return True
def load_config_data(self, arg_config_data):
self.master_config = arg_config_data["master_config"]
self.iss = self.master_config.get("iss", {})
self.generator = self.master_config.get("generator", {})
self.rtl = self.master_config.get("rtl", {})
self.performance = self.master_config.get(CtrlItmKeys.performance,
CtrlItmDefs.performance)
self.regression = self.master_config.get(CtrlItmKeys.regression,
CtrlItmDefs.regression)
Msg.user("iss : %s" % (str(self.iss)))
Msg.user("generator : %s" % (str(self.generator)))
Msg.user("rtl : %s" % (str(self.rtl)))
Msg.user("performance: %s" % (str(self.performance)))
Msg.user("regression : %s" % (str(self.regression)))
return True
# resolves the the test_base, initial control directory, and populates
# options dictionay populated with these three values if failure an
# Exception is raised
def initialize_directories(self):
# extract the initial control file information
# self.fctrl_dir is now the fully qualified path to the first control
# file, thus
self.fctrl_dir, self.fctrl_name = self.locate_control_file()
self.m_app_info.mTestBaseDir = self.locate_directory(
CmdLine.Switches[CmdLine.test_base],
EnVars.test_base,
self.fctrl_dir
if self.fctrl_dir is not None else Defaults.test_base,
)
Msg.user("Module Path : %s" % (str(self.module_dir)),
"INITIAL_DIRS")
Msg.user("Main Control File: %s" % (str(self.fctrl_name)),
"INITIAL_DIRS")
Msg.user("Main Control Dir : %s" % (str(self.fctrl_dir)),
"INITIAL_DIRS")
Msg.user(
"Test Root : %s" % (str(self.m_app_info.mTestBaseDir)),
"INITIAL_DIRS",
)
def locate_control_file(self):
# populate the initial control file, if none is specified then use
# the default
my_fctrl_path = self.option_def(CmdLine.Switches[CmdLine.control_name],
CtrlItmDefs.fctrl_name)
Msg.user("Control Path: %s (1)" % (str(my_fctrl_path)), "INITIAL_DIRS")
# if the control file contains a path then split that into the
# directory and the file
my_fctrl_dir, my_fctrl_file = PathUtils.split_path(my_fctrl_path)
Msg.user(
"Control File Split, Directory: %s, FileName: %s" %
(str(my_fctrl_dir), str(my_fctrl_file)),
"INITIAL_DIRS",
)
# important to realize that if the default control file is used, it is
# necessary to find the right file
if my_fctrl_dir is None:
# if the control file does not contain a path then need to assume
# the default path of
my_fctrl_dir = self.locate_directory(
CmdLine.Switches[CmdLine.control_dir],
EnVars.test_base,
Defaults.test_base,
)
else:
my_fctrl_dir = PathUtils.include_trailing_path_delimiter(
PathUtils.real_path(my_fctrl_dir))
return my_fctrl_dir, my_fctrl_file
def locate_directory(self, arg_cmd_switch, arg_envar, arg_default):
Msg.dbg("arg_cmd_switch[%s], arg_envar[%s], arg_default[%s]" %
(str(arg_cmd_switch), str(arg_envar), arg_default))
my_tmp = self.option_def(arg_cmd_switch, None)
# Msg.user( "Result Path: %s" % ( str( my_tmp )))
if my_tmp is None:
if arg_envar is not None:
# Not passed on the command line check for envar and the
# default
my_tmp = SysUtils.envar(arg_envar, arg_default, False)
else:
my_tmp = arg_default
# If a relative path has been provided either in the environmental
# var or as the default that path needs to be appended to the
# module path. Since all full paths begin with a path delimiter
# this is a valid test
if my_tmp[0] != "/":
my_tmp = (PathUtils.include_trailing_path_delimiter(
self.module_dir) + my_tmp)
# OK here is where it gets a bit tricky, when passed on the command
# line the path is calculated from the current directory, in all other
# cases from the module path. Since the app should be in the initial
# directory calculating the real path should resolve to a fully
# qualified path. To remove all indirection use real path
my_tmp = PathUtils.real_path(my_tmp)
# At this point the path should be a fully qualified path
# Msg.user( "Result Path: %s" % ( str( my_tmp )))
#
Msg.user("Result Path: %s" % (str(my_tmp)))
if not PathUtils.valid_path(my_tmp):
raise FileNotFoundError(
"Initial Directory for %s Resolution Failed[%s] could not be"
"located" % (arg_cmd_switch, my_tmp))
if not PathUtils.check_exe(my_tmp):
my_tmp = PathUtils.include_trailing_path_delimiter(my_tmp)
return my_tmp
# set up director and archive new directories
def initialize_output(self):
self.mode = self.option_def(CmdLine.Switches[CmdLine.mode], None)
self.output_root = PathUtils.exclude_trailing_path_delimiter(
self.option_def(CmdLine.Switches[CmdLine.target_dir],
PathUtils.current_dir()))
Msg.user("Output Root: %s" % (str(self.output_root)))
# check launcher type here since we need to know if we are running
# with LSF
Msg.user("Before Launcher Type", "MASTERRUN")
self.launcher_type = self.option_def(
CmdLine.Switches[CmdLine.run_launcher], Defaults.run_launcher)
Msg.user("Launcher Type: %s" % (str(self.launcher_type)), "MASTERRUN")
# ok the root output directory has been established.
# next check to see if there is an expiration if there is handle that
# and exit the session
my_expire = self.option_def(CmdLine.Switches[CmdLine.expire], None)
my_session_type = (Formats.perf_output_dir if SysUtils.found(
self.mode.find(Modes.perf)) else Formats.regress_output_dir)
if my_expire is not None:
self.handle_expire(my_expire, my_session_type)
raise Exception(
"Problem with handle_expire, should have terminated .....")
# Continuing create the full output directory which if exists should
# be archived or removed
my_output_base = Formats.main_output_dir % self.output_root
self.output_dir = "%s/%s/" % (
PathUtils.exclude_trailing_path_delimiter(my_output_base),
PathUtils.exclude_trailing_path_delimiter(my_session_type),
)
Msg.user("Target Output Dir: %s" % (str(self.output_dir)))
mod_time = None
# if there is no expire setting then
if PathUtils.check_dir(self.output_dir):
# check modification time of the directory, if it is created very
# recently, delay a bit when running on LSF.
# since client machines might hold a stale directory handler still.
mod_time = PathUtils.time_modified(self.output_dir)
if self.option_def(CmdLine.Switches[CmdLine.no_archive],
Defaults.no_archive):
PathUtils.rmdir(self.output_dir,
True) # remove output directory tree
else:
PathUtils.archive_dir(self.output_dir)
PathUtils.mkdir(self.output_dir)
if mod_time is not None:
self.waitForLfs(mod_time)
return True
# Wait a bit for LSF to expire stale file handle for regression directory
# if running with LSF
def waitForLfs(self, aModTime):
if self.launcher_type == LauncherType.Lsf:
time_diff = int(DateTime.Time() - aModTime)
if time_diff < MasterRun.cLsfWaitTime:
sec_delay = MasterRun.cLsfWaitTime - time_diff
Msg.info("Using LSF, delaying %d seconds so that stale "
"output/regression file handle will expire..." %
sec_delay)
SysUtils.sleep_seconds_with_progress(sec_delay)
Msg.info("Waiting done, resumed master run")
def writeVersionInfo(self):
out_line_fmt = (
"{}, scm_system: {}, revision number: {}, location: {}, url: {}\n")
version_info = ""
for app_tag, app_config in self.m_app_info.mTagToApp.items():
Msg.user("app_tag: %s, app_config: %s" % (app_tag, app_config))
version_data = app_config.parameter("version")
for item in version_data:
if item["status"]:
version_info += out_line_fmt.format(
app_config.name(),
item["scm_type"],
str(item["version"]),
item["folder"],
item["url"],
)
with open(self.output_dir + "version_info.txt", "w+") as outfile:
if version_info:
outfile.write(version_info)
else:
outfile.write("No version information found")
# Call the report methods from each of the sequence apps. Some apps
# report, others pass through
def modulesReport(self):
for app_cfg in self.m_app_info.mSequenceApps:
reporter = app_cfg.createReporter()
reporter.report(self.m_app_info, app_cfg.tag())
@staticmethod
def to_int(a_value):
return int(a_value)
@staticmethod
def to_hex(a_value):
return hex(int(a_value, 0))
# populate the general options
def populate_options(self):
self.options[CtrlItmKeys.fdir] = self.fctrl_dir
self.options[CtrlItmKeys.no_sim] = self.option_def(
CmdLine.Switches[CmdLine.no_sim], CtrlItmDefs.no_sim)
self.options[CtrlItmKeys.num_chips] = self.option_def(
CmdLine.Switches[CmdLine.num_chips],
CtrlItmDefs.num_chips,
self.to_int,
)
self.options[CtrlItmKeys.num_cores] = self.option_def(
CmdLine.Switches[CmdLine.num_cores],
CtrlItmDefs.num_cores,
self.to_int,
)
self.options[CtrlItmKeys.num_threads] = self.option_def(
CmdLine.Switches[CmdLine.num_threads],
CtrlItmDefs.num_threads,
self.to_int,
)
self.options[CtrlItmKeys.min_instr] = self.option_def(
CmdLine.Switches[CmdLine.min_instr],
CtrlItmDefs.min_instr,
self.to_int,
)
self.options[CtrlItmKeys.max_instr] = self.option_def(
CmdLine.Switches[CmdLine.max_instr],
CtrlItmDefs.max_instr,
self.to_int,
)
self.options[CtrlItmKeys.timeout] = self.option_def(
CmdLine.Switches[CmdLine.timeout], CtrlItmDefs.timeout,
self.to_int)
self.options[CtrlItmKeys.seed] = self.option_def(
CmdLine.Switches[CmdLine.seed], CtrlItmDefs.seed, self.to_hex)
self.options[CtrlItmKeys.suffix] = Defaults.suffix
self.max_fails = self.option_def(
CmdLine.Switches[CmdLine.max_fails],
Defaults.max_fails,
self.to_int,
)
if self.max_fails > 0:
self.is_term_proc = self.query_terminated
self.on_fail_proc = self.handle_on_fail
def process_general_options(self):
# run options
self.num_runs = self.option_def(CmdLine.Switches[CmdLine.num_runs],
Defaults.num_runs, self.to_int)
self.sum_level = self.option_def(CmdLine.Switches[CmdLine.sum_level],
SummaryLevel.Fail, self.to_int)
Msg.user("process-max: %d" % (self.m_app_info.mProcessMax), "MASTER")
# create the proper summary
def initialize_summary(self):
my_keep = self.option_def(CmdLine.Switches[CmdLine.keep],
Defaults.keep)
clean_up_rules = CleanUpRules(my_keep)
if SysUtils.found(self.mode.find(Modes.perf)):
self.mode = Modes.perf
self.options[CtrlItmKeys.no_sim] = True
self.summary = PerformanceSummary(self.output_dir, clean_up_rules)
elif SysUtils.found(self.mode.find(Modes.regress)):
self.mode = Modes.regress
self.summary = RegressionSummary(self.output_dir, clean_up_rules)
else:
self.mode = Modes.count
self.m_app_info.mMode = "count"
self.summary = RegressionSummary(self.output_dir, clean_up_rules)
if self.summary is not None:
self.summary.set_on_fail_proc(self.on_fail_proc)
self.summary.set_is_term_proc(self.is_term_proc)
def initialize_process_queue(self):
global workers_done_event
self.process_queue = ProcessQueue()
self.process_queue.process_cmd = self.process_cmd
self.process_queue.processor_name = self.processor_name
self.process_queue.summary = self.summary
self.process_queue.process_max = self.m_app_info.mProcessMax
self.process_queue.launcher_type = self.launcher_type
Msg.user("Done Event: %s" % (str(workers_done_event)), "MAIN")
self.process_queue.done_event = workers_done_event
# Msg.user( "Done Event: %s" % (str( workers_done_event)), "MAIN")
self.process_queue.open_queue()
# self.process_queue.open_queue(self.process_cmd, self.summary,
# self.process_max, workers_done_event,
# self.process_launcher )
def initialize_processor_cmd(self):
# the default task processor is "forrest_run.py"
# the default directory is the same directory as the master_run
# the processor can be replaced with a command line argument which may
# or may not contain a path
# if it does not contain a path then the default path will be used
# if a directory is passed on the command line in all cases that will
# be the location of the processor
my_run_dir = None
my_run_name = None
my_tmp_name = None
my_tmp_path = None
my_run_path = self.option_def(CmdLine.Switches[CmdLine.run_name], None)
if my_run_path is not None:
my_run_dir, my_run_name = PathUtils.split_path(my_run_path)
Msg.user(
"Client Dir: %s, Client Name: %s (1)" %
(str(my_run_dir), str(my_run_name)),
"PROCESS_CMD",
)
if my_run_dir is None:
my_tmp_path = self.locate_directory(
CmdLine.Switches[CmdLine.run_dir],
EnVars.run_path,
self.module_dir,
)
if PathUtils.check_exe(my_tmp_path):
my_run_dir, my_tmp_name = PathUtils.split_path(my_tmp_path)
else:
my_run_dir = my_tmp_path
if my_run_name is None:
my_run_name = (my_tmp_name
if my_tmp_name is not None else Defaults.run_name)
my_process_cmd = PathUtils.real_path(
PathUtils.append_path(
PathUtils.include_trailing_path_delimiter(my_run_dir),
my_run_name,
))
Msg.user("Process Cmd: %s (1)" % (str(my_process_cmd)), "PROCESS_CMD")
my_msg_lev = self.option_def(CmdLine.Switches[CmdLine.client_lev],
None)
if my_msg_lev is not None:
if my_msg_lev:
my_process_cmd += Msg.get_level_as_str()
else:
my_process_cmd += " -l " + my_msg_lev
Msg.user("Process Cmd: %s" % (str(my_process_cmd)),
"PROCESS_CMD")
if self.m_app_info.mConfigPath is not None:
my_process_cmd += " -w %s" % self.m_app_info.mConfigPath
my_process_cmd += " -f %s"
self.processor_name = my_run_name.replace(".py",
"").replace("_run", "")
self.process_cmd = my_process_cmd
Msg.dbg("Process Cmd: %s" % (str(self.process_cmd)))
# Msg.dbg( "Process Cmd: %s" % (self.process_cmd))
# An expire value has been found
# There are several possibilities
# 1. "clean" was pass on the command line, in this case remove the output
# directory which should the last directory that received output for
# that mode
# 2. "purge" was passed on the command line, in this case purge all
# output directories for that mode
# 3. a zero (0) was passed on the command line
# 4. a none zero integer was passed on the command line, purge all
# directories related to the mode
# if none of these are found and exception is raised in all cases
# master_run terminates immediately
def handle_expire(self, arg_expire, arg_mask):
try:
Msg.user(
"Expire: %s, Mask: %s" % (str(arg_expire), str(arg_mask)),
"EXPIRE",
)
my_output_base = Formats.main_output_dir % self.output_root
Msg.user("Expire [1], Output Base: %s" % (str(my_output_base)),
"EXPIRE")
Msg.info("Output Directories Cleanup, Please wait ...")
if int(arg_expire) == Expire.all:
Msg.info("Building Directory List, [%s]" % (my_output_base))
my_dirs = PathUtils.list_dirs(my_output_base)
Msg.dbg("All Dirs: %s" % (str(my_dirs)), "EXPIRE")
for my_dir in my_dirs:
if my_dir.startswith(arg_mask):
my_full_dir = "%s%s" % (
PathUtils.include_trailing_path_delimiter(
my_output_base),
my_dir,
)
Msg.info("Removing: %s" % (my_full_dir))
PathUtils.rmdir(my_full_dir, True)
else:
Msg.info("Checking for Expired Directories: %s" %
(my_full_dir))
my_expiredate = DateTime.DateDelta(int(my_expire))
PathUtils.expire(my_full_dir, my_expiredate)
except Exception as ex:
Msg.error_trace()
Msg.err(str(ex))
finally:
Msg.info("Operation Complete, Restart Master Run to continue ...")
sys.exit(1)
def run_mode(self):
return self.mode
def process(self):
for my_ndx in range(self.ctrl_item.iterations):
if self.is_terminated():
break
try:
my_item_ndx = 0
for my_item_dict in self.fcontrol:
try:
if self.is_terminated():
break
my_item_ndx += 1
my_ctrl_item = ControlItem()
my_ctrl_item.parent_fctrl = self.parent_fctrl
my_ctrl_item.fctrl_item = str(my_item_dict)
try:
my_ctrl_item.load(
self.mAppsInfo,
my_item_dict,
self.ctrl_item,
)
except BaseException:
raise
my_item_type = my_ctrl_item.item_type()
my_controller = None
if my_item_type == ControlItemType.TaskItem:
my_controller = TaskController(
self.mProcessQueue, self.mAppsInfo)
elif my_item_type == ControlItemType.FileItem:
my_controller = FileController(
self.mProcessQueue,
self.mAppsInfo,
self.mControlFileLocals,
)
else:
raise Exception('"' + my_fctrl_name +
'": Unknown Item Type ...\n'
"Unable to Process ... ")
if my_controller.load(my_ctrl_item):
my_controller.set_on_fail_proc(self.on_fail_proc)
my_controller.set_is_term_proc(self.is_term_proc)
my_controller.process()
except TypeError as arg_ex:
Msg.err(str(arg_ex))
my_err_queue_item = SummaryErrorQueueItem({
"error":
"Item #%s Contains an Invalid "
"Type" % (str(my_item_ndx)),
"message":
arg_ex,
"path":
self.ctrl_item.file_path(),
"type":
str(type(arg_ex)),
})
if self.mProcessQueue.summary is not None:
self.mProcessQueue.summary.queue.enqueue(
my_err_queue_item)
Msg.blank()
except FileNotFoundError as arg_ex:
Msg.err(str(arg_ex))
my_err_queue_item = SummaryErrorQueueItem({
"error":
arg_ex,
"message":
"Control File Not Found ...",
"path":
self.ctrl_item.file_path(),
"type":
str(type(arg_ex)),
})
if self.mProcessQueue.summary is not None:
self.mProcessQueue.summary.queue.enqueue(
my_err_queue_item)
Msg.blank()
except Exception as arg_ex:
Msg.error_trace(str(arg_ex))
Msg.err(str(arg_ex))
Msg.blank()
finally:
my_controller = None
my_item_dict = None
except Exception as arg_ex:
Msg.error_trace("[ERROR] - " + str(arg_ex))
Msg.err(str(arg_ex))
return True