def execute( self ):
my_result = None
test_passed = True
try:
self.build_cmd()
self.copyWavesFsdbDoFile()
self.outputRerunScript()
# report the command line
Msg.info( "RTLCommand = " + str( { "rtl-command": self.rtl_cmd } ))
# execute the simulation
my_result = SysUtils.exec_process( self.rtl_cmd, self.rtl_log, self.rtl_log, self.ctrl_item.timeout, True )
Msg.user( "Results: %s" % ( str( my_result )), "RTLEX-RESULT" "")
my_extract_results = self.extract_results( my_result, "./" + self.rtl_log, None )
# report the results
Msg.info( "RTLResult = " + str( my_extract_results ))
except Exception as arg_ex:
Msg.error_trace( "RTL Process Failure" )
Msg.err( "RTL did not properly execute, Reason: %s" % ( str( arg_ex )))
return False
finally:
pass
#return SysUtils.success( int(my_result[ RtlResult.process_retcode ]) )
return test_passed
def load(self, aAppsInfo, aItemDict, aParentItem=None):
# Msg.lout( aItemDict, "dbg", "Loaded Control Item Values" )
self.validate_item_data(aAppsInfo, aItemDict)
# fname is the key in the control file
self.fctrl_name = aItemDict.get(CtrlItmKeys.fname, CtrlItmDefs.fctrl_name)
work_dir = self.load_item_values(aAppsInfo, aItemDict, aParentItem)
try:
self.resolve_file_location(aAppsInfo, work_dir)
except BaseException:
raise
Msg.user(
"File Directory: %s, File Name: %s, Work Directory: %s"
% (self.fctrl_dir, self.fctrl_name, work_dir)
)
# grouped options
#
if aParentItem is None:
self.performance = aItemDict.get(CtrlItmKeys.performance, CtrlItmDefs.performance)
self.regression = aItemDict.get(CtrlItmKeys.regression, CtrlItmDefs.regression)
else:
self.performance = aItemDict.get(CtrlItmKeys.performance, aParentItem.performance)
self.regression = aItemDict.get(CtrlItmKeys.regression, aParentItem.regression)
for seq_app_cfg in aAppsInfo.mAllAppsOrder[1:]:
self.processAppControlData(seq_app_cfg, aItemDict, aParentItem)
def process_executors(self):
my_ret_val = False
try:
for my_executor in self.executors:
Msg.user(
"ExecuteController::process_executors( my_executor: %s )" %
str(type(my_executor)),
"EXE-CTRL",
)
my_executor.pre()
if not my_executor.execute():
Msg.user("Executor returning False", "EXE-CTRL")
break
my_executor.post()
my_ret_val = True
except Exception as arg_ex:
Msg.error_trace()
self.report_error(arg_ex)
finally:
return my_ret_val
def load_executors(self):
try:
# for app_cfg in self.mAppsInfo.mSequenceApps:
for seq_app_cfg in self.mAppsInfo.mSequenceApps:
my_executor = seq_app_cfg.createExecutor()
Msg.user("ExecuteController::load_executors( 2 )")
my_executor.load(self.ctrl_item)
Msg.user("ExecuteController::load_executors( 1 )")
if not my_executor.skip():
Msg.user("ExecuteController::load_executors( 3 )")
my_executor.set_frun(self.frun)
# Msg.user( "ExecuteController::load_executors( 4 )" )
my_executor.set_task_file(self.task_file)
# Msg.user( "ExecuteController::load_executors( 5 )" )
my_executor.set_task_name(self.task_name)
# Msg.user( "ExecuteController::load_executors( 5.1 )" )
self.executors.append(my_executor)
except Exception as arg_ex:
Msg.user("ExecuteController::load_executors( 14 )")
Msg.error_trace()
self.report_error(arg_ex)
finally:
Msg.user("ExecuteController::load_executors( 13 )")
return True
def locate_control_file(self):
# populate the initial control file, if none is specified then use
# the default
my_fctrl_path = self.option_def(CmdLine.Switches[CmdLine.control_name],
CtrlItmDefs.fctrl_name)
Msg.user("Control Path: %s (1)" % (str(my_fctrl_path)), "INITIAL_DIRS")
# if the control file contains a path then split that into the
# directory and the file
my_fctrl_dir, my_fctrl_file = PathUtils.split_path(my_fctrl_path)
Msg.user(
"Control File Split, Directory: %s, FileName: %s" %
(str(my_fctrl_dir), str(my_fctrl_file)),
"INITIAL_DIRS",
)
# important to realize that if the default control file is used, it is
# necessary to find the right file
if my_fctrl_dir is None:
# if the control file does not contain a path then need to assume
# the default path of
my_fctrl_dir = self.locate_directory(
CmdLine.Switches[CmdLine.control_dir],
EnVars.test_base,
Defaults.test_base,
)
else:
my_fctrl_dir = PathUtils.include_trailing_path_delimiter(
PathUtils.real_path(my_fctrl_dir))
return my_fctrl_dir, my_fctrl_file
def rename_elfs(self, arg_suffix):
# before proceeding it is necessary to remove any existing renamed files to eliminate the possibility of causing
# a chain and messing up the results
for my_mask in ["*.ELF", "*.S", "*.img"]:
my_match_files = PathUtils.list_files(my_mask)
Msg.lout(my_match_files, "user", "Simulate File List")
for my_src_file in my_match_files:
Msg.user(
"Match File: %s, Suffix: %s" %
(str(my_src_file), str(arg_suffix)), "MATCH")
if SysUtils.found(
my_src_file.find("_%s_force" % (str(arg_suffix)))):
PathUtils.remove(my_src_file)
continue
# Now rename all the files that are found
for my_mask in ["*.ELF", "*.S", "*.img"]:
my_match_files = PathUtils.list_files(my_mask)
for my_src_file in my_match_files:
my_tgt_file = my_src_file.replace(
"_force", "_%s_force" % (str(arg_suffix)))
PathUtils.rename(my_src_file, my_tgt_file)
# raise Exception("Type %s test case for base test: %s not found." % (arg_extension, arg_test_name))
return True
def __init__(
self,
arg_summary,
done_semaphore,
peer_count,
arg_queue_item,
aProcessorName,
aProcessCmd,
aUseLsf,
):
# acquire the semaphore here
done_semaphore.acquire(True)
# once acquired, increment the active peer thread count
peer_count.add(1)
self.done_semaphore = done_semaphore
self.queue_item = arg_queue_item
self.thread_count = peer_count
self.mProcessorName = aProcessorName
self.mProcessCmd = aProcessCmd
self.mUseLsf = aUseLsf
self.summary = arg_summary
# We do not want the thread to launch until we've loaded all the
# properties
Msg.user("Thread Id: %s __init__" % (str(id(self))), "WORK-THREAD")
super().__init__(True)
self.on_done = None
self._launcher = None
def locate_directory( self, arg_cmd_switch, arg_envar, arg_default ):
Msg.dbg( "arg_cmd_switch[%s], arg_envar[%s], arg_default[%s]" % (str(arg_cmd_switch), str(arg_envar), arg_default))
my_tmp = self.option_def( arg_cmd_switch, None)
# Msg.user( "Result Path: %s" % ( str( my_tmp )))
if my_tmp is None:
if arg_envar is not None:
# Not passed on the command line check for envar and the default
my_tmp = SysUtils.envar( arg_envar, arg_default, False )
else:
my_tmp = arg_default
# If a relative path has been provided either in the environmental var or as the default that path needs
# to be appended to the module path. Since all full paths begin with a path delimiter this is a valid test
if my_tmp[0] != "/":
my_tmp = PathUtils.include_trailing_path_delimiter( self.module_dir ) + my_tmp
# OK here is where it gets a bit tricky, when passed on the command line the path is calculated from the
# current directory, in all other cases from the module path. Since the app should be in the initial directory
# calculating the real path should resolve to a fully qualified path. To remove all indirection use real path
my_tmp = PathUtils.real_path( my_tmp )
# At this point the path should be a fully qualified path
# Msg.user( "Result Path: %s" % ( str( my_tmp )))
#
Msg.user( "Result Path: %s" % ( str( my_tmp )))
if not PathUtils.valid_path( my_tmp ):
raise FileNotFoundError( "Initial Directory for %s Resolution Failed[%s] could not be located" % ( arg_cmd_switch, my_tmp ))
if not PathUtils.check_exe( my_tmp ):
my_tmp = PathUtils.include_trailing_path_delimiter( my_tmp )
return my_tmp
def clean_up(self):
clean_up_rules = self.summary.cleanUpRules
if self.passed and not clean_up_rules.shouldKeepAll():
# list all the files and delete them except for _def_frun.py and
# the processor log (forrest.log)
my_dir = PathUtils.include_trailing_path_delimiter(self.work_dir)
Msg.user("Dir: %s" % (str(my_dir)), "FILES-TO-REMOVE")
my_file_path = str(my_dir) + "*"
Msg.user("Path: %s" % (my_file_path), "FILES-TO-REMOVE")
my_file_list = PathUtils.list_files(my_file_path)
Msg.user("Files: %s" % (str(my_file_list)), "FILES-TO-REMOVE")
process_log_base = PathUtils.base_name(self.process_log)
clean_up_rules.setBaseNamesToKeep(
["_def_frun.py", "PASS", process_log_base])
for my_file in my_file_list:
if clean_up_rules.shouldKeepFile(my_file):
Msg.user("File: %s KEPT" % clean_up_rules.lastBaseName())
else:
Msg.user("File: %s REMOVED" %
clean_up_rules.lastBaseName())
PathUtils.remove(my_file)
def load_process_line(self, arg_line):
Msg.user("Process Result Line: %s" % (str(arg_line)), "SUM-TUPLE")
my_val = None
try:
my_glb, my_loc = SysUtils.exec_content(arg_line, True)
except SyntaxError as arg_ex:
return
except:
raise
# summary tuples are initialized in the __init__ as are the element indexes
# the idea is to the tuple list for a match on the key. When one is found
# the callback proc that is referenced is executed with the line dictionary
# retrieved and the flags are updated as to what this summary item contains
for (my_key, my_proc, my_mask) in self.sum_tups:
# Msg.user( "Key: %s, Proc: %s, Mask: %s" % (str( my_key ), str( my_proc is not None ) , (str( my_mask ))), "SUM-TUPLE" )
my_result = my_loc.get(my_key, None)
# Msg.user( "Key: %s, Result: %s" % ( str( my_key ), str( my_result )), "SUM-TUPLE" )
if my_result is None:
# nothing was returned continue causes the next element to be checked
# if there is a next element
continue
my_proc(my_result)
self.detail_flags |= int(my_mask)
break
return True
def load_process_line(self, arg_line):
Msg.user("Process Result Line: %s" % (str(arg_line)), "SUM-TUPLE")
if arg_line[0] == "[":
return
my_val = None
try:
my_glb, my_loc = SysUtils.exec_content(arg_line, True)
except (SyntaxError, TypeError) as arg_ex:
return
except BaseException:
raise
# summary tuples are initialized in the __init__ as are the element
# indexes the idea is to the tuple list for a match on the key. When
# one is found the callback proc that is referenced is executed with
# the line dictionary retrieved and the flags are updated as to what
# this summary item contains
for (my_key, my_proc, my_mask) in self.sum_tups:
my_result = my_loc.get(my_key, None)
if my_result is None:
# nothing was returned continue causes the next element to be
# checked if there is a next element
continue
my_proc(my_result)
self.detail_flags |= int(my_mask)
break
return True
def process_general_options(self):
# run options
self.num_runs = self.option_def(CmdLine.Switches[CmdLine.num_runs],
Defaults.num_runs, self.to_int)
self.sum_level = self.option_def(CmdLine.Switches[CmdLine.sum_level],
SummaryLevel.Fail, self.to_int)
Msg.user("process-max: %d" % (self.m_app_info.mProcessMax), "MASTER")
def skip(self):
if not self.ctrl_item.compile.get('run', False):
Msg.user('[CompileExecutor::skip] skipping since run is not set to True...')
return True
Msg.user('[CompileExecutor::skip] not skipping')
return False
def load(self):
# Msg.dbg( "ForrestRun::load" )
my_frun_path = self.option_def(CmdLine.Switches[CmdLine.control_name],
None)
if my_frun_path is None:
raise Exception(
"F-Run Control File Not Found on the Forrest Run Command Line: Given Path: %s",
str((my_frun_path)))
# good to here
self.locate_frun(my_frun_path)
Msg.user("File Path: %s" % (my_frun_path))
my_content = open(self.frun_name).read()
my_glb, my_loc = SysUtils.exec_content(my_content)
Msg.dbg(str(my_loc))
self.fcontrol = my_loc["control_items"]
my_ctrl_dict = self.fcontrol[0]
my_ctrl_item = ControlItem()
my_ctrl_item.load(self._mAppsInfo, my_ctrl_dict)
# Msg.lout( my_ctrl_dict, "user", "Forrest Parent Data ...." )
self.check_simulator()
self.fctrl = ExecuteController(self._mAppsInfo)
self.fctrl.set_frun(self.frun_name)
self.fctrl.load(my_ctrl_item)
def open_queue( self ):
# Create the Process Thread and lock ???
Msg.user( "Done Event: %s" %str( self.done_event ), "PROCESS-QUEUE" )
self.process_thread = ProcessThread( self, self.summary, self.process_max, self.done_event )
Msg.user( "F-Run Command Line: %s" % (self.process_cmd ), "PROCESS-QUEUE")
def locate_frun(self, arg_frun_path):
Msg.user("Directory set to %s" % (PathUtils.current_dir()))
# if the control file contains a path then split that into the directory and the file
my_frun_dir, my_frun_name = PathUtils.split_path(arg_frun_path)
# always convert to full path
my_cur_dir = PathUtils.real_path(PathUtils.current_dir())
# gots to have a source directory as part of the file name
if my_frun_dir is None:
my_frun_dir = my_cur_dir
else:
# always convert to full path. If the frun was loaded correctly then we can conclude that
# the path tendered is either a relative path from the starting directory or a full path
# to that file. If it is not a full path then it will need to be converted to a full path
# and all links removed
my_frun_dir = PathUtils.real_path(my_frun_dir)
# Msg.user( "FRun Dir: %s, FRun Name: %s, Cur Dir: %s" % ( str( my_frun_dir ), str( my_frun_name ), my_cur_dir ), "FRUN-DIR")
# change into the directory to generate and simulate
if not PathUtils.chdir(my_frun_dir):
raise Exception("F-Run Directory[%s] Not Found" %
(str(my_frun_dir)))
self.frun_name = my_frun_name
self.frun_dir = my_frun_dir
def build(self):
"""build the command line launcher with initial values"""
my_cmd = self.process_cmd % (self.frun_path)
Msg.user("Process Command: %s" % (str(my_cmd)), "LAUNCHER")
my_log = (PathUtils.include_trailing_path_delimiter(self.frun_dir) +
self.process_log)
return my_cmd, my_log
def post(self):
if self.ctrl_item.compile.get('mp'):
lsu_folder = PathUtils.real_path('%s/../../../rtl/lsu' % self.mMakefilePath)
cmd = 'svn revert %s/lsu_scb_retire_ctrl.vp %s/lsu_scb_sca_array.vp %s/lsu_scb_scd_array.vp' % (lsu_folder, lsu_folder, lsu_folder)
Msg.user('MP post-compile command = %s' % cmd)
result = SysUtils.exec_process(cmd, self.log, self.elog, self.ctrl_item.timeout, True)
Msg.user('MP post-compile command = %s' % str(result))
def extract_results(self, arg_result, arg_log, arg_elog):
# extract information from the generate log
my_result, my_error = self.query_logs(arg_log, arg_elog)
Msg.user("Process: %s" % (str(arg_result)), "RTL-SIM")
Msg.user("Log[%s]: []" % (str(arg_log)), "RTL-SIM")
process_ret_code = int(arg_result[RtlResult.process_retcode])
test_passed = my_result[RtlResult.rtl_passed]
if ((process_ret_code == 0) and not test_passed):
Msg.warn(
"[RTL-SIM] Test Passed=%s, but process return code is %d, changing return code to 1"
% (str(test_passed), process_ret_code))
process_ret_code = 1
my_process_data = {
RtlKeys.rtl_retcode: process_ret_code,
RtlKeys.rtl_stdout: str(arg_result[RtlResult.process_stdout]),
RtlKeys.rtl_stderr: str(arg_result[RtlResult.process_stderr]),
RtlKeys.rtl_start: str(arg_result[RtlResult.process_start]),
RtlKeys.rtl_end: str(arg_result[RtlResult.process_end]),
RtlKeys.rtl_count: int(my_result[RtlResult.rtl_cycle_count]),
RtlKeys.rtl_message: str(my_result[RtlResult.rtl_message])
# {{{TODO}}} following line deprecated
,
RtlKeys.rtl_log: arg_log
}
return my_process_data
def process_general_options(self):
# run options
self.num_runs = self.option_def(CmdLine.Switches[CmdLine.num_runs],
Defaults.num_runs, self.to_int)
self.sum_level = self.option_def(CmdLine.Switches[CmdLine.sum_level],
SummaryLevel.Fail, self.to_int)
#my_usr_lbl = Msg.set_label( "user", "PROCESS" )
Msg.user("process-max: %d" % (self._mAppsInfo.mProcessMax), "MASTER")
def skip(self):
if not self.ctrl_item.fruntoctrl.get("run", False):
Msg.user(
"[FrunToCtrlExecutor::skip] skipping - run is not True...")
return True
Msg.user("[FrunToCtrlExecutor::skip] not skipping")
return False
def skip(self):
if not self.use_rtl():
Msg.user(
"[RtlExecutor::skip] skipping due to no rtl information specified"
)
return True
return False
def skip(self):
if not self.ctrl_item.compile_rtl.get("run", False):
Msg.user(
"[CompileExecutor::skip] skipping since 'run' is not True...")
return True
Msg.user("[CompileExecutor::skip] not skipping")
return False
def load_signaled(self, arg_dict):
Msg.user(str(arg_dict), "SIGNALED")
try:
self.signal_id = arg_dict["retcode"]
self.signal_message = arg_dict["message"]
except BaseException:
self.signal_id = -1
Msg.err("Signal Info Corrupt")
def launch(self):
self.process_cmd, my_process_log = self.build()
my_timeout = self.timeout # cif not self.red_zone else None
self.process_result = SysUtils.exec_process(self.process_cmd,
my_process_log,
my_process_log, my_timeout)
Msg.user("Process Results: %s" % (str(self.process_result)),
"LSF-LAUNCHER")
def skip(self):
if not self.ctrl_item.fruntoctrl.get('run', False):
Msg.user(
"[FrunToCtrlExecutor::skip] skipping since run is not set to True..."
)
return True
Msg.user("[FrunToCtrlExecutor::skip] not skipping")
return False
def launch( self ):
Msg.user( "Timeout: %s" % ( str( self.timeout )), "STD-LAUNCHER" )
self.process_cmd, self.process_log = self.build()
Msg.user( "Process Cmd: %s, Process Log: %s" % ( str( self.process_cmd ), str( self.process_log )), "STD-LAUNCHER" )
Msg.user( "Launcher Id 1: %s" % ( str( id( self ))), "STD-LAUNCHER" )
# enable timeout but only trigger shutdown of spawned process allow that process to kill the child processes.
self.process_result = SysUtils.exec_process( self.process_cmd, self.process_log, self.process_log, self.timeout, False, self.set_process )
Msg.user( "Launcher Id 2: %s" % ( str( id( self ))), "STD-LAUNCHER" )
Msg.user( "Process Results: %s" % ( str( self.process_result )), "STD-LAUNCHER" )
def handle_on_fail( self, arg_sender ):
Msg.user( "Fail from: %s" % ( str( arg_sender )))
with self.crit_sec:
if self.terminated:
return
self.max_fails -= 1
if self.max_fails > 0:
return
Msg.info("####\n#### Reached max fails limit, setting terminated status flag and triggering regression shutdown.\n####\n")
self.terminated = True
def post(self):
if self.ctrl_item.compile_rtl.get("mp"):
lsu_folder = PathUtils.real_path("%s/../../../rtl/lsu" %
self.mMakefilePath)
cmd = ("svn revert %s/lsu_scb_retire_ctrl.vp "
"%s/lsu_scb_sca_array.vp %s/lsu_scb_scd_array.vp" %
(lsu_folder, lsu_folder, lsu_folder))
Msg.user("MP post-compile command = %s" % cmd)
result = SysUtils.exec_process(cmd, self.log, self.elog,
self.ctrl_item.timeout, True)
Msg.user("MP post-compile command = %s" % str(result))
def initialize_task(self):
# Msg.user( "ExecuteController::initialize_task(1)" )
my_task_file = PathUtils.append_path(self.ctrl_item.fctrl_dir,
self.ctrl_item.fctrl_name)
# Msg.user( "ExecuteController::initialize_task(2)" )
my_tmp, my_task_ndx = PathUtils.split_path(self.ctrl_item.fctrl_dir)
# Msg.user( "ExecuteController::initialize_task(3)" )
my_task_name = self.ctrl_item.fctrl_name.replace(".py", "")
# Msg.user( "ExecuteController::initialize_task(5)" )
Msg.user("Task File: %s, Task Name: %s, Task Index: %s" %
(my_task_file, my_task_name, my_task_ndx))
return (my_task_file, my_task_name, my_task_ndx)
