def load_config(self, arg_cfg_file):
# ok there is a config file to load, there are two possibilities,
# neither require path manipulation
# 1. the config file is specified as a relative path from the launch
# directory
# 2. the config file is specified as a fully qualified path
try:
# load the config file if it exists
my_content = open(arg_cfg_file).read()
except Exception as arg_ex:
Msg.err("Unable to open Config File: %s" % (str(arg_cfg_file)))
raise
try:
my_glb, my_loc = SysUtils.exec_content(my_content)
self.load_config_data(my_loc)
except Exception as arg_ex:
Msg.err("Unable to Process Config File, Message: %s" %
(str(arg_ex)))
raise
return True
def locate_control_file(self):
# populate the initial control file, if none is specified then use
# the default
my_fctrl_path = self.option_def(CmdLine.Switches[CmdLine.control_name],
CtrlItmDefs.fctrl_name)
Msg.user("Control Path: %s (1)" % (str(my_fctrl_path)), "INITIAL_DIRS")
# if the control file contains a path then split that into the
# directory and the file
my_fctrl_dir, my_fctrl_file = PathUtils.split_path(my_fctrl_path)
Msg.user(
"Control File Split, Directory: %s, FileName: %s" %
(str(my_fctrl_dir), str(my_fctrl_file)),
"INITIAL_DIRS",
)
# important to realize that if the default control file is used, it is
# necessary to find the right file
if my_fctrl_dir is None:
# if the control file does not contain a path then need to assume
# the default path of
my_fctrl_dir = self.locate_directory(
CmdLine.Switches[CmdLine.control_dir],
EnVars.test_base,
Defaults.test_base,
)
else:
my_fctrl_dir = PathUtils.include_trailing_path_delimiter(
PathUtils.real_path(my_fctrl_dir))
return my_fctrl_dir, my_fctrl_file
def load(self, aAppsInfo, aItemDict, aParentItem=None):
# Msg.lout( aItemDict, "dbg", "Loaded Control Item Values" )
self.validate_item_data(aAppsInfo, aItemDict)
# fname is the key in the control file
self.fctrl_name = aItemDict.get(CtrlItmKeys.fname, CtrlItmDefs.fctrl_name)
work_dir = self.load_item_values(aAppsInfo, aItemDict, aParentItem)
try:
self.resolve_file_location(aAppsInfo, work_dir)
except BaseException:
raise
Msg.user(
"File Directory: %s, File Name: %s, Work Directory: %s"
% (self.fctrl_dir, self.fctrl_name, work_dir)
)
# grouped options
#
if aParentItem is None:
self.performance = aItemDict.get(CtrlItmKeys.performance, CtrlItmDefs.performance)
self.regression = aItemDict.get(CtrlItmKeys.regression, CtrlItmDefs.regression)
else:
self.performance = aItemDict.get(CtrlItmKeys.performance, aParentItem.performance)
self.regression = aItemDict.get(CtrlItmKeys.regression, aParentItem.regression)
for seq_app_cfg in aAppsInfo.mAllAppsOrder[1:]:
self.processAppControlData(seq_app_cfg, aItemDict, aParentItem)
def load(self):
# Msg.user( "MasterRun::load" )
self.init_all()
# create the top level FileController
self.fctrl = FileController(self.process_queue, self.m_app_info)
# Msg.lout( self.options, "user", "Initial Option Values" )
self.item_data[CtrlItmKeys.fname] = self.fctrl_name
self.item_data[CtrlItmKeys.options] = self.options
if self.rtl is not None:
self.item_data["rtl"] = self.rtl
try:
self.ctrl_item.load(self.m_app_info, self.item_data)
except BaseException:
Msg.err("Unable to load initial control item.")
raise
# populate the controller
if not self.fctrl.load(self.ctrl_item):
raise LoadError("Unable to load initial Control File .... ")
# initialize the callbacks, if these are None then these are ignored
self.fctrl.set_on_fail_proc(self.on_fail_proc)
self.fctrl.set_is_term_proc(self.is_term_proc)
def commit_generate(self):
self.force_result = SysUtils.ifthen(
SysUtils.success(self.force_retcode), "PASS", "FAIL"
)
self.force_level = SummaryLevel.Any
if SysUtils.failed(self.force_retcode):
self.force_level = SummaryLevel.Fail
Msg.lout(self, "user", "Performance Summary Item Commit Generate")
if SysUtils.success(self.force_retcode):
Msg.info(
"Instructions: %d, Default: %d, Secondary: %d, "
"Elapsed Time: %0.5f Seconds\n\n"
% (
self.total,
self.default,
self.secondary,
self.force_end - self.force_start,
)
)
return 1
return 0
def test_summary(aParameters):
forrest_log = aParameters.forrest_log
msg_level = aParameters.msg_lev
if msg_level is not None:
Msg.set_level(Msg.translate_levelstr(msg_level))
print("Forrest log file is: %s" % forrest_log)
work_dir, my_tmp = PathUtils.split_path(forrest_log)
frun_path = PathUtils.append_path(
PathUtils.include_trailing_path_delimiter(work_dir), "_def_frun.py")
# test - timeout
summary_queue_args = {
"frun-path":
frun_path,
"process-log":
forrest_log,
"process-result": (
0,
None,
"Process Timeout Occurred",
1555792446.313606,
1555792446.313606,
SysUtils.PROCESS_TIMEOUT,
),
}
summary_queue_item = SummaryQueueItem(summary_queue_args)
summary_item = SummaryItem({})
summary_item.load(summary_queue_item)
def locate_directory( self, arg_cmd_switch, arg_envar, arg_default ):
Msg.dbg( "arg_cmd_switch[%s], arg_envar[%s], arg_default[%s]" % (str(arg_cmd_switch), str(arg_envar), arg_default))
my_tmp = self.option_def( arg_cmd_switch, None)
# Msg.user( "Result Path: %s" % ( str( my_tmp )))
if my_tmp is None:
if arg_envar is not None:
# Not passed on the command line check for envar and the default
my_tmp = SysUtils.envar( arg_envar, arg_default, False )
else:
my_tmp = arg_default
# If a relative path has been provided either in the environmental var or as the default that path needs
# to be appended to the module path. Since all full paths begin with a path delimiter this is a valid test
if my_tmp[0] != "/":
my_tmp = PathUtils.include_trailing_path_delimiter( self.module_dir ) + my_tmp
# OK here is where it gets a bit tricky, when passed on the command line the path is calculated from the
# current directory, in all other cases from the module path. Since the app should be in the initial directory
# calculating the real path should resolve to a fully qualified path. To remove all indirection use real path
my_tmp = PathUtils.real_path( my_tmp )
# At this point the path should be a fully qualified path
# Msg.user( "Result Path: %s" % ( str( my_tmp )))
#
Msg.user( "Result Path: %s" % ( str( my_tmp )))
if not PathUtils.valid_path( my_tmp ):
raise FileNotFoundError( "Initial Directory for %s Resolution Failed[%s] could not be located" % ( arg_cmd_switch, my_tmp ))
if not PathUtils.check_exe( my_tmp ):
my_tmp = PathUtils.include_trailing_path_delimiter( my_tmp )
return my_tmp
def rename_elfs(self, arg_suffix):
# before proceeding it is necessary to remove any existing renamed files to eliminate the possibility of causing
# a chain and messing up the results
for my_mask in ["*.ELF", "*.S", "*.img"]:
my_match_files = PathUtils.list_files(my_mask)
Msg.lout(my_match_files, "user", "Simulate File List")
for my_src_file in my_match_files:
Msg.user(
"Match File: %s, Suffix: %s" %
(str(my_src_file), str(arg_suffix)), "MATCH")
if SysUtils.found(
my_src_file.find("_%s_force" % (str(arg_suffix)))):
PathUtils.remove(my_src_file)
continue
# Now rename all the files that are found
for my_mask in ["*.ELF", "*.S", "*.img"]:
my_match_files = PathUtils.list_files(my_mask)
for my_src_file in my_match_files:
my_tgt_file = my_src_file.replace(
"_force", "_%s_force" % (str(arg_suffix)))
PathUtils.rename(my_src_file, my_tgt_file)
# raise Exception("Type %s test case for base test: %s not found." % (arg_extension, arg_test_name))
return True
def load_process_line(self, arg_line):
Msg.user("Process Result Line: %s" % (str(arg_line)), "SUM-TUPLE")
my_val = None
try:
my_glb, my_loc = SysUtils.exec_content(arg_line, True)
except SyntaxError as arg_ex:
return
except:
raise
# summary tuples are initialized in the __init__ as are the element indexes
# the idea is to the tuple list for a match on the key. When one is found
# the callback proc that is referenced is executed with the line dictionary
# retrieved and the flags are updated as to what this summary item contains
for (my_key, my_proc, my_mask) in self.sum_tups:
# Msg.user( "Key: %s, Proc: %s, Mask: %s" % (str( my_key ), str( my_proc is not None ) , (str( my_mask ))), "SUM-TUPLE" )
my_result = my_loc.get(my_key, None)
# Msg.user( "Key: %s, Result: %s" % ( str( my_key ), str( my_result )), "SUM-TUPLE" )
if my_result is None:
# nothing was returned continue causes the next element to be checked
# if there is a next element
continue
my_proc(my_result)
self.detail_flags |= int(my_mask)
break
return True
def process_task(self, arg_task_file, aTaskDir):
try:
# get the subdirectory index
my_ndx = self.mAppsInfo.getNextIndex(aTaskDir)
# form sub task directory
sub_task_dir = PathUtils.append_path(
PathUtils.include_trailing_path_delimiter(aTaskDir),
"%05d" % my_ndx)
# save the task template file name with path to the control item
self.ctrl_item.fname = arg_task_file
# prepare control item content, TODO don't really need it.
my_content = self.ctrl_item.prepare(self.mAppsInfo, arg_task_file)
my_queue_item = ProcessQueueItem(sub_task_dir, self.ctrl_item,
self.mAppsInfo, my_content)
self.mProcessQueue.enqueue(my_queue_item)
except Exception as arg_ex:
Msg.error_trace()
Msg.err(str(arg_ex))
# reraise to prevent adding to summary instance
raise
finally:
pass
def execute( self ):
my_result = None
test_passed = True
try:
self.build_cmd()
self.copyWavesFsdbDoFile()
self.outputRerunScript()
# report the command line
Msg.info( "RTLCommand = " + str( { "rtl-command": self.rtl_cmd } ))
# execute the simulation
my_result = SysUtils.exec_process( self.rtl_cmd, self.rtl_log, self.rtl_log, self.ctrl_item.timeout, True )
Msg.user( "Results: %s" % ( str( my_result )), "RTLEX-RESULT" "")
my_extract_results = self.extract_results( my_result, "./" + self.rtl_log, None )
# report the results
Msg.info( "RTLResult = " + str( my_extract_results ))
except Exception as arg_ex:
Msg.error_trace( "RTL Process Failure" )
Msg.err( "RTL did not properly execute, Reason: %s" % ( str( arg_ex )))
return False
finally:
pass
#return SysUtils.success( int(my_result[ RtlResult.process_retcode ]) )
return test_passed
def process_general_options(self):
# run options
self.num_runs = self.option_def(CmdLine.Switches[CmdLine.num_runs],
Defaults.num_runs, self.to_int)
self.sum_level = self.option_def(CmdLine.Switches[CmdLine.sum_level],
SummaryLevel.Fail, self.to_int)
Msg.user("process-max: %d" % (self.m_app_info.mProcessMax), "MASTER")
def load_process_line(self, arg_line):
Msg.user("Process Result Line: %s" % (str(arg_line)), "SUM-TUPLE")
if arg_line[0] == "[":
return
my_val = None
try:
my_glb, my_loc = SysUtils.exec_content(arg_line, True)
except (SyntaxError, TypeError) as arg_ex:
return
except BaseException:
raise
# summary tuples are initialized in the __init__ as are the element
# indexes the idea is to the tuple list for a match on the key. When
# one is found the callback proc that is referenced is executed with
# the line dictionary retrieved and the flags are updated as to what
# this summary item contains
for (my_key, my_proc, my_mask) in self.sum_tups:
my_result = my_loc.get(my_key, None)
if my_result is None:
# nothing was returned continue causes the next element to be
# checked if there is a next element
continue
my_proc(my_result)
self.detail_flags |= int(my_mask)
break
return True
def skip(self):
if not self.ctrl_item.compile.get('run', False):
Msg.user('[CompileExecutor::skip] skipping since run is not set to True...')
return True
Msg.user('[CompileExecutor::skip] not skipping')
return False
def __init__(
self,
arg_summary,
done_semaphore,
peer_count,
arg_queue_item,
aProcessorName,
aProcessCmd,
aUseLsf,
):
# acquire the semaphore here
done_semaphore.acquire(True)
# once acquired, increment the active peer thread count
peer_count.add(1)
self.done_semaphore = done_semaphore
self.queue_item = arg_queue_item
self.thread_count = peer_count
self.mProcessorName = aProcessorName
self.mProcessCmd = aProcessCmd
self.mUseLsf = aUseLsf
self.summary = arg_summary
# We do not want the thread to launch until we've loaded all the
# properties
Msg.user("Thread Id: %s __init__" % (str(id(self))), "WORK-THREAD")
super().__init__(True)
self.on_done = None
self._launcher = None
def clean_up(self):
clean_up_rules = self.summary.cleanUpRules
if self.passed and not clean_up_rules.shouldKeepAll():
# list all the files and delete them except for _def_frun.py and
# the processor log (forrest.log)
my_dir = PathUtils.include_trailing_path_delimiter(self.work_dir)
Msg.user("Dir: %s" % (str(my_dir)), "FILES-TO-REMOVE")
my_file_path = str(my_dir) + "*"
Msg.user("Path: %s" % (my_file_path), "FILES-TO-REMOVE")
my_file_list = PathUtils.list_files(my_file_path)
Msg.user("Files: %s" % (str(my_file_list)), "FILES-TO-REMOVE")
process_log_base = PathUtils.base_name(self.process_log)
clean_up_rules.setBaseNamesToKeep(
["_def_frun.py", "PASS", process_log_base])
for my_file in my_file_list:
if clean_up_rules.shouldKeepFile(my_file):
Msg.user("File: %s KEPT" % clean_up_rules.lastBaseName())
else:
Msg.user("File: %s REMOVED" %
clean_up_rules.lastBaseName())
PathUtils.remove(my_file)
def dir_count(arg_class, arg_path=None):
try:
return len(PathUtils.list_dirs(arg_path))
except:
Msg.error_trace()
print("Exception")
return 0
def remove(arg_class, arg_path, arg_force=False):
try:
os.remove(arg_path)
Msg.dbg("Success, File Removed: %s" % (arg_path))
except OSError as arg_ex:
Msg.Err(str(arg_exe))
return False
def open_queue( self ):
# Create the Process Thread and lock ???
Msg.user( "Done Event: %s" %str( self.done_event ), "PROCESS-QUEUE" )
self.process_thread = ProcessThread( self, self.summary, self.process_max, self.done_event )
Msg.user( "F-Run Command Line: %s" % (self.process_cmd ), "PROCESS-QUEUE")
def load(self):
# Msg.dbg( "ForrestRun::load" )
my_frun_path = self.option_def(CmdLine.Switches[CmdLine.control_name],
None)
if my_frun_path is None:
raise Exception(
"F-Run Control File Not Found on the Forrest Run Command Line: Given Path: %s",
str((my_frun_path)))
# good to here
self.locate_frun(my_frun_path)
Msg.user("File Path: %s" % (my_frun_path))
my_content = open(self.frun_name).read()
my_glb, my_loc = SysUtils.exec_content(my_content)
Msg.dbg(str(my_loc))
self.fcontrol = my_loc["control_items"]
my_ctrl_dict = self.fcontrol[0]
my_ctrl_item = ControlItem()
my_ctrl_item.load(self._mAppsInfo, my_ctrl_dict)
# Msg.lout( my_ctrl_dict, "user", "Forrest Parent Data ...." )
self.check_simulator()
self.fctrl = ExecuteController(self._mAppsInfo)
self.fctrl.set_frun(self.frun_name)
self.fctrl.load(my_ctrl_item)
def build(self):
"""build the command line launcher with initial values"""
my_cmd = self.process_cmd % (self.frun_path)
Msg.user("Process Command: %s" % (str(my_cmd)), "LAUNCHER")
my_log = (PathUtils.include_trailing_path_delimiter(self.frun_dir) +
self.process_log)
return my_cmd, my_log
def locate_frun(self, arg_frun_path):
Msg.user("Directory set to %s" % (PathUtils.current_dir()))
# if the control file contains a path then split that into the directory and the file
my_frun_dir, my_frun_name = PathUtils.split_path(arg_frun_path)
# always convert to full path
my_cur_dir = PathUtils.real_path(PathUtils.current_dir())
# gots to have a source directory as part of the file name
if my_frun_dir is None:
my_frun_dir = my_cur_dir
else:
# always convert to full path. If the frun was loaded correctly then we can conclude that
# the path tendered is either a relative path from the starting directory or a full path
# to that file. If it is not a full path then it will need to be converted to a full path
# and all links removed
my_frun_dir = PathUtils.real_path(my_frun_dir)
# Msg.user( "FRun Dir: %s, FRun Name: %s, Cur Dir: %s" % ( str( my_frun_dir ), str( my_frun_name ), my_cur_dir ), "FRUN-DIR")
# change into the directory to generate and simulate
if not PathUtils.chdir(my_frun_dir):
raise Exception("F-Run Directory[%s] Not Found" %
(str(my_frun_dir)))
self.frun_name = my_frun_name
self.frun_dir = my_frun_dir
def process_summary(self, arg_sum_level=SummaryLevel.Fail):
# Msg.user( "process_summary()", "REG-SUMMARY" )
my_utcdt = DateTime.UTCNow()
my_file_name = "%sregression_summary.log" % (
PathUtils().include_trailing_path_delimiter(self.summary_dir)
)
# Msg.user( "Master Log File: %s" % ( my_file_name ))
my_ofile = None
myLines = []
# First try to open file
with open(my_file_name, "w") as my_ofile:
try:
my_ofile.write("Date: %s\n" % (DateTime.DateAsStr(my_utcdt)))
my_ofile.write("Time: %s\n" % (DateTime.TimeAsStr(my_utcdt)))
self.process_errors(my_ofile)
my_instr_count, my_cycle_count = self.process_summary_tasks(
my_ofile, arg_sum_level
)
self.process_summary_totals(my_ofile, my_instr_count, my_cycle_count)
except Exception as arg_ex:
Msg.error_trace()
Msg.err("Processing Summary, " + str(arg_ex))
finally:
my_ofile.close()
def process_summary(self, sum_level=SummaryLevel.Fail):
my_file_name = "%sperformance_summary.log" % (
PathUtils().include_trailing_path_delimiter(self.summary_dir))
Msg.dbg("Master Log File: %s" % (my_file_name))
my_utcdt = DateTime.UTCNow()
my_ofile = None
try:
# First try to open file
with open(my_file_name, "w") as my_ofile:
my_ofile.write("Date: %s\n" % (DateTime.DateAsStr(my_utcdt)))
my_ofile.write("Time: %s\n" % (DateTime.TimeAsStr(my_utcdt)))
self.process_errors(my_ofile)
my_total_count, my_total_elapsed = self.process_groups(
my_ofile)
Msg.blank("info")
my_line = "Total Instructions Generated: %3d\n" % (
my_total_count)
my_line += "Total Elapsed Time: %0.3f\n" % (my_total_elapsed)
my_line += "Overall Instructions per Second: %0.3f\n" % (
SysUtils.ifthen(bool(my_total_elapsed),
my_total_count / my_total_elapsed, 0))
Msg.info(my_line)
my_ofile.write(my_line)
except Exception as arg_ex:
Msg.error_trace()
Msg.err("Error Processing Summary, " + str(arg_ex))
finally:
my_ofile.close()
def process_group_items(self, arg_ofile, arg_items):
# Msg.trace("PerformanceSummaryItem::process_group_items")
my_grp_count = 0
my_grp_elapsed = 0
try:
for my_item in arg_items:
my_item_elapsed = my_item.force_end - my_item.force_start
my_item_count = my_item.total
my_grp_elapsed += my_item_elapsed
my_grp_count += my_item_count
my_line = "\nTask: %s, Instructions: %d, Elapsed: %0.3f\n" % (
my_item.task_id, my_item_count, my_item_elapsed)
arg_ofile.write(my_line)
# Msg.dbg( my_line )
#end: for my_task in my_group["tasks"]:
except Exception as arg_ex:
Msg.error_trace()
Msg.err("Error Processing Summary, Reason: %s" % (str(arg_ex)))
return my_grp_count, my_grp_elapsed
def post(self):
if self.ctrl_item.compile.get('mp'):
lsu_folder = PathUtils.real_path('%s/../../../rtl/lsu' % self.mMakefilePath)
cmd = 'svn revert %s/lsu_scb_retire_ctrl.vp %s/lsu_scb_sca_array.vp %s/lsu_scb_scd_array.vp' % (lsu_folder, lsu_folder, lsu_folder)
Msg.user('MP post-compile command = %s' % cmd)
result = SysUtils.exec_process(cmd, self.log, self.elog, self.ctrl_item.timeout, True)
Msg.user('MP post-compile command = %s' % str(result))
def __init__( self, aAppsInfo):
Msg.dbg( "Controller::__init__()" )
self.ctrl_item = None
self.on_fail_proc = None
self.is_term_proc = None
self.mAppsInfo = aAppsInfo
self.crit_sec = HiCriticalSection()
def skip(self):
if not self.ctrl_item.compile_rtl.get("run", False):
Msg.user(
"[CompileExecutor::skip] skipping since 'run' is not True...")
return True
Msg.user("[CompileExecutor::skip] not skipping")
return False
def init_thread( self ):
Msg.info( "HiEvent: Creating Test Thread ..." )
myThreadProcs = { "on-start" : self.thread_start # start thread sequence (outside thread space)
, "on-execute" : self.thread_execute # thread termination handler (inside thread space)
, "on-done" : self.thread_done # thread terminated handler (outside thread space)
, "on-finished": self.thread_finished # thread before finished handler (inside thread space)
}
self.thread = ThreadFactory( "EventTestThread", True, myThreadProcs )
def skip(self):
if not self.use_rtl():
Msg.user(
"[RtlExecutor::skip] skipping due to no rtl information specified"
)
return True
return False
