def process_summary(self, sum_level=SummaryLevel.Fail):
my_file_name = "%sperformance_summary.log" % (
PathUtils().include_trailing_path_delimiter(self.summary_dir))
Msg.dbg("Master Log File: %s" % (my_file_name))
my_utcdt = DateTime.UTCNow()
my_ofile = None
try:
# First try to open file
with open(my_file_name, "w") as my_ofile:
my_ofile.write("Date: %s\n" % (DateTime.DateAsStr(my_utcdt)))
my_ofile.write("Time: %s\n" % (DateTime.TimeAsStr(my_utcdt)))
self.process_errors(my_ofile)
my_total_count, my_total_elapsed = self.process_groups(
my_ofile)
Msg.blank("info")
my_line = "Total Instructions Generated: %3d\n" % (
my_total_count)
my_line += "Total Elapsed Time: %0.3f\n" % (my_total_elapsed)
my_line += "Overall Instructions per Second: %0.3f\n" % (
SysUtils.ifthen(bool(my_total_elapsed),
my_total_count / my_total_elapsed, 0))
Msg.info(my_line)
my_ofile.write(my_line)
except Exception as arg_ex:
Msg.error_trace()
Msg.err("Error Processing Summary, " + str(arg_ex))
finally:
my_ofile.close()
def process_summary(self, arg_sum_level=SummaryLevel.Fail):
# Msg.user( "process_summary()", "REG-SUMMARY" )
my_utcdt = DateTime.UTCNow()
my_file_name = "%sregression_summary.log" % (
PathUtils().include_trailing_path_delimiter(self.summary_dir)
)
# Msg.user( "Master Log File: %s" % ( my_file_name ))
my_ofile = None
myLines = []
# First try to open file
with open(my_file_name, "w") as my_ofile:
try:
my_ofile.write("Date: %s\n" % (DateTime.DateAsStr(my_utcdt)))
my_ofile.write("Time: %s\n" % (DateTime.TimeAsStr(my_utcdt)))
self.process_errors(my_ofile)
my_instr_count, my_cycle_count = self.process_summary_tasks(
my_ofile, arg_sum_level
)
self.process_summary_totals(my_ofile, my_instr_count, my_cycle_count)
except Exception as arg_ex:
Msg.error_trace()
Msg.err("Processing Summary, " + str(arg_ex))
finally:
my_ofile.close()
def __init__(self, arg_summary_path, arg_keep):
super().__init__(arg_summary_path, arg_keep)
self.gen_total = 0
self.gen_passed = 0
self.iss_total = 0
self.iss_passed = 0
self.task_total = 0
self.start_time = DateTime.Time()
def waitForLfs(self, aModTime):
if self.launcher_type == LauncherType.Lsf:
time_diff = int ( DateTime.Time() - aModTime )
if time_diff < MasterRun.cLsfWaitTime:
sec_delay = MasterRun.cLsfWaitTime - time_diff
Msg.info ("Using LSF, delaying %d seconds so that stale output/regression file handle will expire..." % sec_delay)
SysUtils.sleep_seconds_with_progress(sec_delay)
Msg.info ("Waiting done, resumed master run")
def process_summary_totals( self, arg_ofile, arg_instr_count, arg_cycle_count ):
Msg.blank("info")
# generator totals
my_gfails = self.gen_total - self.gen_passed
my_lines = "\nGenerate : %3d\n" % ( self.gen_total )
my_lines += "Generate Fails: %3d\n" % ( my_gfails )
my_lines += "Generate Success Rate : %5.2f%%\n" % ( SysUtils.percent( self.gen_passed, self.gen_total ))
my_lines += "\n"
my_sfails = 0
if self.iss_total >0:
# ISS sim totals
my_sfails = self.iss_total - self.iss_passed
my_lines += "ISS Sim : %3d\n" % ( self.iss_total )
my_lines += "ISS Sim Fails: %3d\n" % ( my_sfails )
my_lines += "ISS Sim Success Rate : %5.2f%%\n" % ( SysUtils.percent( self.iss_passed, self.iss_total ))
my_lines += "\n"
my_lines += "Total Instructions Emulated: %3d\n" % ( arg_instr_count )
my_lines += "\n"
my_rfails = 0
if self.rtl_total >0:
# RTL sim totals
my_rfails = self.rtl_total - self.rtl_passed
my_lines += "RTL Sim : %3d\n" % ( self.rtl_total )
my_lines += "RTL Sim Fails: %3d\n" % ( my_rfails )
my_lines += "RTL Sim Success Rate : %5.2f%%\n" % ( SysUtils.percent( self.rtl_passed, self.rtl_total ))
my_lines += "\n"
my_lines += "Total Cycle Count: %3d\n" % ( arg_cycle_count )
my_lines += "\n"
my_cfails = 0
if self.trace_cmp_total > 0:
# simulation totals
my_cfails = self.trace_cmp_total - self.trace_cmp_passed
my_lines += "Compared : %3d\n" % ( self.trace_cmp_total )
my_lines += "Compare Fails: %3d\n" % ( my_cfails )
my_lines += "Compare Success Rate : %5.2f%%\n" % ( SysUtils.percent( self.trace_cmp_passed, self.trace_cmp_total ))
my_lines += "\n"
# task totals
my_task_total = ( self.task_total)
my_total_fails = my_gfails + my_sfails + my_cfails + my_rfails
my_lines += "Total Tasks : %3d\n" % ( my_task_total )
my_lines += "Task Fails : %3d\n" % ( my_total_fails )
my_lines += "Task Success Rate: %5.2f%%\n" % ( SysUtils.percent( my_task_total - my_total_fails, ( my_task_total )))
my_lines += "\n"
my_lines += "Total Run Time: %0.5f Seconds" % ( DateTime.Time() - self.start_time )
Msg.blank("info")
Msg.info( my_lines )
Msg.blank("info")
arg_ofile.write( my_lines )
def __init__(self, arg_summary_path, arg_keep):
super().__init__(arg_summary_path, arg_keep)
self.gen_total = 0
self.gen_passed = 0
self.iss_total = 0
self.iss_passed = 0
self.rtl_total = 0
self.rtl_passed = 0
self.trace_cmp_total = 0
self.trace_cmp_passed = 0
self.task_total = 0
self.start_time = DateTime.Time()
self.total_cycle_count = 0
self.total_instruction_count = 0
def archive_dir(cls, arg_srcdir):
try:
# get the base name
my_basename = "%s" % (str(DateTime.YMD()))
Msg.dbg("Base Name: %s" % (my_basename))
# set the directory mask
my_srcdir = PathUtils.exclude_trailing_path_delimiter(arg_srcdir)
my_srcmask = "%s_%s_???" % (my_srcdir, my_basename)
Msg.dbg("Directory Mask: %s" % (my_srcmask))
# list any previous copies
my_dirlist = sorted(PathUtils.list_files(my_srcmask))
Msg.lout(my_dirlist, "dbg")
my_findex = 0
# there are only two possiblities here
# 1. there is only one match, in which case the mask does not
# include a number
# 2. there are more than one match in which case the last match
# should contain a number
if len(my_dirlist) > 0:
# remove the wildcards
my_srcmask = my_srcmask.replace("???", "")
Msg.dbg("New File Mask: %s" % (my_srcmask))
my_tmp = my_dirlist[-1]
Msg.dbg("Last Filename: %s" % (my_tmp))
my_tmp = my_tmp.replace(my_srcmask, "")
Msg.dbg("My Index Last Filename: %s" % (my_tmp))
my_findex = int(my_tmp) + 1
Msg.dbg("My New Index Filename: %s" % (my_findex))
# get the target name
my_tgtdir = "%s_%s_%0.3d" % (my_srcdir, my_basename, my_findex)
Msg.dbg("Target Directory: %s" % (my_tgtdir))
return PathUtils.move(my_srcdir, my_tgtdir)
except Exception as arg_ex:
Msg.error_trace(str(arg_ex))
Msg.err(str(arg_ex))
return False
def handle_expire(self, arg_expire, arg_mask):
try:
Msg.user(
"Expire: %s, Mask: %s" % (str(arg_expire), str(arg_mask)),
"EXPIRE",
)
my_output_base = Formats.main_output_dir % self.output_root
Msg.user("Expire [1], Output Base: %s" % (str(my_output_base)),
"EXPIRE")
Msg.info("Output Directories Cleanup, Please wait ...")
if int(arg_expire) == Expire.all:
Msg.info("Building Directory List, [%s]" % (my_output_base))
my_dirs = PathUtils.list_dirs(my_output_base)
Msg.dbg("All Dirs: %s" % (str(my_dirs)), "EXPIRE")
for my_dir in my_dirs:
if my_dir.startswith(arg_mask):
my_full_dir = "%s%s" % (
PathUtils.include_trailing_path_delimiter(
my_output_base),
my_dir,
)
Msg.info("Removing: %s" % (my_full_dir))
PathUtils.rmdir(my_full_dir, True)
else:
Msg.info("Checking for Expired Directories: %s" %
(my_full_dir))
my_expiredate = DateTime.DateDelta(int(my_expire))
PathUtils.expire(my_full_dir, my_expiredate)
except Exception as ex:
Msg.error_trace()
Msg.err(str(ex))
finally:
Msg.info("Operation Complete, Restart Master Run to continue ...")
sys.exit(1)
def exec_process(arg_class,
arg_cmd,
arg_fout=None,
arg_ferr=None,
arg_timeout=None,
arg_kill=False,
arg_set_process=None):
#def exec_process( arg_class, arg_cmd, arg_fout = None, arg_ferr = None, arg_timeout = None, arg_kill_all = True ):
my_process_cmd = arg_cmd
Msg.dbg(my_process_cmd)
my_fout, my_ferr = SysUtils.open_output(arg_fout, arg_ferr)
my_result = None
my_stdout = None
my_stderr = None
Msg.user(
"SysUtils.exec_process( ... , arg_set_process: %s" %
(str(bool(arg_set_process is not None))), "SYS-UTILS")
Msg.flush()
try:
from common.datetime_utils import DateTime
my_process = subprocess.Popen(my_process_cmd,
stdout=my_fout,
stderr=my_ferr,
shell=True)
my_start_time = DateTime.Time()
my_pid = my_process.pid
if arg_timeout is None or not SysUtils.is_numeric(arg_timeout):
Msg.user("Exec PID[NONE]: %s" % (str(my_pid)))
my_result = my_process.communicate()
Msg.user("Done PID[NONE]: %s" % (str(my_pid)))
else:
Msg.user("Timeout: %s" % (str(arg_timeout)))
try:
my_pgid = os.getpgid(my_pid)
my_parent_pgid = os.getpgid(0)
# use callback to save an instance of the process to allow
if arg_set_process is not None:
arg_set_process(my_process)
# Msg.dbg( "Exec Cmd: %s" % ( str( arg_cmd )))
# Msg.dbg( "PID: %s, PGID: %s, P-PGID: %s, Timeout: %s" % ( str( my_pid ), str( my_pgid ), str( my_parent_pgid ), str(arg_timeout)))
# Msg.flush()
my_result = my_process.communicate(timeout=arg_timeout)
#my_pgid = os.getpgid(my_pid)
# Msg.user( "Done PID: %s, PGID: %s, Timeout: %s" % ( str( my_pid ), str( my_pgid ), str(arg_timeout)))
except TimeoutExpired:
try:
# Msg.user( "Timeout occurred ...." )
my_parent_pgid = os.getpgid(0)
my_pgid = os.getpgid(my_pid)
# os.setpgid( os.getpid(), os.getpid())
# on timeout kill the spawned process and all related sub-processes
if arg_kill:
# Msg.dbg( "Killing Everything ..." )
# os.killpg( os.getpgid( my_pid ), signal.SIGTERM )
os.killpg(os.getpgid(my_pid), signal.SIGKILL)
# Trigger Shutdown of spawned processes
else:
Msg.error_trace()
Msg.err("Process Did Not Execute Properly: %s" %
(my_process_cmd))
# my_process.kill()
os.kill(my_pid, signal.SIGTERM)
except OSError as arg_ex:
# Msg.dbg( "Spawned Killing encountered OS Error ... " )
Msg.error(str(arg_ex))
finally:
# Msg.dbg( "Timeout Occurred ... " )
my_result = my_process.communicate()
Msg.flush()
return (my_process.returncode, None,
"Process Timeout Occurred", my_start_time,
my_start_time)
except Exception as arg_ex:
Msg.err(str(arg_ex))
Msg.error_trace()
except:
Msg.error_trace()
finally:
Msg.user("[1] SysUtils::exec_process")
# Msg.user( "SysUtils.exec_process[10]", "test" )
pass
my_end_time = DateTime.Time()
# Msg.dbg( "SysUtils.exec_process, my_result: %s" % ( str( my_result )))
# Msg.dbg( "Return Code: %d" % ( my_process.returncode ))
if my_result[0] is not None:
my_stdout = my_result[0].decode("utf-8")
if my_result[1] is not None:
my_stderr = my_result[1].decode("utf-8")
return (my_process.returncode, my_stdout, my_stderr, my_start_time,
my_end_time)
finally:
if not (my_fout == PIPE or my_fout.closed): my_fout.close()
if not (my_ferr == PIPE or my_ferr.closed): my_ferr.close()
return (SysUtils.PROCESS_UNABLE_TO_SPAWN, None,
"Unable to Spawn Process ....", 0, None, None)
def exec_process(
cls,
arg_cmd,
arg_fout=None,
arg_ferr=None,
arg_timeout=None,
arg_kill=False,
arg_set_process=None,
):
my_process_cmd = arg_cmd
Msg.dbg(my_process_cmd)
my_fout, my_ferr = SysUtils.open_output(arg_fout, arg_ferr)
my_result = None
my_stdout = None
my_stderr = None
Msg.user(
"SysUtils.exec_process( ... , arg_set_process: %s" %
(str(bool(arg_set_process is not None))),
"SYS-UTILS",
)
Msg.flush()
try:
from common.datetime_utils import DateTime
my_process = subprocess.Popen(shlex.split(my_process_cmd),
stdout=my_fout,
stderr=my_ferr)
my_start_time = DateTime.Time()
my_pid = my_process.pid
if arg_timeout is None or not SysUtils.is_numeric(arg_timeout):
Msg.user("Exec PID[NONE]: %s" % (str(my_pid)))
my_result = my_process.communicate()
Msg.user("Done PID[NONE]: %s" % (str(my_pid)))
else:
Msg.user("Timeout: %s" % (str(arg_timeout)))
try:
my_pgid = os.getpgid(my_pid)
my_parent_pgid = os.getpgid(0)
# use callback to save an instance of the process to allow
if arg_set_process is not None:
arg_set_process(my_process)
my_result = my_process.communicate(timeout=arg_timeout)
except subprocess.TimeoutExpired:
try:
Msg.err(
"Timeout after %d seconds, terminating process: %s"
% (arg_timeout, my_process_cmd))
my_parent_pgid = os.getpgid(0)
my_pgid = os.getpgid(my_pid)
# os.setpgid( os.getpid(), os.getpid())
# on timeout kill the spawned process and all related
# sub-processes
if arg_kill:
# Msg.dbg( "Killing Everything ..." )
os.killpg(os.getpgid(my_pid), signal.SIGKILL)
# Trigger Shutdown of spawned processes
else:
Msg.error_trace()
# my_process.kill()
os.kill(my_pid, signal.SIGTERM)
except OSError as arg_ex:
# Msg.dbg( "Spawned Killing encountered OS Error." )
Msg.error(str(arg_ex))
finally:
# Msg.dbg( "Timeout Occurred ... " )
my_result = my_process.communicate()
Msg.flush()
return (
my_process.returncode,
None,
"Process Timeout Occurred",
my_start_time,
my_start_time,
SysUtils.PROCESS_TIMEOUT,
)
except Exception as arg_ex:
Msg.err(str(arg_ex))
Msg.error_trace()
except BaseException:
Msg.error_trace()
finally:
Msg.user("[1] SysUtils::exec_process")
my_end_time = DateTime.Time()
if my_result[0] is not None:
my_stdout = my_result[0].decode("utf-8")
if my_result[1] is not None:
my_stderr = my_result[1].decode("utf-8")
return (
my_process.returncode,
my_stdout,
my_stderr,
my_start_time,
my_end_time,
SysUtils.NORMAL,
)
finally:
if not (my_fout == subprocess.PIPE or my_fout.closed):
my_fout.close()
if not (my_ferr == subprocess.PIPE or my_ferr.closed):
my_ferr.close()
return (
0,
None,
"Unable to Spawn Process ....",
0,
None,
None,
SysUtils.PROCESS_UNABLE_TO_SPAWN,
)