def tcase_check_preqs(tcase: TestCase, preqs: dict,
tsuite_name: str) -> Tuple[int, bool]:
"""
Retrieve metrics and check them against the given preqs.
Logs results to tcase.analysis_log_fpath.
"""
tc_err = 0
test_metrics = _get_metrics(tcase.aux_root)
if not test_metrics:
cij.info(f"{tsuite_name}/{tcase.name} no measurements found")
return tc_err, True
# Check performance requirements against measured metrics
with open(tcase.analysis_log_fpath, 'w', encoding="UTF-8") as alog:
for metrics in test_metrics:
checked_preqs = check_preqs(preqs, metrics)
for cpreq in checked_preqs:
cij.emph(f"{cpreq.key}: {cpreq.msg}", rval=int(cpreq.error))
cij.emph(f"{cpreq.key}: {cpreq.msg} {cpreq.ctx}",
rval=int(cpreq.error),
file=alog)
tc_err += sum(cpreq.error for cpreq in checked_preqs)
return tc_err, False
def execute(cmd=None, shell=True, echo=True):
"""
Execute the given 'cmd'
@returns (rcode, stdout, stderr)
"""
if echo:
cij.emph("cij.util.execute: shell: %r, cmd: %r" % (shell, cmd))
rcode = 1
stdout, stderr = ("", "")
if cmd:
if shell:
cmd = " ".join(cmd)
proc = Popen(cmd,
stdout=PIPE,
stderr=PIPE,
shell=shell,
close_fds=True)
stdout, stderr = proc.communicate()
rcode = proc.returncode
if rcode and echo:
cij.warn("cij.util.execute: stdout: %s" % stdout)
cij.err("cij.util.execute: stderr: %s" % stderr)
cij.err("cij.util.execute: rcode: %s" % rcode)
return rcode, stdout, stderr
def __run(self, shell=True, echo=True):
"""Run DMESG job"""
if env():
return 1
cij.emph("cij.dmesg.start: shell: %r, cmd: %r" % (shell, self.__prefix + self.__suffix))
return cij.ssh.command(self.__prefix, shell, echo, self.__suffix)
def run(self, shell=True, cmdline=False, echo=True):
"""Run FIO job"""
if env():
return 1
cmd = ["fio"] + self.__parse_parms()
if cmdline:
cij.emph("cij.fio.run: shell: %r, cmd: %r" % (shell, cmd))
return cij.ssh.command(cmd, shell, echo)
def enter():
"""Enter the test, check requirements and setup aux. environment"""
if cij.ssh.env():
tfail("cij.test: invalid SSH environment")
for req in REQS:
if getattr(cij, req).env():
tfail()
cij.emph("cij.test: entering test")
def script_run(trun: TestRun, script: Runnable):
"""Execute a script or testcase"""
if trun.args.verbose:
cij.emph("rnr:script:run { script: %s }" % script)
cij.emph("rnr:script:run:evars: %s" % script.evars)
launchers = {".py": "python", ".sh": "source"}
ext = os.path.splitext(script.fpath)[-1]
if ext not in launchers.keys():
cij.err("rnr:script:run { invalid script.fpath: %r }" % script.fpath)
return 1
launch = launchers[ext]
with open(script.log_fpath, "a", encoding="UTF-8") as log_fd:
log_fd.write("# script_fpath: %r\n" % script.fpath)
log_fd.flush()
script.stamp["begin"] = time.time()
cmd = [
'bash', '-c',
'CIJ_ROOT=$(cij_root) && '
'source $CIJ_ROOT/modules/cijoe.sh && '
'source %s && '
'CIJ_TEST_RES_ROOT="%s" %s %s ' %
(trun.args.env_fpath, script.res_root, launch, script.fpath)
]
if trun.args.verbose > 1:
cij.emph("rnr:script:run { cmd: %r }" % " ".join(cmd))
evars = os.environ.copy()
evars.update({k: str(script.evars[k]) for k in script.evars})
with Popen(cmd,
stdout=log_fd,
stderr=STDOUT,
cwd=script.res_root,
env=evars) as process:
process.wait()
script.rcode = process.returncode
script.stamp["end"] = time.time()
script.wallc = script.stamp["end"] - script.stamp["begin"]
if trun.args.verbose:
cij.emph("rnr:script:run { wallc: %02f }" %
(script.wallc if script.wallc is not None else 0.0))
cij.emph("rnr:script:run { rcode: %r } " % script.rcode, script.rcode)
return script.rcode
def script_run(trun, script):
"""Execute a script or testcase"""
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:script:run { script: %s }" % script)
cij.emph("rnr:script:run:evars: %s" % script["evars"])
launchers = {".py": "python", ".sh": "source"}
ext = os.path.splitext(script["fpath"])[-1]
if not ext in launchers.keys():
cij.err("rnr:script:run { invalid script[\"fpath\"]: %r }" %
script["fpath"])
return 1
launch = launchers[ext]
with open(script["log_fpath"], "a") as log_fd:
log_fd.write("# script_fpath: %r\n" % script["fpath"])
log_fd.flush()
bgn = time.time()
cmd = [
'bash', '-c',
'CIJ_ROOT=$(cij_root) && '
'source $CIJ_ROOT/modules/cijoe.sh && '
'source %s && '
'CIJ_TEST_RES_ROOT="%s" %s %s ' %
(trun["conf"]["ENV_FPATH"], script["res_root"], launch,
script["fpath"])
]
if trun["conf"]["VERBOSE"] > 1:
cij.emph("rnr:script:run { cmd: %r }" % " ".join(cmd))
evars = os.environ.copy()
evars.update({k: str(script["evars"][k]) for k in script["evars"]})
process = Popen(cmd,
stdout=log_fd,
stderr=STDOUT,
cwd=script["res_root"],
env=evars)
process.wait()
script["rcode"] = process.returncode
script["wallc"] = time.time() - bgn
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:script:run { wallc: %02f }" % script["wallc"])
cij.emph("rnr:script:run { rcode: %r } " % script["rcode"],
script["rcode"])
return script["rcode"]
def trun_exit(trun):
"""Triggers when exiting the given testrun"""
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:trun:exit")
rcode = 0
for hook in reversed(trun["hooks"]["exit"]): # EXIT-hooks
rcode = script_run(trun, hook)
if rcode:
break
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:trun::exit { rcode: %r }" % rcode, rcode)
return rcode
def tsuite_enter(trun, tsuite):
"""Triggers when entering the given testsuite"""
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:tsuite:enter { name: %r }" % tsuite["name"])
rcode = 0
for hook in tsuite["hooks"]["enter"]: # ENTER-hooks
rcode = script_run(trun, hook)
if rcode:
break
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:tsuite:enter { rcode: %r } " % rcode, rcode)
return rcode
def tcase_exit(trun, tsuite, tcase):
"""..."""
#pylint: disable=locally-disabled, unused-argument
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:tcase:exit { fname: %r }" % tcase["fname"])
rcode = 0
for hook in reversed(tcase["hooks"]["exit"]): # tcase EXIT-hooks
rcode = script_run(trun, hook)
if rcode:
break
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:tcase:exit { rcode: %r }" % rcode, rcode)
return rcode
def remove():
"""Remove LNVM device"""
if env():
cij.err("cij.lnvm.create: Invalid LNVM ENV")
return 1
lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)
cij.emph("lnvm.remove: LNVM_DEV_NAME: %s" % lnvm["DEV_NAME"])
cmd = ["nvme lnvm remove -n %s" % (lnvm["DEV_NAME"])]
rcode, _, _ = cij.ssh.command(cmd, shell=True)
if rcode:
cij.err("cij.lnvm.remove: FAILED")
return 1
return 0
def trun_enter(trun):
"""Triggers when entering the given testrun"""
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:trun::enter")
trun["stamp"]["begin"] = int(time.time()) # Record start timestamp
rcode = 0
for hook in trun["hooks"]["enter"]: # ENTER-hooks
rcode = script_run(trun, hook)
if rcode:
break
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:trun::enter { rcode: %r }" % rcode, rcode)
return rcode
def recover():
"""Recover LNVM device"""
if env():
cij.err("cij.lnvm.create: Invalid LNVM ENV")
return 1
nvme = cij.env_to_dict("NVME", ["DEV_NAME"])
lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)
cij.emph("lnvm.recover: LNVM_DEV_NAME: %s" % lnvm["DEV_NAME"])
cmd = ["nvme lnvm create -d %s -n %s -t %s -b %s -e %s" % (
nvme["DEV_NAME"], lnvm["DEV_NAME"], lnvm["DEV_TYPE"], lnvm["BGN"], lnvm["END"])]
rcode, _, _ = cij.ssh.command(cmd, shell=True)
if rcode:
cij.err("cij.lnvm.recover: FAILED")
return 1
return 0
def enter(self, trun):
"""Called by runner before invoking the runnable"""
if trun.args.verbose:
cij.emph("rnr:enter { ident: %r }" % self.ident)
self.stamp["begin"] = time.time()
rcode = 0
for hook in self.hooks["enter"]: # ENTER-hooks
rcode = script_run(trun, hook)
if rcode:
break
self.entered = not rcode
if trun.args.verbose:
cij.emph("rnr:enter { ident: %r, rcode: %r } " %
(self.ident, rcode))
return rcode
def main(args):
"""Main entry point"""
trun = cij.runner.trun_from_file(args.trun_fpath)
# pylint: disable=no-member
rehome(trun.args.output, args.output, trun)
postprocess(trun)
cij.emph("main: reports are uses tmpl_fpath: %r" % args.tmpl_fpath)
cij.emph("main: reports are here args.output: %r" % args.output)
_, ext = os.path.splitext(args.tmpl_fpath)
html_fpath = os.path.join(args.output, "".join([args.tmpl_name, ext]))
cij.emph("html_fpath: %r" % html_fpath)
try: # Create and store HTML report
with open(html_fpath, 'w', encoding="UTF-8") as html_file:
html_file.write(dset_to_html(trun, args.tmpl_fpath))
except (IOError, OSError, ValueError) as exc:
traceback.print_exc()
cij.err("rprtr:main: exc: %s" % exc)
return 1
return 0
def exit(self, trun):
"""Called by runner after invoking the runnable"""
if trun.args.verbose:
cij.emph("rnr:exit: { ident: %r }" % self.ident)
rcode = 0
if self.entered:
for hook in reversed(self.hooks["exit"]): # EXIT-hooks
rcode = script_run(trun, hook)
if rcode:
break
self.stamp["end"] = time.time()
if self.wallc is None:
self.wallc = self.stamp["end"] - self.stamp["begin"]
if trun.args.verbose:
cij.emph("rnr:exit: { ident: %r, rcode: %r }" %
(self.ident, rcode))
return 0
def tcase_enter(trun, tsuite, tcase):
"""
setup res_root and aux_root, log info and run tcase-enter-hooks
@returns 0 when all hooks succeed, some value othervise
"""
#pylint: disable=locally-disabled, unused-argument
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:tcase:enter")
cij.emph("rnr:tcase:enter { fname: %r }" % tcase["fname"])
cij.emph("rnr:tcase:enter { log_fpath: %r }" % tcase["log_fpath"])
rcode = 0
for hook in tcase["hooks"]["enter"]: # tcase ENTER-hooks
rcode = script_run(trun, hook)
if rcode:
break
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:tcase:exit: { rcode: %r }" % rcode, rcode)
return rcode
def main(args):
"""Main entry point"""
trun = cij.runner.trun_from_file(args.trun_fpath)
rehome(trun["conf"]["OUTPUT"], args.output, trun)
postprocess(trun)
cij.emph("main: reports are uses tmpl_fpath: %r" % args.tmpl_fpath)
cij.emph("main: reports are here args.output: %r" % args.output)
html_fpath = os.sep.join([args.output, "%s.html" % args.tmpl_name])
cij.emph("html_fpath: %r" % html_fpath)
try: # Create and store HTML report
with open(html_fpath, 'w') as html_file:
html_file.write(dset_to_html(trun, args.tmpl_fpath))
except (IOError, OSError, ValueError) as exc:
import traceback
traceback.print_exc()
cij.err("rprtr:main: exc: %s" % exc)
return 1
return 0
def trun_emph(trun: TestRun):
"""Print essential info on"""
if trun.args.verbose > 1: # Print environment variables
cij.emph("rnr:conf {")
conf_dict = dataclasses.asdict(trun.conf)
for var in sorted(conf_dict.keys()):
cij.emph(" % 16s: %r" % (var, conf_dict[var]))
cij.emph("}")
cij.emph("rnr:args {")
args_dict = vars(trun.args)
for var in sorted(args_dict.keys()):
cij.emph(" % 16s: %r" % (var, args_dict[var]))
cij.emph("}")
if trun.args.verbose:
cij.emph("rnr:INFO {")
cij.emph(" output: %r" % trun.args.output)
cij.emph(" yml_fpath: %r" % yml_fpath(trun.args.output))
cij.emph("}")
def main(args, conf):
"""CIJ Test Runner main entry point"""
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# There are a lot of branches and statements here... but that is fine.
fpath = yml_fpath(args.output)
if os.path.exists(fpath): # YAML exists, we exit, it might be RUNNING!
cij.err("main:FAILED { fpath: %r }, exists" % fpath)
return 1
trun: TestRun
try:
trun = trun_setup(args, conf) # Construct 'trun' from args and conf
except CIJError as ex:
cij.err("main:FAILED to start testrun: %s" % ex)
trun_to_file(trun) # Persist trun
trun_to_junitfile(trun) # Persist as jUNIT XML
trun.enter(trun)
for tplan in (tp for tp in trun.testplans if trun.entered):
tplan.enter(trun)
for tsuite in (ts for ts in tplan.testsuites if tplan.entered):
tsuite.enter(trun)
for tcase in (tc for tc in tsuite.testcases if tsuite.entered):
if (args.testcase_match is None
or args.testcase_match in tcase.name):
tcase.enter(trun)
if tcase.entered:
script_run(trun, tcase)
tcase.exit(trun)
tcase.status = Status.Pass
if tcase.rcode is None or tcase.rcode:
tcase.status = Status.Fail
cij.err(f"main:{tcase.ident} failed.")
cij.err(f"See log: `less {tcase.log_fpath}`")
tsuite.progress[Status.Unkn] -= 1 # Update progress
tplan.progress[Status.Unkn] -= 1
trun.progress[Status.Unkn] -= 1
tsuite.progress[tcase.status] += 1 # Update progress
tplan.progress[tcase.status] += 1
trun.progress[tcase.status] += 1
if tcase.status == Status.Fail: # Propagate failure
trun.status = tplan.status = tsuite.status = tcase.status
trun_to_file(trun) # Persist trun
trun_to_junitfile(trun) # Persist as jUNIT XML
if tsuite.exit(trun) or not tsuite.entered:
trun.status = tplan.status = tsuite.status = Status.Fail
if tsuite.status == Status.Unkn and tsuite.entered:
tsuite.status = Status.Pass
trun_to_file(trun) # Persist trun
trun_to_junitfile(trun) # Persist as jUNIT XML
if tplan.exit(trun) or not tplan.entered:
trun.status = tplan.status = Status.Fail
if tplan.status == Status.Unkn and tplan.entered:
tplan.status = Status.Pass
trun_to_file(trun) # Persist trun
trun_to_junitfile(trun) # Persist as jUNIT XML
if trun.exit(trun) or not trun.entered:
trun.status = Status.Fail
if trun.status == Status.Unkn and trun.entered:
trun.status = Status.Pass
trun_to_file(trun) # PERSIST
trun_to_junitfile(trun) # Persist as jUNIT XML
cij.emph("rnr:main:progress %r" % trun.progress)
cij.emph("rnr:main:trun %r" % trun.status, trun.status != Status.Pass)
return trun.progress[Status.Fail]
def main(conf):
"""CIJ Test Runner main entry point"""
fpath = yml_fpath(conf["OUTPUT"])
if os.path.exists(fpath): # YAML exists, we exit, it might be RUNNING!
cij.err("main:FAILED { fpath: %r }, exists" % fpath)
return 1
trun = trun_setup(conf) # Construct 'trun' from 'conf'
if not trun:
return 1
trun_to_file(trun) # Persist trun
trun_to_junitfile(trun) # Persist as jUNIT XML
trun_emph(trun) # Print trun before run
tr_err = 0
tr_ent_err = trun_enter(trun)
for tsuite in (ts for ts in trun["testsuites"] if not tr_ent_err):
ts_err = 0
ts_ent_err = tsuite_enter(trun, tsuite)
for tcase in (tc for tc in tsuite["testcases"] if not ts_ent_err):
tc_err = tcase_enter(trun, tsuite, tcase)
if not tc_err:
tc_err += script_run(trun, tcase)
tc_err += tcase_exit(trun, tsuite, tcase)
tcase["status"] = "FAIL" if tc_err else "PASS"
trun["progress"][tcase["status"]] += 1 # Update progress
trun["progress"]["UNKN"] -= 1
ts_err += tc_err # Accumulate errors
trun_to_file(trun) # Persist trun
trun_to_junitfile(trun) # Persist as jUNIT XML
if not ts_ent_err:
ts_err += tsuite_exit(trun, tsuite)
ts_err += ts_ent_err # Accumulate errors
tr_err += ts_err
tsuite["status"] = "FAIL" if ts_err else "PASS"
cij.emph("rnr:tsuite %r" % tsuite["status"], tsuite["status"] != "PASS")
if not tr_ent_err:
trun_exit(trun)
tr_err += tr_ent_err
trun["status"] = "FAIL" if tr_err else "PASS"
trun["stamp"]["end"] = int(time.time()) + 1 # END STAMP
trun_to_file(trun) # PERSIST
trun_to_junitfile(trun) # Persist as jUNIT XML
cij.emph("rnr:main:progress %r" % trun["progress"])
cij.emph("rnr:main:trun %r" % trun["status"], trun["status"] != "PASS")
return trun["progress"]["UNKN"] + trun["progress"]["FAIL"]
def trun_emph(trun):
"""Print essential info on"""
if trun["conf"]["VERBOSE"] > 1: # Print environment variables
cij.emph("rnr:CONF {")
for cvar in sorted(trun["conf"].keys()):
cij.emph(" % 16s: %r" % (cvar, trun["conf"][cvar]))
cij.emph("}")
if trun["conf"]["VERBOSE"]:
cij.emph("rnr:INFO {")
cij.emph(" OUTPUT: %r" % trun["conf"]["OUTPUT"])
cij.emph(" yml_fpath: %r" % yml_fpath(trun["conf"]["OUTPUT"]))
cij.emph("}")
