def tcase_check_preqs(tcase: TestCase, preqs: dict, tsuite_name: str) -> Tuple[int, bool]: """ Retrieve metrics and check them against the given preqs. Logs results to tcase.analysis_log_fpath. """ tc_err = 0 test_metrics = _get_metrics(tcase.aux_root) if not test_metrics: cij.info(f"{tsuite_name}/{tcase.name} no measurements found") return tc_err, True # Check performance requirements against measured metrics with open(tcase.analysis_log_fpath, 'w', encoding="UTF-8") as alog: for metrics in test_metrics: checked_preqs = check_preqs(preqs, metrics) for cpreq in checked_preqs: cij.emph(f"{cpreq.key}: {cpreq.msg}", rval=int(cpreq.error)) cij.emph(f"{cpreq.key}: {cpreq.msg} {cpreq.ctx}", rval=int(cpreq.error), file=alog) tc_err += sum(cpreq.error for cpreq in checked_preqs) return tc_err, False
def execute(cmd=None, shell=True, echo=True): """ Execute the given 'cmd' @returns (rcode, stdout, stderr) """ if echo: cij.emph("cij.util.execute: shell: %r, cmd: %r" % (shell, cmd)) rcode = 1 stdout, stderr = ("", "") if cmd: if shell: cmd = " ".join(cmd) proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=shell, close_fds=True) stdout, stderr = proc.communicate() rcode = proc.returncode if rcode and echo: cij.warn("cij.util.execute: stdout: %s" % stdout) cij.err("cij.util.execute: stderr: %s" % stderr) cij.err("cij.util.execute: rcode: %s" % rcode) return rcode, stdout, stderr
def __run(self, shell=True, echo=True): """Run DMESG job""" if env(): return 1 cij.emph("cij.dmesg.start: shell: %r, cmd: %r" % (shell, self.__prefix + self.__suffix)) return cij.ssh.command(self.__prefix, shell, echo, self.__suffix)
def run(self, shell=True, cmdline=False, echo=True): """Run FIO job""" if env(): return 1 cmd = ["fio"] + self.__parse_parms() if cmdline: cij.emph("cij.fio.run: shell: %r, cmd: %r" % (shell, cmd)) return cij.ssh.command(cmd, shell, echo)
def enter(): """Enter the test, check requirements and setup aux. environment""" if cij.ssh.env(): tfail("cij.test: invalid SSH environment") for req in REQS: if getattr(cij, req).env(): tfail() cij.emph("cij.test: entering test")
def script_run(trun: TestRun, script: Runnable): """Execute a script or testcase""" if trun.args.verbose: cij.emph("rnr:script:run { script: %s }" % script) cij.emph("rnr:script:run:evars: %s" % script.evars) launchers = {".py": "python", ".sh": "source"} ext = os.path.splitext(script.fpath)[-1] if ext not in launchers.keys(): cij.err("rnr:script:run { invalid script.fpath: %r }" % script.fpath) return 1 launch = launchers[ext] with open(script.log_fpath, "a", encoding="UTF-8") as log_fd: log_fd.write("# script_fpath: %r\n" % script.fpath) log_fd.flush() script.stamp["begin"] = time.time() cmd = [ 'bash', '-c', 'CIJ_ROOT=$(cij_root) && ' 'source $CIJ_ROOT/modules/cijoe.sh && ' 'source %s && ' 'CIJ_TEST_RES_ROOT="%s" %s %s ' % (trun.args.env_fpath, script.res_root, launch, script.fpath) ] if trun.args.verbose > 1: cij.emph("rnr:script:run { cmd: %r }" % " ".join(cmd)) evars = os.environ.copy() evars.update({k: str(script.evars[k]) for k in script.evars}) with Popen(cmd, stdout=log_fd, stderr=STDOUT, cwd=script.res_root, env=evars) as process: process.wait() script.rcode = process.returncode script.stamp["end"] = time.time() script.wallc = script.stamp["end"] - script.stamp["begin"] if trun.args.verbose: cij.emph("rnr:script:run { wallc: %02f }" % (script.wallc if script.wallc is not None else 0.0)) cij.emph("rnr:script:run { rcode: %r } " % script.rcode, script.rcode) return script.rcode
def script_run(trun, script): """Execute a script or testcase""" if trun["conf"]["VERBOSE"]: cij.emph("rnr:script:run { script: %s }" % script) cij.emph("rnr:script:run:evars: %s" % script["evars"]) launchers = {".py": "python", ".sh": "source"} ext = os.path.splitext(script["fpath"])[-1] if not ext in launchers.keys(): cij.err("rnr:script:run { invalid script[\"fpath\"]: %r }" % script["fpath"]) return 1 launch = launchers[ext] with open(script["log_fpath"], "a") as log_fd: log_fd.write("# script_fpath: %r\n" % script["fpath"]) log_fd.flush() bgn = time.time() cmd = [ 'bash', '-c', 'CIJ_ROOT=$(cij_root) && ' 'source $CIJ_ROOT/modules/cijoe.sh && ' 'source %s && ' 'CIJ_TEST_RES_ROOT="%s" %s %s ' % (trun["conf"]["ENV_FPATH"], script["res_root"], launch, script["fpath"]) ] if trun["conf"]["VERBOSE"] > 1: cij.emph("rnr:script:run { cmd: %r }" % " ".join(cmd)) evars = os.environ.copy() evars.update({k: str(script["evars"][k]) for k in script["evars"]}) process = Popen(cmd, stdout=log_fd, stderr=STDOUT, cwd=script["res_root"], env=evars) process.wait() script["rcode"] = process.returncode script["wallc"] = time.time() - bgn if trun["conf"]["VERBOSE"]: cij.emph("rnr:script:run { wallc: %02f }" % script["wallc"]) cij.emph("rnr:script:run { rcode: %r } " % script["rcode"], script["rcode"]) return script["rcode"]
def trun_exit(trun): """Triggers when exiting the given testrun""" if trun["conf"]["VERBOSE"]: cij.emph("rnr:trun:exit") rcode = 0 for hook in reversed(trun["hooks"]["exit"]): # EXIT-hooks rcode = script_run(trun, hook) if rcode: break if trun["conf"]["VERBOSE"]: cij.emph("rnr:trun::exit { rcode: %r }" % rcode, rcode) return rcode
def tsuite_enter(trun, tsuite): """Triggers when entering the given testsuite""" if trun["conf"]["VERBOSE"]: cij.emph("rnr:tsuite:enter { name: %r }" % tsuite["name"]) rcode = 0 for hook in tsuite["hooks"]["enter"]: # ENTER-hooks rcode = script_run(trun, hook) if rcode: break if trun["conf"]["VERBOSE"]: cij.emph("rnr:tsuite:enter { rcode: %r } " % rcode, rcode) return rcode
def tcase_exit(trun, tsuite, tcase): """...""" #pylint: disable=locally-disabled, unused-argument if trun["conf"]["VERBOSE"]: cij.emph("rnr:tcase:exit { fname: %r }" % tcase["fname"]) rcode = 0 for hook in reversed(tcase["hooks"]["exit"]): # tcase EXIT-hooks rcode = script_run(trun, hook) if rcode: break if trun["conf"]["VERBOSE"]: cij.emph("rnr:tcase:exit { rcode: %r }" % rcode, rcode) return rcode
def remove(): """Remove LNVM device""" if env(): cij.err("cij.lnvm.create: Invalid LNVM ENV") return 1 lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED) cij.emph("lnvm.remove: LNVM_DEV_NAME: %s" % lnvm["DEV_NAME"]) cmd = ["nvme lnvm remove -n %s" % (lnvm["DEV_NAME"])] rcode, _, _ = cij.ssh.command(cmd, shell=True) if rcode: cij.err("cij.lnvm.remove: FAILED") return 1 return 0
def trun_enter(trun): """Triggers when entering the given testrun""" if trun["conf"]["VERBOSE"]: cij.emph("rnr:trun::enter") trun["stamp"]["begin"] = int(time.time()) # Record start timestamp rcode = 0 for hook in trun["hooks"]["enter"]: # ENTER-hooks rcode = script_run(trun, hook) if rcode: break if trun["conf"]["VERBOSE"]: cij.emph("rnr:trun::enter { rcode: %r }" % rcode, rcode) return rcode
def recover(): """Recover LNVM device""" if env(): cij.err("cij.lnvm.create: Invalid LNVM ENV") return 1 nvme = cij.env_to_dict("NVME", ["DEV_NAME"]) lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED) cij.emph("lnvm.recover: LNVM_DEV_NAME: %s" % lnvm["DEV_NAME"]) cmd = ["nvme lnvm create -d %s -n %s -t %s -b %s -e %s" % ( nvme["DEV_NAME"], lnvm["DEV_NAME"], lnvm["DEV_TYPE"], lnvm["BGN"], lnvm["END"])] rcode, _, _ = cij.ssh.command(cmd, shell=True) if rcode: cij.err("cij.lnvm.recover: FAILED") return 1 return 0
def enter(self, trun): """Called by runner before invoking the runnable""" if trun.args.verbose: cij.emph("rnr:enter { ident: %r }" % self.ident) self.stamp["begin"] = time.time() rcode = 0 for hook in self.hooks["enter"]: # ENTER-hooks rcode = script_run(trun, hook) if rcode: break self.entered = not rcode if trun.args.verbose: cij.emph("rnr:enter { ident: %r, rcode: %r } " % (self.ident, rcode)) return rcode
def main(args): """Main entry point""" trun = cij.runner.trun_from_file(args.trun_fpath) # pylint: disable=no-member rehome(trun.args.output, args.output, trun) postprocess(trun) cij.emph("main: reports are uses tmpl_fpath: %r" % args.tmpl_fpath) cij.emph("main: reports are here args.output: %r" % args.output) _, ext = os.path.splitext(args.tmpl_fpath) html_fpath = os.path.join(args.output, "".join([args.tmpl_name, ext])) cij.emph("html_fpath: %r" % html_fpath) try: # Create and store HTML report with open(html_fpath, 'w', encoding="UTF-8") as html_file: html_file.write(dset_to_html(trun, args.tmpl_fpath)) except (IOError, OSError, ValueError) as exc: traceback.print_exc() cij.err("rprtr:main: exc: %s" % exc) return 1 return 0
def exit(self, trun): """Called by runner after invoking the runnable""" if trun.args.verbose: cij.emph("rnr:exit: { ident: %r }" % self.ident) rcode = 0 if self.entered: for hook in reversed(self.hooks["exit"]): # EXIT-hooks rcode = script_run(trun, hook) if rcode: break self.stamp["end"] = time.time() if self.wallc is None: self.wallc = self.stamp["end"] - self.stamp["begin"] if trun.args.verbose: cij.emph("rnr:exit: { ident: %r, rcode: %r }" % (self.ident, rcode)) return 0
def tcase_enter(trun, tsuite, tcase): """ setup res_root and aux_root, log info and run tcase-enter-hooks @returns 0 when all hooks succeed, some value othervise """ #pylint: disable=locally-disabled, unused-argument if trun["conf"]["VERBOSE"]: cij.emph("rnr:tcase:enter") cij.emph("rnr:tcase:enter { fname: %r }" % tcase["fname"]) cij.emph("rnr:tcase:enter { log_fpath: %r }" % tcase["log_fpath"]) rcode = 0 for hook in tcase["hooks"]["enter"]: # tcase ENTER-hooks rcode = script_run(trun, hook) if rcode: break if trun["conf"]["VERBOSE"]: cij.emph("rnr:tcase:exit: { rcode: %r }" % rcode, rcode) return rcode
def main(args): """Main entry point""" trun = cij.runner.trun_from_file(args.trun_fpath) rehome(trun["conf"]["OUTPUT"], args.output, trun) postprocess(trun) cij.emph("main: reports are uses tmpl_fpath: %r" % args.tmpl_fpath) cij.emph("main: reports are here args.output: %r" % args.output) html_fpath = os.sep.join([args.output, "%s.html" % args.tmpl_name]) cij.emph("html_fpath: %r" % html_fpath) try: # Create and store HTML report with open(html_fpath, 'w') as html_file: html_file.write(dset_to_html(trun, args.tmpl_fpath)) except (IOError, OSError, ValueError) as exc: import traceback traceback.print_exc() cij.err("rprtr:main: exc: %s" % exc) return 1 return 0
def trun_emph(trun: TestRun): """Print essential info on""" if trun.args.verbose > 1: # Print environment variables cij.emph("rnr:conf {") conf_dict = dataclasses.asdict(trun.conf) for var in sorted(conf_dict.keys()): cij.emph(" % 16s: %r" % (var, conf_dict[var])) cij.emph("}") cij.emph("rnr:args {") args_dict = vars(trun.args) for var in sorted(args_dict.keys()): cij.emph(" % 16s: %r" % (var, args_dict[var])) cij.emph("}") if trun.args.verbose: cij.emph("rnr:INFO {") cij.emph(" output: %r" % trun.args.output) cij.emph(" yml_fpath: %r" % yml_fpath(trun.args.output)) cij.emph("}")
def main(args, conf): """CIJ Test Runner main entry point""" # pylint: disable=too-many-branches # pylint: disable=too-many-statements # There are a lot of branches and statements here... but that is fine. fpath = yml_fpath(args.output) if os.path.exists(fpath): # YAML exists, we exit, it might be RUNNING! cij.err("main:FAILED { fpath: %r }, exists" % fpath) return 1 trun: TestRun try: trun = trun_setup(args, conf) # Construct 'trun' from args and conf except CIJError as ex: cij.err("main:FAILED to start testrun: %s" % ex) trun_to_file(trun) # Persist trun trun_to_junitfile(trun) # Persist as jUNIT XML trun.enter(trun) for tplan in (tp for tp in trun.testplans if trun.entered): tplan.enter(trun) for tsuite in (ts for ts in tplan.testsuites if tplan.entered): tsuite.enter(trun) for tcase in (tc for tc in tsuite.testcases if tsuite.entered): if (args.testcase_match is None or args.testcase_match in tcase.name): tcase.enter(trun) if tcase.entered: script_run(trun, tcase) tcase.exit(trun) tcase.status = Status.Pass if tcase.rcode is None or tcase.rcode: tcase.status = Status.Fail cij.err(f"main:{tcase.ident} failed.") cij.err(f"See log: `less {tcase.log_fpath}`") tsuite.progress[Status.Unkn] -= 1 # Update progress tplan.progress[Status.Unkn] -= 1 trun.progress[Status.Unkn] -= 1 tsuite.progress[tcase.status] += 1 # Update progress tplan.progress[tcase.status] += 1 trun.progress[tcase.status] += 1 if tcase.status == Status.Fail: # Propagate failure trun.status = tplan.status = tsuite.status = tcase.status trun_to_file(trun) # Persist trun trun_to_junitfile(trun) # Persist as jUNIT XML if tsuite.exit(trun) or not tsuite.entered: trun.status = tplan.status = tsuite.status = Status.Fail if tsuite.status == Status.Unkn and tsuite.entered: tsuite.status = Status.Pass trun_to_file(trun) # Persist trun trun_to_junitfile(trun) # Persist as jUNIT XML if tplan.exit(trun) or not tplan.entered: trun.status = tplan.status = Status.Fail if tplan.status == Status.Unkn and tplan.entered: tplan.status = Status.Pass trun_to_file(trun) # Persist trun trun_to_junitfile(trun) # Persist as jUNIT XML if trun.exit(trun) or not trun.entered: trun.status = Status.Fail if trun.status == Status.Unkn and trun.entered: trun.status = Status.Pass trun_to_file(trun) # PERSIST trun_to_junitfile(trun) # Persist as jUNIT XML cij.emph("rnr:main:progress %r" % trun.progress) cij.emph("rnr:main:trun %r" % trun.status, trun.status != Status.Pass) return trun.progress[Status.Fail]
def main(conf): """CIJ Test Runner main entry point""" fpath = yml_fpath(conf["OUTPUT"]) if os.path.exists(fpath): # YAML exists, we exit, it might be RUNNING! cij.err("main:FAILED { fpath: %r }, exists" % fpath) return 1 trun = trun_setup(conf) # Construct 'trun' from 'conf' if not trun: return 1 trun_to_file(trun) # Persist trun trun_to_junitfile(trun) # Persist as jUNIT XML trun_emph(trun) # Print trun before run tr_err = 0 tr_ent_err = trun_enter(trun) for tsuite in (ts for ts in trun["testsuites"] if not tr_ent_err): ts_err = 0 ts_ent_err = tsuite_enter(trun, tsuite) for tcase in (tc for tc in tsuite["testcases"] if not ts_ent_err): tc_err = tcase_enter(trun, tsuite, tcase) if not tc_err: tc_err += script_run(trun, tcase) tc_err += tcase_exit(trun, tsuite, tcase) tcase["status"] = "FAIL" if tc_err else "PASS" trun["progress"][tcase["status"]] += 1 # Update progress trun["progress"]["UNKN"] -= 1 ts_err += tc_err # Accumulate errors trun_to_file(trun) # Persist trun trun_to_junitfile(trun) # Persist as jUNIT XML if not ts_ent_err: ts_err += tsuite_exit(trun, tsuite) ts_err += ts_ent_err # Accumulate errors tr_err += ts_err tsuite["status"] = "FAIL" if ts_err else "PASS" cij.emph("rnr:tsuite %r" % tsuite["status"], tsuite["status"] != "PASS") if not tr_ent_err: trun_exit(trun) tr_err += tr_ent_err trun["status"] = "FAIL" if tr_err else "PASS" trun["stamp"]["end"] = int(time.time()) + 1 # END STAMP trun_to_file(trun) # PERSIST trun_to_junitfile(trun) # Persist as jUNIT XML cij.emph("rnr:main:progress %r" % trun["progress"]) cij.emph("rnr:main:trun %r" % trun["status"], trun["status"] != "PASS") return trun["progress"]["UNKN"] + trun["progress"]["FAIL"]
def trun_emph(trun): """Print essential info on""" if trun["conf"]["VERBOSE"] > 1: # Print environment variables cij.emph("rnr:CONF {") for cvar in sorted(trun["conf"].keys()): cij.emph(" % 16s: %r" % (cvar, trun["conf"][cvar])) cij.emph("}") if trun["conf"]["VERBOSE"]: cij.emph("rnr:INFO {") cij.emph(" OUTPUT: %r" % trun["conf"]["OUTPUT"]) cij.emph(" yml_fpath: %r" % yml_fpath(trun["conf"]["OUTPUT"])) cij.emph("}")