def __init__(self, flush=False): """Initialises the test runner""" # Thread.__init__(self) logger.log( TermColor.colorText("NIGHTMARE I", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("s of ", TermColor.White) + TermColor.colorText("G", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("enerous ", TermColor.White) + TermColor.colorText("H", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("elp when ", TermColor.White) + TermColor.colorText("T", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("esting; ", TermColor.White) + TermColor.colorText("M", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("ay ", TermColor.White) + TermColor.colorText("A", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("rnold be ", TermColor.White) + TermColor.colorText("R", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("emembered ", TermColor.White) + TermColor.colorText("E", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("ternally", TermColor.White) ) logger.log("Welcome to nightmare Version {}".format(version.Version)) if flush: logger.flush(quiet=False) self.options = dict() self.testCount = 0 self.runsuite = None self.finished = None
def loadSuite(self, fname=None): """Loads a python based suite from a file""" if fname is not None: self.options['bench'] = fname if self.options['bench'] is not None and self.options['bench'] != "" and os.path.exists(self.options['bench']): logger.log("\nReading testfile ...") if self.options['relative']: os.chdir(os.path.dirname(os.path.abspath(self.options['bench']))) logger.log("Current Working Dir is: {}".format(os.getcwd())) self.options['bench'] = os.path.basename(self.options['bench']) if self.options['arnold']: self.runsuite = self.loadArnold() else: self.runsuite = self.loadPython() if self.runsuite is not None: self.runsuite.options['commands'] = self.options['commands'] self.runsuite.setAll( state=TestState.InfoOnly if self.options['info'] else TestState.Waiting, pipe=self.options['pipe'], out=self.options['output'], diff=self.options['diff'], timeout=self.options['timeout'], linesep=self.options['linesep'], ignoreEmptyLines=self.options['ignoreEmptyLines'] ) self.testCount = len(self.runsuite.testList) logger.log("I have loaded {} Testcase{}".format(self.testCount, "s" if self.testCount > 0 else "")) else: logger.log("Sorry, but I failed to load the requested suite") else: logger.log("Sorry, but I couldn't find the file '{}'".format(self.options['bench'])) logger.flush(self.options['quiet']) return self.runsuite
def run(self, quiet=False, tests=[]): """ Runs the whole suite of tests @type quiet: Boolean @param quiet: Flag, passed along to the logger """ self.success = 0 self.failed = 0 self.count = 0 self.error = 0 self.lastResult = TestState.Waiting for t in self._getTests(tests): self.lastResult = t.run() if t.descr is not None: logger.log("{}[{: 03}] {} - {}: {}".format( TermColor.colorText("Test", TermColor.Purple), self.count, t.name, t.descr, TestState.toString(t.state))) else: logger.log("{}[{: 03}] {}: {}".format( TermColor.colorText("Test", TermColor.Purple), self.count, t.name, TestState.toString(t.state))) if self.options['commands']: logger.log(" --> {}".format(t.cmd), showTime=False) logger.flush(quiet) if self.lastResult in [TestState.Success, TestState.Clean]: self.success += 1 elif self.lastResult == TestState.Fail: self.failed += 1 elif self.lastResult == TestState.Error: self.error += 1 elif self.lastResult == TestState.Timeout: self.timedout += 1 elif self.lastResult == TestState.SegFault: self.segfaults += 1 elif self.lastResult == TestState.Assertion: self.assertions += 1 self.count = self.count + 1 yield t if self.lastResult != TestState.Disabled: if (self.mode == TestSuiteMode.BreakOnFail) and ( self.lastResult != TestState.Success): break if (self.mode == TestSuiteMode.BreakOnError) and ( self.lastResult == TestState.Error): break raise StopIteration()
def run(self): """Thread run function""" if self.options['length']: print len(self.runsuite.getTests()) elif len(self.options['save']) == 1: logger.log("Saving Suite to {}".format(self.options['save'][0])) self.saveToFile(self.options['save'][0]) else: logger.flush(self.options['quiet']) self.runsuite.setMode(self.options['mode']) for test in self.runsuite.run(self.options['quiet'], tests=self.options['test']): yield test self.runsuite.stats(self.options['quiet']) if self.finished is not None: self.finished() logger.flush(self.options['quiet']) raise StopIteration()
def run(self, quiet=False, tests=[]): """ Runs the whole suite of tests @type quiet: Boolean @param quiet: Flag, passed along to the logger """ self.success = 0 self.failed = 0 self.count = 0 self.error = 0 self.lastResult = TestState.Waiting for t in self._getTests(tests): self.lastResult = t.run() if t.descr is not None: logger.log("{}[{: 03}] {} - {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, t.descr, TestState.toString(t.state))) else: logger.log("{}[{: 03}] {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, TestState.toString(t.state))) if self.options['commands']: logger.log(" --> {}".format(t.cmd), showTime=False) logger.flush(quiet) if self.lastResult in [TestState.Success, TestState.Clean]: self.success += 1 elif self.lastResult == TestState.Fail: self.failed += 1 elif self.lastResult == TestState.Error: self.error += 1 elif self.lastResult == TestState.Timeout: self.timedout += 1 elif self.lastResult == TestState.SegFault: self.segfaults += 1 elif self.lastResult == TestState.Assertion: self.assertions += 1 self.count = self.count + 1 yield t if self.lastResult != TestState.Disabled: if (self.mode == TestSuiteMode.BreakOnFail) and (self.lastResult != TestState.Success): break if (self.mode == TestSuiteMode.BreakOnError) and (self.lastResult == TestState.Error): break raise StopIteration()
def loadSuite(self, fname = None): """Loads a python based suite from a file""" if fname is not None: self.file = fname logger.log("\nReading testfile ...") if self.file is not None and self.file != "" and os.path.exists(self.file): glb = {"__builtins__":__builtins__, "math":math, "Test":Test, "Suite":TestSuite, "Mode":TestSuiteMode, "State":TestState} ctx = {self.suite:None, "DUT":None} execfile(self.file, glb, ctx) if (self.suite in ctx): self.runsuite = None if (ctx[self.suite] != None): if ctx[self.suite].__class__ == TestSuite: self.runsuite = ctx[self.suite] self.runsuite.setDUT(self.DUT) if self.mode is None: self.mode =self.runsuite.mode elif self.runsuite.mode is None: self.runsuite.mode = self.mode else: self.runsuite = TestSuite(*ctx[self.suite], **{'DUT':self.DUT, 'mode':self.mode}) self.runsuite.setAll( state=TestState.InfoOnly if self.infoOnly else TestState.Waiting, pipe=self.pipe, out=self.out, timeout = self.timeout, linesep = self.linesep ) self.testCount = len(self.runsuite.testList) if 'DUT' in ctx and ctx['DUT'] is not None and self.DUT is None: self.setDUT(ctx['DUT']) logger.log("I could load {} Testcase".format(self.testCount)) else: logger.log("Sorry, but I can't find any tests inside the suite '{}'".format(self.suite)) else: logger.log("Sorry, but there was no test-suite in the file") else: logger.log("Sorry, but I couldn't find the file '{}'".format(self.file)) logger.flush(self.quiet) return self.runsuite
def runAll(self, quiet = False): """ Runs the whole suite of tests @type quiet: Boolean @param quiet: Flag, passed along to the logger """ self.success = 0 self.failed = 0 self.count = 0 self.error = 0 self.lastResult = TestState.Waiting for t in self.testList: self.count = self.count + 1 self.lastResult = t.run() if t.descr is not None: logger.log("Test[{:02}] {} - {}: {}".format(self.count, t.name, t.descr, TestState.toString(t.state))) else: logger.log("Test[{:02}] {}: {}".format(self.count, t.name, TestState.toString(t.state))) logger.flush(quiet) if self.lastResult == TestState.Success: self.success += 1 elif self.lastResult == TestState.Fail: self.failed += 1 elif self.lastResult == TestState.Error: self.error += 1 elif self.lastResult == TestState.Timeout: self.timedout += 1 elif self.lastResult == TestState.SegFault: self.segfaults += 1 elif self.lastResult == TestState.Assertion: self.assertions += 1 yield t if self.lastResult != TestState.Disabled: if (self.mode == TestSuiteMode.BreakOnFail) and (self.lastResult != TestState.Success): break if (self.mode == TestSuiteMode.BreakOnError) and (self.lastResult == TestState.Error): break raise StopIteration()
def run(self): """Thread run function""" if self.lengthOnly: print len(self.runsuite.getTests()) else: logger.flush(self.quiet) self.runsuite.setMode(self.mode) if (self.test == -1): for test in self.runsuite.runAll(self.quiet): yield test self.runsuite.stats(self.quiet) logger.flush(self.quiet) else: self.runsuite.runOne(self.test) logger.flush(self.quiet) if self.finished != None: self.finished() #Thread.__init__(self) # This looks like a real dirty hack :/ raise StopIteration()
def parseArgv(self): """Parses the argument vector""" args = argparse.ArgumentParser(description="A test tool for non-interactive commandline programms") group = args.add_argument_group("Test selection") group.add_argument("--bench", action="store", nargs=1, help="File which contains the testbench.") group.add_argument("--suite", action="store", nargs=1, help="Use testsuite SUITE from the testbench.", metavar="SUITE") group.add_argument("--dut", "--DUT", action="store", nargs=1, help="Set the device under test.") group.add_argument("--test", action="store", nargs="+", type=int, help="Run only the specified tests") group.add_argument("--timeout", action="store", nargs=1, type=float, help="Set a global timeout for all tests.") group.add_argument("--arnold", "-a", action="store_true", default=False, dest="arnold", help="Use the arnold mode (requires pyparsing module)") group.add_argument("--save", action="store", nargs=1, help="Save the testsuite as FILE", metavar="FILE") group = args.add_argument_group("Output Control") group.add_argument("--limit", action="store", nargs=1, type=int, default=2000, help="Set a (soft) limit for a number of Bytes, after which output piping will we stopped. Checks are made after each line.") group.add_argument("--quiet", "-q", action="store_const", const=True, default=False, dest="quiet", help="Quiet mode. There will be no output except results.") group.add_argument("--verbose", "-v", action="store_const", const=False, dest="quiet", help="Verbose mode. The program gets chatty (default).") group.add_argument("--commands", "-C", action="store_true", default=False, dest="commands", help="Show the command executed for each test.") group.add_argument("--length", "-l", action="store_true", default=False, dest="length", help="Print only the number of tests in the suite.") group.add_argument("--info-only", "-i", action="store_true", default=False, dest="info", help="Display only test information, but don't run them.") group.add_argument("--pipe-streams", "-p", action="store_true", default=None, dest="pipe", help="Redirect DUT output to their respective streams.") group.add_argument("--output-fails", "-o", action="store_true", default=None, dest="output", help="Redirect DUT output from failed tests to their respective streams.") group.add_argument("--unify-fails", "-u", action="store_true", default=None, dest="diff", help="Display the unified diff of output and expectation.") group.add_argument("--no-color", action="store_false", default=True, dest="color", help="Don't use any colored output.") group = args.add_argument_group("Test Flow") group.add_argument("--continue", "-c", action="store_const", const=TestSuiteMode.Continuous, dest="mode", help="Continuous mode (Don't halt on failed tests).") group.add_argument("--error", "-e", action="store_const", const=TestSuiteMode.BreakOnError, dest="mode", help="Same as '-c', but will halt if an error occurs.") group.add_argument("--ignoreEmptyLines", "-L", action="store_true", default=None, dest="ignoreEmptyLines", help="Ignore empty lines") group.add_argument("--relative", "-r", action="store_true", default=False, dest="relative", help="Use a path relative to the testbench path.") group.add_argument("--cr", action="store_const", const="\r", dest="linesep", help="Force the line separation character (Mac OS).") group.add_argument("--ln", action="store_const", const="\n", dest="linesep", help="Force the line separation character (Unix / Mac OS-X).") group.add_argument("--crln", action="store_const", const="\r\n", dest="linesep", help="Force the line separation character (Windows).") args.add_argument("--gui", action="store_true", default=False, dest="gui", help="Use the GUI (experimental and unstable).") args.add_argument("--no-gui", action="store_true", default=False, dest="gui", help="Don't use the GUI.") args.add_argument("--version", action="store_const", const=True, default=False, help="Display version information") args.set_defaults(linesep=os.linesep, bench=[""], save=[], suite=["suite"], dut=[None], timeout=[None], test=[]) self.options.update(vars(args.parse_args())) self.options['bench'] = self.options['bench'][0] self.options['suite'] = self.options['suite'][0] self.options['dut'] = self.options['dut'][0] self.options['timeout'] = self.options['timeout'][0] logMessages = [ ('mode', lambda v: "I'm running in continuous mode now" if v == TestSuiteMode.Continuous else "I'm running in continuous mode now, but will halt if an error occurs" if v == TestSuiteMode.BreakOnError else "I will halt on first fail."), ('suite', lambda v: "I'm using the testsuite '{}'".format(v)), ('test', lambda v: "I'm only running test {}".format(v) if len(v) > 0 else ""), ('bench', lambda v: "I'm using testbench '{}'".format(v)), ('timeout', lambda v: "Setting global timeout to {}".format(v)), ('dut', lambda v: "Device under Test is: {}".format(v)), ('commands', lambda v: "I will print every command I'll exceute." if v else ""), ('length', lambda v: "I will only print the number of tests" if v else ""), ('info', lambda v: "I will only print the test information." if v else ""), ('pipe', lambda v: "I will pipe all tests outputs to their respective streams" if v else ""), ('output', lambda v: "I will pipe failed tests outputs to their respective streams" if v else ""), ('diff', lambda v: "I will show the differences in output and expectations" if v else ""), ] for option, msgFunc in logMessages: if self.options[option] is not None: msg = msgFunc(self.options[option]) if len(msg) > 0: logger.log("\t{}".format(msg)) logger.flush(self.options['quiet'])