def __init__(self, flush=False):
"""Initialises the test runner"""
# Thread.__init__(self)
logger.log(
TermColor.colorText("NIGHTMARE I", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("s of ", TermColor.White) +
TermColor.colorText("G", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("enerous ", TermColor.White) +
TermColor.colorText("H", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("elp when ", TermColor.White) +
TermColor.colorText("T", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("esting; ", TermColor.White) +
TermColor.colorText("M", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("ay ", TermColor.White) +
TermColor.colorText("A", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("rnold be ", TermColor.White) +
TermColor.colorText("R", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("emembered ", TermColor.White) +
TermColor.colorText("E", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("ternally", TermColor.White)
)
logger.log("Welcome to nightmare Version {}".format(version.Version))
if flush:
logger.flush(quiet=False)
self.options = dict()
self.testCount = 0
self.runsuite = None
self.finished = None
def loadSuite(self, fname=None):
"""Loads a python based suite from a file"""
if fname is not None:
self.options['bench'] = fname
if self.options['bench'] is not None and self.options['bench'] != "" and os.path.exists(self.options['bench']):
logger.log("\nReading testfile ...")
if self.options['relative']:
os.chdir(os.path.dirname(os.path.abspath(self.options['bench'])))
logger.log("Current Working Dir is: {}".format(os.getcwd()))
self.options['bench'] = os.path.basename(self.options['bench'])
if self.options['arnold']:
self.runsuite = self.loadArnold()
else:
self.runsuite = self.loadPython()
if self.runsuite is not None:
self.runsuite.options['commands'] = self.options['commands']
self.runsuite.setAll(
state=TestState.InfoOnly if self.options['info'] else TestState.Waiting,
pipe=self.options['pipe'],
out=self.options['output'],
diff=self.options['diff'],
timeout=self.options['timeout'],
linesep=self.options['linesep'],
ignoreEmptyLines=self.options['ignoreEmptyLines']
)
self.testCount = len(self.runsuite.testList)
logger.log("I have loaded {} Testcase{}".format(self.testCount, "s" if self.testCount > 0 else ""))
else:
logger.log("Sorry, but I failed to load the requested suite")
else:
logger.log("Sorry, but I couldn't find the file '{}'".format(self.options['bench']))
logger.flush(self.options['quiet'])
return self.runsuite
def run(self, quiet=False, tests=[]):
"""
Runs the whole suite of tests
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
self.success = 0
self.failed = 0
self.count = 0
self.error = 0
self.lastResult = TestState.Waiting
for t in self._getTests(tests):
self.lastResult = t.run()
if t.descr is not None:
logger.log("{}[{: 03}] {} - {}: {}".format(
TermColor.colorText("Test", TermColor.Purple), self.count,
t.name, t.descr, TestState.toString(t.state)))
else:
logger.log("{}[{: 03}] {}: {}".format(
TermColor.colorText("Test", TermColor.Purple), self.count,
t.name, TestState.toString(t.state)))
if self.options['commands']:
logger.log(" --> {}".format(t.cmd), showTime=False)
logger.flush(quiet)
if self.lastResult in [TestState.Success, TestState.Clean]:
self.success += 1
elif self.lastResult == TestState.Fail:
self.failed += 1
elif self.lastResult == TestState.Error:
self.error += 1
elif self.lastResult == TestState.Timeout:
self.timedout += 1
elif self.lastResult == TestState.SegFault:
self.segfaults += 1
elif self.lastResult == TestState.Assertion:
self.assertions += 1
self.count = self.count + 1
yield t
if self.lastResult != TestState.Disabled:
if (self.mode == TestSuiteMode.BreakOnFail) and (
self.lastResult != TestState.Success):
break
if (self.mode == TestSuiteMode.BreakOnError) and (
self.lastResult == TestState.Error):
break
raise StopIteration()
def run(self):
"""Thread run function"""
if self.options['length']:
print len(self.runsuite.getTests())
elif len(self.options['save']) == 1:
logger.log("Saving Suite to {}".format(self.options['save'][0]))
self.saveToFile(self.options['save'][0])
else:
logger.flush(self.options['quiet'])
self.runsuite.setMode(self.options['mode'])
for test in self.runsuite.run(self.options['quiet'], tests=self.options['test']):
yield test
self.runsuite.stats(self.options['quiet'])
if self.finished is not None:
self.finished()
logger.flush(self.options['quiet'])
raise StopIteration()
def run(self, quiet=False, tests=[]):
"""
Runs the whole suite of tests
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
self.success = 0
self.failed = 0
self.count = 0
self.error = 0
self.lastResult = TestState.Waiting
for t in self._getTests(tests):
self.lastResult = t.run()
if t.descr is not None:
logger.log("{}[{: 03}] {} - {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, t.descr, TestState.toString(t.state)))
else:
logger.log("{}[{: 03}] {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, TestState.toString(t.state)))
if self.options['commands']:
logger.log(" --> {}".format(t.cmd), showTime=False)
logger.flush(quiet)
if self.lastResult in [TestState.Success, TestState.Clean]:
self.success += 1
elif self.lastResult == TestState.Fail:
self.failed += 1
elif self.lastResult == TestState.Error:
self.error += 1
elif self.lastResult == TestState.Timeout:
self.timedout += 1
elif self.lastResult == TestState.SegFault:
self.segfaults += 1
elif self.lastResult == TestState.Assertion:
self.assertions += 1
self.count = self.count + 1
yield t
if self.lastResult != TestState.Disabled:
if (self.mode == TestSuiteMode.BreakOnFail) and (self.lastResult != TestState.Success):
break
if (self.mode == TestSuiteMode.BreakOnError) and (self.lastResult == TestState.Error):
break
raise StopIteration()
def loadSuite(self, fname = None):
"""Loads a python based suite from a file"""
if fname is not None:
self.file = fname
logger.log("\nReading testfile ...")
if self.file is not None and self.file != "" and os.path.exists(self.file):
glb = {"__builtins__":__builtins__, "math":math, "Test":Test, "Suite":TestSuite, "Mode":TestSuiteMode, "State":TestState}
ctx = {self.suite:None, "DUT":None}
execfile(self.file, glb, ctx)
if (self.suite in ctx):
self.runsuite = None
if (ctx[self.suite] != None):
if ctx[self.suite].__class__ == TestSuite:
self.runsuite = ctx[self.suite]
self.runsuite.setDUT(self.DUT)
if self.mode is None:
self.mode =self.runsuite.mode
elif self.runsuite.mode is None:
self.runsuite.mode = self.mode
else:
self.runsuite = TestSuite(*ctx[self.suite], **{'DUT':self.DUT, 'mode':self.mode})
self.runsuite.setAll(
state=TestState.InfoOnly if self.infoOnly else TestState.Waiting,
pipe=self.pipe,
out=self.out,
timeout = self.timeout,
linesep = self.linesep
)
self.testCount = len(self.runsuite.testList)
if 'DUT' in ctx and ctx['DUT'] is not None and self.DUT is None:
self.setDUT(ctx['DUT'])
logger.log("I could load {} Testcase".format(self.testCount))
else:
logger.log("Sorry, but I can't find any tests inside the suite '{}'".format(self.suite))
else:
logger.log("Sorry, but there was no test-suite in the file")
else:
logger.log("Sorry, but I couldn't find the file '{}'".format(self.file))
logger.flush(self.quiet)
return self.runsuite
def runAll(self, quiet = False):
"""
Runs the whole suite of tests
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
self.success = 0
self.failed = 0
self.count = 0
self.error = 0
self.lastResult = TestState.Waiting
for t in self.testList:
self.count = self.count + 1
self.lastResult = t.run()
if t.descr is not None:
logger.log("Test[{:02}] {} - {}: {}".format(self.count, t.name, t.descr, TestState.toString(t.state)))
else:
logger.log("Test[{:02}] {}: {}".format(self.count, t.name, TestState.toString(t.state)))
logger.flush(quiet)
if self.lastResult == TestState.Success:
self.success += 1
elif self.lastResult == TestState.Fail:
self.failed += 1
elif self.lastResult == TestState.Error:
self.error += 1
elif self.lastResult == TestState.Timeout:
self.timedout += 1
elif self.lastResult == TestState.SegFault:
self.segfaults += 1
elif self.lastResult == TestState.Assertion:
self.assertions += 1
yield t
if self.lastResult != TestState.Disabled:
if (self.mode == TestSuiteMode.BreakOnFail) and (self.lastResult != TestState.Success):
break
if (self.mode == TestSuiteMode.BreakOnError) and (self.lastResult == TestState.Error):
break
raise StopIteration()
def run(self): """Thread run function""" if self.lengthOnly: print len(self.runsuite.getTests()) else: logger.flush(self.quiet) self.runsuite.setMode(self.mode) if (self.test == -1): for test in self.runsuite.runAll(self.quiet): yield test self.runsuite.stats(self.quiet) logger.flush(self.quiet) else: self.runsuite.runOne(self.test) logger.flush(self.quiet) if self.finished != None: self.finished() #Thread.__init__(self) # This looks like a real dirty hack :/ raise StopIteration()
def parseArgv(self):
"""Parses the argument vector"""
args = argparse.ArgumentParser(description="A test tool for non-interactive commandline programms")
group = args.add_argument_group("Test selection")
group.add_argument("--bench", action="store", nargs=1, help="File which contains the testbench.")
group.add_argument("--suite", action="store", nargs=1, help="Use testsuite SUITE from the testbench.", metavar="SUITE")
group.add_argument("--dut", "--DUT", action="store", nargs=1, help="Set the device under test.")
group.add_argument("--test", action="store", nargs="+", type=int, help="Run only the specified tests")
group.add_argument("--timeout", action="store", nargs=1, type=float, help="Set a global timeout for all tests.")
group.add_argument("--arnold", "-a", action="store_true", default=False, dest="arnold", help="Use the arnold mode (requires pyparsing module)")
group.add_argument("--save", action="store", nargs=1, help="Save the testsuite as FILE", metavar="FILE")
group = args.add_argument_group("Output Control")
group.add_argument("--limit", action="store", nargs=1, type=int, default=2000, help="Set a (soft) limit for a number of Bytes, after which output piping will we stopped. Checks are made after each line.")
group.add_argument("--quiet", "-q", action="store_const", const=True, default=False, dest="quiet", help="Quiet mode. There will be no output except results.")
group.add_argument("--verbose", "-v", action="store_const", const=False, dest="quiet", help="Verbose mode. The program gets chatty (default).")
group.add_argument("--commands", "-C", action="store_true", default=False, dest="commands", help="Show the command executed for each test.")
group.add_argument("--length", "-l", action="store_true", default=False, dest="length", help="Print only the number of tests in the suite.")
group.add_argument("--info-only", "-i", action="store_true", default=False, dest="info", help="Display only test information, but don't run them.")
group.add_argument("--pipe-streams", "-p", action="store_true", default=None, dest="pipe", help="Redirect DUT output to their respective streams.")
group.add_argument("--output-fails", "-o", action="store_true", default=None, dest="output", help="Redirect DUT output from failed tests to their respective streams.")
group.add_argument("--unify-fails", "-u", action="store_true", default=None, dest="diff", help="Display the unified diff of output and expectation.")
group.add_argument("--no-color", action="store_false", default=True, dest="color", help="Don't use any colored output.")
group = args.add_argument_group("Test Flow")
group.add_argument("--continue", "-c", action="store_const", const=TestSuiteMode.Continuous, dest="mode", help="Continuous mode (Don't halt on failed tests).")
group.add_argument("--error", "-e", action="store_const", const=TestSuiteMode.BreakOnError, dest="mode", help="Same as '-c', but will halt if an error occurs.")
group.add_argument("--ignoreEmptyLines", "-L", action="store_true", default=None, dest="ignoreEmptyLines", help="Ignore empty lines")
group.add_argument("--relative", "-r", action="store_true", default=False, dest="relative", help="Use a path relative to the testbench path.")
group.add_argument("--cr", action="store_const", const="\r", dest="linesep", help="Force the line separation character (Mac OS).")
group.add_argument("--ln", action="store_const", const="\n", dest="linesep", help="Force the line separation character (Unix / Mac OS-X).")
group.add_argument("--crln", action="store_const", const="\r\n", dest="linesep", help="Force the line separation character (Windows).")
args.add_argument("--gui", action="store_true", default=False, dest="gui", help="Use the GUI (experimental and unstable).")
args.add_argument("--no-gui", action="store_true", default=False, dest="gui", help="Don't use the GUI.")
args.add_argument("--version", action="store_const", const=True, default=False, help="Display version information")
args.set_defaults(linesep=os.linesep, bench=[""], save=[], suite=["suite"], dut=[None], timeout=[None], test=[])
self.options.update(vars(args.parse_args()))
self.options['bench'] = self.options['bench'][0]
self.options['suite'] = self.options['suite'][0]
self.options['dut'] = self.options['dut'][0]
self.options['timeout'] = self.options['timeout'][0]
logMessages = [
('mode', lambda v: "I'm running in continuous mode now"
if v == TestSuiteMode.Continuous
else "I'm running in continuous mode now, but will halt if an error occurs"
if v == TestSuiteMode.BreakOnError
else "I will halt on first fail."),
('suite', lambda v: "I'm using the testsuite '{}'".format(v)),
('test', lambda v: "I'm only running test {}".format(v) if len(v) > 0 else ""),
('bench', lambda v: "I'm using testbench '{}'".format(v)),
('timeout', lambda v: "Setting global timeout to {}".format(v)),
('dut', lambda v: "Device under Test is: {}".format(v)),
('commands', lambda v: "I will print every command I'll exceute." if v else ""),
('length', lambda v: "I will only print the number of tests" if v else ""),
('info', lambda v: "I will only print the test information." if v else ""),
('pipe', lambda v: "I will pipe all tests outputs to their respective streams" if v else ""),
('output', lambda v: "I will pipe failed tests outputs to their respective streams" if v else ""),
('diff', lambda v: "I will show the differences in output and expectations" if v else ""),
]
for option, msgFunc in logMessages:
if self.options[option] is not None:
msg = msgFunc(self.options[option])
if len(msg) > 0:
logger.log("\t{}".format(msg))
logger.flush(self.options['quiet'])
