def main():
TermColor.init()
# Check whether to load the default testbench
if not argvContains("--bench=") and os.path.isfile("testbench.py"):
sys.argv.append("--bench=testbench.py")
if "-h" in sys.argv or "--help" in sys.argv or "-?" in sys.argv:
printHelp()
if "--no-gui" in sys.argv:
# Capt. Obvious: We're running in console mode
runner = TestRunner()
runner.parseArgv()
suite = runner.loadSuite()
for testcase in runner.run():
pass
if not runner.lengthOnly and not runner.infoOnly and runner.test == -1:
print "{:2.2f}%".format(suite.getRate())
sys.exit(suite.lastResult if suite.lastResult not in [TestState.Waiting,TestState.InfoOnly] else 0)
else:
from pyTestGui import TestRunnerGui
if len(sys.argv) > 1 and not sys.argv[1].startswith("-") and os.path.exists(sys.argv[1]):
sys.argv[1] = '--bench='+sys.argv[1]
gui = TestRunnerGui()
gui.buildWindow()
gui.show()
def pipeOutputStream(self, stream, lines, color):
bytes = 0
for line in lines:
bytes += len(line)
stream.write(TermColor.colorText(line + " ", fg=color) + "\n")
if bytes > self.pipeLimit:
stream.write(
TermColor.colorText("Stopped after {} Bytes".format(bytes),
fg=TermColor.Yellow) + "\n")
break
def lineComparison(self, expLines, outLines, stream=""):
same = True
if self.ignoreEmptyLines:
while expLines.count("") > 0:
expLines.remove("")
while outLines.count("") > 0:
outLines.remove("")
for line in difflib.unified_diff(expLines, outLines, stream,
"expectation"):
col = TermColor.White
uline = unicode(
line, encoding="utf-8",
errors="ignore") if type(line) is not unicode else line
if uline.startswith(" + "):
same = False
col = TermColor.Green
elif uline.startswith("-"):
same = False
col = TermColor.Red
elif uline.startswith("@"):
same = False
col = TermColor.Cyan
if self.diff:
logger.log(TermColor.colorText(uline.rstrip(), col))
return same
def run(self, quiet=False, tests=[]):
"""
Runs the whole suite of tests
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
self.success = 0
self.failed = 0
self.count = 0
self.error = 0
self.lastResult = TestState.Waiting
for t in self._getTests(tests):
self.lastResult = t.run()
if t.descr is not None:
logger.log("{}[{: 03}] {} - {}: {}".format(
TermColor.colorText("Test", TermColor.Purple), self.count,
t.name, t.descr, TestState.toString(t.state)))
else:
logger.log("{}[{: 03}] {}: {}".format(
TermColor.colorText("Test", TermColor.Purple), self.count,
t.name, TestState.toString(t.state)))
if self.options['commands']:
logger.log(" --> {}".format(t.cmd), showTime=False)
logger.flush(quiet)
if self.lastResult in [TestState.Success, TestState.Clean]:
self.success += 1
elif self.lastResult == TestState.Fail:
self.failed += 1
elif self.lastResult == TestState.Error:
self.error += 1
elif self.lastResult == TestState.Timeout:
self.timedout += 1
elif self.lastResult == TestState.SegFault:
self.segfaults += 1
elif self.lastResult == TestState.Assertion:
self.assertions += 1
self.count = self.count + 1
yield t
if self.lastResult != TestState.Disabled:
if (self.mode == TestSuiteMode.BreakOnFail) and (
self.lastResult != TestState.Success):
break
if (self.mode == TestSuiteMode.BreakOnError) and (
self.lastResult == TestState.Error):
break
raise StopIteration()
def run(self, quiet=False, tests=[]):
"""
Runs the whole suite of tests
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
self.success = 0
self.failed = 0
self.count = 0
self.error = 0
self.lastResult = TestState.Waiting
for t in self._getTests(tests):
self.lastResult = t.run()
if t.descr is not None:
logger.log("{}[{: 03}] {} - {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, t.descr, TestState.toString(t.state)))
else:
logger.log("{}[{: 03}] {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, TestState.toString(t.state)))
if self.options['commands']:
logger.log(" --> {}".format(t.cmd), showTime=False)
logger.flush(quiet)
if self.lastResult in [TestState.Success, TestState.Clean]:
self.success += 1
elif self.lastResult == TestState.Fail:
self.failed += 1
elif self.lastResult == TestState.Error:
self.error += 1
elif self.lastResult == TestState.Timeout:
self.timedout += 1
elif self.lastResult == TestState.SegFault:
self.segfaults += 1
elif self.lastResult == TestState.Assertion:
self.assertions += 1
self.count = self.count + 1
yield t
if self.lastResult != TestState.Disabled:
if (self.mode == TestSuiteMode.BreakOnFail) and (self.lastResult != TestState.Success):
break
if (self.mode == TestSuiteMode.BreakOnError) and (self.lastResult == TestState.Error):
break
raise StopIteration()
def stats(self, quiet=False):
"""
Generate and write the stats
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
if (self.lastResult != TestState.InfoOnly):
logger.log("I ran {} out of {} tests in total".format(self.count, len(self.testList)))
fails = self.count - self.success
logger.log(TermColor.colorText("\tSuccess: {}".format(self.success), TermColor.Green))
if (self.failed > 0):
logger.log(TermColor.colorText("\tFailed: {}".format(self.failed), TermColor.Red))
if (self.error > 0):
logger.log(TermColor.colorText("\tErrors: {}".format(self.error), TermColor.Yellow))
if (self.assertions > 0):
logger.log(TermColor.colorText("\tAssertions: {}".format(self.assertions), TermColor.Yellow))
if (self.segfaults > 0):
logger.log(TermColor.colorText("\tSegFaults: {}".format(self.segfaults), TermColor.Yellow))
if (self.timedout > 0):
logger.log(TermColor.colorText("\tTimeouts: {}".format(self.timedout), TermColor.Purple))
# A little bit of fun
if (self.success == len(self) and self.count > 3):
logger.log("\tCongratulations, you passed all tests!")
logger.log("\t`grep` yourself a refreshing " + TermColor.colorText("Beer", TermColor.Yellow, style=TermColor.Bold))
logger.log("")
logger.log(" \033[1;37m,%%%%.\033[0m")
logger.log(" \033[1;37mi\033[36m====\033[1;37mi\033[1;36m_\033[0m")
logger.log(" \033[1;36m|\033[1;33m####\033[36m| |\033[0m")
logger.log(" \033[1;36m|\033[1;33m####\033[36m|-'\033[0m")
logger.log(" \033[1;36m`-==-'\033[0m")
logger.log("")
elif (self.success == 0 and self.count > 3 and self.failed > 0):
logger.log("\tWhat is wrong with you, not even a single test?")
elif fails > 0 and self.count > 3:
if self.assertions / fails > 0.6:
logger.log("\tYou do realise that assertions do not replace error handling?")
elif self.assertions / fails > 0.3:
logger.log("\tWe're a bit lazy with calling environments, aren't we?")
if self.segfaults / fails > 0.6:
logger.log("\tMay the CPU-Gods have mercy with that poor memory management units soul!")
elif self.segfaults / fails > 0.3:
logger.log("\tYou know, memory garbage doesn't collect itself?!")
return self.calcRate()
def runCmd(self, command):
if "$DUT" in command:
if self.DUT is None:
self.state = TestState.Error
return
else:
_cmd = Command(cmd=str(command).replace("$DUT", self.DUT))
else:
_cmd = Command(cmd=str(command))
cmdRet = _cmd.execute(self.timeout)
if cmdRet == TestState.Success:
if self.binary:
self.output = _cmd.out
self.error = _cmd.err
else:
self.output = _cmd.out.decode(encoding="utf8", errors="ignore")
self.error = _cmd.err.decode(encoding="utf8", errors="ignore")
self.retCode = _cmd.ret
if (self.check(self.expectRetCode, self.retCode)
and self.check(self.expectStdout, self.output, "stdout")
and self.check(self.expectStderr, self.error, "stderr")):
self.state = TestState.Success
else:
if 'Assertion' in self.error or 'assertion' in self.error:
self.state = TestState.Assertion
elif "stackdump" in self.error or "coredump" in self.error or "Segmentation Fault" in self.error or self.retCode < 0:
self.state = TestState.SegFault
else:
self.state = TestState.Fail
if (self.pipe) or (self.outputOnFail
and self.state is TestState.Fail):
sys.stdout.write(
TermColor.colorText("{}".format(self.retCode),
fg=TermColor.Yellow) + " ")
self.pipeOutputStream(sys.stdout, self.output.splitlines(),
TermColor.Green)
self.pipeOutputStream(sys.stderr, self.error.splitlines(),
TermColor.Red)
else:
self.state = cmdRet
def stats(self, quiet = False):
"""
Generate and write the stats
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
logger.log("I ran {} out of {} tests in total".format(self.count, len(self.testList)))
logger.log(TermColor.colorText("\tSuccess: {}".format(self.success), TermColor.Green))
if (self.failed > 0):
logger.log(TermColor.colorText("\tFailed: {}".format(self.failed), TermColor.Red))
if (self.error > 0):
logger.log(TermColor.colorText("\tErrors: {}".format(self.error), TermColor.Yellow))
if (self.assertions > 0):
logger.log(TermColor.colorText("\tAssertions: {}".format(self.assertions), TermColor.Yellow))
if (self.segfaults > 0):
logger.log(TermColor.colorText("\tSegFaults: {}".format(self.segfaults), TermColor.Yellow))
if (self.timedout > 0):
logger.log(TermColor.colorText("\tTimeouts: {}".format(self.timedout), TermColor.Purple))
if (self.error == 0) and (self.failed == 0) and (self.timedout == 0):
logger.log("\tCongratulations, you passed all tests!")
return self.calcRate()
def toString(state):
"""
Converts the enumeration value into a string
@type state: int
@param state: Enumeration value
"""
if state == TestState.Waiting:
return TermColor.colorText(" WAITING ", TermColor.White)
if state == TestState.Success:
return TermColor.colorText(" SUCCESS ",
fg=TermColor.Black,
bg=TermColor.Green)
if state == TestState.Fail:
return TermColor.colorText(" FAIL ",
fg=TermColor.Black,
bg=TermColor.Red)
if state == TestState.Error:
return TermColor.colorText(" ERROR ",
fg=TermColor.Black,
bg=TermColor.Red,
style=TermColor.Bold)
if state == TestState.SegFault:
return TermColor.colorText(" SEGFAULT ",
fg=TermColor.Black,
bg=TermColor.Yellow)
if state == TestState.Assertion:
return TermColor.colorText(" ASSERTION ",
fg=TermColor.Black,
bg=TermColor.Yellow,
style=TermColor.Bold)
if state == TestState.InfoOnly:
return TermColor.colorText(" INFO ", TermColor.White)
if state == TestState.Timeout:
return TermColor.colorText(" TIMEOUT ", TermColor.Purple)
if state == TestState.Disabled:
return TermColor.colorText(" DISABLED ", TermColor.Blue)
if state == TestState.Clean:
return TermColor.colorText(" CLEAN ",
fg=TermColor.White,
bg=TermColor.Green,
style=TermColor.Bold)
if state == TestState.BadWord:
return TermColor.colorText(" BADWORD ",
fg=TermColor.Yellow,
bg=TermColor.Red,
style=TermColor.Bold)
return TermColor.colorText(" UNKNOWN ", TermColor.Yellow)
def stats(self, quiet=False):
"""
Generate and write the stats
@type quiet: Boolean
@param quiet: Flag, passed along to the logger
"""
if (self.lastResult != TestState.InfoOnly):
logger.log("I ran {} out of {} tests in total".format(
self.count, len(self.testList)))
fails = self.count - self.success
logger.log(
TermColor.colorText("\tSuccess: {}".format(self.success),
TermColor.Green))
if (self.failed > 0):
logger.log(
TermColor.colorText("\tFailed: {}".format(self.failed),
TermColor.Red))
if (self.error > 0):
logger.log(
TermColor.colorText("\tErrors: {}".format(self.error),
TermColor.Yellow))
if (self.assertions > 0):
logger.log(
TermColor.colorText(
"\tAssertions: {}".format(self.assertions),
TermColor.Yellow))
if (self.segfaults > 0):
logger.log(
TermColor.colorText(
"\tSegFaults: {}".format(self.segfaults),
TermColor.Yellow))
if (self.timedout > 0):
logger.log(
TermColor.colorText("\tTimeouts: {}".format(self.timedout),
TermColor.Purple))
# A little bit of fun
if (self.success == len(self) and self.count > 3):
logger.log("\tCongratulations, you passed all tests!")
logger.log("\t`grep` yourself a refreshing " +
TermColor.colorText(
"Beer", TermColor.Yellow, style=TermColor.Bold))
logger.log("")
logger.log(" \033[1;37m,%%%%.\033[0m")
logger.log(
" \033[1;37mi\033[36m====\033[1;37mi\033[1;36m_\033[0m"
)
logger.log(
" \033[1;36m|\033[1;33m####\033[36m| |\033[0m"
)
logger.log(
" \033[1;36m|\033[1;33m####\033[36m|-'\033[0m"
)
logger.log(" \033[1;36m`-==-'\033[0m")
logger.log("")
elif (self.success == 0 and self.count > 3 and self.failed > 0):
logger.log("\tWhat is wrong with you, not even a single test?")
elif fails > 0 and self.count > 3:
if self.assertions / fails > 0.6:
logger.log(
"\tYou do realise that assertions do not replace error handling?"
)
elif self.assertions / fails > 0.3:
logger.log(
"\tWe're a bit lazy with calling environments, aren't we?"
)
if self.segfaults / fails > 0.6:
logger.log(
"\tMay the CPU-Gods have mercy with that poor memory management units soul!"
)
elif self.segfaults / fails > 0.3:
logger.log(
"\tYou know, memory garbage doesn't collect itself?!")
return self.calcRate()
def __init__(self, flush=False):
"""Initialises the test runner"""
# Thread.__init__(self)
logger.log(
TermColor.colorText("NIGHTMARE I", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("s of ", TermColor.White) +
TermColor.colorText("G", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("enerous ", TermColor.White) +
TermColor.colorText("H", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("elp when ", TermColor.White) +
TermColor.colorText("T", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("esting; ", TermColor.White) +
TermColor.colorText("M", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("ay ", TermColor.White) +
TermColor.colorText("A", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("rnold be ", TermColor.White) +
TermColor.colorText("R", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("emembered ", TermColor.White) +
TermColor.colorText("E", TermColor.Red, style=TermColor.Bold) +
TermColor.colorText("ternally", TermColor.White)
)
logger.log("Welcome to nightmare Version {}".format(version.Version))
if flush:
logger.flush(quiet=False)
self.options = dict()
self.testCount = 0
self.runsuite = None
self.finished = None
