def pipeOutputStream(self, stream, lines, color): bytes = 0 for line in lines: bytes += len(line) stream.write(TermColor.colorText(line + " ", fg=color) + "\n") if bytes > self.pipeLimit: stream.write( TermColor.colorText("Stopped after {} Bytes".format(bytes), fg=TermColor.Yellow) + "\n") break
def lineComparison(self, expLines, outLines, stream=""): same = True if self.ignoreEmptyLines: while expLines.count("") > 0: expLines.remove("") while outLines.count("") > 0: outLines.remove("") for line in difflib.unified_diff(expLines, outLines, stream, "expectation"): col = TermColor.White uline = unicode( line, encoding="utf-8", errors="ignore") if type(line) is not unicode else line if uline.startswith(" + "): same = False col = TermColor.Green elif uline.startswith("-"): same = False col = TermColor.Red elif uline.startswith("@"): same = False col = TermColor.Cyan if self.diff: logger.log(TermColor.colorText(uline.rstrip(), col)) return same
def run(self, quiet=False, tests=[]): """ Runs the whole suite of tests @type quiet: Boolean @param quiet: Flag, passed along to the logger """ self.success = 0 self.failed = 0 self.count = 0 self.error = 0 self.lastResult = TestState.Waiting for t in self._getTests(tests): self.lastResult = t.run() if t.descr is not None: logger.log("{}[{: 03}] {} - {}: {}".format( TermColor.colorText("Test", TermColor.Purple), self.count, t.name, t.descr, TestState.toString(t.state))) else: logger.log("{}[{: 03}] {}: {}".format( TermColor.colorText("Test", TermColor.Purple), self.count, t.name, TestState.toString(t.state))) if self.options['commands']: logger.log(" --> {}".format(t.cmd), showTime=False) logger.flush(quiet) if self.lastResult in [TestState.Success, TestState.Clean]: self.success += 1 elif self.lastResult == TestState.Fail: self.failed += 1 elif self.lastResult == TestState.Error: self.error += 1 elif self.lastResult == TestState.Timeout: self.timedout += 1 elif self.lastResult == TestState.SegFault: self.segfaults += 1 elif self.lastResult == TestState.Assertion: self.assertions += 1 self.count = self.count + 1 yield t if self.lastResult != TestState.Disabled: if (self.mode == TestSuiteMode.BreakOnFail) and ( self.lastResult != TestState.Success): break if (self.mode == TestSuiteMode.BreakOnError) and ( self.lastResult == TestState.Error): break raise StopIteration()
def run(self, quiet=False, tests=[]): """ Runs the whole suite of tests @type quiet: Boolean @param quiet: Flag, passed along to the logger """ self.success = 0 self.failed = 0 self.count = 0 self.error = 0 self.lastResult = TestState.Waiting for t in self._getTests(tests): self.lastResult = t.run() if t.descr is not None: logger.log("{}[{: 03}] {} - {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, t.descr, TestState.toString(t.state))) else: logger.log("{}[{: 03}] {}: {}".format(TermColor.colorText("Test", TermColor.Purple), self.count, t.name, TestState.toString(t.state))) if self.options['commands']: logger.log(" --> {}".format(t.cmd), showTime=False) logger.flush(quiet) if self.lastResult in [TestState.Success, TestState.Clean]: self.success += 1 elif self.lastResult == TestState.Fail: self.failed += 1 elif self.lastResult == TestState.Error: self.error += 1 elif self.lastResult == TestState.Timeout: self.timedout += 1 elif self.lastResult == TestState.SegFault: self.segfaults += 1 elif self.lastResult == TestState.Assertion: self.assertions += 1 self.count = self.count + 1 yield t if self.lastResult != TestState.Disabled: if (self.mode == TestSuiteMode.BreakOnFail) and (self.lastResult != TestState.Success): break if (self.mode == TestSuiteMode.BreakOnError) and (self.lastResult == TestState.Error): break raise StopIteration()
def stats(self, quiet=False): """ Generate and write the stats @type quiet: Boolean @param quiet: Flag, passed along to the logger """ if (self.lastResult != TestState.InfoOnly): logger.log("I ran {} out of {} tests in total".format(self.count, len(self.testList))) fails = self.count - self.success logger.log(TermColor.colorText("\tSuccess: {}".format(self.success), TermColor.Green)) if (self.failed > 0): logger.log(TermColor.colorText("\tFailed: {}".format(self.failed), TermColor.Red)) if (self.error > 0): logger.log(TermColor.colorText("\tErrors: {}".format(self.error), TermColor.Yellow)) if (self.assertions > 0): logger.log(TermColor.colorText("\tAssertions: {}".format(self.assertions), TermColor.Yellow)) if (self.segfaults > 0): logger.log(TermColor.colorText("\tSegFaults: {}".format(self.segfaults), TermColor.Yellow)) if (self.timedout > 0): logger.log(TermColor.colorText("\tTimeouts: {}".format(self.timedout), TermColor.Purple)) # A little bit of fun if (self.success == len(self) and self.count > 3): logger.log("\tCongratulations, you passed all tests!") logger.log("\t`grep` yourself a refreshing " + TermColor.colorText("Beer", TermColor.Yellow, style=TermColor.Bold)) logger.log("") logger.log(" \033[1;37m,%%%%.\033[0m") logger.log(" \033[1;37mi\033[36m====\033[1;37mi\033[1;36m_\033[0m") logger.log(" \033[1;36m|\033[1;33m####\033[36m| |\033[0m") logger.log(" \033[1;36m|\033[1;33m####\033[36m|-'\033[0m") logger.log(" \033[1;36m`-==-'\033[0m") logger.log("") elif (self.success == 0 and self.count > 3 and self.failed > 0): logger.log("\tWhat is wrong with you, not even a single test?") elif fails > 0 and self.count > 3: if self.assertions / fails > 0.6: logger.log("\tYou do realise that assertions do not replace error handling?") elif self.assertions / fails > 0.3: logger.log("\tWe're a bit lazy with calling environments, aren't we?") if self.segfaults / fails > 0.6: logger.log("\tMay the CPU-Gods have mercy with that poor memory management units soul!") elif self.segfaults / fails > 0.3: logger.log("\tYou know, memory garbage doesn't collect itself?!") return self.calcRate()
def runCmd(self, command): if "$DUT" in command: if self.DUT is None: self.state = TestState.Error return else: _cmd = Command(cmd=str(command).replace("$DUT", self.DUT)) else: _cmd = Command(cmd=str(command)) cmdRet = _cmd.execute(self.timeout) if cmdRet == TestState.Success: if self.binary: self.output = _cmd.out self.error = _cmd.err else: self.output = _cmd.out.decode(encoding="utf8", errors="ignore") self.error = _cmd.err.decode(encoding="utf8", errors="ignore") self.retCode = _cmd.ret if (self.check(self.expectRetCode, self.retCode) and self.check(self.expectStdout, self.output, "stdout") and self.check(self.expectStderr, self.error, "stderr")): self.state = TestState.Success else: if 'Assertion' in self.error or 'assertion' in self.error: self.state = TestState.Assertion elif "stackdump" in self.error or "coredump" in self.error or "Segmentation Fault" in self.error or self.retCode < 0: self.state = TestState.SegFault else: self.state = TestState.Fail if (self.pipe) or (self.outputOnFail and self.state is TestState.Fail): sys.stdout.write( TermColor.colorText("{}".format(self.retCode), fg=TermColor.Yellow) + " ") self.pipeOutputStream(sys.stdout, self.output.splitlines(), TermColor.Green) self.pipeOutputStream(sys.stderr, self.error.splitlines(), TermColor.Red) else: self.state = cmdRet
def stats(self, quiet = False): """ Generate and write the stats @type quiet: Boolean @param quiet: Flag, passed along to the logger """ logger.log("I ran {} out of {} tests in total".format(self.count, len(self.testList))) logger.log(TermColor.colorText("\tSuccess: {}".format(self.success), TermColor.Green)) if (self.failed > 0): logger.log(TermColor.colorText("\tFailed: {}".format(self.failed), TermColor.Red)) if (self.error > 0): logger.log(TermColor.colorText("\tErrors: {}".format(self.error), TermColor.Yellow)) if (self.assertions > 0): logger.log(TermColor.colorText("\tAssertions: {}".format(self.assertions), TermColor.Yellow)) if (self.segfaults > 0): logger.log(TermColor.colorText("\tSegFaults: {}".format(self.segfaults), TermColor.Yellow)) if (self.timedout > 0): logger.log(TermColor.colorText("\tTimeouts: {}".format(self.timedout), TermColor.Purple)) if (self.error == 0) and (self.failed == 0) and (self.timedout == 0): logger.log("\tCongratulations, you passed all tests!") return self.calcRate()
def toString(state): """ Converts the enumeration value into a string @type state: int @param state: Enumeration value """ if state == TestState.Waiting: return TermColor.colorText(" WAITING ", TermColor.White) if state == TestState.Success: return TermColor.colorText(" SUCCESS ", fg=TermColor.Black, bg=TermColor.Green) if state == TestState.Fail: return TermColor.colorText(" FAIL ", fg=TermColor.Black, bg=TermColor.Red) if state == TestState.Error: return TermColor.colorText(" ERROR ", fg=TermColor.Black, bg=TermColor.Red, style=TermColor.Bold) if state == TestState.SegFault: return TermColor.colorText(" SEGFAULT ", fg=TermColor.Black, bg=TermColor.Yellow) if state == TestState.Assertion: return TermColor.colorText(" ASSERTION ", fg=TermColor.Black, bg=TermColor.Yellow, style=TermColor.Bold) if state == TestState.InfoOnly: return TermColor.colorText(" INFO ", TermColor.White) if state == TestState.Timeout: return TermColor.colorText(" TIMEOUT ", TermColor.Purple) if state == TestState.Disabled: return TermColor.colorText(" DISABLED ", TermColor.Blue) if state == TestState.Clean: return TermColor.colorText(" CLEAN ", fg=TermColor.White, bg=TermColor.Green, style=TermColor.Bold) if state == TestState.BadWord: return TermColor.colorText(" BADWORD ", fg=TermColor.Yellow, bg=TermColor.Red, style=TermColor.Bold) return TermColor.colorText(" UNKNOWN ", TermColor.Yellow)
def stats(self, quiet=False): """ Generate and write the stats @type quiet: Boolean @param quiet: Flag, passed along to the logger """ if (self.lastResult != TestState.InfoOnly): logger.log("I ran {} out of {} tests in total".format( self.count, len(self.testList))) fails = self.count - self.success logger.log( TermColor.colorText("\tSuccess: {}".format(self.success), TermColor.Green)) if (self.failed > 0): logger.log( TermColor.colorText("\tFailed: {}".format(self.failed), TermColor.Red)) if (self.error > 0): logger.log( TermColor.colorText("\tErrors: {}".format(self.error), TermColor.Yellow)) if (self.assertions > 0): logger.log( TermColor.colorText( "\tAssertions: {}".format(self.assertions), TermColor.Yellow)) if (self.segfaults > 0): logger.log( TermColor.colorText( "\tSegFaults: {}".format(self.segfaults), TermColor.Yellow)) if (self.timedout > 0): logger.log( TermColor.colorText("\tTimeouts: {}".format(self.timedout), TermColor.Purple)) # A little bit of fun if (self.success == len(self) and self.count > 3): logger.log("\tCongratulations, you passed all tests!") logger.log("\t`grep` yourself a refreshing " + TermColor.colorText( "Beer", TermColor.Yellow, style=TermColor.Bold)) logger.log("") logger.log(" \033[1;37m,%%%%.\033[0m") logger.log( " \033[1;37mi\033[36m====\033[1;37mi\033[1;36m_\033[0m" ) logger.log( " \033[1;36m|\033[1;33m####\033[36m| |\033[0m" ) logger.log( " \033[1;36m|\033[1;33m####\033[36m|-'\033[0m" ) logger.log(" \033[1;36m`-==-'\033[0m") logger.log("") elif (self.success == 0 and self.count > 3 and self.failed > 0): logger.log("\tWhat is wrong with you, not even a single test?") elif fails > 0 and self.count > 3: if self.assertions / fails > 0.6: logger.log( "\tYou do realise that assertions do not replace error handling?" ) elif self.assertions / fails > 0.3: logger.log( "\tWe're a bit lazy with calling environments, aren't we?" ) if self.segfaults / fails > 0.6: logger.log( "\tMay the CPU-Gods have mercy with that poor memory management units soul!" ) elif self.segfaults / fails > 0.3: logger.log( "\tYou know, memory garbage doesn't collect itself?!") return self.calcRate()
def __init__(self, flush=False): """Initialises the test runner""" # Thread.__init__(self) logger.log( TermColor.colorText("NIGHTMARE I", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("s of ", TermColor.White) + TermColor.colorText("G", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("enerous ", TermColor.White) + TermColor.colorText("H", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("elp when ", TermColor.White) + TermColor.colorText("T", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("esting; ", TermColor.White) + TermColor.colorText("M", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("ay ", TermColor.White) + TermColor.colorText("A", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("rnold be ", TermColor.White) + TermColor.colorText("R", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("emembered ", TermColor.White) + TermColor.colorText("E", TermColor.Red, style=TermColor.Bold) + TermColor.colorText("ternally", TermColor.White) ) logger.log("Welcome to nightmare Version {}".format(version.Version)) if flush: logger.flush(quiet=False) self.options = dict() self.testCount = 0 self.runsuite = None self.finished = None