def _run(self): # Start the program: self.pWrap = self._createPWrap( [join(".", self.executionDirectory, self.executable)]) self._startPWrap(self.pWrap) # Wait for child beeing ready: self.__waitForInput("Enter Rot:") # Send rot: self.pWrap.writeStdin("{}\n".format(self.rot)) sleep(0.25) self.__waitForInput("Enter text:") # Send input text: self.pWrap.writeStdin("{}\n".format(self.input)) # Compare result: self.__waitForInput(self.__getExpectedRotX(self.rot, self.input)) # Wait reading until the programm terminates: printTester("Waiting for the programm to terminate...") if (not self.pWrap.waitUntilTerminationReading(3)): printTester("Programm did not terminate - killing it!") self.pWrap.kill() self._failWith("Programm did not terminate at the end.") # Always cleanup to make sure all threads get joined: self.pWrap.cleanup()
def run(self): """ Starts the tester and runs all tests added via "addTest(test: AbstractTest)". """ printTester("Running: {}".format(self.name)) # A dictionary of test results: # Test name -> result testResults: Dict[str, Result] = dict() for name, test in self.tests.items(): # Reset the tester output cache: clearTesterOutputCache() printTester( "Running test case '{}' with a {} second timeout...".format( name, test.timeoutSec)) test.start(testResults, self.suite) printTester("Finished test case '{}' in {} seconds.".format( name, (test.case.time.total_seconds()))) # Store the tester output in the test case: test.case.testerOutput = self.name + "\n" + getTesterOutput() # Update test results: testResults[name] = test.case.result
def __markAsFailed(self, msg: str): """ Marks the current test case as failed and loads all stdout and stderr. """ self.case.message = msg self.case.result = Result.FAILURE self.case.stdout = self._loadFullStdout() self.case.stderr = self._loadFullStderr() printTester("Test {} failed with: {}".format(self.name, msg))
def __getChildrenCount(self): pid = self.pWrap.getPID() # If the process has terminated, the process creation will fail: try: process: Process = Process(pid) except NoSuchProcess: return 0 count: int = len(process.children()) printTester("Parent process has PID: " + str(pid) + " and " + str(count) + " child processes.") return count
def _startPWrap(self, pWrap: PWrap): """ Starts the PWrap execution. Handels FileNotFoundError if for example the executable was not found or does not exist. """ try: pWrap.start() except FileNotFoundError as e: printTester(str(e)) self._failWith("File not found for execution. Did compiling fail?")
def _failWith(self, msg: str): """ Marks the current test as failed with the given message. Stores the complete stderr and stdout output from the run. """ self.case.message = msg self.case.result = Result.FAILURE self.case.stdout = self._loadFullStdout() self.case.stderr = self._loadFullStderr() self._onFailed() printTester("Test {} failed with: {}".format(self.name, msg)) raise TestFailedError("{} failed.".format(self.name))
def _run(self): # Start the program: self.pWrap = self._createPWrap([join(".", self.executionDirectory, self.executable)]) self._startPWrap(self.pWrap) # Wait for child being ready: printTester("Waiting for: 'Hello world!'") expected: str = "Hello world!" while True: if self.pWrap.hasTerminated() and not self.pWrap.canReadLineStdout(): self._progTerminatedUnexpectedly() # Read a single line form the program output: line: str = self.pWrap.readLineStdout() # Perform a "student save" compare: if studSaveStrComp(expected, line): break else: printTester(f"Expected '{expected}' but received read '{line}'") # Wait reading until the program terminates: printTester("Waiting for the program to terminate...") if not self.pWrap.waitUntilTerminationReading(3): printTester("Program did not terminate - killing it!") self.pWrap.kill() self._failWith("Program did not terminate at the end.") # Always cleanup to make sure all threads get joined: self.pWrap.cleanup()
def __waitForInput(self, expected: str): printTester("Waiting for: '{}'".format(expected)) while True: if self.pWrap.hasTerminated( ) and not self.pWrap.canReadLineStdout(): self._progTerminatedUnexpectedly() # Read a single line form the programm output: line: str = self.pWrap.readLineStdout() # Perform a "student save" compare: if studSaveStrComp(expected, line): break else: printTester("Expected '{}' but received '{}'".format( expected, line))
def _timeout(self, msg: str = ""): """ Marks the current test as failed with the given optional message. Stores the complete stderr and stdout output from the run. Should be called once a test timeout occurred. """ self.case.message = msg self.case.result = Result.FAILURE self.case.stdout = self._loadFullStdout() self.case.stderr = self._loadFullStderr() if msg: printTester("'{}' triggered a timeout with message: {}".format( self.name, msg)) else: printTester("'{}' triggered a timeout.".format(self.name))
def __matchChildBye(self, line: str, pids: List[int]): # Get child PID: pidRegex = r"\D*(\d+)\D*" m = search(pidRegex, line) if m is None: printTester("No PID found in string: '{}'".format(line)) return False childPid = int(m.group(1)) # Child PID # Check if the processes PID is one of those that said hello: if not childPid in pids: self._failWith("PID {} unknown. We are only aware of: {}".format(childPid, str(pids))) pids.remove(childPid) # Check if string read matches: expected: str = "Child with PID {} terminated.".format(childPid) if not studSaveStrComp(expected, line): printTester("Expected: '{}' but received: '{}'".format(expected, line)) return False return True
def __matchChildHello(self, line: str): # Get PPID: ppid = self.pWrap.getPID() try: process: Process = Process(ppid) except NoSuchProcess: self.__progTerminatedUnexpectedly() # Get child PID: pidRegex = r"\D*(\d+)\D*(" + str(ppid) + r")" m = search(pidRegex, line) if m is None: printTester("No PID found in string: '{}'".format(line)) return False childPid = int(m.group(1)) # Child PID # Check if string read matches: expected: str = "I'm your child! PID: {}, PPID: {}".format(childPid, ppid) if not studSaveStrComp(expected, line): printTester("Expected: '{}' but received: '{}'".format(expected, line)) return False # Check if process exists for the child PID: if not self.__existsChildWithPid(childPid): printTester("No child process with PID {} exists".format(childPid)) return False return True
def _startPWrap(self, pWrap: PWrap): """ Starts the PWrap execution. Handels FileNotFoundError if for example the executable was not found or does not exist. """ try: pWrap.start() except FileNotFoundError as fe: printTester(str(fe)) self._failWith("File not found for execution. Did compiling fail?") except NotADirectoryError as de: printTester(str(de)) self._failWith("Directory '{}' does not exist.".format(pWrap.cwd)) except PermissionError as pe: printTester(str(pe)) self._failWith( "Missing file execution permission. Make sure it has execute rights (chmod +x <FILE_NAME>)." )
def run(self): """ Starts the tester and runs all tests added via "addTest(test: AbstractTest)". """ setStdoutLimitEnabled(False) printTester(f"Running: {self.name}") # A dictionary of test results: # Test name -> result testResults: Dict[str, Result] = dict() for name, test in self.tests.items(): if test.timeoutSec >= 0: printTester("Running test case '{}' with a {} second timeout...".format( name, test.timeoutSec)) else: printTester( f"Running test case '{name}' with no timeout...") # Reset the tester output cache: resetStdoutLimit() setStdoutLimitEnabled(True) clearTesterOutputCache() test.start(testResults, self.suite) setStdoutLimitEnabled(False) printTester("Finished test case '{}' in {} seconds.".format( name, test.case.time.total_seconds())) # Store the tester output in the test case: test.case.testerOutput = self.name + "\n" + getTesterOutput() # Update test results: testResults[name] = test.case.result self.__printResult()
def start(self, testResults: Dict[str, Result], suite: TestSuite): """ Starts the test run. --- testResults: Dict[str, Result] All test results up to this point. suite: TestSuite The test suite where this test should get added to. """ self.suite = suite self.case = TestCase(self.name) # Check if all test requirements (other tests) are fulfilled: if not self.__checkTestRequirements(testResults): printTester( "Skipping test case '{}' not all requirements ({}) are fulfilled" .format(self.name, str(self.requirements))) self.case.message = "Test requires other test cases to succeed first ({})".format( str(self.requirements)) self.case.result = Result.SKIPPED self.case.stdout = "" self.case.stderr = "" self.case.time = timedelta() self.suite.addCase(self.case) return startTime: datetime = datetime.now() self._initOutputDirectory() if self.timeoutSec > 0: # Run with timeout: with self.__timeout(self.timeoutSec): try: self._run() except TestFailedError: printTester("'{}' failed.".format(self.name)) except TimeoutError: self._timeout() except Exception as e: self.__markAsFailed( "'{}' had an internal error. {}.\nPlease report this on Moodle (Detailfragen zu Programmieraufgaben)!" .format(self.name, str(e))) print_exc() self._onFailed() else: # Run without timeout: try: self._run() except TestFailedError: printTester("'{}' failed.".format(self.name)) except Exception as e: self.__markAsFailed( "'{}' had an internal error. {}.\nPlease report this on Moodle (Detailfragen zu Programmieraufgaben)!" .format(self.name, str(e))) print_exc() self._onFailed() self.case.time = datetime.now() - startTime self.suite.addCase(self.case)
def _run(self): # Start the program: self.pWrap = self._createPWrap([join(".", self.makefileLocation, "main")]) self._startPWrap(self.pWrap) # Wait to make sure the child processes has started: sleep(0.5) if self.pWrap.hasTerminated(): self.__progTerminatedUnexpectedly() # Wait for child beeing ready: printTester("Waiting for: 'Enter process count:'") while True: if self.pWrap.hasTerminated(): self.__progTerminatedUnexpectedly() if studSaveStrComp("Enter process count:", self.pWrap.readLineStdout()): self.pWrap.writeStdin("{}\n".format(self.count)) break # Give the programm some time to start it's child processes: sleep(0.25) printTester("Waiting for: \"I'm your child! PID: %i, PPID: %i\\n\"") for i in range(0, self.count): while True: if self.pWrap.hasTerminated(): self.__progTerminatedUnexpectedly() if self.__matchChildHello(self.pWrap.readLineStdout()): break # Get all child process PIDs: pids: List[int] = self.__getChildPids() # Send 'END' to terminate processes: printTester("Sending 'END's...") for i in range(0, self.count): if self.pWrap.hasTerminated(): self.__progTerminatedUnexpectedly() # Check child process count: self.__testChildProcessCount(self.count - i) self.pWrap.writeStdin("END\n") sleep(0.25) printTester("Waiting for: \"Child with PID %i terminated.\\n\"") while True: if self.pWrap.hasTerminated() and not self.pWrap.canReadLineStdout(): self.__progTerminatedUnexpectedly() if self.__matchChildBye(self.pWrap.readLineStdout(), pids): break # Check if all children terminated: self.__testChildProcessCount(0) # Wait reading until the programm terminates: printTester("Waiting for the programm to terminate...") if(not self.pWrap.waitUntilTerminationReading(3)): printTester("Programm did not terminate - killing it!") pWrap.kill() if not self.pWrap.hasTerminated(): self._failWith("Programm did not terminate at the end.") # Allways cleanup to make sure all threads get joined: self.pWrap.cleanup()