def _run(self):
# Start the program:
self.pWrap = self._createPWrap(
[join(".", self.executionDirectory, self.executable)])
self._startPWrap(self.pWrap)
# Wait for child beeing ready:
self.__waitForInput("Enter Rot:")
# Send rot:
self.pWrap.writeStdin("{}\n".format(self.rot))
sleep(0.25)
self.__waitForInput("Enter text:")
# Send input text:
self.pWrap.writeStdin("{}\n".format(self.input))
# Compare result:
self.__waitForInput(self.__getExpectedRotX(self.rot, self.input))
# Wait reading until the programm terminates:
printTester("Waiting for the programm to terminate...")
if (not self.pWrap.waitUntilTerminationReading(3)):
printTester("Programm did not terminate - killing it!")
self.pWrap.kill()
self._failWith("Programm did not terminate at the end.")
# Always cleanup to make sure all threads get joined:
self.pWrap.cleanup()
def run(self):
"""
Starts the tester and runs all tests added via "addTest(test: AbstractTest)".
"""
printTester("Running: {}".format(self.name))
# A dictionary of test results:
# Test name -> result
testResults: Dict[str, Result] = dict()
for name, test in self.tests.items():
# Reset the tester output cache:
clearTesterOutputCache()
printTester(
"Running test case '{}' with a {} second timeout...".format(
name, test.timeoutSec))
test.start(testResults, self.suite)
printTester("Finished test case '{}' in {} seconds.".format(
name, (test.case.time.total_seconds())))
# Store the tester output in the test case:
test.case.testerOutput = self.name + "\n" + getTesterOutput()
# Update test results:
testResults[name] = test.case.result
def __markAsFailed(self, msg: str):
"""
Marks the current test case as failed and loads all stdout and stderr.
"""
self.case.message = msg
self.case.result = Result.FAILURE
self.case.stdout = self._loadFullStdout()
self.case.stderr = self._loadFullStderr()
printTester("Test {} failed with: {}".format(self.name, msg))
def __getChildrenCount(self):
pid = self.pWrap.getPID()
# If the process has terminated, the process creation will fail:
try:
process: Process = Process(pid)
except NoSuchProcess:
return 0
count: int = len(process.children())
printTester("Parent process has PID: " + str(pid) + " and " + str(count) + " child processes.")
return count
def _startPWrap(self, pWrap: PWrap):
"""
Starts the PWrap execution.
Handels FileNotFoundError if for example the executable was not found or does not exist.
"""
try:
pWrap.start()
except FileNotFoundError as e:
printTester(str(e))
self._failWith("File not found for execution. Did compiling fail?")
def _failWith(self, msg: str):
"""
Marks the current test as failed with the given message.
Stores the complete stderr and stdout output from the run.
"""
self.case.message = msg
self.case.result = Result.FAILURE
self.case.stdout = self._loadFullStdout()
self.case.stderr = self._loadFullStderr()
self._onFailed()
printTester("Test {} failed with: {}".format(self.name, msg))
raise TestFailedError("{} failed.".format(self.name))
def _run(self):
# Start the program:
self.pWrap = self._createPWrap([join(".", self.executionDirectory, self.executable)])
self._startPWrap(self.pWrap)
# Wait for child being ready:
printTester("Waiting for: 'Hello world!'")
expected: str = "Hello world!"
while True:
if self.pWrap.hasTerminated() and not self.pWrap.canReadLineStdout():
self._progTerminatedUnexpectedly()
# Read a single line form the program output:
line: str = self.pWrap.readLineStdout()
# Perform a "student save" compare:
if studSaveStrComp(expected, line):
break
else:
printTester(f"Expected '{expected}' but received read '{line}'")
# Wait reading until the program terminates:
printTester("Waiting for the program to terminate...")
if not self.pWrap.waitUntilTerminationReading(3):
printTester("Program did not terminate - killing it!")
self.pWrap.kill()
self._failWith("Program did not terminate at the end.")
# Always cleanup to make sure all threads get joined:
self.pWrap.cleanup()
def __waitForInput(self, expected: str):
printTester("Waiting for: '{}'".format(expected))
while True:
if self.pWrap.hasTerminated(
) and not self.pWrap.canReadLineStdout():
self._progTerminatedUnexpectedly()
# Read a single line form the programm output:
line: str = self.pWrap.readLineStdout()
# Perform a "student save" compare:
if studSaveStrComp(expected, line):
break
else:
printTester("Expected '{}' but received '{}'".format(
expected, line))
def _timeout(self, msg: str = ""):
"""
Marks the current test as failed with the given optional message.
Stores the complete stderr and stdout output from the run.
Should be called once a test timeout occurred.
"""
self.case.message = msg
self.case.result = Result.FAILURE
self.case.stdout = self._loadFullStdout()
self.case.stderr = self._loadFullStderr()
if msg:
printTester("'{}' triggered a timeout with message: {}".format(
self.name, msg))
else:
printTester("'{}' triggered a timeout.".format(self.name))
def __matchChildBye(self, line: str, pids: List[int]):
# Get child PID:
pidRegex = r"\D*(\d+)\D*"
m = search(pidRegex, line)
if m is None:
printTester("No PID found in string: '{}'".format(line))
return False
childPid = int(m.group(1)) # Child PID
# Check if the processes PID is one of those that said hello:
if not childPid in pids:
self._failWith("PID {} unknown. We are only aware of: {}".format(childPid, str(pids)))
pids.remove(childPid)
# Check if string read matches:
expected: str = "Child with PID {} terminated.".format(childPid)
if not studSaveStrComp(expected, line):
printTester("Expected: '{}' but received: '{}'".format(expected, line))
return False
return True
def __matchChildHello(self, line: str):
# Get PPID:
ppid = self.pWrap.getPID()
try:
process: Process = Process(ppid)
except NoSuchProcess:
self.__progTerminatedUnexpectedly()
# Get child PID:
pidRegex = r"\D*(\d+)\D*(" + str(ppid) + r")"
m = search(pidRegex, line)
if m is None:
printTester("No PID found in string: '{}'".format(line))
return False
childPid = int(m.group(1)) # Child PID
# Check if string read matches:
expected: str = "I'm your child! PID: {}, PPID: {}".format(childPid, ppid)
if not studSaveStrComp(expected, line):
printTester("Expected: '{}' but received: '{}'".format(expected, line))
return False
# Check if process exists for the child PID:
if not self.__existsChildWithPid(childPid):
printTester("No child process with PID {} exists".format(childPid))
return False
return True
def _startPWrap(self, pWrap: PWrap):
"""
Starts the PWrap execution.
Handels FileNotFoundError if for example the executable was not found or does not exist.
"""
try:
pWrap.start()
except FileNotFoundError as fe:
printTester(str(fe))
self._failWith("File not found for execution. Did compiling fail?")
except NotADirectoryError as de:
printTester(str(de))
self._failWith("Directory '{}' does not exist.".format(pWrap.cwd))
except PermissionError as pe:
printTester(str(pe))
self._failWith(
"Missing file execution permission. Make sure it has execute rights (chmod +x <FILE_NAME>)."
)
def run(self):
"""
Starts the tester and runs all tests added via "addTest(test: AbstractTest)".
"""
setStdoutLimitEnabled(False)
printTester(f"Running: {self.name}")
# A dictionary of test results:
# Test name -> result
testResults: Dict[str, Result] = dict()
for name, test in self.tests.items():
if test.timeoutSec >= 0:
printTester("Running test case '{}' with a {} second timeout...".format(
name, test.timeoutSec))
else:
printTester(
f"Running test case '{name}' with no timeout...")
# Reset the tester output cache:
resetStdoutLimit()
setStdoutLimitEnabled(True)
clearTesterOutputCache()
test.start(testResults, self.suite)
setStdoutLimitEnabled(False)
printTester("Finished test case '{}' in {} seconds.".format(
name, test.case.time.total_seconds()))
# Store the tester output in the test case:
test.case.testerOutput = self.name + "\n" + getTesterOutput()
# Update test results:
testResults[name] = test.case.result
self.__printResult()
def start(self, testResults: Dict[str, Result], suite: TestSuite):
"""
Starts the test run.
---
testResults: Dict[str, Result]
All test results up to this point.
suite: TestSuite
The test suite where this test should get added to.
"""
self.suite = suite
self.case = TestCase(self.name)
# Check if all test requirements (other tests) are fulfilled:
if not self.__checkTestRequirements(testResults):
printTester(
"Skipping test case '{}' not all requirements ({}) are fulfilled"
.format(self.name, str(self.requirements)))
self.case.message = "Test requires other test cases to succeed first ({})".format(
str(self.requirements))
self.case.result = Result.SKIPPED
self.case.stdout = ""
self.case.stderr = ""
self.case.time = timedelta()
self.suite.addCase(self.case)
return
startTime: datetime = datetime.now()
self._initOutputDirectory()
if self.timeoutSec > 0:
# Run with timeout:
with self.__timeout(self.timeoutSec):
try:
self._run()
except TestFailedError:
printTester("'{}' failed.".format(self.name))
except TimeoutError:
self._timeout()
except Exception as e:
self.__markAsFailed(
"'{}' had an internal error. {}.\nPlease report this on Moodle (Detailfragen zu Programmieraufgaben)!"
.format(self.name, str(e)))
print_exc()
self._onFailed()
else:
# Run without timeout:
try:
self._run()
except TestFailedError:
printTester("'{}' failed.".format(self.name))
except Exception as e:
self.__markAsFailed(
"'{}' had an internal error. {}.\nPlease report this on Moodle (Detailfragen zu Programmieraufgaben)!"
.format(self.name, str(e)))
print_exc()
self._onFailed()
self.case.time = datetime.now() - startTime
self.suite.addCase(self.case)
def _run(self):
# Start the program:
self.pWrap = self._createPWrap([join(".", self.makefileLocation, "main")])
self._startPWrap(self.pWrap)
# Wait to make sure the child processes has started:
sleep(0.5)
if self.pWrap.hasTerminated():
self.__progTerminatedUnexpectedly()
# Wait for child beeing ready:
printTester("Waiting for: 'Enter process count:'")
while True:
if self.pWrap.hasTerminated():
self.__progTerminatedUnexpectedly()
if studSaveStrComp("Enter process count:", self.pWrap.readLineStdout()):
self.pWrap.writeStdin("{}\n".format(self.count))
break
# Give the programm some time to start it's child processes:
sleep(0.25)
printTester("Waiting for: \"I'm your child! PID: %i, PPID: %i\\n\"")
for i in range(0, self.count):
while True:
if self.pWrap.hasTerminated():
self.__progTerminatedUnexpectedly()
if self.__matchChildHello(self.pWrap.readLineStdout()):
break
# Get all child process PIDs:
pids: List[int] = self.__getChildPids()
# Send 'END' to terminate processes:
printTester("Sending 'END's...")
for i in range(0, self.count):
if self.pWrap.hasTerminated():
self.__progTerminatedUnexpectedly()
# Check child process count:
self.__testChildProcessCount(self.count - i)
self.pWrap.writeStdin("END\n")
sleep(0.25)
printTester("Waiting for: \"Child with PID %i terminated.\\n\"")
while True:
if self.pWrap.hasTerminated() and not self.pWrap.canReadLineStdout():
self.__progTerminatedUnexpectedly()
if self.__matchChildBye(self.pWrap.readLineStdout(), pids):
break
# Check if all children terminated:
self.__testChildProcessCount(0)
# Wait reading until the programm terminates:
printTester("Waiting for the programm to terminate...")
if(not self.pWrap.waitUntilTerminationReading(3)):
printTester("Programm did not terminate - killing it!")
pWrap.kill()
if not self.pWrap.hasTerminated():
self._failWith("Programm did not terminate at the end.")
# Allways cleanup to make sure all threads get joined:
self.pWrap.cleanup()
