def runTests(cmdPrefix, testFile, timeLimit):
startTime = datetime.now()
# testProc = Popen(cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'], stdout=PIPE, stderr=PIPE)
#
# timeoutReached = False
# while testProc.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# testProc.kill()
# timeoutReached = True
# break
testProc = Command(
cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'])
timeoutReached, testOut, testError = testProc.run(timeout=int(timeLimit),
env=environ)
if timeoutReached:
summary = {}
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = timeoutReached
summary['died'] = False
summary['generalError'] = ""
summary['rawOut'] = ""
summary['rawErr'] = ""
return summary, {}
#testOut, testError = testProc.communicate()
summary = {}
summary['rawOut'] = testOut
summary['rawErr'] = testError
summary['timeout'] = False
testOut = testError.split('\n')
failedTests = {}
for line in testOut:
if re.match(r"^% No tests to run$", line):
#Compilation error
summary['died'] = True
summary['totalTests'] = 0
summary['failedTests'] = 0
if re.match("\ttest \w*: [(failed)(received error)]", line):
failedTests[line[6:line.find(":")]] = {'hint': ""}
summary['totalTests'] = "N/A"
summary['failedTests'] = "N/A"
summary['died'] = False
return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
startTime = datetime.now()
# testProc = Popen(cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'], stdout=PIPE, stderr=PIPE)
#
# timeoutReached = False
# while testProc.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# testProc.kill()
# timeoutReached = True
# break
testProc = Command(cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'])
timeoutReached, testOut, testError = testProc.run(timeout=int(timeLimit), env=environ)
if timeoutReached:
summary = {}
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = timeoutReached
summary['died'] = False
summary['generalError'] = ""
summary['rawOut'] = ""
summary['rawErr'] = ""
return summary, {}
#testOut, testError = testProc.communicate()
summary = {}
summary['rawOut'] = testOut
summary['rawErr'] = testError
summary['timeout'] = False
testOut = testError.split('\n')
failedTests = {}
for line in testOut:
if re.match(r"^% No tests to run$", line):
#Compilation error
summary['died'] = True
summary['totalTests'] = 0
summary['failedTests'] = 0
if re.match("\ttest \w*: [(failed)(received error)]", line):
failedTests[line[6:line.find(":")]] = {'hint':""}
summary['totalTests'] = "N/A"
summary['failedTests'] = "N/A"
summary['died'] = False
return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
startTime = datetime.now()
testProc = Command(cmdPrefix + ['python', testFile])
timeoutReached, testOut, testError = \
testProc.run(timeout=int(timeLimit), env=environ)
#Check for a timeout
if timeoutReached:
print "Timeout reached"
summary = {}
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = timeoutReached
summary['died'] = False
summary['rawOut'] = ""
summary['rawErr'] = ""
return summary, {}
#testOut, testError = testProc.communicate()
summary = {}
summary['rawOut'] = testOut
summary['rawErr'] = testError
#Parse the results
testSummarySearch = re.search("Ran ([0-9]+) tests? in", testError)
#If we don't find the test summary the tests died so we report that
if not testSummarySearch:
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = timeoutReached
summary['died'] = True
return summary, {}
# Get the list of failed tests
failedSections = testError.split('='*70)
splitList = []
for section in failedSections:
splitList.append(section.split('-'*70))
#flatten the lists
sections = list(itertools.chain.from_iterable(splitList))
#Create the summary
summary['totalTests'] = int(testSummarySearch.group(1))
failedSections = sections[1:-1]
summary['failedTests'] = len(failedSections)/2
summary['died'] = False
summary['timeout'] = False
#Extract the messages from the failed tests
failedTests = {}
for i in range(0, len(failedSections), 2):
header = failedSections[i]
headerParts = header.split()
tname = headerParts[1]
pyunitmsg = failedSections[i+1]
failedTests[tname] = {'hint': pyunitmsg}
return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
# Racket tests require some manipulation of the input and test files so that
# it works nicely. We do that here
with open(testFile, 'r') as testF:
testText = testF.read()
studentFileName = re.search(r'^[^;]*\(include "([^"]+)"\)',\
testText, re.MULTILINE).group(1)
#Remove the #lang racket from the student file so that the import in the test
#will work
try:
studentFile = open(studentFileName, 'r')
studentFileText = studentFile.read()
studentFile.close()
studentFileText = re.sub(r'(#lang +racket)', r';\1\n', studentFileText)
studentFile = open(studentFileName, 'w')
studentFile.write(studentFileText)
studentFile.close()
except Exception as e:
return {'timeout': False, 'died': True, 'rawErr': str(e)}, {}
#Put a random string in the test file so that we can tell where the tests
#begin.
randline = ''.join(
random.choice(string.letters + string.digits) for _ in range(10))
testText = re.sub(r'(\(check-[^ ]+)',
'(displayln "' + randline +
r'" (current-error-port))\n\1',
testText,
count=1)
with open(testFile, 'w') as testF:
testF.write(testText)
#End of manipulation of files
#Run the tests
testProc = Command(cmdPrefix + ['racket', testFile])
timeout, testOut, testError = testProc.run(timeout=int(timeLimit),
env=environ)
if timeout:
print testError
print "Timeout reached"
summary = {}
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = True
summary['died'] = False
summary['rawOut'] = ""
summary['rawErr'] = ""
return summary, {}
# startTime = datetime.now()
# testProc = Popen(cmdPrefix + ['/usr/bin/racket', testFile],\
# stdout=PIPE, stderr=PIPE)
#
# while testProc.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# testProc.kill()
# #Report a timeout
# return {'timeout':True, 'died':False}, {}
summary = {}
#testOut, testError = testProc.communicate()
summary['rawOut'] = testOut
summary['rawErr'] = testError
if testProc.returncode == -1:
return {
'timeout': False,
'died': True,
'rawErr': testError,
'rawOut': testOut
}, {}
try:
testResults = testError.split(randline)[1]
# if there were no errors, result is printed to stdout
testSummarySearch = re.search("([0-9]+) test\(s\) run", testOut)
# Otherwise, result is printed to stderr
if (testSummarySearch == None):
testSummarySearch = re.search("([0-9]+) test\(s\) run", testError)
except IndexError:
# b. If this line was never seen, then something bad happened
return {
'timeout':
False,
'died':
True,
'rawErr':
"Could not parse the test output:\n" + testOut + "\n\n" + testError
}, {}
else:
failedTests = {}
for failedCaseMatch in re.finditer('-' * 20 + r'\n(.*?)\n' + '-' * 20,
testResults,
flags=re.DOTALL):
failedCase = failedCaseMatch.group(1)
err = re.search(r'\nmessage: *"([^:]*):? *(.*)"\n', failedCase)
if err:
testname = err.group(1)
msg = err.group(2)
failedTests[testname] = {'hint': msg}
summary['died'] = False
summary['timeout'] = False
if testSummarySearch != None:
summary['totalTests'] = int(testSummarySearch.group(1))
else:
summary['totalTests'] = ""
summary['failedTests'] = len(failedTests.keys())
return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
cmdCompile = ["javac", "-cp", "/usr/share/java/junit4.jar:.", testFile]
#Not using cmdPrefix because we need to write files to the directory
compileProc = Popen(cmdCompile, stderr=PIPE)
_, compOut = compileProc.communicate()
if compOut:
#If the compiler produced output on stderr we have an issue so report it
summary = {}
summary['died'] = True
summary['rawErr'] = compOut
summary['timeout'] = False
return summary, {}
className = testFile[:-5]
pathName = os.getcwd()+className
cmdRun = ["/usr/bin/java", "-cp", "/usr/share/java/junit4.jar:.", "org.junit.runner.JUnitCore", className]
# startTime = datetime.now()
#
# runner = Popen(cmdPrefix+cmdRun,stdout=PIPE, stderr=PIPE)
#
# while runner.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# runner.kill()
# return {'timeout':True, 'died':False}, {}
# break
#
# stdout,stderr = runner.communicate()
runner = Command(cmdPrefix+cmdRun)
timeout, stdout, stderr = runner.run(timeout=int(timeLimit), env=environ)
if timeout:
return {'timeout':True, 'died':False}, {}
summary = {}
summary['rawOut'] = stdout
summary['rawErr'] = stderr
#Find all error reports
#failedTests= re.finditer("[0-9]*\) (.+)\(\w+\)\n(?:(?:java\.lang\..+Error)|(?:org\.junit\..+Failure)): (.*)", stdout)
testSplit = re.split(r"[0-9]*\) (.+)\(\w+\)", stdout)
#Slice the summary off
testSplit = testSplit[1:]
failedTests = {}
#Parse error reports for outputting to json
for i in range(0, len(testSplit), 2):
# testName = fail.group(1)
# report = {}
# message = fail.group(2)
# hint = message
# expact = re.search("expected:\<(.+)\> but was:\<(.+)\>", message)
#
# if expact:
# hint = message[0:message.find("expected")-1]
# report['expected'] = expact.group(1)
# report['returned'] = expact.group(2)
report = {}
report['hint'] = testSplit[i+1]
failedTests[testSplit[i]] = report
#Creates summary info
lastLine = re.search("OK \(([0-9]*) tests\)", stdout)
if lastLine:
summary['totalTests'] = int(lastLine.group(1))
summary['failedTests'] = 0
else:
lastLine = re.search("FAILURES[!]+\nTests run: ([0-9]*),[ ]+Failures: ([0-9]*)", stdout)
summary['totalTests'] = int(lastLine.group(1))
summary['failedTests'] = int(lastLine.group(2))
summary['timeout'] = False
summary['died'] = False
return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
# Racket tests require some manipulation of the input and test files so that
# it works nicely. We do that here
with open(testFile, "r") as testF:
testText = testF.read()
studentFileName = re.search(r'^[^;]*\(include "([^"]+)"\)', testText, re.MULTILINE).group(1)
# Remove the #lang racket from the student file so that the import in the test
# will work
try:
studentFile = open(studentFileName, "r")
studentFileText = studentFile.read()
studentFile.close()
studentFileText = re.sub(r"(#lang +racket)", r";\1\n", studentFileText)
studentFile = open(studentFileName, "w")
studentFile.write(studentFileText)
studentFile.close()
except Exception as e:
return {"timeout": False, "died": True, "rawErr": str(e)}, {}
# Put a random string in the test file so that we can tell where the tests
# begin.
randline = "".join(random.choice(string.letters + string.digits) for _ in range(10))
testText = re.sub(r"(\(check-[^ ]+)", '(displayln "' + randline + r'" (current-error-port))\n\1', testText, count=1)
with open(testFile, "w") as testF:
testF.write(testText)
# End of manipulation of files
# Run the tests
testProc = Command(cmdPrefix + ["/usr/bin/racket", testFile])
timeout, testOut, testError = testProc.run(timeout=int(timeLimit), env=environ)
if timeout:
print testError
print "Timeout reached"
summary = {}
summary["totalTests"] = 0
summary["failedTests"] = 0
summary["timeout"] = True
summary["died"] = False
summary["rawOut"] = ""
summary["rawErr"] = ""
return summary, {}
# startTime = datetime.now()
# testProc = Popen(cmdPrefix + ['/usr/bin/racket', testFile],\
# stdout=PIPE, stderr=PIPE)
#
# while testProc.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# testProc.kill()
# #Report a timeout
# return {'timeout':True, 'died':False}, {}
summary = {}
# testOut, testError = testProc.communicate()
summary["rawOut"] = testOut
summary["rawErr"] = testError
if testProc.returncode == -1:
return {"timeout": False, "died": True, "rawErr": testError, "rawOut": testOut}, {}
try:
testResults = testError.split(randline)[1]
# if there were no errors, result is printed to stdout
testSummarySearch = re.search("([0-9]+) test\(s\) run", testOut)
# Otherwise, result is printed to stderr
if testSummarySearch == None:
testSummarySearch = re.search("([0-9]+) test\(s\) run", testError)
except IndexError:
# b. If this line was never seen, then something bad happened
return (
{
"timeout": False,
"died": True,
"rawErr": "Could not parse the test output:\n" + testOut + "\n\n" + testError,
},
{},
)
else:
failedTests = {}
for failedCaseMatch in re.finditer("-" * 20 + r"\n(.*?)\n" + "-" * 20, testResults, flags=re.DOTALL):
failedCase = failedCaseMatch.group(1)
err = re.search(r'\nmessage: *"([^:]*):? *(.*)"\n', failedCase)
if err:
testname = err.group(1)
msg = err.group(2)
failedTests[testname] = {"hint": msg}
summary["died"] = False
summary["timeout"] = False
if testSummarySearch != None:
summary["totalTests"] = int(testSummarySearch.group(1))
else:
summary["totalTests"] = ""
summary["failedTests"] = len(failedTests.keys())
return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
# Racket tests require some manipulation of the input and test files so that
# it works nicely. We do that here
with open(testFile, 'r') as testF:
testText = testF.read()
studentFileName = re.search(r'^[^;]*\(include "([^"]+)"\)',\
testText, re.MULTILINE).group(1)
#Remove the #lang racket from the student file so that the import in the test
#will work
try:
studentFile = open(studentFileName, 'r')
studentFileText = studentFile.read()
studentFile.close()
studentFileText = re.sub(r'(#lang +racket)', r';\1\n', studentFileText)
studentFile = open(studentFileName, 'w')
studentFile.write(studentFileText)
studentFile.close()
except Exception as e:
return {'timeout':False, 'died':True, 'rawErr': str(e)}, {}
#Put a random string in the test file so that we can tell where the tests
#begin.
randline = ''.join(random.choice(string.letters + string.digits) for _ in range(10))
testText = re.sub(r'(\(check-[^ ]+)',
'(displayln "' +
randline +
r'" (current-error-port))\n\1',
testText, count=1)
with open(testFile, 'w') as testF:
testF.write(testText)
#End of manipulation of files
#Run the tests
testProc = Command(cmdPrefix + ['/usr/bin/racket', testFile])
timeout, testOut, testError = testProc.run(timeout=int(timeLimit), env=environ)
if timeout:
return {'timeout':True, 'died':False}, {}
# startTime = datetime.now()
# testProc = Popen(cmdPrefix + ['/usr/bin/racket', testFile],\
# stdout=PIPE, stderr=PIPE)
#
# while testProc.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# testProc.kill()
# #Report a timeout
# return {'timeout':True, 'died':False}, {}
summary = {}
#testOut, testError = testProc.communicate()
summary['rawOut'] = testOut
summary['rawErr'] = testError
if testProc.returncode != 0:
return {'timeout':False, 'died':True, 'rawErr': testError, 'rawOut':testOut}, {}
try:
testResults = testError.split(randline)[1]
except IndexError:
# b. If this line was never seen, then something bad happened
return {'timeout':False, 'died':True, 'rawErr': "Could not parse the test output:\n" + testOut + "\n\n" + testError}, {}
else:
failedTests = {}
for failedCaseMatch in re.finditer('-'*20 + r'\n(.*?)\n' + '-'*20, testResults, flags=re.DOTALL):
failedCase = failedCaseMatch.group(1)
err = re.search(r'\nmessage: *"([^:]*):? *(.*)"\n', failedCase)
if err:
testname = err.group(1)
msg = err.group(2)
failedTests[testname] = {'hint': msg}
summary['died'] = False
summary['timeout'] = False
summary['totalTests'] = 0
summary['failedTests'] = len(failedTests.keys())
return summary, failedTests
def run_tests(command_prefix, test_file, time_limit):
"""Wrapper for "runTests" that adheres to PEP8 variable naming
conventions."""
try:
with open(test_file) as f:
try:
tests = parse_test_file_contents(f.read())
except JFLAPTestFileParseError as e:
error = ("Could not parse test file '{}': {}"
.format(test_file, e.message))
raise CouldNotRunJFLAPTestsError(error)
# Determine the containing directory and literal filename
# for the test file.
directory, test_filename = os.path.split(test_file)
if not directory:
directory = "."
# Find all the JFLAP files that the test_file might
# possibly be testing.
jflap_filenames = []
for file in os.listdir(directory):
if file.endswith(".jff"):
jflap_filenames.append(file)
# If there's only one JFLAP file, nothing fancy is needed,
# because there's only one choice.
if len(jflap_filenames) == 1:
jflap_filename = jflap_filenames[0]
else:
# Otherwise, we need to try to guess which file to
# test by swapping out the extension of the test file
# for ".jff".
try:
extensionIndex = test_filename.rindex(".")
except ValueError:
error = ("Test file '{}' does not have an extension"
.format(test_filename))
raise CouldNotRunJFLAPTestsError(error)
jflap_filename = test_filename[:extensionIndex] + ".jff"
if jflap_filename not in jflap_filenames:
# We could try harder to find a match, or allow
# specifying the filename of the corresponding
# JFLAP file within the test file, but to be
# honest that wouldn't really provide much help
# (because if you can put the name of the JFLAP
# file in the test file, you can just name the
# test file instead).
error = ("Test file '{}' does not match any of the"
" available JFLAP files, which are: {}"
.format(test_filename,
", ".join("'{}'".format(file)
for file in jflap_filenames)))
raise CouldNotRunJFLAPTestsError(error)
jflap_file = os.path.join(directory, jflap_filename)
failedTests = {}
all_stdout = ""
all_stderr = ""
# To ensure that this process does not take longer
# than the provided time limit, we divide the time
# equally among each test.
if time_limit:
timeout = time_limit / len(tests)
else:
timeout = None
# We'll also need to figure out the directory containing
# this Python file, so we can find jflaplib-cli.jar.
script_directory = os.path.split(__file__)[0]
for word, should_accept in tests.items():
# See [1] for the source code of jflaplib-cli.jar.
# Note that the command-line parsing library used by
# jflaplib-cli, JCommander, has an odd quirk in the
# way it parses arguments. First it trims whitespace
# from both ends of each argument, and then it removes
# a pair of double quotes if one exists. So, to ensure
# an argument is interpreted literally, we just wrap
# it in double quotes! See [2] for discussion of this
# issue.
#
# [1]: https://github.com/raxod502/jflap-lib
# [2]: https://github.com/cbeust/jcommander/issues/306
command = Command(command_prefix +
["java",
# The following system property
# prevents the Java process from
# showing up in the Mac app
# switcher, which is extremely
# annoying.
"-Dapple.awt.UIElement=true",
"-jar",
os.path.join(script_directory,
"jflaplib-cli.jar"),
"run",
jflap_file,
'"{}"'.format(word)])
timed_out, stdout, stderr = command.run(
timeout=timeout,
env=os.environ)
all_stdout += stdout
all_stderr += stderr
if timed_out:
error = ("Timed out (took more than {} seconds)"
.format(timeout))
failedTests[word] = {"hint": error}
else:
# jflaplib-cli should print "true" or "false",
# depending on whether the NFA or Turing
# machine accepted or rejected the input. But
# we handle all the possible edge cases here,
# just in case.
contains_true = "true" in stdout
contains_false = "false" in stdout
if contains_true and contains_false:
stdout = stdout.strip()
if not stdout:
stdout = "(none)"
stderr = stderr.strip()
if not stderr:
stderr = "(none)"
error = ("JFLAP reported both 'accept' and"
" 'reject', output: {}; error: {}"
.format(stdout.strip(), stderr.strip()))
failedTests[word] = {"hint": error}
elif not (contains_true or contains_false):
stdout = stdout.strip()
if not stdout:
stdout = "(none)"
stderr = stderr.strip()
if not stderr:
stderr = "(none)"
error = ("JFLAP reported neither 'accept' nor"
" 'reject', output: {}; error: {}"
.format(stdout.strip(), stderr.strip()))
failedTests[word] = {"hint": error}
elif contains_true is not should_accept:
error = ("This word should have been {}, but it"
" was {}"
.format(result_to_str(should_accept),
result_to_str(contains_true)))
failedTests[word] = {"hint": error}
summary = {"died": False,
"timeout": False,
"totalTests": len(tests),
"failedTests": len(failedTests),
"rawOut": all_stdout,
"rawErr": all_stderr}
return summary, failedTests
except Exception as e:
if isinstance(e, CouldNotRunJFLAPTestsError):
# If the error is a CouldNotRunJFLAPTestsError, then this
# code generated the error message and included all
# necessary information. So we can just return the
# message.
error = e.message
else:
# Otherwise, there was an unexpected error, and we'll
# provide the whole stack trace for debugging purposes.
# This is obviously very bad from a security perspective,
# but worrying about it would be like making sure to turn
# out the lights when the building is on fire, given the
# security of the rest of this website.
error = traceback.format_exc()
summary = {"died": True,
"timeout": False,
"totalTests": 0,
"failedTests": 0,
"rawOut": "",
"rawErr": error}
failedTests = {}
return summary, failedTests
def run_tests(command_prefix, test_file, time_limit):
"""Wrapper for "runTests" that adheres to PEP8 variable naming
conventions.
"""
try:
with open(test_file) as f:
# Split the time limit 1-1-2 among finding the HMMM
# program, assembling it, and running the test cases.
get_hmmm_timeout = time_limit and time_limit / 4
assembly_timeout = time_limit and time_limit / 4
total_test_timeout = time_limit and time_limit / 2
# Get the text of the HMMM program (if a unique match can
# be found).
program = get_hmmm_program(command_prefix,
test_file,
get_hmmm_timeout)
# Get the paths to the assembler and simulator. We have to
# shell out here because they are written in Python 3!
script_directory = os.path.split(__file__)[0]
hmmmAssembler = os.path.join(script_directory, "hmmmAssembler.py")
hmmmSimulator = os.path.join(script_directory, "hmmmSimulator.py")
# Normalize the paths to the assembler and simulator,
# since our working directory could be somewhere totally
# unrelated when we invoke them.
hmmmAssembler = os.path.realpath(hmmmAssembler)
hmmmSimulator = os.path.realpath(hmmmSimulator)
# Get the path to the test file folder.
test_directory, test_filename = os.path.split(test_file)
# If the test file is specified only as a filename, then
# the test_directory is the current directory.
if not test_directory:
test_directory = "."
# Run the assembler.
command = Command(command_prefix +
["python3",
hmmmAssembler,
"--program-text",
program])
result = command.run(compatibility=False,
cwd=test_directory,
timeout=assembly_timeout)
return_code, stdout, stderr, timed_out = result
if return_code != 0:
error = ("Assembly completed unsuccessfully{}"
.format(" (timed out)" if timed_out else ""))
info = []
if stdout.strip():
info.append("Output:\n{}".format(stdout.strip()))
if stderr.strip():
info.append("Error:\n{}".format(stderr.strip()))
if info:
error += "\n"
error += "\n".join(info)
raise CouldNotRunHmmmTestsError(error)
# Parse the test cases from the test file.
test_cases = {} # discard duplicate test cases
for line in f:
test_case = parse_test_case(line)
if test_case:
test_cases[format_test_case(test_case)] = test_case
# Run the test cases using the simulator, collecting
# stdout and stderr.
failed_tests = {}
all_stdout = ""
all_stderr = ""
if test_cases:
timeout = (total_test_timeout and
total_test_timeout / len(test_cases))
for name, test_case in test_cases.items():
command = Command(command_prefix +
["python3",
hmmmSimulator,
"--test-case",
repr(test_case)])
result = command.run(compatibility=False,
cwd=test_directory,
timeout=timeout)
return_code, stdout, stderr, timed_out = result
all_stdout += stdout
all_stderr += stderr
# Messages designed to be read by hmmmgrader.py are
# delimited by double brackets [[ like this ]].
outputs = re.findall(r"\[\[ (.+?) \]\]", stdout)
if "test case passed" in outputs:
continue
found_failure = False
for output in outputs:
match = re.match(r"test case failed: (.+)", output)
if match:
error = match.group(1)
# Capitalize the error message.
error = error[:1].upper() + error[1:]
failed_tests[name] = {"hint": error}
found_failure = True
break
if found_failure:
continue
error = ("simulation completed unsuccessfully{}"
.format(" (timed out)" if timed_out else ""))
info = []
if stdout.strip():
info.append("Output:\n{}".format(stdout.strip()))
if stderr.strip():
info.append("Error:\n{}".format(stderr.strip()))
if info:
error += "\n"
error += "\n".join(info)
failed_tests[name] = {"hint": error}
summary = {"died": False,
"timeout": False,
"totalTests": len(test_cases),
"failedTests": len(failed_tests),
"rawOut": all_stdout,
"rawErr": all_stderr}
return summary, failed_tests
except Exception as e:
if isinstance(e, CouldNotRunHmmmTestsError):
# If the error is a CouldNotRunHmmmTestsError, then this
# code generated the error message and included all
# necessary information. So we can just return the
# message.
error = e.message
else:
# Otherwise, there was an unexpected error, and we'll
# provide the whole stack trace for debugging purposes.
# This is obviously very bad from a security perspective,
# but worrying about it would be like making sure to turn
# out the lights when the building is on fire, given the
# security of the rest of this website.
error = traceback.format_exc()
summary = {"died": True,
"timeout": False,
"totalTests": 0,
"failedTests": 0,
"rawOut": "",
"rawErr": error}
failedTests = {}
return summary, failedTests
def get_hmmm_program(command_prefix, test_file, time_limit):
"""Attempts to find the HMMM program that should be tested by the
test_file, according to a number of heuristics.
Returns the text of the program, or throws a
CouldNotRunHmmmTestsError if an appropriate program could not be
found.
"""
# To start, we determine the directory and filename of the test
# file, and get the list of files in that directory.
test_directory, test_filename = os.path.split(test_file)
# If the test_file is specified as just a filename, then its
# directory is the current directory.
if not test_directory:
test_directory = "."
# Get the names of all the files in the same directory as the test
# file.
filenames = os.listdir(test_directory)
# First we look for .hmmm files.
hmmm_filenames = []
for file_ in filenames:
if file_.endswith(".hmmm"):
hmmm_filenames.append(file_)
# If there's only one .hmmm file, we have found our candidate.
if len(hmmm_filenames) == 1:
with open(hmmm_filenames[0]) as f:
return f.read()
# Except in the special case of there only being one .hmmm file,
# we're going to need to know the name of the HMMM program to be
# tested. The only way to determine this is to strip the extension
# from the test file (presuming, of course, that it has one!).
try:
extension_index = test_filename.rindex(".")
except ValueError:
error = ("Test file '{}' does not have an extension"
.format(test_filename))
raise CouldNotRunHmmmTestsError(error)
program_name = test_filename[:extension_index]
# Presuming that there are any .hmmm files, we'll try to guess the
# file by swapping out the extension on the test file.
if hmmm_filenames:
hmmm_filename = program_name + ".hmmm"
if hmmm_filename in hmmm_filenames:
with open(hmmm_filename) as f:
return f.read()
else:
error = ("Test file '{}' does not match any of the available"
" .hmmm files, which are: {}"
.format(test_filename,
", ".join("'{}'".format(file_)
for file_ in hmmm_filenames)))
raise CouldNotRunHmmmTestsError(error)
# If there aren't any .hmmm files, we'll check any available .py
# files next.
py_filenames = []
for filename in filenames:
file_ = os.path.join(test_directory, filename)
if not filename.endswith(".py"):
continue
# To filter out irrelevant .py files (possibly autograder
# code), we check to make sure the ones we find define a
# "Hmmm" function, which seems to be the template used in CS5
# classes (whereas CS42 classes just create .hmmm files, thus
# obviating the problem of distinguishing between different
# types of .py files).
with open(file_) as f:
# Split the string literal so that the check doesn't match
# this file (hmmmgrader.py)!
if "def " "Hmmm" not in f.read():
continue
py_filenames.append(filename)
if not py_filenames:
error = ("no .hmmm or .py files with HMMM programs available")
raise CouldNotRunHmmmTestsError(error)
# We define a function to run using "python3 -c" that will print a
# dictionary containing all the multiline strings defined in a
# module.
extraction_command = r"""\
import {0}
programs = {{}} # needed to escape the braces from str.format
for name in dir({0}):
# Ignore items that are in all modules (including the __builtins__
# module).
if name not in dir(__builtins__):
item = eval("{0}." + name)
# Only accept multiline strings.
if isinstance(item, str) and "\n" in item:
programs[name] = item
print(repr(programs))\
"""
# The time limit is split equally among each of the subprocess
# calls, to ensure that we don't accidentally take longer than
# allowed in total.
timeout = time_limit and time_limit / len(py_filenames)
# Now we extract all possible HMMM programs from all the valid .py
# files that we found, and place them into a single dictionary
# (allowing for multiple programs by the same name, and retaining
# the information about which programs were found in which files).
programs = {}
# All the information about errors while opening files goes into
# another dictionary, for possible error reporting.
error_data = {}
for file_ in py_filenames:
# Trim the .py extension.
module_name = file_[:-3]
command = Command(command_prefix +
["python3",
"-c",
extraction_command.format(module_name)])
result = command.run(compatibility=False,
cwd=test_directory,
timeout=timeout)
return_code, stdout, stderr, timed_out = result
if return_code == 0:
new_programs = eval(stdout)
for name, program in new_programs.items():
if name not in programs:
programs[name] = []
programs[name].append((file_, program))
else:
error_data[file_] = result
# Now we look at the collected programs and see if we can uniquely
# identify one that matches the desired program. Collecting lists
# of all the names of programs and all the files that were
# investigated allows for more useful error reporting.
all_names = set()
all_files = set()
for name, candidates in programs.items():
all_names.add(name)
for candidate in candidates:
all_files.add(candidates[0])
# If only one program is defined, then we'll choose it regardless
# of whether its name matches the test file.
if len(all_names) == 1:
program_name = next(iter(all_names))
if program_name in programs:
# Keep in mind that a program by the same name might be
# defined in multiple .py files, so we need to check for that.
candidates = programs[program_name]
if len(candidates) == 1:
return candidates[0][1]
error = ("program '{}' was defined in multiple files ({})"
.format(program_name,
", ".join("'{}'".format(pair[0])
for pair in candidates)))
raise CouldNotRunHmmmTestsError(error)
# If we have more than one program defined, and none of them
# matches the test file name (which defines the initial value of
# program_name), then we have an error. We report it with as much
# diagnostic information as possible.
error = ("test file '{}' does not match any of the available HMMM programs"
.format(test_filename))
if all_names:
error += (" (programs {} defined in files {})"
.format(", ".join("'{}'".format(name)
for name in all_names),
", ".join("'{}'".format(file_)
for file_ in all_files)))
if len(all_files) < len(py_filenames):
error += (" (could not open files {})"
.format(", ".join("'{}'".format(file_)
for file_ in (set(py_filenames) -
all_files))))
if error_data:
error += ("; errors (returncode, stdout, stderr, timed_out)"
" while opening files: {}"
.format(repr(error_data)))
raise CouldNotRunHmmmTestsError(error)
def runTests(cmdPrefix, testFile, timeLimit):
cmdCompile = ["javac", "-cp", "/usr/share/java/junit4.jar:.", testFile]
#Not using cmdPrefix because we need to write files to the directory
compileProc = Popen(cmdCompile, stderr=PIPE)
_, compOut = compileProc.communicate()
if compOut:
#If the compiler produced output on stderr we have an issue so report it
summary = {}
summary['died'] = True
summary['rawErr'] = compOut
summary['timeout'] = False
return summary, {}
className = testFile[:-5]
pathName = os.getcwd() + className
cmdRun = [
"/usr/bin/java", "-cp", "/usr/share/java/junit4.jar:.",
"org.junit.runner.JUnitCore", className
]
# startTime = datetime.now()
#
# runner = Popen(cmdPrefix+cmdRun,stdout=PIPE, stderr=PIPE)
#
# while runner.poll() is None:
# currentTime = datetime.now()
# delta = currentTime - startTime
# if delta.total_seconds() > timeLimit:
# runner.kill()
# return {'timeout':True, 'died':False}, {}
# break
#
# stdout,stderr = runner.communicate()
runner = Command(cmdPrefix + cmdRun)
timeout, stdout, stderr = runner.run(timeout=int(timeLimit), env=environ)
if timeout:
return {'timeout': True, 'died': False}, {}
summary = {}
summary['rawOut'] = stdout
summary['rawErr'] = stderr
#Find all error reports
#failedTests= re.finditer("[0-9]*\) (.+)\(\w+\)\n(?:(?:java\.lang\..+Error)|(?:org\.junit\..+Failure)): (.*)", stdout)
testSplit = re.split(r"[0-9]*\) (.+)\(\w+\)", stdout)
#Slice the summary off
testSplit = testSplit[1:]
failedTests = {}
#Parse error reports for outputting to json
for i in range(0, len(testSplit), 2):
# testName = fail.group(1)
# report = {}
# message = fail.group(2)
# hint = message
# expact = re.search("expected:\<(.+)\> but was:\<(.+)\>", message)
#
# if expact:
# hint = message[0:message.find("expected")-1]
# report['expected'] = expact.group(1)
# report['returned'] = expact.group(2)
report = {}
report['hint'] = testSplit[i + 1]
failedTests[testSplit[i]] = report
#Creates summary info
lastLine = re.search("OK \(([0-9]*) tests\)", stdout)
if lastLine:
summary['totalTests'] = int(lastLine.group(1))
summary['failedTests'] = 0
else:
lastLine = re.search(
"FAILURES[!]+\nTests run: ([0-9]*),[ ]+Failures: ([0-9]*)", stdout)
summary['totalTests'] = int(lastLine.group(1))
summary['failedTests'] = int(lastLine.group(2))
summary['timeout'] = False
summary['died'] = False
return summary, failedTests
def run_tests(command_prefix, test_file, time_limit):
"""Wrapper for "runTests" that adheres to PEP8 variable naming
conventions.
"""
try:
with open(test_file) as f:
# Split the time limit 1-1-2 among finding the HMMM
# program, assembling it, and running the test cases.
get_hmmm_timeout = time_limit and time_limit / 4
assembly_timeout = time_limit and time_limit / 4
total_test_timeout = time_limit and time_limit / 2
# Get the text of the HMMM program (if a unique match can
# be found).
program = get_hmmm_program(command_prefix, test_file,
get_hmmm_timeout)
# Get the paths to the assembler and simulator. We have to
# shell out here because they are written in Python 3!
script_directory = os.path.split(__file__)[0]
hmmmAssembler = os.path.join(script_directory, "hmmmAssembler.py")
hmmmSimulator = os.path.join(script_directory, "hmmmSimulator.py")
# Normalize the paths to the assembler and simulator,
# since our working directory could be somewhere totally
# unrelated when we invoke them.
hmmmAssembler = os.path.realpath(hmmmAssembler)
hmmmSimulator = os.path.realpath(hmmmSimulator)
# Get the path to the test file folder.
test_directory, test_filename = os.path.split(test_file)
# If the test file is specified only as a filename, then
# the test_directory is the current directory.
if not test_directory:
test_directory = "."
# Run the assembler.
command = Command(
command_prefix +
["python3", hmmmAssembler, "--program-text", program])
result = command.run(compatibility=False,
cwd=test_directory,
timeout=assembly_timeout)
return_code, stdout, stderr, timed_out = result
if return_code != 0:
error = ("Assembly completed unsuccessfully{}".format(
" (timed out)" if timed_out else ""))
info = []
if stdout.strip():
info.append("Output:\n{}".format(stdout.strip()))
if stderr.strip():
info.append("Error:\n{}".format(stderr.strip()))
if info:
error += "\n"
error += "\n".join(info)
raise CouldNotRunHmmmTestsError(error)
# Parse the test cases from the test file.
test_cases = {} # discard duplicate test cases
for line in f:
test_case = parse_test_case(line)
if test_case:
test_cases[format_test_case(test_case)] = test_case
# Run the test cases using the simulator, collecting
# stdout and stderr.
failed_tests = {}
all_stdout = ""
all_stderr = ""
if test_cases:
timeout = (total_test_timeout
and total_test_timeout / len(test_cases))
for name, test_case in test_cases.items():
command = Command(
command_prefix +
["python3", hmmmSimulator, "--test-case",
repr(test_case)])
result = command.run(compatibility=False,
cwd=test_directory,
timeout=timeout)
return_code, stdout, stderr, timed_out = result
all_stdout += stdout
all_stderr += stderr
# Messages designed to be read by hmmmgrader.py are
# delimited by double brackets [[ like this ]].
outputs = re.findall(r"\[\[ (.+?) \]\]", stdout)
if "test case passed" in outputs:
continue
found_failure = False
for output in outputs:
match = re.match(r"test case failed: (.+)", output)
if match:
error = match.group(1)
# Capitalize the error message.
error = error[:1].upper() + error[1:]
failed_tests[name] = {"hint": error}
found_failure = True
break
if found_failure:
continue
error = ("simulation completed unsuccessfully{}".format(
" (timed out)" if timed_out else ""))
info = []
if stdout.strip():
info.append("Output:\n{}".format(stdout.strip()))
if stderr.strip():
info.append("Error:\n{}".format(stderr.strip()))
if info:
error += "\n"
error += "\n".join(info)
failed_tests[name] = {"hint": error}
summary = {
"died": False,
"timeout": False,
"totalTests": len(test_cases),
"failedTests": len(failed_tests),
"rawOut": all_stdout,
"rawErr": all_stderr
}
return summary, failed_tests
except Exception as e:
if isinstance(e, CouldNotRunHmmmTestsError):
# If the error is a CouldNotRunHmmmTestsError, then this
# code generated the error message and included all
# necessary information. So we can just return the
# message.
error = e.message
else:
# Otherwise, there was an unexpected error, and we'll
# provide the whole stack trace for debugging purposes.
# This is obviously very bad from a security perspective,
# but worrying about it would be like making sure to turn
# out the lights when the building is on fire, given the
# security of the rest of this website.
error = traceback.format_exc()
summary = {
"died": True,
"timeout": False,
"totalTests": 0,
"failedTests": 0,
"rawOut": "",
"rawErr": error
}
failedTests = {}
return summary, failedTests
def get_hmmm_program(command_prefix, test_file, time_limit):
"""Attempts to find the HMMM program that should be tested by the
test_file, according to a number of heuristics.
Returns the text of the program, or throws a
CouldNotRunHmmmTestsError if an appropriate program could not be
found.
"""
# To start, we determine the directory and filename of the test
# file, and get the list of files in that directory.
test_directory, test_filename = os.path.split(test_file)
# If the test_file is specified as just a filename, then its
# directory is the current directory.
if not test_directory:
test_directory = "."
# Get the names of all the files in the same directory as the test
# file.
filenames = os.listdir(test_directory)
# First we look for .hmmm files.
hmmm_filenames = []
for file_ in filenames:
if file_.endswith(".hmmm"):
hmmm_filenames.append(file_)
# If there's only one .hmmm file, we have found our candidate.
if len(hmmm_filenames) == 1:
with open(hmmm_filenames[0]) as f:
return f.read()
# Except in the special case of there only being one .hmmm file,
# we're going to need to know the name of the HMMM program to be
# tested. The only way to determine this is to strip the extension
# from the test file (presuming, of course, that it has one!).
try:
extension_index = test_filename.rindex(".")
except ValueError:
error = (
"Test file '{}' does not have an extension".format(test_filename))
raise CouldNotRunHmmmTestsError(error)
program_name = test_filename[:extension_index]
# Presuming that there are any .hmmm files, we'll try to guess the
# file by swapping out the extension on the test file.
if hmmm_filenames:
hmmm_filename = program_name + ".hmmm"
if hmmm_filename in hmmm_filenames:
with open(hmmm_filename) as f:
return f.read()
else:
error = ("Test file '{}' does not match any of the available"
" .hmmm files, which are: {}".format(
test_filename,
", ".join("'{}'".format(file_)
for file_ in hmmm_filenames)))
raise CouldNotRunHmmmTestsError(error)
# If there aren't any .hmmm files, we'll check any available .py
# files next.
py_filenames = []
for filename in filenames:
file_ = os.path.join(test_directory, filename)
if not filename.endswith(".py"):
continue
# To filter out irrelevant .py files (possibly autograder
# code), we check to make sure the ones we find define a
# "Hmmm" function, which seems to be the template used in CS5
# classes (whereas CS42 classes just create .hmmm files, thus
# obviating the problem of distinguishing between different
# types of .py files).
with open(file_) as f:
# Split the string literal so that the check doesn't match
# this file (hmmmgrader.py)!
if "def " "Hmmm" not in f.read():
continue
py_filenames.append(filename)
if not py_filenames:
error = ("no .hmmm or .py files with HMMM programs available")
raise CouldNotRunHmmmTestsError(error)
# We define a function to run using "python3 -c" that will print a
# dictionary containing all the multiline strings defined in a
# module.
extraction_command = r"""\
import {0}
programs = {{}} # needed to escape the braces from str.format
for name in dir({0}):
# Ignore items that are in all modules (including the __builtins__
# module).
if name not in dir(__builtins__):
item = eval("{0}." + name)
# Only accept multiline strings.
if isinstance(item, str) and "\n" in item:
programs[name] = item
print(repr(programs))\
"""
# The time limit is split equally among each of the subprocess
# calls, to ensure that we don't accidentally take longer than
# allowed in total.
timeout = time_limit and time_limit / len(py_filenames)
# Now we extract all possible HMMM programs from all the valid .py
# files that we found, and place them into a single dictionary
# (allowing for multiple programs by the same name, and retaining
# the information about which programs were found in which files).
programs = {}
# All the information about errors while opening files goes into
# another dictionary, for possible error reporting.
error_data = {}
for file_ in py_filenames:
# Trim the .py extension.
module_name = file_[:-3]
command = Command(
command_prefix +
["python3", "-c",
extraction_command.format(module_name)])
result = command.run(compatibility=False,
cwd=test_directory,
timeout=timeout)
return_code, stdout, stderr, timed_out = result
if return_code == 0:
new_programs = eval(stdout)
for name, program in new_programs.items():
if name not in programs:
programs[name] = []
programs[name].append((file_, program))
else:
error_data[file_] = result
# Now we look at the collected programs and see if we can uniquely
# identify one that matches the desired program. Collecting lists
# of all the names of programs and all the files that were
# investigated allows for more useful error reporting.
all_names = set()
all_files = set()
for name, candidates in programs.items():
all_names.add(name)
for candidate in candidates:
all_files.add(candidates[0])
# If only one program is defined, then we'll choose it regardless
# of whether its name matches the test file.
if len(all_names) == 1:
program_name = next(iter(all_names))
if program_name in programs:
# Keep in mind that a program by the same name might be
# defined in multiple .py files, so we need to check for that.
candidates = programs[program_name]
if len(candidates) == 1:
return candidates[0][1]
error = ("program '{}' was defined in multiple files ({})".format(
program_name,
", ".join("'{}'".format(pair[0]) for pair in candidates)))
raise CouldNotRunHmmmTestsError(error)
# If we have more than one program defined, and none of them
# matches the test file name (which defines the initial value of
# program_name), then we have an error. We report it with as much
# diagnostic information as possible.
error = ("test file '{}' does not match any of the available HMMM programs"
.format(test_filename))
if all_names:
error += (" (programs {} defined in files {})".format(
", ".join("'{}'".format(name) for name in all_names),
", ".join("'{}'".format(file_) for file_ in all_files)))
if len(all_files) < len(py_filenames):
error += (" (could not open files {})".format(", ".join(
"'{}'".format(file_)
for file_ in (set(py_filenames) - all_files))))
if error_data:
error += ("; errors (returncode, stdout, stderr, timed_out)"
" while opening files: {}".format(repr(error_data)))
raise CouldNotRunHmmmTestsError(error)
def runTests(cmdPrefix, testFile, timeLimit):
startTime = datetime.now()
runCommand = 'python3' # change to just python for using python 2
testProc = Command(cmdPrefix + [runCommand, testFile])
timeoutReached, testOut, testError = \
testProc.run(timeout=int(timeLimit), env=environ)
#Check for a timeout
if timeoutReached:
print "Timeout reached"
summary = {}
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = timeoutReached
summary['died'] = False
summary['rawOut'] = ""
summary['rawErr'] = ""
return summary, {}
#testOut, testError = testProc.communicate()
summary = {}
summary['rawOut'] = testOut
summary['rawErr'] = testError
#Parse the results
testSummarySearch = re.search("Ran ([0-9]+) tests? in", testError)
#If we don't find the test summary the tests died so we report that
if not testSummarySearch:
summary['totalTests'] = 0
summary['failedTests'] = 0
summary['timeout'] = timeoutReached
summary['died'] = True
return summary, {}
# Get the list of failed tests
failedSections = testError.split('=' * 70)
splitList = []
for section in failedSections:
splitList.append(section.split('-' * 70))
#flatten the lists
sections = list(itertools.chain.from_iterable(splitList))
#Create the summary
summary['totalTests'] = int(testSummarySearch.group(1))
failedSections = sections[1:-1]
summary['failedTests'] = len(failedSections) / 2
summary['died'] = False
summary['timeout'] = False
#Extract the messages from the failed tests
failedTests = {}
for i in range(0, len(failedSections), 2):
header = failedSections[i]
headerParts = header.split()
tname = headerParts[1]
pyunitmsg = failedSections[i + 1]
failedTests[tname] = {'hint': pyunitmsg}
return summary, failedTests