Ejemplo n.º 1
0
def runTests(cmdPrefix, testFile, timeLimit):
    startTime = datetime.now()
    # testProc = Popen(cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'], stdout=PIPE, stderr=PIPE)
    #
    # timeoutReached = False
    # while testProc.poll() is None:
    #   currentTime = datetime.now()
    #   delta = currentTime - startTime
    #   if delta.total_seconds() > timeLimit:
    #     testProc.kill()
    #     timeoutReached = True
    #     break

    testProc = Command(
        cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'])

    timeoutReached, testOut, testError = testProc.run(timeout=int(timeLimit),
                                                      env=environ)

    if timeoutReached:
        summary = {}
        summary['totalTests'] = 0
        summary['failedTests'] = 0
        summary['timeout'] = timeoutReached
        summary['died'] = False
        summary['generalError'] = ""
        summary['rawOut'] = ""
        summary['rawErr'] = ""

        return summary, {}

    #testOut, testError = testProc.communicate()

    summary = {}
    summary['rawOut'] = testOut
    summary['rawErr'] = testError
    summary['timeout'] = False

    testOut = testError.split('\n')
    failedTests = {}
    for line in testOut:
        if re.match(r"^% No tests to run$", line):
            #Compilation error
            summary['died'] = True
            summary['totalTests'] = 0
            summary['failedTests'] = 0

        if re.match("\ttest \w*: [(failed)(received error)]", line):
            failedTests[line[6:line.find(":")]] = {'hint': ""}

    summary['totalTests'] = "N/A"
    summary['failedTests'] = "N/A"
    summary['died'] = False

    return summary, failedTests
def runTests(cmdPrefix, testFile, timeLimit):
  startTime = datetime.now()
  # testProc = Popen(cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'], stdout=PIPE, stderr=PIPE)
  #
  # timeoutReached = False
  # while testProc.poll() is None:
  #   currentTime = datetime.now()
  #   delta = currentTime - startTime
  #   if delta.total_seconds() > timeLimit:
  #     testProc.kill()
  #     timeoutReached = True
  #     break

  testProc = Command(cmdPrefix + ['swipl', '-s', testFile, '-g', 'run_tests', '-t', 'halt'])

  timeoutReached, testOut, testError = testProc.run(timeout=int(timeLimit), env=environ)

  if timeoutReached:
    summary = {}
    summary['totalTests'] = 0
    summary['failedTests'] = 0
    summary['timeout'] = timeoutReached
    summary['died'] = False
    summary['generalError'] = ""
    summary['rawOut'] = ""
    summary['rawErr'] = ""

    return summary, {}

  #testOut, testError = testProc.communicate()

  summary = {}
  summary['rawOut'] = testOut
  summary['rawErr'] = testError
  summary['timeout'] = False

  testOut = testError.split('\n')
  failedTests = {}
  for line in testOut:
    if re.match(r"^% No tests to run$", line):
      #Compilation error
      summary['died'] = True
      summary['totalTests'] = 0
      summary['failedTests'] = 0

    if re.match("\ttest \w*: [(failed)(received error)]", line):
      failedTests[line[6:line.find(":")]] = {'hint':""}

  summary['totalTests'] = "N/A"
  summary['failedTests'] = "N/A"
  summary['died'] = False

  return summary, failedTests
Ejemplo n.º 3
0
def runTests(cmdPrefix, testFile, timeLimit):
  startTime = datetime.now()

  testProc = Command(cmdPrefix + ['python', testFile])

  timeoutReached, testOut, testError = \
    testProc.run(timeout=int(timeLimit), env=environ)

  #Check for a timeout
  if timeoutReached:
    print "Timeout reached"
    summary = {}
    summary['totalTests'] = 0
    summary['failedTests'] = 0
    summary['timeout'] = timeoutReached
    summary['died'] = False
    summary['rawOut'] = ""
    summary['rawErr'] = ""

    return summary, {}

  #testOut, testError = testProc.communicate()

  summary = {}
  summary['rawOut'] = testOut
  summary['rawErr'] = testError

  #Parse the results
  testSummarySearch = re.search("Ran ([0-9]+) tests? in", testError)

  #If we don't find the test summary the tests died so we report that
  if not testSummarySearch:
    summary['totalTests'] = 0
    summary['failedTests'] = 0
    summary['timeout'] = timeoutReached
    summary['died'] = True

    return summary, {}

  # Get the list of failed tests

  failedSections = testError.split('='*70)

  splitList = []
  for section in failedSections:
    splitList.append(section.split('-'*70))

  #flatten the lists
  sections = list(itertools.chain.from_iterable(splitList))

  #Create the summary
  summary['totalTests'] = int(testSummarySearch.group(1))

  failedSections = sections[1:-1]
  summary['failedTests'] = len(failedSections)/2
  summary['died'] = False
  summary['timeout'] = False

  #Extract the messages from the failed tests
  failedTests = {}
  for i in range(0, len(failedSections), 2):
    header = failedSections[i]
    headerParts = header.split()
    tname = headerParts[1]

    pyunitmsg = failedSections[i+1]

    failedTests[tname] = {'hint': pyunitmsg}

  return summary, failedTests
Ejemplo n.º 4
0
def runTests(cmdPrefix, testFile, timeLimit):
    # Racket tests require some manipulation of the input and test files so that
    # it works nicely. We do that here
    with open(testFile, 'r') as testF:
        testText = testF.read()
        studentFileName = re.search(r'^[^;]*\(include "([^"]+)"\)',\
                                    testText, re.MULTILINE).group(1)

    #Remove the #lang racket from the student file so that the import in the test
    #will work
    try:
        studentFile = open(studentFileName, 'r')
        studentFileText = studentFile.read()
        studentFile.close()

        studentFileText = re.sub(r'(#lang +racket)', r';\1\n', studentFileText)

        studentFile = open(studentFileName, 'w')
        studentFile.write(studentFileText)
        studentFile.close()
    except Exception as e:
        return {'timeout': False, 'died': True, 'rawErr': str(e)}, {}

    #Put a random string in the test file so that we can tell where the tests
    #begin.

    randline = ''.join(
        random.choice(string.letters + string.digits) for _ in range(10))
    testText = re.sub(r'(\(check-[^ ]+)',
                      '(displayln "' + randline +
                      r'" (current-error-port))\n\1',
                      testText,
                      count=1)

    with open(testFile, 'w') as testF:
        testF.write(testText)

    #End of manipulation of files
    #Run the tests

    testProc = Command(cmdPrefix + ['racket', testFile])

    timeout, testOut, testError = testProc.run(timeout=int(timeLimit),
                                               env=environ)

    if timeout:
        print testError
        print "Timeout reached"
        summary = {}
        summary['totalTests'] = 0
        summary['failedTests'] = 0
        summary['timeout'] = True
        summary['died'] = False
        summary['rawOut'] = ""
        summary['rawErr'] = ""

        return summary, {}

    # startTime = datetime.now()
    # testProc = Popen(cmdPrefix + ['/usr/bin/racket', testFile],\
    #                 stdout=PIPE, stderr=PIPE)
    #
    # while testProc.poll() is None:
    #   currentTime = datetime.now()
    #   delta = currentTime - startTime
    #   if delta.total_seconds() > timeLimit:
    #     testProc.kill()
    #     #Report a timeout
    #     return {'timeout':True, 'died':False}, {}

    summary = {}

    #testOut, testError = testProc.communicate()

    summary['rawOut'] = testOut
    summary['rawErr'] = testError

    if testProc.returncode == -1:
        return {
            'timeout': False,
            'died': True,
            'rawErr': testError,
            'rawOut': testOut
        }, {}

    try:
        testResults = testError.split(randline)[1]
        # if there were no errors, result is printed to stdout
        testSummarySearch = re.search("([0-9]+) test\(s\) run", testOut)
        # Otherwise, result is printed to stderr
        if (testSummarySearch == None):
            testSummarySearch = re.search("([0-9]+) test\(s\) run", testError)

    except IndexError:
        # b. If this line was never seen, then something bad happened
        return {
            'timeout':
            False,
            'died':
            True,
            'rawErr':
            "Could not parse the test output:\n" + testOut + "\n\n" + testError
        }, {}
    else:
        failedTests = {}
        for failedCaseMatch in re.finditer('-' * 20 + r'\n(.*?)\n' + '-' * 20,
                                           testResults,
                                           flags=re.DOTALL):
            failedCase = failedCaseMatch.group(1)
            err = re.search(r'\nmessage: *"([^:]*):? *(.*)"\n', failedCase)
            if err:
                testname = err.group(1)
                msg = err.group(2)
                failedTests[testname] = {'hint': msg}

        summary['died'] = False
        summary['timeout'] = False
        if testSummarySearch != None:
            summary['totalTests'] = int(testSummarySearch.group(1))
        else:
            summary['totalTests'] = ""
        summary['failedTests'] = len(failedTests.keys())

        return summary, failedTests
Ejemplo n.º 5
0
def runTests(cmdPrefix, testFile, timeLimit):
  cmdCompile = ["javac", "-cp", "/usr/share/java/junit4.jar:.", testFile]
  #Not using cmdPrefix because we need to write files to the directory
  compileProc = Popen(cmdCompile, stderr=PIPE)
  _, compOut = compileProc.communicate()

  if compOut:
    #If the compiler produced output on stderr we have an issue so report it
    summary = {}
    summary['died'] = True
    summary['rawErr'] = compOut
    summary['timeout'] = False
    return summary, {}

  className = testFile[:-5]
  pathName = os.getcwd()+className
  cmdRun = ["/usr/bin/java", "-cp", "/usr/share/java/junit4.jar:.", "org.junit.runner.JUnitCore", className]

  # startTime = datetime.now()
  #
  # runner = Popen(cmdPrefix+cmdRun,stdout=PIPE, stderr=PIPE)
  #
  # while runner.poll() is None:
  #   currentTime = datetime.now()
  #   delta = currentTime - startTime
  #   if delta.total_seconds() > timeLimit:
  #     runner.kill()
  #     return {'timeout':True, 'died':False}, {}
  #     break
  #
  # stdout,stderr = runner.communicate()

  runner = Command(cmdPrefix+cmdRun)

  timeout, stdout, stderr = runner.run(timeout=int(timeLimit), env=environ)

  if timeout:
    return {'timeout':True, 'died':False}, {}

  summary = {}
  summary['rawOut'] = stdout
  summary['rawErr'] = stderr

  #Find all error reports
  #failedTests= re.finditer("[0-9]*\) (.+)\(\w+\)\n(?:(?:java\.lang\..+Error)|(?:org\.junit\..+Failure)): (.*)", stdout)
  testSplit = re.split(r"[0-9]*\) (.+)\(\w+\)", stdout)
  #Slice the summary off
  testSplit = testSplit[1:]

  failedTests = {}
  #Parse error reports for outputting to json
  for i in range(0, len(testSplit), 2):
      # testName = fail.group(1)
      # report = {}
      # message = fail.group(2)
      # hint = message
      # expact = re.search("expected:\<(.+)\> but was:\<(.+)\>", message)
      #
      # if expact:
      #     hint = message[0:message.find("expected")-1]
      #     report['expected'] = expact.group(1)
      #     report['returned'] = expact.group(2)
      report = {}
      report['hint'] = testSplit[i+1]
      failedTests[testSplit[i]] = report

  #Creates summary info
  lastLine = re.search("OK \(([0-9]*) tests\)", stdout)
  if lastLine:
      summary['totalTests'] = int(lastLine.group(1))
      summary['failedTests'] = 0
  else:
      lastLine = re.search("FAILURES[!]+\nTests run: ([0-9]*),[ ]+Failures: ([0-9]*)", stdout)
      summary['totalTests'] = int(lastLine.group(1))
      summary['failedTests'] = int(lastLine.group(2))

  summary['timeout'] = False
  summary['died'] = False

  return summary, failedTests
Ejemplo n.º 6
0
def runTests(cmdPrefix, testFile, timeLimit):
    # Racket tests require some manipulation of the input and test files so that
    # it works nicely. We do that here
    with open(testFile, "r") as testF:
        testText = testF.read()
        studentFileName = re.search(r'^[^;]*\(include "([^"]+)"\)', testText, re.MULTILINE).group(1)

    # Remove the #lang racket from the student file so that the import in the test
    # will work
    try:
        studentFile = open(studentFileName, "r")
        studentFileText = studentFile.read()
        studentFile.close()

        studentFileText = re.sub(r"(#lang +racket)", r";\1\n", studentFileText)

        studentFile = open(studentFileName, "w")
        studentFile.write(studentFileText)
        studentFile.close()
    except Exception as e:
        return {"timeout": False, "died": True, "rawErr": str(e)}, {}

    # Put a random string in the test file so that we can tell where the tests
    # begin.

    randline = "".join(random.choice(string.letters + string.digits) for _ in range(10))
    testText = re.sub(r"(\(check-[^ ]+)", '(displayln "' + randline + r'" (current-error-port))\n\1', testText, count=1)

    with open(testFile, "w") as testF:
        testF.write(testText)

    # End of manipulation of files
    # Run the tests

    testProc = Command(cmdPrefix + ["/usr/bin/racket", testFile])

    timeout, testOut, testError = testProc.run(timeout=int(timeLimit), env=environ)

    if timeout:
        print testError
        print "Timeout reached"
        summary = {}
        summary["totalTests"] = 0
        summary["failedTests"] = 0
        summary["timeout"] = True
        summary["died"] = False
        summary["rawOut"] = ""
        summary["rawErr"] = ""

        return summary, {}

    # startTime = datetime.now()
    # testProc = Popen(cmdPrefix + ['/usr/bin/racket', testFile],\
    #                 stdout=PIPE, stderr=PIPE)
    #
    # while testProc.poll() is None:
    #   currentTime = datetime.now()
    #   delta = currentTime - startTime
    #   if delta.total_seconds() > timeLimit:
    #     testProc.kill()
    #     #Report a timeout
    #     return {'timeout':True, 'died':False}, {}

    summary = {}

    # testOut, testError = testProc.communicate()

    summary["rawOut"] = testOut
    summary["rawErr"] = testError

    if testProc.returncode == -1:
        return {"timeout": False, "died": True, "rawErr": testError, "rawOut": testOut}, {}

    try:
        testResults = testError.split(randline)[1]
        # if there were no errors, result is printed to stdout
        testSummarySearch = re.search("([0-9]+) test\(s\) run", testOut)
        # Otherwise, result is printed to stderr
        if testSummarySearch == None:
            testSummarySearch = re.search("([0-9]+) test\(s\) run", testError)

    except IndexError:
        # b. If this line was never seen, then something bad happened
        return (
            {
                "timeout": False,
                "died": True,
                "rawErr": "Could not parse the test output:\n" + testOut + "\n\n" + testError,
            },
            {},
        )
    else:
        failedTests = {}
        for failedCaseMatch in re.finditer("-" * 20 + r"\n(.*?)\n" + "-" * 20, testResults, flags=re.DOTALL):
            failedCase = failedCaseMatch.group(1)
            err = re.search(r'\nmessage: *"([^:]*):? *(.*)"\n', failedCase)
            if err:
                testname = err.group(1)
                msg = err.group(2)
                failedTests[testname] = {"hint": msg}

        summary["died"] = False
        summary["timeout"] = False
        if testSummarySearch != None:
            summary["totalTests"] = int(testSummarySearch.group(1))
        else:
            summary["totalTests"] = ""
        summary["failedTests"] = len(failedTests.keys())

        return summary, failedTests
Ejemplo n.º 7
0
def runTests(cmdPrefix, testFile, timeLimit):
  # Racket tests require some manipulation of the input and test files so that
  # it works nicely. We do that here
  with open(testFile, 'r') as testF:
    testText = testF.read()
    studentFileName = re.search(r'^[^;]*\(include "([^"]+)"\)',\
                                testText, re.MULTILINE).group(1)

  #Remove the #lang racket from the student file so that the import in the test
  #will work
  try:
    studentFile = open(studentFileName, 'r')
    studentFileText = studentFile.read()
    studentFile.close()

    studentFileText = re.sub(r'(#lang +racket)', r';\1\n', studentFileText)

    studentFile = open(studentFileName, 'w')
    studentFile.write(studentFileText)
    studentFile.close()
  except Exception as e:
    return {'timeout':False, 'died':True, 'rawErr': str(e)}, {}

  #Put a random string in the test file so that we can tell where the tests
  #begin.

  randline = ''.join(random.choice(string.letters + string.digits) for _ in range(10))
  testText = re.sub(r'(\(check-[^ ]+)',
                       '(displayln "' +
                        randline +
                        r'" (current-error-port))\n\1',
                        testText, count=1)

  with open(testFile, 'w') as testF:
    testF.write(testText)

  #End of manipulation of files
  #Run the tests

  testProc = Command(cmdPrefix + ['/usr/bin/racket', testFile])

  timeout, testOut, testError = testProc.run(timeout=int(timeLimit), env=environ)

  if timeout:
    return {'timeout':True, 'died':False}, {}

  # startTime = datetime.now()
  # testProc = Popen(cmdPrefix + ['/usr/bin/racket', testFile],\
  #                 stdout=PIPE, stderr=PIPE)
  #
  # while testProc.poll() is None:
  #   currentTime = datetime.now()
  #   delta = currentTime - startTime
  #   if delta.total_seconds() > timeLimit:
  #     testProc.kill()
  #     #Report a timeout
  #     return {'timeout':True, 'died':False}, {}

  summary = {}

  #testOut, testError = testProc.communicate()

  summary['rawOut'] = testOut
  summary['rawErr'] = testError

  if testProc.returncode != 0:
    return {'timeout':False, 'died':True, 'rawErr': testError, 'rawOut':testOut}, {}

  try:
    testResults = testError.split(randline)[1]
  except IndexError:
    # b. If this line was never seen, then something bad happened
    return {'timeout':False, 'died':True, 'rawErr': "Could not parse the test output:\n" + testOut + "\n\n" + testError}, {}
  else:
    failedTests = {}
    for failedCaseMatch in re.finditer('-'*20 + r'\n(.*?)\n' + '-'*20, testResults, flags=re.DOTALL):
      failedCase = failedCaseMatch.group(1)
      err = re.search(r'\nmessage: *"([^:]*):? *(.*)"\n', failedCase)
      if err:
        testname = err.group(1)
        msg = err.group(2)
        failedTests[testname] = {'hint': msg}

    summary['died'] = False
    summary['timeout'] = False
    summary['totalTests'] = 0
    summary['failedTests'] = len(failedTests.keys())

    return summary, failedTests
Ejemplo n.º 8
0
def run_tests(command_prefix, test_file, time_limit):
    """Wrapper for "runTests" that adheres to PEP8 variable naming
    conventions."""
    try:
        with open(test_file) as f:
            try:
                tests = parse_test_file_contents(f.read())
            except JFLAPTestFileParseError as e:
                error = ("Could not parse test file '{}': {}"
                         .format(test_file, e.message))
                raise CouldNotRunJFLAPTestsError(error)
            # Determine the containing directory and literal filename
            # for the test file.
            directory, test_filename = os.path.split(test_file)
            if not directory:
                directory = "."
            # Find all the JFLAP files that the test_file might
            # possibly be testing.
            jflap_filenames = []
            for file in os.listdir(directory):
                if file.endswith(".jff"):
                    jflap_filenames.append(file)
            # If there's only one JFLAP file, nothing fancy is needed,
            # because there's only one choice.
            if len(jflap_filenames) == 1:
                jflap_filename = jflap_filenames[0]
            else:
                # Otherwise, we need to try to guess which file to
                # test by swapping out the extension of the test file
                # for ".jff".
                try:
                    extensionIndex = test_filename.rindex(".")
                except ValueError:
                    error = ("Test file '{}' does not have an extension"
                             .format(test_filename))
                    raise CouldNotRunJFLAPTestsError(error)
                jflap_filename = test_filename[:extensionIndex] + ".jff"
                if jflap_filename not in jflap_filenames:
                    # We could try harder to find a match, or allow
                    # specifying the filename of the corresponding
                    # JFLAP file within the test file, but to be
                    # honest that wouldn't really provide much help
                    # (because if you can put the name of the JFLAP
                    # file in the test file, you can just name the
                    # test file instead).
                    error = ("Test file '{}' does not match any of the"
                             " available JFLAP files, which are: {}"
                             .format(test_filename,
                                     ", ".join("'{}'".format(file)
                                               for file in jflap_filenames)))
                    raise CouldNotRunJFLAPTestsError(error)
            jflap_file = os.path.join(directory, jflap_filename)
            failedTests = {}
            all_stdout = ""
            all_stderr = ""
            # To ensure that this process does not take longer
            # than the provided time limit, we divide the time
            # equally among each test.
            if time_limit:
                timeout = time_limit / len(tests)
            else:
                timeout = None
            # We'll also need to figure out the directory containing
            # this Python file, so we can find jflaplib-cli.jar.
            script_directory = os.path.split(__file__)[0]
            for word, should_accept in tests.items():
                # See [1] for the source code of jflaplib-cli.jar.
                # Note that the command-line parsing library used by
                # jflaplib-cli, JCommander, has an odd quirk in the
                # way it parses arguments. First it trims whitespace
                # from both ends of each argument, and then it removes
                # a pair of double quotes if one exists. So, to ensure
                # an argument is interpreted literally, we just wrap
                # it in double quotes! See [2] for discussion of this
                # issue.
                #
                # [1]: https://github.com/raxod502/jflap-lib
                # [2]: https://github.com/cbeust/jcommander/issues/306
                command = Command(command_prefix +
                                  ["java",
                                   # The following system property
                                   # prevents the Java process from
                                   # showing up in the Mac app
                                   # switcher, which is extremely
                                   # annoying.
                                   "-Dapple.awt.UIElement=true",
                                   "-jar",
                                   os.path.join(script_directory,
                                                "jflaplib-cli.jar"),
                                   "run",
                                   jflap_file,
                                   '"{}"'.format(word)])
                timed_out, stdout, stderr = command.run(
                    timeout=timeout,
                    env=os.environ)
                all_stdout += stdout
                all_stderr += stderr
                if timed_out:
                    error = ("Timed out (took more than {} seconds)"
                             .format(timeout))
                    failedTests[word] = {"hint": error}
                else:
                    # jflaplib-cli should print "true" or "false",
                    # depending on whether the NFA or Turing
                    # machine accepted or rejected the input. But
                    # we handle all the possible edge cases here,
                    # just in case.
                    contains_true = "true" in stdout
                    contains_false = "false" in stdout
                    if contains_true and contains_false:
                        stdout = stdout.strip()
                        if not stdout:
                            stdout = "(none)"
                        stderr = stderr.strip()
                        if not stderr:
                            stderr = "(none)"
                        error = ("JFLAP reported both 'accept' and"
                                 " 'reject', output: {}; error: {}"
                                 .format(stdout.strip(), stderr.strip()))
                        failedTests[word] = {"hint": error}
                    elif not (contains_true or contains_false):
                        stdout = stdout.strip()
                        if not stdout:
                            stdout = "(none)"
                        stderr = stderr.strip()
                        if not stderr:
                            stderr = "(none)"
                        error = ("JFLAP reported neither 'accept' nor"
                                 " 'reject', output: {}; error: {}"
                                 .format(stdout.strip(), stderr.strip()))
                        failedTests[word] = {"hint": error}
                    elif contains_true is not should_accept:
                        error = ("This word should have been {}, but it"
                                 " was {}"
                                 .format(result_to_str(should_accept),
                                         result_to_str(contains_true)))
                        failedTests[word] = {"hint": error}
            summary = {"died": False,
                       "timeout": False,
                       "totalTests": len(tests),
                       "failedTests": len(failedTests),
                       "rawOut": all_stdout,
                       "rawErr": all_stderr}
            return summary, failedTests
    except Exception as e:
        if isinstance(e, CouldNotRunJFLAPTestsError):
            # If the error is a CouldNotRunJFLAPTestsError, then this
            # code generated the error message and included all
            # necessary information. So we can just return the
            # message.
            error = e.message
        else:
            # Otherwise, there was an unexpected error, and we'll
            # provide the whole stack trace for debugging purposes.
            # This is obviously very bad from a security perspective,
            # but worrying about it would be like making sure to turn
            # out the lights when the building is on fire, given the
            # security of the rest of this website.
            error = traceback.format_exc()
        summary = {"died": True,
                   "timeout": False,
                   "totalTests": 0,
                   "failedTests": 0,
                   "rawOut": "",
                   "rawErr": error}
        failedTests = {}
        return summary, failedTests
Ejemplo n.º 9
0
def run_tests(command_prefix, test_file, time_limit):
    """Wrapper for "runTests" that adheres to PEP8 variable naming
    conventions.
    """
    try:
        with open(test_file) as f:
            # Split the time limit 1-1-2 among finding the HMMM
            # program, assembling it, and running the test cases.
            get_hmmm_timeout = time_limit and time_limit / 4
            assembly_timeout = time_limit and time_limit / 4
            total_test_timeout = time_limit and time_limit / 2
            # Get the text of the HMMM program (if a unique match can
            # be found).
            program = get_hmmm_program(command_prefix,
                                       test_file,
                                       get_hmmm_timeout)
            # Get the paths to the assembler and simulator. We have to
            # shell out here because they are written in Python 3!
            script_directory = os.path.split(__file__)[0]
            hmmmAssembler = os.path.join(script_directory, "hmmmAssembler.py")
            hmmmSimulator = os.path.join(script_directory, "hmmmSimulator.py")
            # Normalize the paths to the assembler and simulator,
            # since our working directory could be somewhere totally
            # unrelated when we invoke them.
            hmmmAssembler = os.path.realpath(hmmmAssembler)
            hmmmSimulator = os.path.realpath(hmmmSimulator)
            # Get the path to the test file folder.
            test_directory, test_filename = os.path.split(test_file)
            # If the test file is specified only as a filename, then
            # the test_directory is the current directory.
            if not test_directory:
                test_directory = "."
            # Run the assembler.
            command = Command(command_prefix +
                              ["python3",
                               hmmmAssembler,
                               "--program-text",
                               program])
            result = command.run(compatibility=False,
                                 cwd=test_directory,
                                 timeout=assembly_timeout)
            return_code, stdout, stderr, timed_out = result
            if return_code != 0:
                error = ("Assembly completed unsuccessfully{}"
                         .format(" (timed out)" if timed_out else ""))
                info = []
                if stdout.strip():
                    info.append("Output:\n{}".format(stdout.strip()))
                if stderr.strip():
                    info.append("Error:\n{}".format(stderr.strip()))
                if info:
                    error += "\n"
                    error += "\n".join(info)
                raise CouldNotRunHmmmTestsError(error)
            # Parse the test cases from the test file.
            test_cases = {}  # discard duplicate test cases
            for line in f:
                test_case = parse_test_case(line)
                if test_case:
                    test_cases[format_test_case(test_case)] = test_case
            # Run the test cases using the simulator, collecting
            # stdout and stderr.
            failed_tests = {}
            all_stdout = ""
            all_stderr = ""
            if test_cases:
                timeout = (total_test_timeout and
                           total_test_timeout / len(test_cases))
            for name, test_case in test_cases.items():
                command = Command(command_prefix +
                                  ["python3",
                                   hmmmSimulator,
                                   "--test-case",
                                   repr(test_case)])
                result = command.run(compatibility=False,
                                     cwd=test_directory,
                                     timeout=timeout)
                return_code, stdout, stderr, timed_out = result
                all_stdout += stdout
                all_stderr += stderr
                # Messages designed to be read by hmmmgrader.py are
                # delimited by double brackets [[ like this ]].
                outputs = re.findall(r"\[\[ (.+?) \]\]", stdout)
                if "test case passed" in outputs:
                    continue
                found_failure = False
                for output in outputs:
                    match = re.match(r"test case failed: (.+)", output)
                    if match:
                        error = match.group(1)
                        # Capitalize the error message.
                        error = error[:1].upper() + error[1:]
                        failed_tests[name] = {"hint": error}
                        found_failure = True
                        break
                if found_failure:
                    continue
                error = ("simulation completed unsuccessfully{}"
                         .format(" (timed out)" if timed_out else ""))
                info = []
                if stdout.strip():
                    info.append("Output:\n{}".format(stdout.strip()))
                if stderr.strip():
                    info.append("Error:\n{}".format(stderr.strip()))
                if info:
                    error += "\n"
                    error += "\n".join(info)
                failed_tests[name] = {"hint": error}
            summary = {"died": False,
                       "timeout": False,
                       "totalTests": len(test_cases),
                       "failedTests": len(failed_tests),
                       "rawOut": all_stdout,
                       "rawErr": all_stderr}
            return summary, failed_tests
    except Exception as e:
        if isinstance(e, CouldNotRunHmmmTestsError):
            # If the error is a CouldNotRunHmmmTestsError, then this
            # code generated the error message and included all
            # necessary information. So we can just return the
            # message.
            error = e.message
        else:
            # Otherwise, there was an unexpected error, and we'll
            # provide the whole stack trace for debugging purposes.
            # This is obviously very bad from a security perspective,
            # but worrying about it would be like making sure to turn
            # out the lights when the building is on fire, given the
            # security of the rest of this website.
            error = traceback.format_exc()
        summary = {"died": True,
                   "timeout": False,
                   "totalTests": 0,
                   "failedTests": 0,
                   "rawOut": "",
                   "rawErr": error}
        failedTests = {}
        return summary, failedTests
Ejemplo n.º 10
0
def get_hmmm_program(command_prefix, test_file, time_limit):
    """Attempts to find the HMMM program that should be tested by the
    test_file, according to a number of heuristics.

    Returns the text of the program, or throws a
    CouldNotRunHmmmTestsError if an appropriate program could not be
    found.
    """
    # To start, we determine the directory and filename of the test
    # file, and get the list of files in that directory.
    test_directory, test_filename = os.path.split(test_file)
    # If the test_file is specified as just a filename, then its
    # directory is the current directory.
    if not test_directory:
        test_directory = "."
    # Get the names of all the files in the same directory as the test
    # file.
    filenames = os.listdir(test_directory)
    # First we look for .hmmm files.
    hmmm_filenames = []
    for file_ in filenames:
        if file_.endswith(".hmmm"):
            hmmm_filenames.append(file_)
    # If there's only one .hmmm file, we have found our candidate.
    if len(hmmm_filenames) == 1:
        with open(hmmm_filenames[0]) as f:
            return f.read()
    # Except in the special case of there only being one .hmmm file,
    # we're going to need to know the name of the HMMM program to be
    # tested. The only way to determine this is to strip the extension
    # from the test file (presuming, of course, that it has one!).
    try:
        extension_index = test_filename.rindex(".")
    except ValueError:
        error = ("Test file '{}' does not have an extension"
                 .format(test_filename))
        raise CouldNotRunHmmmTestsError(error)
    program_name = test_filename[:extension_index]
    # Presuming that there are any .hmmm files, we'll try to guess the
    # file by swapping out the extension on the test file.
    if hmmm_filenames:
        hmmm_filename = program_name + ".hmmm"
        if hmmm_filename in hmmm_filenames:
            with open(hmmm_filename) as f:
                return f.read()
        else:
            error = ("Test file '{}' does not match any of the available"
                     " .hmmm files, which are: {}"
                     .format(test_filename,
                             ", ".join("'{}'".format(file_)
                                       for file_ in hmmm_filenames)))
            raise CouldNotRunHmmmTestsError(error)
    # If there aren't any .hmmm files, we'll check any available .py
    # files next.
    py_filenames = []
    for filename in filenames:
        file_ = os.path.join(test_directory, filename)
        if not filename.endswith(".py"):
            continue
        # To filter out irrelevant .py files (possibly autograder
        # code), we check to make sure the ones we find define a
        # "Hmmm" function, which seems to be the template used in CS5
        # classes (whereas CS42 classes just create .hmmm files, thus
        # obviating the problem of distinguishing between different
        # types of .py files).
        with open(file_) as f:
            # Split the string literal so that the check doesn't match
            # this file (hmmmgrader.py)!
            if "def " "Hmmm" not in f.read():
                continue
        py_filenames.append(filename)
    if not py_filenames:
        error = ("no .hmmm or .py files with HMMM programs available")
        raise CouldNotRunHmmmTestsError(error)
    # We define a function to run using "python3 -c" that will print a
    # dictionary containing all the multiline strings defined in a
    # module.
    extraction_command = r"""\
import {0}
programs = {{}} # needed to escape the braces from str.format
for name in dir({0}):
    # Ignore items that are in all modules (including the __builtins__
    # module).
    if name not in dir(__builtins__):
        item = eval("{0}." + name)
        # Only accept multiline strings.
        if isinstance(item, str) and "\n" in item:
            programs[name] = item
print(repr(programs))\
"""
    # The time limit is split equally among each of the subprocess
    # calls, to ensure that we don't accidentally take longer than
    # allowed in total.
    timeout = time_limit and time_limit / len(py_filenames)
    # Now we extract all possible HMMM programs from all the valid .py
    # files that we found, and place them into a single dictionary
    # (allowing for multiple programs by the same name, and retaining
    # the information about which programs were found in which files).
    programs = {}
    # All the information about errors while opening files goes into
    # another dictionary, for possible error reporting.
    error_data = {}
    for file_ in py_filenames:
        # Trim the .py extension.
        module_name = file_[:-3]
        command = Command(command_prefix +
                          ["python3",
                           "-c",
                           extraction_command.format(module_name)])
        result = command.run(compatibility=False,
                             cwd=test_directory,
                             timeout=timeout)
        return_code, stdout, stderr, timed_out = result
        if return_code == 0:
            new_programs = eval(stdout)
            for name, program in new_programs.items():
                if name not in programs:
                    programs[name] = []
                programs[name].append((file_, program))
        else:
            error_data[file_] = result
    # Now we look at the collected programs and see if we can uniquely
    # identify one that matches the desired program. Collecting lists
    # of all the names of programs and all the files that were
    # investigated allows for more useful error reporting.
    all_names = set()
    all_files = set()
    for name, candidates in programs.items():
        all_names.add(name)
        for candidate in candidates:
            all_files.add(candidates[0])
    # If only one program is defined, then we'll choose it regardless
    # of whether its name matches the test file.
    if len(all_names) == 1:
        program_name = next(iter(all_names))
    if program_name in programs:
        # Keep in mind that a program by the same name might be
        # defined in multiple .py files, so we need to check for that.
        candidates = programs[program_name]
        if len(candidates) == 1:
            return candidates[0][1]
        error = ("program '{}' was defined in multiple files ({})"
                 .format(program_name,
                         ", ".join("'{}'".format(pair[0])
                                   for pair in candidates)))
        raise CouldNotRunHmmmTestsError(error)
    # If we have more than one program defined, and none of them
    # matches the test file name (which defines the initial value of
    # program_name), then we have an error. We report it with as much
    # diagnostic information as possible.
    error = ("test file '{}' does not match any of the available HMMM programs"
             .format(test_filename))
    if all_names:
        error += (" (programs {} defined in files {})"
                  .format(", ".join("'{}'".format(name)
                                    for name in all_names),
                          ", ".join("'{}'".format(file_)
                                    for file_ in all_files)))
    if len(all_files) < len(py_filenames):
        error += (" (could not open files {})"
                  .format(", ".join("'{}'".format(file_)
                                    for file_ in (set(py_filenames) -
                                                  all_files))))
    if error_data:
        error += ("; errors (returncode, stdout, stderr, timed_out)"
                  " while opening files: {}"
                  .format(repr(error_data)))
    raise CouldNotRunHmmmTestsError(error)
Ejemplo n.º 11
0
def runTests(cmdPrefix, testFile, timeLimit):
    cmdCompile = ["javac", "-cp", "/usr/share/java/junit4.jar:.", testFile]
    #Not using cmdPrefix because we need to write files to the directory
    compileProc = Popen(cmdCompile, stderr=PIPE)
    _, compOut = compileProc.communicate()

    if compOut:
        #If the compiler produced output on stderr we have an issue so report it
        summary = {}
        summary['died'] = True
        summary['rawErr'] = compOut
        summary['timeout'] = False
        return summary, {}

    className = testFile[:-5]
    pathName = os.getcwd() + className
    cmdRun = [
        "/usr/bin/java", "-cp", "/usr/share/java/junit4.jar:.",
        "org.junit.runner.JUnitCore", className
    ]

    # startTime = datetime.now()
    #
    # runner = Popen(cmdPrefix+cmdRun,stdout=PIPE, stderr=PIPE)
    #
    # while runner.poll() is None:
    #   currentTime = datetime.now()
    #   delta = currentTime - startTime
    #   if delta.total_seconds() > timeLimit:
    #     runner.kill()
    #     return {'timeout':True, 'died':False}, {}
    #     break
    #
    # stdout,stderr = runner.communicate()

    runner = Command(cmdPrefix + cmdRun)

    timeout, stdout, stderr = runner.run(timeout=int(timeLimit), env=environ)

    if timeout:
        return {'timeout': True, 'died': False}, {}

    summary = {}
    summary['rawOut'] = stdout
    summary['rawErr'] = stderr

    #Find all error reports
    #failedTests= re.finditer("[0-9]*\) (.+)\(\w+\)\n(?:(?:java\.lang\..+Error)|(?:org\.junit\..+Failure)): (.*)", stdout)
    testSplit = re.split(r"[0-9]*\) (.+)\(\w+\)", stdout)
    #Slice the summary off
    testSplit = testSplit[1:]

    failedTests = {}
    #Parse error reports for outputting to json
    for i in range(0, len(testSplit), 2):
        # testName = fail.group(1)
        # report = {}
        # message = fail.group(2)
        # hint = message
        # expact = re.search("expected:\<(.+)\> but was:\<(.+)\>", message)
        #
        # if expact:
        #     hint = message[0:message.find("expected")-1]
        #     report['expected'] = expact.group(1)
        #     report['returned'] = expact.group(2)
        report = {}
        report['hint'] = testSplit[i + 1]
        failedTests[testSplit[i]] = report

    #Creates summary info
    lastLine = re.search("OK \(([0-9]*) tests\)", stdout)
    if lastLine:
        summary['totalTests'] = int(lastLine.group(1))
        summary['failedTests'] = 0
    else:
        lastLine = re.search(
            "FAILURES[!]+\nTests run: ([0-9]*),[ ]+Failures: ([0-9]*)", stdout)
        summary['totalTests'] = int(lastLine.group(1))
        summary['failedTests'] = int(lastLine.group(2))

    summary['timeout'] = False
    summary['died'] = False

    return summary, failedTests
Ejemplo n.º 12
0
def run_tests(command_prefix, test_file, time_limit):
    """Wrapper for "runTests" that adheres to PEP8 variable naming
    conventions.
    """
    try:
        with open(test_file) as f:
            # Split the time limit 1-1-2 among finding the HMMM
            # program, assembling it, and running the test cases.
            get_hmmm_timeout = time_limit and time_limit / 4
            assembly_timeout = time_limit and time_limit / 4
            total_test_timeout = time_limit and time_limit / 2
            # Get the text of the HMMM program (if a unique match can
            # be found).
            program = get_hmmm_program(command_prefix, test_file,
                                       get_hmmm_timeout)
            # Get the paths to the assembler and simulator. We have to
            # shell out here because they are written in Python 3!
            script_directory = os.path.split(__file__)[0]
            hmmmAssembler = os.path.join(script_directory, "hmmmAssembler.py")
            hmmmSimulator = os.path.join(script_directory, "hmmmSimulator.py")
            # Normalize the paths to the assembler and simulator,
            # since our working directory could be somewhere totally
            # unrelated when we invoke them.
            hmmmAssembler = os.path.realpath(hmmmAssembler)
            hmmmSimulator = os.path.realpath(hmmmSimulator)
            # Get the path to the test file folder.
            test_directory, test_filename = os.path.split(test_file)
            # If the test file is specified only as a filename, then
            # the test_directory is the current directory.
            if not test_directory:
                test_directory = "."
            # Run the assembler.
            command = Command(
                command_prefix +
                ["python3", hmmmAssembler, "--program-text", program])
            result = command.run(compatibility=False,
                                 cwd=test_directory,
                                 timeout=assembly_timeout)
            return_code, stdout, stderr, timed_out = result
            if return_code != 0:
                error = ("Assembly completed unsuccessfully{}".format(
                    " (timed out)" if timed_out else ""))
                info = []
                if stdout.strip():
                    info.append("Output:\n{}".format(stdout.strip()))
                if stderr.strip():
                    info.append("Error:\n{}".format(stderr.strip()))
                if info:
                    error += "\n"
                    error += "\n".join(info)
                raise CouldNotRunHmmmTestsError(error)
            # Parse the test cases from the test file.
            test_cases = {}  # discard duplicate test cases
            for line in f:
                test_case = parse_test_case(line)
                if test_case:
                    test_cases[format_test_case(test_case)] = test_case
            # Run the test cases using the simulator, collecting
            # stdout and stderr.
            failed_tests = {}
            all_stdout = ""
            all_stderr = ""
            if test_cases:
                timeout = (total_test_timeout
                           and total_test_timeout / len(test_cases))
            for name, test_case in test_cases.items():
                command = Command(
                    command_prefix +
                    ["python3", hmmmSimulator, "--test-case",
                     repr(test_case)])
                result = command.run(compatibility=False,
                                     cwd=test_directory,
                                     timeout=timeout)
                return_code, stdout, stderr, timed_out = result
                all_stdout += stdout
                all_stderr += stderr
                # Messages designed to be read by hmmmgrader.py are
                # delimited by double brackets [[ like this ]].
                outputs = re.findall(r"\[\[ (.+?) \]\]", stdout)
                if "test case passed" in outputs:
                    continue
                found_failure = False
                for output in outputs:
                    match = re.match(r"test case failed: (.+)", output)
                    if match:
                        error = match.group(1)
                        # Capitalize the error message.
                        error = error[:1].upper() + error[1:]
                        failed_tests[name] = {"hint": error}
                        found_failure = True
                        break
                if found_failure:
                    continue
                error = ("simulation completed unsuccessfully{}".format(
                    " (timed out)" if timed_out else ""))
                info = []
                if stdout.strip():
                    info.append("Output:\n{}".format(stdout.strip()))
                if stderr.strip():
                    info.append("Error:\n{}".format(stderr.strip()))
                if info:
                    error += "\n"
                    error += "\n".join(info)
                failed_tests[name] = {"hint": error}
            summary = {
                "died": False,
                "timeout": False,
                "totalTests": len(test_cases),
                "failedTests": len(failed_tests),
                "rawOut": all_stdout,
                "rawErr": all_stderr
            }
            return summary, failed_tests
    except Exception as e:
        if isinstance(e, CouldNotRunHmmmTestsError):
            # If the error is a CouldNotRunHmmmTestsError, then this
            # code generated the error message and included all
            # necessary information. So we can just return the
            # message.
            error = e.message
        else:
            # Otherwise, there was an unexpected error, and we'll
            # provide the whole stack trace for debugging purposes.
            # This is obviously very bad from a security perspective,
            # but worrying about it would be like making sure to turn
            # out the lights when the building is on fire, given the
            # security of the rest of this website.
            error = traceback.format_exc()
        summary = {
            "died": True,
            "timeout": False,
            "totalTests": 0,
            "failedTests": 0,
            "rawOut": "",
            "rawErr": error
        }
        failedTests = {}
        return summary, failedTests
Ejemplo n.º 13
0
def get_hmmm_program(command_prefix, test_file, time_limit):
    """Attempts to find the HMMM program that should be tested by the
    test_file, according to a number of heuristics.

    Returns the text of the program, or throws a
    CouldNotRunHmmmTestsError if an appropriate program could not be
    found.
    """
    # To start, we determine the directory and filename of the test
    # file, and get the list of files in that directory.
    test_directory, test_filename = os.path.split(test_file)
    # If the test_file is specified as just a filename, then its
    # directory is the current directory.
    if not test_directory:
        test_directory = "."
    # Get the names of all the files in the same directory as the test
    # file.
    filenames = os.listdir(test_directory)
    # First we look for .hmmm files.
    hmmm_filenames = []
    for file_ in filenames:
        if file_.endswith(".hmmm"):
            hmmm_filenames.append(file_)
    # If there's only one .hmmm file, we have found our candidate.
    if len(hmmm_filenames) == 1:
        with open(hmmm_filenames[0]) as f:
            return f.read()
    # Except in the special case of there only being one .hmmm file,
    # we're going to need to know the name of the HMMM program to be
    # tested. The only way to determine this is to strip the extension
    # from the test file (presuming, of course, that it has one!).
    try:
        extension_index = test_filename.rindex(".")
    except ValueError:
        error = (
            "Test file '{}' does not have an extension".format(test_filename))
        raise CouldNotRunHmmmTestsError(error)
    program_name = test_filename[:extension_index]
    # Presuming that there are any .hmmm files, we'll try to guess the
    # file by swapping out the extension on the test file.
    if hmmm_filenames:
        hmmm_filename = program_name + ".hmmm"
        if hmmm_filename in hmmm_filenames:
            with open(hmmm_filename) as f:
                return f.read()
        else:
            error = ("Test file '{}' does not match any of the available"
                     " .hmmm files, which are: {}".format(
                         test_filename,
                         ", ".join("'{}'".format(file_)
                                   for file_ in hmmm_filenames)))
            raise CouldNotRunHmmmTestsError(error)
    # If there aren't any .hmmm files, we'll check any available .py
    # files next.
    py_filenames = []
    for filename in filenames:
        file_ = os.path.join(test_directory, filename)
        if not filename.endswith(".py"):
            continue
        # To filter out irrelevant .py files (possibly autograder
        # code), we check to make sure the ones we find define a
        # "Hmmm" function, which seems to be the template used in CS5
        # classes (whereas CS42 classes just create .hmmm files, thus
        # obviating the problem of distinguishing between different
        # types of .py files).
        with open(file_) as f:
            # Split the string literal so that the check doesn't match
            # this file (hmmmgrader.py)!
            if "def " "Hmmm" not in f.read():
                continue
        py_filenames.append(filename)
    if not py_filenames:
        error = ("no .hmmm or .py files with HMMM programs available")
        raise CouldNotRunHmmmTestsError(error)
    # We define a function to run using "python3 -c" that will print a
    # dictionary containing all the multiline strings defined in a
    # module.
    extraction_command = r"""\
import {0}
programs = {{}} # needed to escape the braces from str.format
for name in dir({0}):
    # Ignore items that are in all modules (including the __builtins__
    # module).
    if name not in dir(__builtins__):
        item = eval("{0}." + name)
        # Only accept multiline strings.
        if isinstance(item, str) and "\n" in item:
            programs[name] = item
print(repr(programs))\
"""
    # The time limit is split equally among each of the subprocess
    # calls, to ensure that we don't accidentally take longer than
    # allowed in total.
    timeout = time_limit and time_limit / len(py_filenames)
    # Now we extract all possible HMMM programs from all the valid .py
    # files that we found, and place them into a single dictionary
    # (allowing for multiple programs by the same name, and retaining
    # the information about which programs were found in which files).
    programs = {}
    # All the information about errors while opening files goes into
    # another dictionary, for possible error reporting.
    error_data = {}
    for file_ in py_filenames:
        # Trim the .py extension.
        module_name = file_[:-3]
        command = Command(
            command_prefix +
            ["python3", "-c",
             extraction_command.format(module_name)])
        result = command.run(compatibility=False,
                             cwd=test_directory,
                             timeout=timeout)
        return_code, stdout, stderr, timed_out = result
        if return_code == 0:
            new_programs = eval(stdout)
            for name, program in new_programs.items():
                if name not in programs:
                    programs[name] = []
                programs[name].append((file_, program))
        else:
            error_data[file_] = result
    # Now we look at the collected programs and see if we can uniquely
    # identify one that matches the desired program. Collecting lists
    # of all the names of programs and all the files that were
    # investigated allows for more useful error reporting.
    all_names = set()
    all_files = set()
    for name, candidates in programs.items():
        all_names.add(name)
        for candidate in candidates:
            all_files.add(candidates[0])
    # If only one program is defined, then we'll choose it regardless
    # of whether its name matches the test file.
    if len(all_names) == 1:
        program_name = next(iter(all_names))
    if program_name in programs:
        # Keep in mind that a program by the same name might be
        # defined in multiple .py files, so we need to check for that.
        candidates = programs[program_name]
        if len(candidates) == 1:
            return candidates[0][1]
        error = ("program '{}' was defined in multiple files ({})".format(
            program_name,
            ", ".join("'{}'".format(pair[0]) for pair in candidates)))
        raise CouldNotRunHmmmTestsError(error)
    # If we have more than one program defined, and none of them
    # matches the test file name (which defines the initial value of
    # program_name), then we have an error. We report it with as much
    # diagnostic information as possible.
    error = ("test file '{}' does not match any of the available HMMM programs"
             .format(test_filename))
    if all_names:
        error += (" (programs {} defined in files {})".format(
            ", ".join("'{}'".format(name) for name in all_names),
            ", ".join("'{}'".format(file_) for file_ in all_files)))
    if len(all_files) < len(py_filenames):
        error += (" (could not open files {})".format(", ".join(
            "'{}'".format(file_)
            for file_ in (set(py_filenames) - all_files))))
    if error_data:
        error += ("; errors (returncode, stdout, stderr, timed_out)"
                  " while opening files: {}".format(repr(error_data)))
    raise CouldNotRunHmmmTestsError(error)
Ejemplo n.º 14
0
def runTests(cmdPrefix, testFile, timeLimit):
    startTime = datetime.now()
    runCommand = 'python3'  # change to just python for using python 2
    testProc = Command(cmdPrefix + [runCommand, testFile])

    timeoutReached, testOut, testError = \
      testProc.run(timeout=int(timeLimit), env=environ)

    #Check for a timeout
    if timeoutReached:
        print "Timeout reached"
        summary = {}
        summary['totalTests'] = 0
        summary['failedTests'] = 0
        summary['timeout'] = timeoutReached
        summary['died'] = False
        summary['rawOut'] = ""
        summary['rawErr'] = ""

        return summary, {}

    #testOut, testError = testProc.communicate()

    summary = {}
    summary['rawOut'] = testOut
    summary['rawErr'] = testError

    #Parse the results
    testSummarySearch = re.search("Ran ([0-9]+) tests? in", testError)

    #If we don't find the test summary the tests died so we report that
    if not testSummarySearch:
        summary['totalTests'] = 0
        summary['failedTests'] = 0
        summary['timeout'] = timeoutReached
        summary['died'] = True

        return summary, {}

    # Get the list of failed tests

    failedSections = testError.split('=' * 70)

    splitList = []
    for section in failedSections:
        splitList.append(section.split('-' * 70))

    #flatten the lists
    sections = list(itertools.chain.from_iterable(splitList))

    #Create the summary
    summary['totalTests'] = int(testSummarySearch.group(1))

    failedSections = sections[1:-1]
    summary['failedTests'] = len(failedSections) / 2
    summary['died'] = False
    summary['timeout'] = False

    #Extract the messages from the failed tests
    failedTests = {}
    for i in range(0, len(failedSections), 2):
        header = failedSections[i]
        headerParts = header.split()
        tname = headerParts[1]

        pyunitmsg = failedSections[i + 1]

        failedTests[tname] = {'hint': pyunitmsg}

    return summary, failedTests