Example #1
0
def runTests(tests, pythonVers=None, source=None, pythonPath=None, testKey='test'):
    """
    run tests

    tests is a list of strings that contains the names of the tests to be
        run. If the test is a local python file, then it should be named
        <testKey>.<test>.py. If it is a directory, it should be named <test>.
    pythonCmd is the full python command to be used to run the test files. This
        can be used to specify optimization flags to python or the desired
        python version.
    pythonPath is the python path to be set when running the test files.
    testKey is used for two things. First, it is appended to the name of each
        test file. The idea is that each test file would be paired up with the
        file it tests. If the file being tested is spam.py then the test file
        would be <testKey>.spam.py. By separating the test code from the code
        it tests, we make the initial reading of the code faster when it is not
        being tested. Second, when testing the directory, the test script is
        expected to be named ./<test>/<testKey> (notice that there is no .py
        extension even though this would also be a python file).
    """
    clp.process()
    python = pythonCmd(pythonVers)

    if clp.printTests:
        print("Using %s." % python)

    if clp.coverage is not False:
        if os.path.exists('.coverage') and not clp.parent:
            os.remove('.coverage')
        if not source:
            source = clp.coverage if clp.coverage else ''
        clp.coverage = source
        python = coverageCmd(pythonVers, source)
    pythonPath = ('PYTHONPATH=%s; ' % pythonPath) if pythonPath else ''

    if len(clp.args) == 0:
        clp.args = tests

    failures = False
    numTests = 0
    numTestFailures = 0
    numSuites = 0
    numSuiteFailures = 0
    for test in clp.args:
        name = '%s/%s' % (clp.parent, test) if clp.parent else test
        if os.path.isfile('%s.%s.py' % (testKey, test)):
            summaryFileName = './.%s.%s.sum' % (testKey, test)
            _deleteYamlFile(summaryFileName)
            if clp.printSummary:
                sys.stdout.write(status('%s: ' % name))
                sys.stdout.flush()
            cmd = pythonPath + '%s %s.%s.py %s' % (
                python, testKey, test, _childOpts(test)
            )
            error = _invoke(cmd)
        elif os.path.isdir(test):
            summaryFileName = './%s/.%s.sum' % (test, testKey)
            _deleteYamlFile(summaryFileName)
            cmd = 'cd %s; %s %s %s' % (test, pythonCmd, testKey, _childOpts(test))
            error = _invoke(cmd)
        else:
            print(exception(
                '%s: cannot find test %s, skipping.' % (
                    clp.progName, name
                )
            ))
            numSuites += 1
            numSuiteFailures += 1
            continue
        if error and not clp.coverage is not False:
            # return status of coverage seems broken (sigh)
            print(fail('Failures detected in %s tests.' % name))
            failures = True
        try:
            with open(summaryFileName) as f:
                results = loadSummary(f)
                numTests += results['tests']
                numTestFailures += results['testFailures']
                numSuites += results['suites']
                numSuiteFailures += results['suiteFailures']
        except KeyError:
            sys.exit(
                    exception(
                    '%s: invalid summary file: %s' % (
                        clp.progName, summaryFileName
                    )
                )
            )
        except IOError as err:
            if error:
                numSuites += 1
                numSuiteFailures += 1
            else:
                sys.exit(
                    exception(
                        "%s: summary file '%s': %s." % (
                            clp.progName, summaryFileName, err.strerror
                        )
                    )
                )

    if clp.printSummary and not clp.parent and len(clp.args) > 1:
        preamble = info('Composite results')
        synopsis = '%s of %s test suites failed, %s of %s tests failed.' % (
            numSuiteFailures, numSuites, numTestFailures, numTests
        )
        if numSuiteFailures or numTestFailures:
            print("%s: %s" % (preamble, fail("FAIL: %s" % synopsis)))
        else:
            print("%s: %s" % (preamble, succeed("PASS: %s" % synopsis)))

    try:
        writeSummary(numTests, numTestFailures, numSuites, numSuiteFailures)
    except IOError as err:
        sys.exit(
            exception(
                "%s: summary file '%s': %s." % (
                    clp.progName, summaryFileName, err.strerror
                )
            )
        )

    sys.exit(bool(failures))
Example #2
0
def runTests(tests,
             pythonVers=None,
             source=None,
             pythonPath=None,
             testKey='test'):
    """
    run tests

    tests is a list of strings that contains the names of the tests to be
        run. If the test is a local python file, then it should be named
        <testKey>.<test>.py. If it is a directory, it should be named <test>.
    pythonCmd is the full python command to be used to run the test files. This
        can be used to specify optimization flags to python or the desired
        python version.
    pythonPath is the python path to be set when running the test files.
    testKey is used for two things. First, it is appended to the name of each
        test file. The idea is that each test file would be paired up with the
        file it tests. If the file being tested is spam.py then the test file
        would be <testKey>.spam.py. By separating the test code from the code
        it tests, we make the initial reading of the code faster when it is not
        being tested. Second, when testing the directory, the test script is
        expected to be named ./<test>/<testKey> (notice that there is no .py
        extension even though this would also be a python file).
    """
    clp.process()
    python = pythonCmd(pythonVers)

    if clp.printTests:
        print("Using %s." % python)

    if clp.coverage is not False:
        if os.path.exists('.coverage') and not clp.parent:
            os.remove('.coverage')
        if not source:
            source = clp.coverage if clp.coverage else ''
        clp.coverage = source
        python = coverageCmd(pythonVers, source)
    pythonPath = ('PYTHONPATH=%s; ' % pythonPath) if pythonPath else ''

    if len(clp.args) == 0:
        clp.args = tests

    failures = False
    numTests = 0
    numTestFailures = 0
    numSuites = 0
    numSuiteFailures = 0
    for test in clp.args:
        name = '%s/%s' % (clp.parent, test) if clp.parent else test
        if os.path.isfile('%s.%s.py' % (testKey, test)):
            summaryFileName = './.%s.%s.sum' % (testKey, test)
            _deleteYamlFile(summaryFileName)
            if clp.printSummary:
                sys.stdout.write(status('%s: ' % name))
                sys.stdout.flush()
            cmd = pythonPath + '%s %s.%s.py %s' % (python, testKey, test,
                                                   _childOpts(test))
            error = _invoke(cmd)
        elif os.path.isdir(test):
            summaryFileName = './%s/.%s.sum' % (test, testKey)
            _deleteYamlFile(summaryFileName)
            cmd = 'cd %s; %s %s %s' % (test, python, testKey, _childOpts(test))
            error = _invoke(cmd)
        else:
            print(
                exception('%s: cannot find test %s, skipping.' %
                          (clp.progName, name)))
            numSuites += 1
            numSuiteFailures += 1
            continue
        if error and not clp.coverage is not False:
            # return status of coverage seems broken (sigh)
            print(fail('Failures detected in %s tests.' % name))
            failures = True
        try:
            with open(summaryFileName) as f:
                results = loadSummary(f)
                numTests += results['tests']
                numTestFailures += results['testFailures']
                numSuites += results['suites']
                numSuiteFailures += results['suiteFailures']
        except KeyError:
            sys.exit(
                exception('%s: invalid summary file: %s' %
                          (clp.progName, summaryFileName)))
        except IOError as err:
            if error:
                numSuites += 1
                numSuiteFailures += 1
            else:
                sys.exit(
                    exception("%s: summary file '%s': %s." %
                              (clp.progName, summaryFileName, err.strerror)))

    if clp.printSummary and not clp.parent and len(clp.args) > 1:
        preamble = info('Composite results')
        synopsis = '%s of %s test suites failed, %s of %s tests failed.' % (
            numSuiteFailures, numSuites, numTestFailures, numTests)
        if numSuiteFailures or numTestFailures:
            print("%s: %s" % (preamble, fail("FAIL: %s" % synopsis)))
        else:
            print("%s: %s" % (preamble, succeed("PASS: %s" % synopsis)))

    try:
        writeSummary(numTests, numTestFailures, numSuites, numSuiteFailures)
    except IOError as err:
        sys.exit(
            exception("%s: summary file '%s': %s." %
                      (clp.progName, summaryFileName, err.strerror)))

    sys.exit(bool(failures))
Example #3
0
def runTests(tests, pythonCmd=None, pythonPath=None, testKey='test'):
    """
    run tests

    tests is a list of strings that contains the names of the tests to be
        run. If the test is a local python file, then it should be named
        <testKey>.<test>.py. If it is a directory, it should be named <test>.
    pythonCmd is the full python command to be used to run the test files. This
        can be used to specify optimization flags to python or the desired
        python version.
    pythonPath is the python path to be set when running the test files.
    testKey is used for two things. First, it is appended to the name of each
        test file. The idea is that each test file would be paired up with the
        file it tests. If the file being tested is spam.py then the test file
        would be <testKey>.spam.py. By separating the test code from the code
        it tests, we make the initial reading of the code faster when it is not
        being tested. Second, when testing the directory, the test script is
        expected to be named ./<test>/<testKey> (notice that there is no .py
        extension even though this would also be a python file).
    """
    if not pythonCmd:
        pythonCmd = 'python'
    if coverage:
        pythonCmd = pythonCmd + ' /usr/bin/coverage run -a --branch'
    if pythonPath:
        pythonPath = ('PYTHONPATH=%s; ' % pythonPath)
    else:
        pythonPath = ''

    colors.colorize(colorize and isTTY())
    global args
    if len(args) == 0:
        args = tests

    failures = False
    numTests = 0
    numTestFailures = 0
    numSuites = 0
    numSuiteFailures = 0
    for test in args:
        if parent:
            name = '%s/%s' % (parent, test)
        else:
            name = test
        if os.path.isfile('%s.%s.py' % (testKey, test)):
            summaryFileName = './.%s.%s.sum' % (testKey, test)
            _deleteYamlFile(summaryFileName)
            if printSummary:
                sys.stdout.write(status('%s: ' % name))
                sys.stdout.flush()
            cmd = pythonPath + '%s %s.%s.py %s' % (
                pythonCmd, testKey, test, _childOpts(test)
            )
            error = _invoke(cmd)
        elif os.path.isdir(test):
            summaryFileName = './%s/.%s.sum' % (test, testKey)
            _deleteYamlFile(summaryFileName)
            cmd = 'cd %s; ./%s %s' % (test, testKey, _childOpts(test))
            error = _invoke(cmd)
        else:
            print(exception(
                '%s: cannot find test %s, skipping.' % (
                    progName, name
                )
            ))
            numSuites += 1
            numSuiteFailures += 1
            continue
        if error:
            if not coverage:
                # return status of coverage seems broken (sigh)
                print(fail('Failures detected in %s tests.' % name))
                failures = True
        try:
            f = open(summaryFileName)
            results = loadSummary(f)
            numTests += results['tests']
            numTestFailures += results['testFailures']
            numSuites += results['suites']
            numSuiteFailures += results['suiteFailures']
            f.close()
        except KeyError:
            sys.exit(
                    exception(
                    '%s: invalid summary file: %s' % (
                        progName, summaryFileName
                    )
                )
            )
        except IOError, err:
            if error:
                numSuites += 1
                numSuiteFailures += 1
            else:
                sys.exit(
                    exception(
                        "%s: summary file '%s': %s." % (
                            progName, summaryFileName, err.strerror
                        )
                    )
                )