예제 #1
0
    def test_manifest_subsuites(self):
        """
        test subsuites and conditional subsuites
        """
        relative_path = os.path.join(here, "subsuite.ini")
        manifest = TestManifest(manifests=(relative_path,))
        info = {"foo": "bar"}

        # 6 tests total
        tests = manifest.active_tests(exists=False, **info)
        self.assertEqual(len(tests), 6)

        # only 3 tests for subsuite bar when foo==bar
        tests = manifest.active_tests(exists=False, filters=[subsuite("bar")], **info)
        self.assertEqual(len(tests), 3)

        # only 1 test for subsuite baz, regardless of conditions
        other = {"something": "else"}
        tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **info)
        self.assertEqual(len(tests), 1)
        tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **other)
        self.assertEqual(len(tests), 1)

        # 4 tests match when the condition doesn't match (all tests except
        # the unconditional subsuite)
        info = {"foo": "blah"}
        tests = manifest.active_tests(exists=False, filters=[subsuite()], **info)
        self.assertEqual(len(tests), 5)

        # test for illegal subsuite value
        manifest.tests[0]["subsuite"] = 'subsuite=bar,foo=="bar",type="nothing"'
        with self.assertRaises(ParseError):
            manifest.active_tests(exists=False, filters=[subsuite("foo")], **info)
예제 #2
0
    def test_add_duplicates_to_set(self):
        foo = lambda x, y: x
        bar = lambda x, y: x
        sub = subsuite('foo')
        fl = filterlist([foo, bar, sub])
        self.assertEquals(len(fl), 3)
        self.assertEquals(fl[0], foo)

        with self.assertRaises(ValueError):
            fl.append(foo)

        with self.assertRaises(ValueError):
            fl.append(subsuite('bar'))
예제 #3
0
    def test_subsuite(self):
        sub1 = subsuite()
        sub2 = subsuite('baz')

        tests = deepcopy(self.tests)
        tests = list(sub1(tests, {}))
        self.assertNotIn(self.tests[4], tests)
        self.assertEquals(tests[-1]['name'], 'test5')

        tests = deepcopy(self.tests)
        tests = list(sub2(tests, {}))
        self.assertEquals(len(tests), 1)
        self.assertIn(self.tests[4], tests)
예제 #4
0
    def test_add_duplicates_to_list(self):
        foo = lambda x, y: x
        bar = lambda x, y: x
        sub = subsuite('foo')
        fl = filterlist([foo, bar, sub])
        self.assertEquals(len(fl), 3)
        self.assertEquals(fl[0], foo)

        with self.assertRaises(ValueError):
            fl.append(foo)

        with self.assertRaises(ValueError):
            fl.append(subsuite('bar'))
예제 #5
0
    def test_subsuite(self):
        sub1 = subsuite()
        sub2 = subsuite('baz')

        tests = deepcopy(self.tests)
        tests = list(sub1(tests, {}))
        self.assertNotIn(self.tests[5], tests)
        self.assertEquals(len(tests), len(self.tests) - 1)

        tests = deepcopy(self.tests)
        tests = list(sub2(tests, {}))
        self.assertEquals(len(tests), 1)
        self.assertIn(self.tests[5], tests)
예제 #6
0
    def test_subsuite(self):
        sub1 = subsuite()
        sub2 = subsuite('baz')

        tests = deepcopy(self.tests)
        tests = list(sub1(tests, {}))
        self.assertNotIn(self.tests[5], tests)
        self.assertEquals(len(tests), len(self.tests) - 1)

        tests = deepcopy(self.tests)
        tests = list(sub2(tests, {}))
        self.assertEquals(len(tests), 1)
        self.assertIn(self.tests[5], tests)
예제 #7
0
def test_subsuite(tests):
    sub1 = subsuite()
    sub2 = subsuite("baz")

    ref = deepcopy(tests)
    tests = list(sub1(tests, {}))
    assert ref[5] not in tests
    assert len(tests) == len(ref) - 1

    tests = deepcopy(ref)
    tests = list(sub2(tests, {}))
    assert len(tests) == 1
    assert ref[5] in tests
예제 #8
0
def test_add_duplicates_to_list():
    foo = lambda x, y: x
    bar = lambda x, y: x
    sub = subsuite('foo')
    fl = filterlist([foo, bar, sub])
    assert len(fl) == 3
    assert fl[0] == foo

    with pytest.raises(ValueError):
        fl.append(foo)

    with pytest.raises(ValueError):
        fl.append(subsuite('bar'))
예제 #9
0
    def test_subsuite_condition(self):
        sub1 = subsuite()
        sub2 = subsuite('baz')

        tests = deepcopy(self.tests)

        tests = list(sub1(tests, {'foo': 'bar'}))
        self.assertNotIn(self.tests[4], tests)
        self.assertNotIn(self.tests[5], tests)

        tests = deepcopy(self.tests)
        tests = list(sub2(tests, {'foo': 'bar'}))
        self.assertEquals(len(tests), 2)
        self.assertEquals(tests[0]['name'], 'test4')
        self.assertEquals(tests[1]['name'], 'test5')
예제 #10
0
    def test_subsuite_condition(self):
        sub1 = subsuite()
        sub2 = subsuite('baz')

        tests = deepcopy(self.tests)

        tests = list(sub1(tests, {'foo': 'bar'}))
        self.assertNotIn(self.tests[5], tests)
        self.assertNotIn(self.tests[6], tests)

        tests = deepcopy(self.tests)
        tests = list(sub2(tests, {'foo': 'bar'}))
        self.assertEquals(len(tests), 2)
        self.assertEquals(tests[0]['name'], 'test5')
        self.assertEquals(tests[1]['name'], 'test6')
예제 #11
0
def test_subsuite_condition(tests):
    sub1 = subsuite()
    sub2 = subsuite("baz")

    ref = deepcopy(tests)

    tests = list(sub1(tests, {"foo": "bar"}))
    assert ref[5] not in tests
    assert ref[6] not in tests

    tests = deepcopy(ref)
    tests = list(sub2(tests, {"foo": "bar"}))
    assert len(tests) == 2
    assert tests[0]["name"] == "test5"
    assert tests[1]["name"] == "test6"
예제 #12
0
def test_subsuite_condition(tests):
    sub1 = subsuite()
    sub2 = subsuite('baz')

    ref = deepcopy(tests)

    tests = list(sub1(tests, {'foo': 'bar'}))
    assert ref[5] not in tests
    assert ref[6] not in tests

    tests = deepcopy(ref)
    tests = list(sub2(tests, {'foo': 'bar'}))
    assert len(tests) == 2
    assert tests[0]['name'] == 'test5'
    assert tests[1]['name'] == 'test6'
예제 #13
0
def test_add_duplicates_to_list():
    def foo(x, y):
        return x

    def bar(x, y):
        return x

    sub = subsuite("foo")
    fl = filterlist([foo, bar, sub])
    assert len(fl) == 3
    assert fl[0] == foo

    with pytest.raises(ValueError):
        fl.append(foo)

    with pytest.raises(ValueError):
        fl.append(subsuite("bar"))
예제 #14
0
    def test_manifest_subsuites(self):
        """
        test subsuites and conditional subsuites
        """
        relative_path = os.path.join(here, 'subsuite.ini')
        manifest = TestManifest(manifests=(relative_path,))
        info = {'foo': 'bar'}

        # 6 tests total
        tests = manifest.active_tests(exists=False, **info)
        self.assertEquals(len(tests), 6)

        # only 3 tests for subsuite bar when foo==bar
        tests = manifest.active_tests(exists=False,
                                      filters=[subsuite('bar')],
                                      **info)
        self.assertEquals(len(tests), 3)

        # only 1 test for subsuite baz, regardless of conditions
        other = {'something': 'else'}
        tests = manifest.active_tests(exists=False,
                                      filters=[subsuite('baz')],
                                      **info)
        self.assertEquals(len(tests), 1)
        tests = manifest.active_tests(exists=False,
                                      filters=[subsuite('baz')],
                                      **other)
        self.assertEquals(len(tests), 1)

        # 4 tests match when the condition doesn't match (all tests except
        # the unconditional subsuite)
        info = {'foo': 'blah'}
        tests = manifest.active_tests(exists=False,
                                      filters=[subsuite()],
                                      **info)
        self.assertEquals(len(tests), 5)

        # test for illegal subsuite value
        manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
        with self.assertRaises(ParseError):
            manifest.active_tests(exists=False,
                                  filters=[subsuite('foo')],
                                  **info)
예제 #15
0
def run_python_tests(
    command_context,
    tests=None,
    test_objects=None,
    subsuite=None,
    verbose=False,
    jobs=None,
    exitfirst=False,
    extra=None,
    **kwargs
):

    command_context.activate_virtualenv()
    if test_objects is None:
        from moztest.resolve import TestResolver

        resolver = command_context._spawn(TestResolver)
        # If we were given test paths, try to find tests matching them.
        test_objects = resolver.resolve_tests(paths=tests, flavor="python")
    else:
        # We've received test_objects from |mach test|. We need to ignore
        # the subsuite because python-tests don't use this key like other
        # harnesses do and |mach test| doesn't realize this.
        subsuite = None

    mp = TestManifest()
    mp.tests.extend(test_objects)

    filters = []
    if subsuite == "default":
        filters.append(mpf.subsuite(None))
    elif subsuite:
        filters.append(mpf.subsuite(subsuite))

    tests = mp.active_tests(
        filters=filters,
        disabled=False,
        python=command_context.virtualenv_manager.version_info()[0],
        **mozinfo.info
    )

    if not tests:
        submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
        message = (
            "TEST-UNEXPECTED-FAIL | No tests collected "
            + "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
        )
        command_context.log(logging.WARN, "python-test", {}, message)
        return 1

    parallel = []
    sequential = []
    os.environ.setdefault("PYTEST_ADDOPTS", "")

    if extra:
        os.environ["PYTEST_ADDOPTS"] += " " + " ".join(extra)

    installed_requirements = set()
    for test in tests:
        if (
            test.get("requirements")
            and test["requirements"] not in installed_requirements
        ):
            command_context.virtualenv_manager.install_pip_requirements(
                test["requirements"], quiet=True
            )
            installed_requirements.add(test["requirements"])

    if exitfirst:
        sequential = tests
        os.environ["PYTEST_ADDOPTS"] += " -x"
    else:
        for test in tests:
            if test.get("sequential"):
                sequential.append(test)
            else:
                parallel.append(test)

    jobs = jobs or cpu_count()

    return_code = 0

    def on_test_finished(result):
        output, ret, test_path = result

        for line in output:
            command_context.log(
                logging.INFO, "python-test", {"line": line.rstrip()}, "{line}"
            )

        if ret and not return_code:
            command_context.log(
                logging.ERROR,
                "python-test",
                {"test_path": test_path, "ret": ret},
                "Setting retcode to {ret} from {test_path}",
            )
        return return_code or ret

    with ThreadPoolExecutor(max_workers=jobs) as executor:
        futures = [
            executor.submit(_run_python_test, command_context, test, jobs, verbose)
            for test in parallel
        ]

        try:
            for future in as_completed(futures):
                return_code = on_test_finished(future.result())
        except KeyboardInterrupt:
            # Hack to force stop currently running threads.
            # https://gist.github.com/clchiou/f2608cbe54403edb0b13
            executor._threads.clear()
            thread._threads_queues.clear()
            raise

    for test in sequential:
        return_code = on_test_finished(
            _run_python_test(command_context, test, jobs, verbose)
        )
        if return_code and exitfirst:
            break

    command_context.log(
        logging.INFO,
        "python-test",
        {"return_code": return_code},
        "Return code from mach python-test: {return_code}",
    )
    return return_code
예제 #16
0
    def run_python_tests(self,
                         tests=[],
                         test_objects=None,
                         subsuite=None,
                         verbose=False,
                         stop=False,
                         jobs=1):
        self._activate_virtualenv()

        def find_tests_by_path():
            import glob
            files = []
            for t in tests:
                if t.endswith('.py') and os.path.isfile(t):
                    files.append(t)
                elif os.path.isdir(t):
                    for root, _, _ in os.walk(t):
                        files += glob.glob(mozpath.join(root, 'test*.py'))
                        files += glob.glob(mozpath.join(root, 'unit*.py'))
                else:
                    self.log(logging.WARN, 'python-test', {'test': t},
                             'TEST-UNEXPECTED-FAIL | Invalid test: {test}')
                    if stop:
                        break
            return files

        # Python's unittest, and in particular discover, has problems with
        # clashing namespaces when importing multiple test modules. What follows
        # is a simple way to keep environments separate, at the price of
        # launching Python multiple times. Most tests are run via mozunit,
        # which produces output in the format Mozilla infrastructure expects.
        # Some tests are run via pytest.
        if test_objects is None:
            from moztest.resolve import TestResolver
            resolver = self._spawn(TestResolver)
            if tests:
                # If we were given test paths, try to find tests matching them.
                test_objects = resolver.resolve_tests(paths=tests,
                                                      flavor='python')
            else:
                # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
                test_objects = resolver.resolve_tests(flavor='python')

        mp = TestManifest()
        mp.tests.extend(test_objects)

        filters = []
        if subsuite == 'default':
            filters.append(mpf.subsuite(None))
        elif subsuite:
            filters.append(mpf.subsuite(subsuite))

        tests = mp.active_tests(filters=filters,
                                disabled=False,
                                **mozinfo.info)

        if not tests:
            submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
            message = "TEST-UNEXPECTED-FAIL | No tests collected " + \
                      "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
            self.log(logging.WARN, 'python-test', {}, message)
            return 1

        parallel = []
        sequential = []
        for test in tests:
            if test.get('sequential'):
                sequential.append(test)
            else:
                parallel.append(test)

        self.jobs = jobs
        self.terminate = False
        self.verbose = verbose

        return_code = 0

        def on_test_finished(result):
            output, ret, test_path = result

            for line in output:
                self.log(logging.INFO, 'python-test', {'line': line.rstrip()},
                         '{line}')

            if ret and not return_code:
                self.log(logging.ERROR, 'python-test', {
                    'test_path': test_path,
                    'ret': ret
                }, 'Setting retcode to {ret} from {test_path}')
            return return_code or ret

        with ThreadPoolExecutor(max_workers=self.jobs) as executor:
            futures = [
                executor.submit(self._run_python_test, test['path'])
                for test in parallel
            ]

            try:
                for future in as_completed(futures):
                    return_code = on_test_finished(future.result())
            except KeyboardInterrupt:
                # Hack to force stop currently running threads.
                # https://gist.github.com/clchiou/f2608cbe54403edb0b13
                executor._threads.clear()
                thread._threads_queues.clear()
                raise

        for test in sequential:
            return_code = on_test_finished(self._run_python_test(test['path']))

        self.log(logging.INFO, 'python-test', {'return_code': return_code},
                 'Return code from mach python-test: {return_code}')
        return return_code
예제 #17
0
    def run_python_tests(self,
                         tests=None,
                         test_objects=None,
                         subsuite=None,
                         verbose=False,
                         jobs=None,
                         python=None,
                         exitfirst=False,
                         extra=None,
                         **kwargs):
        self._activate_test_virtualenvs(python)

        if test_objects is None:
            from moztest.resolve import TestResolver
            resolver = self._spawn(TestResolver)
            # If we were given test paths, try to find tests matching them.
            test_objects = resolver.resolve_tests(paths=tests, flavor='python')
        else:
            # We've received test_objects from |mach test|. We need to ignore
            # the subsuite because python-tests don't use this key like other
            # harnesses do and |mach test| doesn't realize this.
            subsuite = None

        mp = TestManifest()
        mp.tests.extend(test_objects)

        filters = []
        if subsuite == 'default':
            filters.append(mpf.subsuite(None))
        elif subsuite:
            filters.append(mpf.subsuite(subsuite))

        tests = mp.active_tests(filters=filters,
                                disabled=False,
                                python=self.virtualenv_manager.version_info[0],
                                **mozinfo.info)

        if not tests:
            submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
            message = "TEST-UNEXPECTED-FAIL | No tests collected " + \
                      "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
            self.log(logging.WARN, 'python-test', {}, message)
            return 1

        parallel = []
        sequential = []
        os.environ.setdefault('PYTEST_ADDOPTS', '')

        if extra:
            os.environ['PYTEST_ADDOPTS'] += " " + " ".join(extra)

        if exitfirst:
            sequential = tests
            os.environ['PYTEST_ADDOPTS'] += " -x"
        else:
            for test in tests:
                if test.get('sequential'):
                    sequential.append(test)
                else:
                    parallel.append(test)

        self.jobs = jobs or cpu_count()
        self.terminate = False
        self.verbose = verbose

        return_code = 0

        def on_test_finished(result):
            output, ret, test_path = result

            for line in output:
                self.log(logging.INFO, 'python-test', {'line': line.rstrip()},
                         '{line}')

            if ret and not return_code:
                self.log(logging.ERROR, 'python-test', {
                    'test_path': test_path,
                    'ret': ret
                }, 'Setting retcode to {ret} from {test_path}')
            return return_code or ret

        with ThreadPoolExecutor(max_workers=self.jobs) as executor:
            futures = [
                executor.submit(self._run_python_test, test)
                for test in parallel
            ]

            try:
                for future in as_completed(futures):
                    return_code = on_test_finished(future.result())
            except KeyboardInterrupt:
                # Hack to force stop currently running threads.
                # https://gist.github.com/clchiou/f2608cbe54403edb0b13
                executor._threads.clear()
                thread._threads_queues.clear()
                raise

        for test in sequential:
            return_code = on_test_finished(self._run_python_test(test))
            if return_code and exitfirst:
                break

        self.log(logging.INFO, 'python-test', {'return_code': return_code},
                 'Return code from mach python-test: {return_code}')
        return return_code
예제 #18
0
    def run_python_tests(self,
                         tests=[],
                         test_objects=None,
                         subsuite=None,
                         verbose=False,
                         stop=False,
                         jobs=1):
        self._activate_virtualenv()

        def find_tests_by_path():
            import glob
            files = []
            for t in tests:
                if t.endswith('.py') and os.path.isfile(t):
                    files.append(t)
                elif os.path.isdir(t):
                    for root, _, _ in os.walk(t):
                        files += glob.glob(mozpath.join(root, 'test*.py'))
                        files += glob.glob(mozpath.join(root, 'unit*.py'))
                else:
                    self.log(logging.WARN, 'python-test',
                                 {'test': t},
                                 'TEST-UNEXPECTED-FAIL | Invalid test: {test}')
                    if stop:
                        break
            return files

        # Python's unittest, and in particular discover, has problems with
        # clashing namespaces when importing multiple test modules. What follows
        # is a simple way to keep environments separate, at the price of
        # launching Python multiple times. Most tests are run via mozunit,
        # which produces output in the format Mozilla infrastructure expects.
        # Some tests are run via pytest.
        if test_objects is None:
            from mozbuild.testing import TestResolver
            resolver = self._spawn(TestResolver)
            if tests:
                # If we were given test paths, try to find tests matching them.
                test_objects = resolver.resolve_tests(paths=tests,
                                                      flavor='python')
            else:
                # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
                test_objects = resolver.resolve_tests(flavor='python')

        mp = TestManifest()
        mp.tests.extend(test_objects)

        if not mp.tests:
            message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
                      '(Not in PYTHON_UNITTEST_MANIFESTS?)'
            self.log(logging.WARN, 'python-test', {}, message)
            return 1

        filters = []
        if subsuite == 'default':
            filters.append(mpf.subsuite(None))
        elif subsuite:
            filters.append(mpf.subsuite(subsuite))

        tests = mp.active_tests(filters=filters, disabled=False, **mozinfo.info)
        parallel = []
        sequential = []
        for test in tests:
            if test.get('sequential'):
                sequential.append(test)
            else:
                parallel.append(test)

        self.jobs = jobs
        self.terminate = False
        self.verbose = verbose

        return_code = 0

        def on_test_finished(result):
            output, ret, test_path = result

            for line in output:
                self.log(logging.INFO, 'python-test', {'line': line.rstrip()}, '{line}')

            if ret and not return_code:
                self.log(logging.ERROR, 'python-test', {'test_path': test_path, 'ret': ret},
                         'Setting retcode to {ret} from {test_path}')
            return return_code or ret

        with ThreadPoolExecutor(max_workers=self.jobs) as executor:
            futures = [executor.submit(self._run_python_test, test['path'])
                       for test in parallel]

            try:
                for future in as_completed(futures):
                    return_code = on_test_finished(future.result())
            except KeyboardInterrupt:
                # Hack to force stop currently running threads.
                # https://gist.github.com/clchiou/f2608cbe54403edb0b13
                executor._threads.clear()
                thread._threads_queues.clear()
                raise

        for test in sequential:
            return_code = on_test_finished(self._run_python_test(test['path']))

        self.log(logging.INFO, 'python-test', {'return_code': return_code},
                 'Return code from mach python-test: {return_code}')
        return return_code