Пример #1
0
    def run_robocop(self, test_path, **kwargs):
        if not kwargs.get('robocopIni'):
            kwargs['robocopIni'] = os.path.join(self.topobjdir, '_tests', 'testing',
                                                'mochitest', 'robocop.ini')

        if not kwargs.get('robocopApk'):
            kwargs['robocopApk'] = os.path.join(self.topobjdir, 'build', 'mobile',
                                                'robocop', 'robocop-debug.apk')

        if isinstance(test_path, basestring):
            test_path = [test_path]

        mochitest = self._spawn(MochitestRunner)
        tests = mochitest.resolve_tests(test_path, cwd=self._mach_context.cwd)
        return mochitest.run_android_test(self._mach_context, tests, 'robocop', **kwargs)
Пример #2
0
    def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
        from mochitest_options import ALL_FLAVORS

        buildapp = None
        for app in SUPPORTED_APPS:
            if is_buildapp_in(app)(self):
                buildapp = app
                break

        flavors = None
        if flavor:
            for fname, fobj in ALL_FLAVORS.iteritems():
                if flavor in fobj['aliases']:
                    if buildapp not in fobj['enabled_apps']:
                        continue
                    flavors = [fname]
                    break
        else:
            flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]

        from mozbuild.controller.building import BuildDriver
        self._ensure_state_subdir_exists('.')

        test_paths = kwargs['test_paths']
        kwargs['test_paths'] = []

        mochitest = self._spawn(MochitestRunner)
        tests = []
        if resolve_tests:
            tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)

        driver = self._spawn(BuildDriver)
        driver.install_tests(tests)

        subsuite = kwargs.get('subsuite')
        if subsuite == 'default':
            kwargs['subsuite'] = None

        suites = defaultdict(list)
        unsupported = set()
        for test in tests:
            # Filter out non-mochitests and unsupported flavors.
            if test['flavor'] not in ALL_FLAVORS:
                continue

            key = (test['flavor'], test.get('subsuite', ''))
            if test['flavor'] not in flavors:
                unsupported.add(key)
                continue

            if subsuite == 'default':
                # "--subsuite default" means only run tests that don't have a subsuite
                if test.get('subsuite'):
                    unsupported.add(key)
                    continue
            elif subsuite and test.get('subsuite', '') != subsuite:
                unsupported.add(key)
                continue

            suites[key].append(test)

        if ('mochitest', 'media') in suites:
            req = os.path.join('testing', 'tools', 'websocketprocessbridge',
                               'websocketprocessbridge_requirements.txt')
            self.virtualenv_manager.activate()
            self.virtualenv_manager.install_pip_requirements(req, require_hashes=False)

            # sys.executable is used to start the websocketprocessbridge, though for some
            # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
            sys.executable = self.virtualenv_manager.python_path

        # This is a hack to introduce an option in mach to not send
        # filtered tests to the mochitest harness. Mochitest harness will read
        # the master manifest in that case.
        if not resolve_tests:
            for flavor in flavors:
                key = (flavor, kwargs.get('subsuite'))
                suites[key] = []

        if not suites:
            # Make it very clear why no tests were found
            if not unsupported:
                print(TESTS_NOT_FOUND.format('\n'.join(
                    sorted(list(test_paths or test_objects)))))
                return 1

            msg = []
            for f, s in unsupported:
                fobj = ALL_FLAVORS[f]
                apps = fobj['enabled_apps']
                name = fobj['aliases'][0]
                if s:
                    name = '{} --subsuite {}'.format(name, s)

                if buildapp not in apps:
                    reason = 'requires {}'.format(' or '.join(apps))
                else:
                    reason = 'excluded by the command line'
                msg.append('    mochitest -f {} ({})'.format(name, reason))
            print(SUPPORTED_TESTS_NOT_FOUND.format(
                buildapp, '\n'.join(sorted(msg))))
            return 1

        if buildapp == 'android':
            from mozrunner.devices.android_device import grant_runtime_permissions
            grant_runtime_permissions(self)
            run_mochitest = mochitest.run_android_test
        else:
            run_mochitest = mochitest.run_desktop_test

        overall = None
        for (flavor, subsuite), tests in sorted(suites.items()):
            fobj = ALL_FLAVORS[flavor]
            msg = fobj['aliases'][0]
            if subsuite:
                msg = '{} with subsuite {}'.format(msg, subsuite)
            print(NOW_RUNNING.format(msg))

            harness_args = kwargs.copy()
            harness_args['subsuite'] = subsuite
            harness_args.update(fobj.get('extra_args', {}))

            result = run_mochitest(
                self._mach_context,
                tests=tests,
                suite=fobj['suite'],
                **harness_args)

            if result:
                overall = result

        # TODO consolidate summaries from all suites
        return overall
Пример #3
0
    def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
        buildapp = None
        for app in SUPPORTED_APPS:
            if is_buildapp_in(app)(self):
                buildapp = app
                break

        flavors = None
        if flavor:
            for fname, fobj in ALL_FLAVORS.iteritems():
                if flavor in fobj['aliases']:
                    if buildapp not in fobj['enabled_apps']:
                        continue
                    flavors = [fname]
                    break
        else:
            flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]

        from mozbuild.controller.building import BuildDriver
        self._ensure_state_subdir_exists('.')

        driver = self._spawn(BuildDriver)
        driver.install_tests(remove=False)

        test_paths = kwargs['test_paths']
        kwargs['test_paths'] = []

        if test_paths and buildapp == 'b2g':
            # In B2G there is often a 'gecko' directory, though topsrcdir is actually
            # elsewhere. This little hack makes test paths like 'gecko/dom' work, even if
            # GECKO_PATH is set in the .userconfig
            gecko_path = mozpath.abspath(mozpath.join(kwargs['b2gPath'], 'gecko'))
            if gecko_path != self.topsrcdir:
                new_paths = []
                for tp in test_paths:
                    if mozpath.abspath(tp).startswith(gecko_path):
                        new_paths.append(mozpath.relpath(tp, gecko_path))
                    else:
                        new_paths.append(tp)
                test_paths = new_paths

        mochitest = self._spawn(MochitestRunner)
        tests = []
        if resolve_tests:
            tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)

        subsuite = kwargs.get('subsuite')
        if subsuite == 'default':
            kwargs['subsuite'] = None

        suites = defaultdict(list)
        unsupported = set()
        for test in tests:
            # Filter out non-mochitests and unsupported flavors.
            if test['flavor'] not in ALL_FLAVORS:
                continue

            key = (test['flavor'], test['subsuite'])
            if test['flavor'] not in flavors:
                unsupported.add(key)
                continue

            if subsuite == 'default':
                # "--subsuite default" means only run tests that don't have a subsuite
                if test['subsuite']:
                    unsupported.add(key)
                    continue
            elif subsuite and test['subsuite'] != subsuite:
                unsupported.add(key)
                continue

            suites[key].append(test)

        # This is a hack to introduce an option in mach to not send
        # filtered tests to the mochitest harness. Mochitest harness will read
        # the master manifest in that case.
        if not resolve_tests:
            for flavor in flavors:
                key = (flavor, kwargs.get('subsuite'))
                suites[key] = []

        if not suites:
            # Make it very clear why no tests were found
            if not unsupported:
                print(TESTS_NOT_FOUND.format('\n'.join(
                    sorted(list(test_paths or test_objects)))))
                return 1

            msg = []
            for f, s in unsupported:
                fobj = ALL_FLAVORS[f]
                apps = fobj['enabled_apps']
                name = fobj['aliases'][0]
                if s:
                    name = '{} --subsuite {}'.format(name, s)

                if buildapp not in apps:
                    reason = 'requires {}'.format(' or '.join(apps))
                else:
                    reason = 'excluded by the command line'
                msg.append('    mochitest -f {} ({})'.format(name, reason))
            print(SUPPORTED_TESTS_NOT_FOUND.format(
                buildapp, '\n'.join(sorted(msg))))
            return 1

        if buildapp in ('b2g', 'b2g_desktop'):
            run_mochitest = mochitest.run_b2g_test
        elif buildapp == 'android':
            run_mochitest = mochitest.run_android_test
        else:
            run_mochitest = mochitest.run_desktop_test

        overall = None
        for (flavor, subsuite), tests in sorted(suites.items()):
            fobj = ALL_FLAVORS[flavor]
            msg = fobj['aliases'][0]
            if subsuite:
                msg = '{} with subsuite {}'.format(msg, subsuite)
            print(NOW_RUNNING.format(msg))

            harness_args = kwargs.copy()
            harness_args['subsuite'] = subsuite
            harness_args.update(fobj.get('extra_args', {}))

            result = run_mochitest(
                self._mach_context,
                tests=tests,
                suite=fobj['suite'],
                **harness_args)

            if result:
                overall = result

        # TODO consolidate summaries from all suites
        return overall
Пример #4
0
    def run_mochitest_general(self,
                              flavor=None,
                              test_objects=None,
                              resolve_tests=True,
                              **kwargs):
        from mochitest_options import ALL_FLAVORS
        from mozlog.commandline import setup_logging
        from mozlog.handlers import StreamHandler
        from moztest.resolve import get_suite_definition

        buildapp = None
        for app in SUPPORTED_APPS:
            if is_buildapp_in(app)(self):
                buildapp = app
                break

        flavors = None
        if flavor:
            for fname, fobj in ALL_FLAVORS.iteritems():
                if flavor in fobj['aliases']:
                    if buildapp not in fobj['enabled_apps']:
                        continue
                    flavors = [fname]
                    break
        else:
            flavors = [
                f for f, v in ALL_FLAVORS.iteritems()
                if buildapp in v['enabled_apps']
            ]

        from mozbuild.controller.building import BuildDriver
        self._ensure_state_subdir_exists('.')

        test_paths = kwargs['test_paths']
        kwargs['test_paths'] = []

        mochitest = self._spawn(MochitestRunner)
        tests = []
        if resolve_tests:
            tests = mochitest.resolve_tests(test_paths,
                                            test_objects,
                                            cwd=self._mach_context.cwd)

        if not kwargs.get('log'):
            # Create shared logger
            format_args = {
                'level': self._mach_context.settings['test']['level']
            }
            if len(tests) == 1:
                format_args['verbose'] = True
                format_args['compact'] = False

            default_format = self._mach_context.settings['test']['format']
            kwargs['log'] = setup_logging('mach-mochitest', kwargs,
                                          {default_format: sys.stdout},
                                          format_args)
            for handler in kwargs['log'].handlers:
                if isinstance(handler, StreamHandler):
                    handler.formatter.inner.summary_on_shutdown = True

        driver = self._spawn(BuildDriver)
        driver.install_tests(tests)

        subsuite = kwargs.get('subsuite')
        if subsuite == 'default':
            kwargs['subsuite'] = None

        suites = defaultdict(list)
        unsupported = set()
        for test in tests:
            # Filter out non-mochitests and unsupported flavors.
            if test['flavor'] not in ALL_FLAVORS:
                continue

            key = (test['flavor'], test.get('subsuite', ''))
            if test['flavor'] not in flavors:
                unsupported.add(key)
                continue

            if subsuite == 'default':
                # "--subsuite default" means only run tests that don't have a subsuite
                if test.get('subsuite'):
                    unsupported.add(key)
                    continue
            elif subsuite and test.get('subsuite', '') != subsuite:
                unsupported.add(key)
                continue

            suites[key].append(test)

        if ('mochitest', 'media') in suites:
            req = os.path.join('testing', 'tools', 'websocketprocessbridge',
                               'websocketprocessbridge_requirements.txt')
            self.virtualenv_manager.activate()
            self.virtualenv_manager.install_pip_requirements(
                req, require_hashes=False)

            # sys.executable is used to start the websocketprocessbridge, though for some
            # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
            sys.executable = self.virtualenv_manager.python_path

        # This is a hack to introduce an option in mach to not send
        # filtered tests to the mochitest harness. Mochitest harness will read
        # the master manifest in that case.
        if not resolve_tests:
            for flavor in flavors:
                key = (flavor, kwargs.get('subsuite'))
                suites[key] = []

        if not suites:
            # Make it very clear why no tests were found
            if not unsupported:
                print(
                    TESTS_NOT_FOUND.format('\n'.join(
                        sorted(list(test_paths or test_objects)))))
                return 1

            msg = []
            for f, s in unsupported:
                fobj = ALL_FLAVORS[f]
                apps = fobj['enabled_apps']
                name = fobj['aliases'][0]
                if s:
                    name = '{} --subsuite {}'.format(name, s)

                if buildapp not in apps:
                    reason = 'requires {}'.format(' or '.join(apps))
                else:
                    reason = 'excluded by the command line'
                msg.append('    mochitest -f {} ({})'.format(name, reason))
            print(
                SUPPORTED_TESTS_NOT_FOUND.format(buildapp,
                                                 '\n'.join(sorted(msg))))
            return 1

        if buildapp == 'android':
            from mozrunner.devices.android_device import grant_runtime_permissions
            from mozrunner.devices.android_device import verify_android_device
            app = kwargs.get('app')
            if not app:
                app = self.substs["ANDROID_PACKAGE_NAME"]
            device_serial = kwargs.get('deviceSerial')

            # verify installation
            verify_android_device(self,
                                  install=True,
                                  xre=False,
                                  network=True,
                                  app=app,
                                  device_serial=device_serial)
            grant_runtime_permissions(self, app, device_serial=device_serial)
            run_mochitest = mochitest.run_android_test
        else:
            run_mochitest = mochitest.run_desktop_test

        overall = None
        for (flavor, subsuite), tests in sorted(suites.items()):
            _, suite = get_suite_definition(flavor, subsuite)
            if 'test_paths' in suite['kwargs']:
                del suite['kwargs']['test_paths']

            harness_args = kwargs.copy()
            harness_args.update(suite['kwargs'])

            result = run_mochitest(self._mach_context,
                                   tests=tests,
                                   **harness_args)

            if result:
                overall = result

            # Halt tests on keyboard interrupt
            if result == -1:
                break

        # Only shutdown the logger if we created it
        if kwargs['log'].name == 'mach-mochitest':
            kwargs['log'].shutdown()

        return overall
Пример #5
0
    def run_mochitest_general(self,
                              flavor=None,
                              test_objects=None,
                              resolve_tests=True,
                              **kwargs):
        buildapp = None
        for app in SUPPORTED_APPS:
            if is_buildapp_in(app)(self):
                buildapp = app
                break

        flavors = None
        if flavor:
            for fname, fobj in ALL_FLAVORS.iteritems():
                if flavor in fobj['aliases']:
                    if buildapp not in fobj['enabled_apps']:
                        continue
                    flavors = [fname]
                    break
        else:
            flavors = [
                f for f, v in ALL_FLAVORS.iteritems()
                if buildapp in v['enabled_apps']
            ]

        from mozbuild.controller.building import BuildDriver
        self._ensure_state_subdir_exists('.')

        test_paths = kwargs['test_paths']
        kwargs['test_paths'] = []

        mochitest = self._spawn(MochitestRunner)
        tests = []
        if resolve_tests:
            tests = mochitest.resolve_tests(test_paths,
                                            test_objects,
                                            cwd=self._mach_context.cwd)

        driver = self._spawn(BuildDriver)
        driver.install_tests(tests)

        subsuite = kwargs.get('subsuite')
        if subsuite == 'default':
            kwargs['subsuite'] = None

        suites = defaultdict(list)
        unsupported = set()
        for test in tests:
            # Filter out non-mochitests and unsupported flavors.
            if test['flavor'] not in ALL_FLAVORS:
                continue

            key = (test['flavor'], test.get('subsuite', ''))
            if test['flavor'] not in flavors:
                unsupported.add(key)
                continue

            if subsuite == 'default':
                # "--subsuite default" means only run tests that don't have a subsuite
                if test.get('subsuite'):
                    unsupported.add(key)
                    continue
            elif subsuite and test.get('subsuite', '') != subsuite:
                unsupported.add(key)
                continue

            suites[key].append(test)

        # This is a hack to introduce an option in mach to not send
        # filtered tests to the mochitest harness. Mochitest harness will read
        # the master manifest in that case.
        if not resolve_tests:
            for flavor in flavors:
                key = (flavor, kwargs.get('subsuite'))
                suites[key] = []

        if not suites:
            # Make it very clear why no tests were found
            if not unsupported:
                print(
                    TESTS_NOT_FOUND.format('\n'.join(
                        sorted(list(test_paths or test_objects)))))
                return 1

            msg = []
            for f, s in unsupported:
                fobj = ALL_FLAVORS[f]
                apps = fobj['enabled_apps']
                name = fobj['aliases'][0]
                if s:
                    name = '{} --subsuite {}'.format(name, s)

                if buildapp not in apps:
                    reason = 'requires {}'.format(' or '.join(apps))
                else:
                    reason = 'excluded by the command line'
                msg.append('    mochitest -f {} ({})'.format(name, reason))
            print(
                SUPPORTED_TESTS_NOT_FOUND.format(buildapp,
                                                 '\n'.join(sorted(msg))))
            return 1

        if buildapp == 'android':
            from mozrunner.devices.android_device import grant_runtime_permissions
            grant_runtime_permissions(self)
            run_mochitest = mochitest.run_android_test
        else:
            run_mochitest = mochitest.run_desktop_test

        overall = None
        for (flavor, subsuite), tests in sorted(suites.items()):
            fobj = ALL_FLAVORS[flavor]
            msg = fobj['aliases'][0]
            if subsuite:
                msg = '{} with subsuite {}'.format(msg, subsuite)
            print(NOW_RUNNING.format(msg))

            harness_args = kwargs.copy()
            harness_args['subsuite'] = subsuite
            harness_args.update(fobj.get('extra_args', {}))

            result = run_mochitest(self._mach_context,
                                   tests=tests,
                                   suite=fobj['suite'],
                                   **harness_args)

            if result:
                overall = result

        # TODO consolidate summaries from all suites
        return overall
Пример #6
0
def run_mochitest_general(command_context,
                          flavor=None,
                          test_objects=None,
                          resolve_tests=True,
                          **kwargs):
    from mochitest_options import ALL_FLAVORS
    from mozlog.commandline import setup_logging
    from mozlog.handlers import StreamHandler
    from moztest.resolve import get_suite_definition

    # TODO: This is only strictly necessary while mochitest is using Python
    # 2 and can be removed once the command is migrated to Python 3.
    command_context.activate_virtualenv()

    buildapp = None
    for app in SUPPORTED_APPS:
        if conditions.is_buildapp_in(command_context, apps=[app]):
            buildapp = app
            break

    flavors = None
    if flavor:
        for fname, fobj in six.iteritems(ALL_FLAVORS):
            if flavor in fobj["aliases"]:
                if buildapp not in fobj["enabled_apps"]:
                    continue
                flavors = [fname]
                break
    else:
        flavors = [
            f for f, v in six.iteritems(ALL_FLAVORS)
            if buildapp in v["enabled_apps"]
        ]

    from mozbuild.controller.building import BuildDriver

    command_context._ensure_state_subdir_exists(".")

    test_paths = kwargs["test_paths"]
    kwargs["test_paths"] = []

    if kwargs.get("debugger", None):
        import mozdebug

        if not mozdebug.get_debugger_info(kwargs.get("debugger")):
            sys.exit(1)

    mochitest = command_context._spawn(MochitestRunner)
    tests = []
    if resolve_tests:
        tests = mochitest.resolve_tests(test_paths,
                                        test_objects,
                                        cwd=command_context._mach_context.cwd)

    if not kwargs.get("log"):
        # Create shared logger
        format_args = {
            "level": command_context._mach_context.settings["test"]["level"]
        }
        if len(tests) == 1:
            format_args["verbose"] = True
            format_args["compact"] = False

        default_format = command_context._mach_context.settings["test"][
            "format"]
        kwargs["log"] = setup_logging("mach-mochitest", kwargs,
                                      {default_format: sys.stdout},
                                      format_args)
        for handler in kwargs["log"].handlers:
            if isinstance(handler, StreamHandler):
                handler.formatter.inner.summary_on_shutdown = True

    driver = command_context._spawn(BuildDriver)
    driver.install_tests()

    subsuite = kwargs.get("subsuite")
    if subsuite == "default":
        kwargs["subsuite"] = None

    suites = defaultdict(list)
    is_webrtc_tag_present = False
    unsupported = set()
    for test in tests:
        # Check if we're running a webrtc test so we can enable webrtc
        # specific test logic later if needed.
        if "webrtc" in test.get("tags", ""):
            is_webrtc_tag_present = True

        # Filter out non-mochitests and unsupported flavors.
        if test["flavor"] not in ALL_FLAVORS:
            continue

        key = (test["flavor"], test.get("subsuite", ""))
        if test["flavor"] not in flavors:
            unsupported.add(key)
            continue

        if subsuite == "default":
            # "--subsuite default" means only run tests that don't have a subsuite
            if test.get("subsuite"):
                unsupported.add(key)
                continue
        elif subsuite and test.get("subsuite", "") != subsuite:
            unsupported.add(key)
            continue

        suites[key].append(test)

    # Only webrtc mochitests in the media suite need the websocketprocessbridge.
    if ("mochitest", "media") in suites and is_webrtc_tag_present:
        req = os.path.join(
            "testing",
            "tools",
            "websocketprocessbridge",
            "websocketprocessbridge_requirements_3.txt",
        )
        command_context.virtualenv_manager.activate()
        command_context.virtualenv_manager.install_pip_requirements(
            req, require_hashes=False)

        # sys.executable is used to start the websocketprocessbridge, though for some
        # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
        sys.executable = command_context.virtualenv_manager.python_path

    # This is a hack to introduce an option in mach to not send
    # filtered tests to the mochitest harness. Mochitest harness will read
    # the master manifest in that case.
    if not resolve_tests:
        for flavor in flavors:
            key = (flavor, kwargs.get("subsuite"))
            suites[key] = []

    if not suites:
        # Make it very clear why no tests were found
        if not unsupported:
            print(
                TESTS_NOT_FOUND.format("\n".join(
                    sorted(list(test_paths or test_objects)))))
            return 1

        msg = []
        for f, s in unsupported:
            fobj = ALL_FLAVORS[f]
            apps = fobj["enabled_apps"]
            name = fobj["aliases"][0]
            if s:
                name = "{} --subsuite {}".format(name, s)

            if buildapp not in apps:
                reason = "requires {}".format(" or ".join(apps))
            else:
                reason = "excluded by the command line"
            msg.append("    mochitest -f {} ({})".format(name, reason))
        print(
            SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg))))
        return 1

    if buildapp == "android":
        from mozrunner.devices.android_device import (
            get_adb_path,
            verify_android_device,
            InstallIntent,
        )

        app = kwargs.get("app")
        if not app:
            app = "org.mozilla.geckoview.test"
        device_serial = kwargs.get("deviceSerial")
        install = InstallIntent.NO if kwargs.get(
            "no_install") else InstallIntent.YES

        # verify installation
        verify_android_device(
            command_context,
            install=install,
            xre=False,
            network=True,
            app=app,
            device_serial=device_serial,
        )

        if not kwargs["adbPath"]:
            kwargs["adbPath"] = get_adb_path(command_context)

        run_mochitest = mochitest.run_android_test
    else:
        run_mochitest = mochitest.run_desktop_test

    overall = None
    for (flavor, subsuite), tests in sorted(suites.items()):
        suite_name, suite = get_suite_definition(flavor, subsuite)
        if "test_paths" in suite["kwargs"]:
            del suite["kwargs"]["test_paths"]

        harness_args = kwargs.copy()
        harness_args.update(suite["kwargs"])
        # Pass in the full suite name as defined in moztest/resolve.py in case
        # chunk-by-runtime is called, in which case runtime information for
        # specific mochitest suite has to be loaded. See Bug 1637463.
        harness_args.update({"suite_name": suite_name})

        result = run_mochitest(command_context._mach_context,
                               tests=tests,
                               **harness_args)

        if result:
            overall = result

        # Halt tests on keyboard interrupt
        if result == -1:
            break

    # Only shutdown the logger if we created it
    if kwargs["log"].name == "mach-mochitest":
        kwargs["log"].shutdown()

    return overall
Пример #7
0
    def run_mochitest_general(self, flavor=None, test_objects=None, **kwargs):
        buildapp = None
        for app in SUPPORTED_APPS:
            if is_buildapp_in(app)(self):
                buildapp = app
                break

        flavors = None
        if flavor:
            for fname, fobj in ALL_FLAVORS.iteritems():
                if flavor in fobj['aliases']:
                    if buildapp not in fobj['enabled_apps']:
                        continue
                    flavors = [fname]
                    break
        else:
            flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]

        from mozbuild.controller.building import BuildDriver
        self._ensure_state_subdir_exists('.')

        driver = self._spawn(BuildDriver)
        driver.install_tests(remove=False)

        test_paths = kwargs['test_paths']
        kwargs['test_paths'] = []

        if test_paths and buildapp == 'b2g':
            # In B2G there is often a 'gecko' directory, though topsrcdir is actually
            # elsewhere. This little hack makes test paths like 'gecko/dom' work, even if
            # GECKO_PATH is set in the .userconfig
            gecko_path = mozpath.abspath(mozpath.join(kwargs['b2gPath'], 'gecko'))
            if gecko_path != self.topsrcdir:
                new_paths = []
                for tp in test_paths:
                    if mozpath.abspath(tp).startswith(gecko_path):
                        new_paths.append(mozpath.relpath(tp, gecko_path))
                    else:
                        new_paths.append(tp)
                test_paths = new_paths

        mochitest = self._spawn(MochitestRunner)
        tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)

        subsuite = kwargs.get('subsuite')
        if subsuite == 'default':
            kwargs['subsuite'] = None

        suites = defaultdict(list)
        unsupported = set()
        for test in tests:
            # Filter out non-mochitests and unsupported flavors.
            if test['flavor'] not in ALL_FLAVORS:
                continue

            key = (test['flavor'], test['subsuite'])
            if test['flavor'] not in flavors:
                unsupported.add(key)
                continue

            if subsuite == 'default':
                # "--subsuite default" means only run tests that don't have a subsuite
                if test['subsuite']:
                    unsupported.add(key)
                    continue
            elif subsuite and test['subsuite'] != subsuite:
                unsupported.add(key)
                continue

            suites[key].append(test)

        if not suites:
            # Make it very clear why no tests were found
            if not unsupported:
                print(TESTS_NOT_FOUND.format('\n'.join(
                    sorted(list(test_paths or test_objects)))))
                return 1

            msg = []
            for f, s in unsupported:
                fobj = ALL_FLAVORS[f]
                apps = fobj['enabled_apps']
                name = fobj['aliases'][0]
                if s:
                    name = '{} --subsuite {}'.format(name, s)

                if buildapp not in apps:
                    reason = 'requires {}'.format(' or '.join(apps))
                else:
                    reason = 'excluded by the command line'
                msg.append('    mochitest -f {} ({})'.format(name, reason))
            print(SUPPORTED_TESTS_NOT_FOUND.format(
                buildapp, '\n'.join(sorted(msg))))
            return 1

        if buildapp in ('b2g', 'b2g_desktop'):
            run_mochitest = mochitest.run_b2g_test
        elif buildapp == 'android':
            run_mochitest = mochitest.run_android_test
        else:
            run_mochitest = mochitest.run_desktop_test

        overall = None
        for (flavor, subsuite), tests in sorted(suites.items()):
            fobj = ALL_FLAVORS[flavor]
            msg = fobj['aliases'][0]
            if subsuite:
                msg = '{} with subsuite {}'.format(msg, subsuite)
            print(NOW_RUNNING.format(msg))

            harness_args = kwargs.copy()
            harness_args['subsuite'] = subsuite
            harness_args.update(fobj.get('extra_args', {}))

            result = run_mochitest(
                self._mach_context,
                tests=tests,
                suite=fobj['suite'],
                **harness_args)

            if result:
                overall = result

        # TODO consolidate summaries from all suites
        return overall