コード例 #1
0
    def test_two_times_more_chunks_than_tests(self):
        # test case for bug 1182817
        tests = self.generate_tests(5)

        total_chunks = 10
        for i in range(1, total_chunks + 1):
            # ensure IndexError is not raised
            chunk_by_slice(i, total_chunks)(tests, {})
コード例 #2
0
    def run_all_combos(self, num_tests, disabled=None):
        tests = self.generate_tests(num_tests, disabled=disabled)

        for total in range(1, num_tests + 1):
            res = []
            res_disabled = []
            for chunk in range(1, total + 1):
                f = chunk_by_slice(chunk, total)
                res.append(list(f(tests, {})))
                if disabled:
                    f.disabled = True
                    res_disabled.append(list(f(tests, {})))

            lengths = [len([t for t in c if 'disabled' not in t]) for c in res]
            # the chunk with the most tests should have at most one more test
            # than the chunk with the least tests
            self.assertLessEqual(max(lengths) - min(lengths), 1)

            # chaining all chunks back together should equal the original list
            # of tests
            self.assertEqual(list(chain.from_iterable(res)), list(tests))

            if disabled:
                lengths = [len(c) for c in res_disabled]
                self.assertLessEqual(max(lengths) - min(lengths), 1)
                self.assertEqual(list(chain.from_iterable(res_disabled)),
                                 list(tests))
コード例 #3
0
ファイル: runreftest.py プロジェクト: yang123vc/tor-browser
    def getActiveTests(self, manifests, options, testDumpFile=None):
        # These prefs will cause reftest.jsm to parse the manifests,
        # dump the resulting tests to a file, and exit.
        prefs = {
            'reftest.manifests': json.dumps(manifests),
            'reftest.manifests.dumpTests': testDumpFile or self.testDumpFile,
        }
        cmdargs = []  # ['-headless']
        status, _, _ = self.runApp(options, cmdargs=cmdargs, prefs=prefs)

        with open(self.testDumpFile, 'r') as fh:
            tests = json.load(fh)

        if os.path.isfile(self.testDumpFile):
            mozfile.remove(self.testDumpFile)

        for test in tests:
            # Name and path are expected by manifestparser, but not used in reftest.
            test['name'] = test['path'] = test['url1']

        mp = TestManifest(strict=False)
        mp.tests = tests

        filters = []
        if options.totalChunks:
            filters.append(mpf.chunk_by_slice(options.thisChunk, options.totalChunks))

        tests = mp.active_tests(exists=False, filters=filters)
        return tests
コード例 #4
0
    def test_chunk_by_slice(self):
        chunk = chunk_by_slice(1, 1)
        self.assertEqual(list(chunk([], {})), [])

        self.run_all_combos(num_tests=1)
        self.run_all_combos(num_tests=10, disabled=[1, 2])

        num_tests = 67
        disabled = list(i for i in xrange(num_tests) if i % 4 == 0)
        self.run_all_combos(num_tests=num_tests, disabled=disabled)
コード例 #5
0
    def runTests(self):
        self.startup()
        if isinstance(self.options.manifestFile, TestManifest):
            mp = self.options.manifestFile
        else:
            mp = TestManifest(strict=False)
            mp.read(self.options.robocopIni)
        filters = []
        if self.options.totalChunks:
            filters.append(
                chunk_by_slice(self.options.thisChunk,
                               self.options.totalChunks))
        robocop_tests = mp.active_tests(exists=False,
                                        filters=filters,
                                        **mozinfo.info)
        if not self.options.autorun:
            # Force a single loop iteration. The iteration will start Fennec and
            # the httpd server, but not actually run a test.
            self.options.test_paths = [robocop_tests[0]['name']]
        active_tests = []
        for test in robocop_tests:
            if self.options.test_paths and test[
                    'name'] not in self.options.test_paths:
                continue
            if 'disabled' in test:
                self.log.info('TEST-INFO | skipping %s | %s' %
                              (test['name'], test['disabled']))
                continue
            active_tests.append(test)

        tests_by_manifest = defaultdict(list)
        for test in active_tests:
            tests_by_manifest[test['manifest']].append(test['name'])
        self.log.suite_start(tests_by_manifest)

        worstTestResult = None
        for test in active_tests:
            result = self.runSingleTest(test)
            if worstTestResult is None or worstTestResult == 0:
                worstTestResult = result
        if worstTestResult is None:
            self.log.warning(
                "No tests run. Did you pass an invalid TEST_PATH?")
            worstTestResult = 1
        else:
            print "INFO | runtests.py | Test summary: start."
            logResult = self.logTestSummary()
            print "INFO | runtests.py | Test summary: end."
            if worstTestResult == 0:
                worstTestResult = logResult
        return worstTestResult
コード例 #6
0
ファイル: runrobocop.py プロジェクト: luke-chang/gecko-1
    def runTests(self):
        self.startup()
        if isinstance(self.options.manifestFile, TestManifest):
            mp = self.options.manifestFile
        else:
            mp = TestManifest(strict=False)
            mp.read(self.options.robocopIni)
        filters = []
        if self.options.totalChunks:
            filters.append(
                chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
        robocop_tests = mp.active_tests(
            exists=False, filters=filters, **mozinfo.info)
        if not self.options.autorun:
            # Force a single loop iteration. The iteration will start Fennec and
            # the httpd server, but not actually run a test.
            self.options.test_paths = [robocop_tests[0]['name']]
        active_tests = []
        for test in robocop_tests:
            if self.options.test_paths and test['name'] not in self.options.test_paths:
                continue
            if 'disabled' in test:
                self.log.info('TEST-INFO | skipping %s | %s' %
                              (test['name'], test['disabled']))
                continue
            active_tests.append(test)

        tests_by_manifest = defaultdict(list)
        for test in active_tests:
            tests_by_manifest[test['manifest']].append(test['name'])
        self.log.suite_start(tests_by_manifest)

        worstTestResult = None
        for test in active_tests:
            result = self.runSingleTest(test)
            if worstTestResult is None or worstTestResult == 0:
                worstTestResult = result
        if worstTestResult is None:
            self.log.warning(
                "No tests run. Did you pass an invalid TEST_PATH?")
            worstTestResult = 1
        else:
            print "INFO | runtests.py | Test summary: start."
            logResult = self.logTestSummary()
            print "INFO | runtests.py | Test summary: end."
            if worstTestResult == 0:
                worstTestResult = logResult
        return worstTestResult
コード例 #7
0
def run_test_harness(options):
    message_logger = MessageLogger(logger=None)
    process_args = {'messageLogger': message_logger}
    auto = RemoteAutomation(None, "fennec", processArgs=process_args)

    if options is None:
        raise ValueError("Invalid options specified, use --help for a list of valid options")

    dm = options.dm
    auto.setDeviceManager(dm)
    mochitest = MochiRemote(auto, dm, options)

    log = mochitest.log
    message_logger.logger = log
    mochitest.message_logger = message_logger

    productPieces = options.remoteProductName.split('.')
    if (productPieces is not None):
        auto.setProduct(productPieces[0])
    else:
        auto.setProduct(options.remoteProductName)
    auto.setAppName(options.remoteappname)

    logParent = os.path.dirname(options.remoteLogFile)
    dm.mkDir(logParent)
    auto.setRemoteLog(options.remoteLogFile)
    auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)

    mochitest.printDeviceInfo()

    # Add Android version (SDK level) to mozinfo so that manifest entries
    # can be conditional on android_version.
    androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk'])
    log.info(
        "Android sdk version '%s'; will use this to filter manifests" %
        str(androidVersion))
    mozinfo.info['android_version'] = androidVersion

    deviceRoot = dm.deviceRoot
    if options.dmdPath:
        dmdLibrary = "libdmd.so"
        dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary)
        dm.removeFile(dmdPathOnDevice)
        dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice)
        options.dmdPath = deviceRoot

    options.dumpOutputDirectory = deviceRoot

    procName = options.app.split('/')[-1]
    dm.killProcess(procName)

    if options.robocopIni != "":
        # turning buffering off as it's not used in robocop
        message_logger.buffering = False

        # sut may wait up to 300 s for a robocop am process before returning
        dm.default_timeout = 320
        if isinstance(options.manifestFile, TestManifest):
            mp = options.manifestFile
        else:
            mp = TestManifest(strict=False)
            mp.read(options.robocopIni)

        filters = []
        if options.totalChunks:
            filters.append(
                chunk_by_slice(options.thisChunk, options.totalChunks))
        robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info)

        options.extraPrefs.append('browser.search.suggest.enabled=true')
        options.extraPrefs.append('browser.search.suggest.prompted=true')
        options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
        options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
        options.extraPrefs.append('browser.snippets.enabled=false')
        options.extraPrefs.append('browser.casting.enabled=true')
        options.extraPrefs.append('extensions.autoupdate.enabled=false')

        if (options.dm_trans == 'adb' and options.robocopApk):
            dm._checkCmd(["install", "-r", options.robocopApk])

        if not options.autorun:
            # Force a single loop iteration. The iteration will start Fennec and
            # the httpd server, but not actually run a test.
            options.test_paths = [robocop_tests[0]['name']]

        retVal = None
        # Filtering tests
        active_tests = []
        for test in robocop_tests:
            if options.test_paths and test['name'] not in options.test_paths:
                continue

            if 'disabled' in test:
                log.info(
                    'TEST-INFO | skipping %s | %s' %
                    (test['name'], test['disabled']))
                continue

            active_tests.append(test)

        log.suite_start([t['name'] for t in active_tests])

        for test in active_tests:
            # When running in a loop, we need to create a fresh profile for
            # each cycle
            if mochitest.localProfile:
                options.profilePath = mochitest.localProfile
                os.system("rm -Rf %s" % options.profilePath)
                options.profilePath = None
                mochitest.localProfile = options.profilePath

            options.app = "am"
            mochitest.nsprLogName = "nspr-%s.log" % test['name']
            if options.autorun:
                # This launches a test (using "am instrument") and instructs
                # Fennec to /quit/ the browser (using Robocop:Quit) and to
                # /finish/ all opened activities.
                options.browserArgs = [
                    "instrument",
                    "-w",
                    "-e", "quit_and_finish", "1",
                    "-e", "deviceroot", deviceRoot,
                    "-e",
                    "class"]
                options.browserArgs.append(
                    "org.mozilla.gecko.tests.%s" %
                    test['name'].split('.java')[0])
                options.browserArgs.append(
                    "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner")
            else:
                # This does not launch a test at all. It launches an activity
                # that starts Fennec and then waits indefinitely, since cat
                # never returns.
                options.browserArgs = ["start",
                                       "-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
                                       "&&", "cat"]
                dm.default_timeout = sys.maxint # Forever.

                mochitest.log.info("")
                mochitest.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
                    (options.remoteWebServer, options.httpPort))
                mochitest.log.info("")

            # If the test is for checking the import from bookmarks then make
            # sure there is data to import
            if test['name'] == "testImportFromAndroid":

                # Get the OS so we can run the insert in the apropriate
                # database and following the correct table schema
                osInfo = dm.getInfo("os")
                devOS = " ".join(osInfo['os'])

                if ("pandaboard" in devOS):
                    delete = [
                        'execsu',
                        'sqlite3',
                        "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
                else:
                    delete = [
                        'execsu',
                        'sqlite3',
                        "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
                if (options.dm_trans == "sut"):
                    dm._runCmds([{"cmd": " ".join(delete)}])

                # Insert the bookmarks
                log.info(
                    "Insert bookmarks in the default android browser database")
                for i in range(20):
                    if ("pandaboard" in devOS):
                        cmd = [
                            'execsu',
                            'sqlite3',
                            "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" +
                            str(
                                30 +
                                i) +
                            ",\"Bookmark" +
                            str(i) +
                            "\",\"http://www.bookmark" +
                            str(i) +
                            ".com\",0,1," +
                            str(
                                100 +
                                i) +
                            ");'"]
                    else:
                        cmd = [
                            'execsu',
                            'sqlite3',
                            "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark" +
                            str(i) +
                            "\",\"http://www.bookmark" +
                            str(i) +
                            ".com\",1);'"]
                    if (options.dm_trans == "sut"):
                        dm._runCmds([{"cmd": " ".join(cmd)}])
            try:
                screenShotDir = "/mnt/sdcard/Robotium-Screenshots"
                dm.removeDir(screenShotDir)
                dm.recordLogcat()
                result = mochitest.runTests(options)
                if result != 0:
                    log.error("runTests() exited with code %s" % result)
                log_result = mochitest.addLogData()
                if result != 0 or log_result != 0:
                    mochitest.printDeviceInfo(printLogcat=True)
                    mochitest.printScreenshots(screenShotDir)
                # Ensure earlier failures aren't overwritten by success on this
                # run
                if retVal is None or retVal == 0:
                    retVal = result
            except:
                log.error(
                    "Automation Error: Exception caught while running tests")
                traceback.print_exc()
                mochitest.stopServers()
                try:
                    mochitest.cleanup(options)
                except devicemanager.DMError:
                    # device error cleaning up... oh well!
                    pass
                retVal = 1
                break
            finally:
                # Clean-up added bookmarks
                if test['name'] == "testImportFromAndroid":
                    if ("pandaboard" in devOS):
                        cmd_del = [
                            'execsu',
                            'sqlite3',
                            "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
                    else:
                        cmd_del = [
                            'execsu',
                            'sqlite3',
                            "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
                    if (options.dm_trans == "sut"):
                        dm._runCmds([{"cmd": " ".join(cmd_del)}])
        if retVal is None:
            log.warning("No tests run. Did you pass an invalid TEST_PATH?")
            retVal = 1
        else:
            # if we didn't have some kind of error running the tests, make
            # sure the tests actually passed
            print "INFO | runtests.py | Test summary: start."
            overallResult = mochitest.printLog()
            print "INFO | runtests.py | Test summary: end."
            if retVal == 0:
                retVal = overallResult
    else:
        mochitest.nsprLogName = "nspr.log"
        try:
            dm.recordLogcat()
            retVal = mochitest.runTests(options)
        except:
            log.error("Automation Error: Exception caught while running tests")
            traceback.print_exc()
            mochitest.stopServers()
            try:
                mochitest.cleanup(options)
            except devicemanager.DMError:
                # device error cleaning up... oh well!
                pass
            retVal = 1

        mochitest.printDeviceInfo(printLogcat=True)

    message_logger.finish()

    return retVal
コード例 #8
0
ファイル: runtestsremote.py プロジェクト: sjack1017/gecko-dev
def main(args):
    message_logger = MessageLogger(logger=None)
    process_args = {'messageLogger': message_logger}
    auto = RemoteAutomation(None, "fennec", processArgs=process_args)

    parser = RemoteOptions(auto)
    structured.commandline.add_logging_group(parser)
    options, args = parser.parse_args(args)

    if (options.dm_trans == "adb"):
        if (options.deviceIP):
            dm = droid.DroidADB(options.deviceIP,
                                options.devicePort,
                                deviceRoot=options.remoteTestRoot)
        elif (options.deviceSerial):
            dm = droid.DroidADB(None,
                                None,
                                deviceSerial=options.deviceSerial,
                                deviceRoot=options.remoteTestRoot)
        else:
            dm = droid.DroidADB(deviceRoot=options.remoteTestRoot)
    else:
        dm = droid.DroidSUT(options.deviceIP,
                            options.devicePort,
                            deviceRoot=options.remoteTestRoot)
    auto.setDeviceManager(dm)
    options = parser.verifyRemoteOptions(options, auto)

    mochitest = MochiRemote(auto, dm, options)

    log = mochitest.log
    message_logger.logger = log
    mochitest.message_logger = message_logger

    if (options is None):
        log.error(
            "Invalid options specified, use --help for a list of valid options"
        )
        return 1

    productPieces = options.remoteProductName.split('.')
    if (productPieces is not None):
        auto.setProduct(productPieces[0])
    else:
        auto.setProduct(options.remoteProductName)
    auto.setAppName(options.remoteappname)

    options = parser.verifyOptions(options, mochitest)
    if (options is None):
        return 1

    logParent = os.path.dirname(options.remoteLogFile)
    dm.mkDir(logParent)
    auto.setRemoteLog(options.remoteLogFile)
    auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)

    mochitest.printDeviceInfo()

    # Add Android version (SDK level) to mozinfo so that manifest entries
    # can be conditional on android_version.
    androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk'])
    log.info("Android sdk version '%s'; will use this to filter manifests" %
             str(androidVersion))
    mozinfo.info['android_version'] = androidVersion

    deviceRoot = dm.deviceRoot
    if options.dmdPath:
        dmdLibrary = "libdmd.so"
        dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary)
        dm.removeFile(dmdPathOnDevice)
        dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice)
        options.dmdPath = deviceRoot

    options.dumpOutputDirectory = deviceRoot

    procName = options.app.split('/')[-1]
    dm.killProcess(procName)

    if options.robocopIni != "":
        # turning buffering off as it's not used in robocop
        message_logger.buffering = False

        # sut may wait up to 300 s for a robocop am process before returning
        dm.default_timeout = 320
        mp = TestManifest(strict=False)
        # TODO: pull this in dynamically
        mp.read(options.robocopIni)

        filters = []
        if options.totalChunks:
            filters.append(
                chunk_by_slice(options.thisChunk, options.totalChunks))
        robocop_tests = mp.active_tests(exists=False,
                                        filters=filters,
                                        **mozinfo.info)

        options.extraPrefs.append('browser.search.suggest.enabled=true')
        options.extraPrefs.append('browser.search.suggest.prompted=true')
        options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
        options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
        options.extraPrefs.append('browser.snippets.enabled=false')
        options.extraPrefs.append('browser.casting.enabled=true')

        if (options.dm_trans == 'adb' and options.robocopApk):
            dm._checkCmd(["install", "-r", options.robocopApk])

        retVal = None
        # Filtering tests
        active_tests = []
        for test in robocop_tests:
            if options.testPath and options.testPath != test['name']:
                continue

            if 'disabled' in test:
                log.info('TEST-INFO | skipping %s | %s' %
                         (test['name'], test['disabled']))
                continue

            active_tests.append(test)

        log.suite_start([t['name'] for t in active_tests])

        for test in active_tests:
            # When running in a loop, we need to create a fresh profile for
            # each cycle
            if mochitest.localProfile:
                options.profilePath = mochitest.localProfile
                os.system("rm -Rf %s" % options.profilePath)
                options.profilePath = None
                mochitest.localProfile = options.profilePath

            options.app = "am"
            options.browserArgs = [
                "instrument", "-w", "-e", "deviceroot", deviceRoot, "-e",
                "class"
            ]
            options.browserArgs.append("org.mozilla.gecko.tests.%s" %
                                       test['name'])
            options.browserArgs.append(
                "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"
            )
            mochitest.nsprLogName = "nspr-%s.log" % test['name']

            # If the test is for checking the import from bookmarks then make
            # sure there is data to import
            if test['name'] == "testImportFromAndroid":

                # Get the OS so we can run the insert in the apropriate
                # database and following the correct table schema
                osInfo = dm.getInfo("os")
                devOS = " ".join(osInfo['os'])

                if ("pandaboard" in devOS):
                    delete = [
                        'execsu', 'sqlite3',
                        "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"
                    ]
                else:
                    delete = [
                        'execsu', 'sqlite3',
                        "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"
                    ]
                if (options.dm_trans == "sut"):
                    dm._runCmds([{"cmd": " ".join(delete)}])

                # Insert the bookmarks
                log.info(
                    "Insert bookmarks in the default android browser database")
                for i in range(20):
                    if ("pandaboard" in devOS):
                        cmd = [
                            'execsu', 'sqlite3',
                            "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values ("
                            + str(30 + i) + ",\"Bookmark" + str(i) +
                            "\",\"http://www.bookmark" + str(i) +
                            ".com\",0,1," + str(100 + i) + ");'"
                        ]
                    else:
                        cmd = [
                            'execsu', 'sqlite3',
                            "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark"
                            + str(i) + "\",\"http://www.bookmark" + str(i) +
                            ".com\",1);'"
                        ]
                    if (options.dm_trans == "sut"):
                        dm._runCmds([{"cmd": " ".join(cmd)}])
            try:
                screenShotDir = "/mnt/sdcard/Robotium-Screenshots"
                dm.removeDir(screenShotDir)
                dm.recordLogcat()
                result = mochitest.runTests(options)
                if result != 0:
                    log.error("runTests() exited with code %s" % result)
                log_result = mochitest.addLogData()
                if result != 0 or log_result != 0:
                    mochitest.printDeviceInfo(printLogcat=True)
                    mochitest.printScreenshots(screenShotDir)
                # Ensure earlier failures aren't overwritten by success on this
                # run
                if retVal is None or retVal == 0:
                    retVal = result
            except:
                log.error(
                    "Automation Error: Exception caught while running tests")
                traceback.print_exc()
                mochitest.stopServers()
                try:
                    mochitest.cleanup(options)
                except devicemanager.DMError:
                    # device error cleaning up... oh well!
                    pass
                retVal = 1
                break
            finally:
                # Clean-up added bookmarks
                if test['name'] == "testImportFromAndroid":
                    if ("pandaboard" in devOS):
                        cmd_del = [
                            'execsu', 'sqlite3',
                            "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"
                        ]
                    else:
                        cmd_del = [
                            'execsu', 'sqlite3',
                            "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"
                        ]
                    if (options.dm_trans == "sut"):
                        dm._runCmds([{"cmd": " ".join(cmd_del)}])
        if retVal is None:
            log.warning("No tests run. Did you pass an invalid TEST_PATH?")
            retVal = 1
        else:
            # if we didn't have some kind of error running the tests, make
            # sure the tests actually passed
            print "INFO | runtests.py | Test summary: start."
            overallResult = mochitest.printLog()
            print "INFO | runtests.py | Test summary: end."
            if retVal == 0:
                retVal = overallResult
    else:
        mochitest.nsprLogName = "nspr.log"
        try:
            dm.recordLogcat()
            retVal = mochitest.runTests(options)
        except:
            log.error("Automation Error: Exception caught while running tests")
            traceback.print_exc()
            mochitest.stopServers()
            try:
                mochitest.cleanup(options)
            except devicemanager.DMError:
                # device error cleaning up... oh well!
                pass
            retVal = 1

        mochitest.printDeviceInfo(printLogcat=True)

    message_logger.finish()

    return retVal