def _find_misc_tests(self, dirs, changed_files, gpu=False): manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] tests_by_path = {} all_disabled = [] for (path, suite) in manifests: if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=True, filters=[], **mozinfo.info) # Remove disabled tests. Also, remove tests with the same path as # disabled tests, even if they are not disabled, since per-test mode # specifies tests by path (it cannot distinguish between two or more # tests with the same path specified in multiple manifests). disabled = [t['relpath'] for t in active if 'disabled' in t] all_disabled += disabled new_by_path = { t['relpath']: (suite, t.get('subsuite')) for t in active if 'disabled' not in t and t['relpath'] not in disabled } tests_by_path.update(new_by_path) self.info( "Per-test run updated with manifest %s (%d active, %d skipped)" % (path, len(new_by_path), len(disabled))) ref_manifests = [ (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest', 'gpu'), # gpu (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest', None), ] sys.path.append(dirs['abs_reftest_dir']) import manifest self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests') for (path, suite, subsuite) in ref_manifests: if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) for t in man.files: relpath = os.path.relpath(t, self.reftest_test_dir) tests_by_path[relpath] = (suite, subsuite) self._map_test_path_to_source(t, relpath) self.info("Per-test run updated with manifest %s (%d tests)" % (path, len(man.files))) suite = 'jsreftest' self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests') path = os.path.join(self.jsreftest_test_dir, 'jstests.list') if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) for t in man.files: # expect manifest test to look like: # ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js" # while the test is in mercurial at: # js/src/tests/test262/.../some_test.js epos = t.find('=') if epos > 0: relpath = t[epos + 1:] test_path = os.path.join(self.jsreftest_test_dir, relpath) relpath = os.path.join('js', 'src', 'tests', relpath) self._map_test_path_to_source(test_path, relpath) tests_by_path.update({relpath: (suite, None)}) else: self.warning("unexpected jsreftest test format: %s" % str(t)) self.info("Per-test run updated with manifest %s (%d tests)" % (path, len(man.files))) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: # manifest paths use os.sep (like backslash on Windows) but # automation-relevance uses posixpath.sep file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if not entry: if file in all_disabled: self.info("'%s' has been skipped on this platform." % file) if os.environ.get('MOZHARNESS_TEST_PATHS', None) is not None: self.fatal( "Per-test run could not find requested test '%s'" % file) continue if gpu and not self._is_gpu_suite(entry[1]): self.info("Per-test run (gpu) discarded non-gpu test %s (%s)" % (file, entry[1])) continue elif not gpu and self._is_gpu_suite(entry[1]): self.info("Per-test run (non-gpu) discarded gpu test %s (%s)" % (file, entry[1])) continue self.info("Per-test run found test %s (%s/%s)" % (file, entry[0], entry[1])) subsuite_mapping = { # Map (<suite>, <subsuite>): <full-suite> # <suite> is associated with a manifest, explicitly in code above # <subsuite> comes from "subsuite" tags in some manifest entries # <full-suite> is a unique id for the suite, matching desktop mozharness configs ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', ('chrome', 'clipboard'): 'chrome-clipboard', ('plain', 'clipboard'): 'plain-clipboard', ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', ('browser-chrome', 'screenshots'): 'browser-chrome-screenshots', ('plain', 'media'): 'mochitest-media', # below should be on test-verify-gpu job ('chrome', 'gpu'): 'chrome-gpu', ('plain', 'gpu'): 'plain-gpu', ('plain', 'webgl1-core'): 'mochitest-webgl1-core', ('plain', 'webgl1-ext'): 'mochitest-webgl1-ext', ('plain', 'webgl2-core'): 'mochitest-webgl2-core', ('plain', 'webgl2-ext'): 'mochitest-webgl2-ext', ('plain', 'webgl2-deqp'): 'mochitest-webgl2-deqp', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] else: suite = entry[0] suite_files = self.suites.get(suite) if not suite_files: suite_files = [] suite_files.append(file) self.suites[suite] = suite_files
def _find_misc_tests(self, dirs, changed_files, gpu=False): manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] tests_by_path = {} for (path, suite) in manifests: if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=True, filters=[], **mozinfo.info) # Remove disabled tests. Also, remove tests with the same path as # disabled tests, even if they are not disabled, since per-test mode # specifies tests by path (it cannot distinguish between two or more # tests with the same path specified in multiple manifests). disabled = [t['relpath'] for t in active if 'disabled' in t] new_by_path = {t['relpath']:(suite,t.get('subsuite')) \ for t in active if 'disabled' not in t and \ t['relpath'] not in disabled} tests_by_path.update(new_by_path) self.info("Per-test run updated with manifest %s" % path) ref_manifests = [ (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest', 'gpu'), #gpu (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest', None), ] sys.path.append(dirs['abs_reftest_dir']) import manifest self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests') for (path, suite, subsuite) in ref_manifests: if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) tests_by_path.update({ os.path.relpath(t, self.reftest_test_dir): (suite, subsuite) for t in man.files }) self.info("Per-test run updated with manifest %s" % path) suite = 'jsreftest' self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests') path = os.path.join(self.jsreftest_test_dir, 'jstests.list') if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) for t in man.files: # expect manifest test to look like: # ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js" # while the test is in mercurial at: # js/src/tests/test262/.../some_test.js epos = t.find('=') if epos > 0: relpath = t[epos + 1:] relpath = os.path.join('js', 'src', 'tests', relpath) tests_by_path.update({relpath: (suite, None)}) else: self.warning("unexpected jsreftest test format: %s" % str(t)) self.info("Per-test run updated with manifest %s" % path) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: # manifest paths use os.sep (like backslash on Windows) but # automation-relevance uses posixpath.sep file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if entry: if gpu and entry[1] not in ['gpu', 'webgl']: continue elif not gpu and entry[1] in ['gpu', 'webgl']: continue self.info("Per-test run found test %s" % file) subsuite_mapping = { ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', ('chrome', 'clipboard'): 'chrome-clipboard', ('plain', 'clipboard'): 'plain-clipboard', ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', ('browser-chrome', 'screenshots'): 'browser-chrome-screenshots', ('plain', 'media'): 'mochitest-media', # below should be on test-verify-gpu job ('browser-chrome', 'gpu'): 'browser-chrome-gpu', ('chrome', 'gpu'): 'chrome-gpu', ('plain', 'gpu'): 'plain-gpu', ('plain', 'webgl'): 'mochitest-gl', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] else: suite = entry[0] suite_files = self.suites.get(suite) if not suite_files: suite_files = [] suite_files.append(file) self.suites[suite] = suite_files
def _find_misc_tests(self, dirs, changed_files, gpu=False): manifests = [ ( os.path.join(dirs["abs_mochitest_dir"], "tests", "mochitest.ini"), "mochitest-plain", ), ( os.path.join(dirs["abs_mochitest_dir"], "chrome", "chrome.ini"), "mochitest-chrome", ), ( os.path.join(dirs["abs_mochitest_dir"], "browser", "browser-chrome.ini"), "mochitest-browser-chrome", ), ( os.path.join(dirs["abs_mochitest_dir"], "a11y", "a11y.ini"), "mochitest-a11y", ), ( os.path.join(dirs["abs_xpcshell_dir"], "tests", "xpcshell.ini"), "xpcshell", ), ] tests_by_path = {} all_disabled = [] for (path, suite) in manifests: if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=True, filters=[], **mozinfo.info) # Remove disabled tests. Also, remove tests with the same path as # disabled tests, even if they are not disabled, since per-test mode # specifies tests by path (it cannot distinguish between two or more # tests with the same path specified in multiple manifests). disabled = [t["relpath"] for t in active if "disabled" in t] all_disabled += disabled new_by_path = { t["relpath"]: (suite, t.get("subsuite"), None) for t in active if "disabled" not in t and t["relpath"] not in disabled } tests_by_path.update(new_by_path) self.info( "Per-test run updated with manifest %s (%d active, %d skipped)" % (path, len(new_by_path), len(disabled))) ref_manifests = [ ( os.path.join( dirs["abs_reftest_dir"], "tests", "layout", "reftests", "reftest.list", ), "reftest", "gpu", ), # gpu ( os.path.join( dirs["abs_reftest_dir"], "tests", "testing", "crashtest", "crashtests.list", ), "crashtest", None, ), ] sys.path.append(dirs["abs_reftest_dir"]) import manifest self.reftest_test_dir = os.path.join(dirs["abs_reftest_dir"], "tests") for (path, suite, subsuite) in ref_manifests: if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) for t in man.tests: relpath = os.path.relpath(t["path"], self.reftest_test_dir) referenced = (t["referenced-test"] if "referenced-test" in t else None) tests_by_path[relpath] = (suite, subsuite, referenced) self._map_test_path_to_source(t["path"], relpath) self.info("Per-test run updated with manifest %s (%d tests)" % (path, len(man.tests))) suite = "jsreftest" self.jsreftest_test_dir = os.path.join(dirs["abs_test_install_dir"], "jsreftest", "tests") path = os.path.join(self.jsreftest_test_dir, "jstests.list") if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) for t in man.files: # expect manifest test to look like: # ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js" # while the test is in mercurial at: # js/src/tests/test262/.../some_test.js epos = t.find("=") if epos > 0: relpath = t[epos + 1:] test_path = os.path.join(self.jsreftest_test_dir, relpath) relpath = os.path.join("js", "src", "tests", relpath) self._map_test_path_to_source(test_path, relpath) tests_by_path.update({relpath: (suite, None, None)}) else: self.warning("unexpected jsreftest test format: %s" % str(t)) self.info("Per-test run updated with manifest %s (%d tests)" % (path, len(man.files))) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: # manifest paths use os.sep (like backslash on Windows) but # automation-relevance uses posixpath.sep file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if not entry: if file in all_disabled: self.info("'%s' has been skipped on this platform." % file) if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None: self.fatal( "Per-test run could not find requested test '%s'" % file) continue if gpu and not self._is_gpu_suite(entry[1]): self.info("Per-test run (gpu) discarded non-gpu test %s (%s)" % (file, entry[1])) continue elif not gpu and self._is_gpu_suite(entry[1]): self.info("Per-test run (non-gpu) discarded gpu test %s (%s)" % (file, entry[1])) continue if entry[2] is not None: # Test name substitution, for reftest reference file handling: # - if both test and reference modified, run the test file # - if only reference modified, run the test file test_file = os.path.join(os.path.dirname(file), os.path.basename(entry[2])) self.info("Per-test run substituting %s for %s" % (test_file, file)) file = test_file self.info("Per-test run found test %s (%s/%s)" % (file, entry[0], entry[1])) subsuite_mapping = { # Map (<suite>, <subsuite>): <full-suite> # <suite> is associated with a manifest, explicitly in code above # <subsuite> comes from "subsuite" tags in some manifest entries # <full-suite> is a unique id for the suite, matching desktop mozharness configs ( "mochitest-browser-chrome", "devtools", None, ): "mochitest-devtools-chrome", ("mochitest-browser-chrome", "remote", None): "mochitest-remote", ( "mochitest-browser-chrome", "screenshots", None, ): "mochitest-browser-chrome-screenshots", # noqa ("mochitest-plain", "media", None): "mochitest-media", # below should be on test-verify-gpu job ("mochitest-chrome", "gpu", None): "mochitest-chrome-gpu", ("mochitest-plain", "gpu", None): "mochitest-plain-gpu", ("mochitest-plain", "webgl1-core", None): "mochitest-webgl1-core", ("mochitest-plain", "webgl1-ext", None): "mochitest-webgl1-ext", ("mochitest-plain", "webgl2-core", None): "mochitest-webgl2-core", ("mochitest-plain", "webgl2-ext", None): "mochitest-webgl2-ext", ("mochitest-plain", "webgl2-deqp", None): "mochitest-webgl2-deqp", ("mochitest-plain", "webgpu", None): "mochitest-webgpu", } if entry in subsuite_mapping: suite = subsuite_mapping[entry] else: suite = entry[0] suite_files = self.suites.get(suite) if not suite_files: suite_files = [] if file not in suite_files: suite_files.append(file) self.suites[suite] = suite_files
def _find_misc_tests(self, dirs, changed_files): manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] tests_by_path = {} for (path, suite) in manifests: if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=False, filters=[], **mozinfo.info) tests_by_path.update( {t['relpath']: (suite, t.get('subsuite')) for t in active}) self.info("Verification updated with manifest %s" % path) ref_manifests = [ (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'), (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'), ] sys.path.append(dirs['abs_reftest_dir']) import manifest self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests') for (path, suite) in ref_manifests: if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) tests_by_path.update({ os.path.relpath(t, self.reftest_test_dir): (suite, None) for t in man.files }) self.info("Verification updated with manifest %s" % path) suite = 'jsreftest' self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests') path = os.path.join(self.jsreftest_test_dir, 'jstests.list') if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) for t in man.files: # expect manifest test to look like: # ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js" # while the test is in mercurial at: # js/src/tests/test262/.../some_test.js epos = t.find('=') if epos > 0: relpath = t[epos + 1:] relpath = os.path.join('js', 'src', 'tests', relpath) tests_by_path.update({relpath: (suite, None)}) else: self.warning("unexpected jsreftest test format: %s" % str(t)) self.info("Verification updated with manifest %s" % path) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: # manifest paths use os.sep (like backslash on Windows) but # automation-relevance uses posixpath.sep file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if entry: self.info("Verification found test %s" % file) subsuite_mapping = { ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', ('chrome', 'clipboard'): 'chrome-clipboard', ('plain', 'clipboard'): 'plain-clipboard', ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', ('browser-chrome', 'gpu'): 'browser-chrome-gpu', ('browser-chrome', 'screenshots'): 'browser-chrome-screenshots', ('chrome', 'gpu'): 'chrome-gpu', ('plain', 'gpu'): 'plain-gpu', ('plain', 'media'): 'mochitest-media', ('plain', 'webgl'): 'mochitest-gl', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] else: suite = entry[0] suite_files = self.verify_suites.get(suite) if not suite_files: suite_files = [] suite_files.append(file) self.verify_suites[suite] = suite_files
def find_tests_for_verification(self, action, success=None): """ For each file modified on this push, determine if the modified file is a test, by searching test manifests. Populate self.verify_suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ if self.config.get('verify') != True: return repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: self.warning("unable to verify tests: no repo or revision!") return [] def get_automationrelevance(): response = self.load_json_url(url) return response dirs = self.query_abs_dirs() mozinfo.find_and_update_from_json(dirs['abs_test_install_dir']) if self.config.get('e10s') == True: mozinfo.update({"e10s": True}) # Additional mozinfo properties like "headless" and "coverage" are # also normally updated dynamically in the harness, but neither of # these apply to the test-verify task. manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] tests_by_path = {} for (path, suite) in manifests: if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=False, filters=[], **mozinfo.info) tests_by_path.update( {t['relpath']: (suite, t.get('subsuite')) for t in active}) self.info("Verification updated with manifest %s" % path) ref_manifests = [ (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'), (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'), # TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', 'jstests.list'), 'jstestbrowser'), ] sys.path.append(dirs['abs_reftest_dir']) import manifest self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests') for (path, suite) in ref_manifests: if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) tests_by_path.update({ os.path.relpath(t, self.reftest_test_dir): (suite, None) for t in man.files }) self.info("Verification updated with manifest %s" % path) # determine which files were changed on this push url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) changed_files = set() for c in contents['changesets']: self.info(" {cset} {desc}".format( cset=c['node'][0:12], desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: # manifest paths use os.sep (like backslash on Windows) but # automation-relevance uses posixpath.sep file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if entry: self.info("Verification found test %s" % file) subsuite_mapping = { ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', ('chrome', 'clipboard'): 'chrome-clipboard', ('plain', 'clipboard'): 'plain-clipboard', ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', ('browser-chrome', 'gpu'): 'browser-chrome-gpu', ('chrome', 'gpu'): 'chrome-gpu', ('plain', 'gpu'): 'plain-gpu', ('plain', 'media'): 'mochitest-media', ('plain', 'webgl'): 'mochitest-gl', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] else: suite = entry[0] suite_files = self.verify_suites.get(suite) if not suite_files: suite_files = [] suite_files.append(file) self.verify_suites[suite] = suite_files self.verify_downloaded = True