def _find_misc_tests(self, dirs, changed_files, gpu=False):
manifests = [
(os.path.join(dirs['abs_mochitest_dir'], 'tests',
'mochitest.ini'), 'plain'),
(os.path.join(dirs['abs_mochitest_dir'], 'chrome',
'chrome.ini'), 'chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'browser',
'browser-chrome.ini'), 'browser-chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'a11y',
'a11y.ini'), 'a11y'),
(os.path.join(dirs['abs_xpcshell_dir'], 'tests',
'xpcshell.ini'), 'xpcshell'),
]
tests_by_path = {}
all_disabled = []
for (path, suite) in manifests:
if os.path.exists(path):
man = TestManifest([path], strict=False)
active = man.active_tests(exists=False,
disabled=True,
filters=[],
**mozinfo.info)
# Remove disabled tests. Also, remove tests with the same path as
# disabled tests, even if they are not disabled, since per-test mode
# specifies tests by path (it cannot distinguish between two or more
# tests with the same path specified in multiple manifests).
disabled = [t['relpath'] for t in active if 'disabled' in t]
all_disabled += disabled
new_by_path = {
t['relpath']: (suite, t.get('subsuite'))
for t in active
if 'disabled' not in t and t['relpath'] not in disabled
}
tests_by_path.update(new_by_path)
self.info(
"Per-test run updated with manifest %s (%d active, %d skipped)"
% (path, len(new_by_path), len(disabled)))
ref_manifests = [
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout',
'reftests',
'reftest.list'), 'reftest', 'gpu'), # gpu
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing',
'crashtest', 'crashtests.list'), 'crashtest', None),
]
sys.path.append(dirs['abs_reftest_dir'])
import manifest
self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests')
for (path, suite, subsuite) in ref_manifests:
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
for t in man.files:
relpath = os.path.relpath(t, self.reftest_test_dir)
tests_by_path[relpath] = (suite, subsuite)
self._map_test_path_to_source(t, relpath)
self.info("Per-test run updated with manifest %s (%d tests)" %
(path, len(man.files)))
suite = 'jsreftest'
self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'],
'jsreftest', 'tests')
path = os.path.join(self.jsreftest_test_dir, 'jstests.list')
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
for t in man.files:
# expect manifest test to look like:
# ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js"
# while the test is in mercurial at:
# js/src/tests/test262/.../some_test.js
epos = t.find('=')
if epos > 0:
relpath = t[epos + 1:]
test_path = os.path.join(self.jsreftest_test_dir, relpath)
relpath = os.path.join('js', 'src', 'tests', relpath)
self._map_test_path_to_source(test_path, relpath)
tests_by_path.update({relpath: (suite, None)})
else:
self.warning("unexpected jsreftest test format: %s" %
str(t))
self.info("Per-test run updated with manifest %s (%d tests)" %
(path, len(man.files)))
# for each changed file, determine if it is a test file, and what suite it is in
for file in changed_files:
# manifest paths use os.sep (like backslash on Windows) but
# automation-relevance uses posixpath.sep
file = file.replace(posixpath.sep, os.sep)
entry = tests_by_path.get(file)
if not entry:
if file in all_disabled:
self.info("'%s' has been skipped on this platform." % file)
if os.environ.get('MOZHARNESS_TEST_PATHS', None) is not None:
self.fatal(
"Per-test run could not find requested test '%s'" %
file)
continue
if gpu and not self._is_gpu_suite(entry[1]):
self.info("Per-test run (gpu) discarded non-gpu test %s (%s)" %
(file, entry[1]))
continue
elif not gpu and self._is_gpu_suite(entry[1]):
self.info("Per-test run (non-gpu) discarded gpu test %s (%s)" %
(file, entry[1]))
continue
self.info("Per-test run found test %s (%s/%s)" %
(file, entry[0], entry[1]))
subsuite_mapping = {
# Map (<suite>, <subsuite>): <full-suite>
# <suite> is associated with a manifest, explicitly in code above
# <subsuite> comes from "subsuite" tags in some manifest entries
# <full-suite> is a unique id for the suite, matching desktop mozharness configs
('browser-chrome', 'clipboard'):
'browser-chrome-clipboard',
('chrome', 'clipboard'):
'chrome-clipboard',
('plain', 'clipboard'):
'plain-clipboard',
('browser-chrome', 'devtools'):
'mochitest-devtools-chrome',
('browser-chrome', 'screenshots'):
'browser-chrome-screenshots',
('plain', 'media'):
'mochitest-media',
# below should be on test-verify-gpu job
('chrome', 'gpu'):
'chrome-gpu',
('plain', 'gpu'):
'plain-gpu',
('plain', 'webgl1-core'):
'mochitest-webgl1-core',
('plain', 'webgl1-ext'):
'mochitest-webgl1-ext',
('plain', 'webgl2-core'):
'mochitest-webgl2-core',
('plain', 'webgl2-ext'):
'mochitest-webgl2-ext',
('plain', 'webgl2-deqp'):
'mochitest-webgl2-deqp',
}
if entry in subsuite_mapping:
suite = subsuite_mapping[entry]
else:
suite = entry[0]
suite_files = self.suites.get(suite)
if not suite_files:
suite_files = []
suite_files.append(file)
self.suites[suite] = suite_files
def _find_misc_tests(self, dirs, changed_files, gpu=False):
manifests = [
(os.path.join(dirs['abs_mochitest_dir'], 'tests',
'mochitest.ini'), 'plain'),
(os.path.join(dirs['abs_mochitest_dir'], 'chrome',
'chrome.ini'), 'chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'browser',
'browser-chrome.ini'), 'browser-chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'a11y',
'a11y.ini'), 'a11y'),
(os.path.join(dirs['abs_xpcshell_dir'], 'tests',
'xpcshell.ini'), 'xpcshell'),
]
tests_by_path = {}
for (path, suite) in manifests:
if os.path.exists(path):
man = TestManifest([path], strict=False)
active = man.active_tests(exists=False,
disabled=True,
filters=[],
**mozinfo.info)
# Remove disabled tests. Also, remove tests with the same path as
# disabled tests, even if they are not disabled, since per-test mode
# specifies tests by path (it cannot distinguish between two or more
# tests with the same path specified in multiple manifests).
disabled = [t['relpath'] for t in active if 'disabled' in t]
new_by_path = {t['relpath']:(suite,t.get('subsuite')) \
for t in active if 'disabled' not in t and \
t['relpath'] not in disabled}
tests_by_path.update(new_by_path)
self.info("Per-test run updated with manifest %s" % path)
ref_manifests = [
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout',
'reftests', 'reftest.list'), 'reftest', 'gpu'), #gpu
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing',
'crashtest', 'crashtests.list'), 'crashtest', None),
]
sys.path.append(dirs['abs_reftest_dir'])
import manifest
self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests')
for (path, suite, subsuite) in ref_manifests:
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
tests_by_path.update({
os.path.relpath(t, self.reftest_test_dir):
(suite, subsuite)
for t in man.files
})
self.info("Per-test run updated with manifest %s" % path)
suite = 'jsreftest'
self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'],
'jsreftest', 'tests')
path = os.path.join(self.jsreftest_test_dir, 'jstests.list')
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
for t in man.files:
# expect manifest test to look like:
# ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js"
# while the test is in mercurial at:
# js/src/tests/test262/.../some_test.js
epos = t.find('=')
if epos > 0:
relpath = t[epos + 1:]
relpath = os.path.join('js', 'src', 'tests', relpath)
tests_by_path.update({relpath: (suite, None)})
else:
self.warning("unexpected jsreftest test format: %s" %
str(t))
self.info("Per-test run updated with manifest %s" % path)
# for each changed file, determine if it is a test file, and what suite it is in
for file in changed_files:
# manifest paths use os.sep (like backslash on Windows) but
# automation-relevance uses posixpath.sep
file = file.replace(posixpath.sep, os.sep)
entry = tests_by_path.get(file)
if entry:
if gpu and entry[1] not in ['gpu', 'webgl']:
continue
elif not gpu and entry[1] in ['gpu', 'webgl']:
continue
self.info("Per-test run found test %s" % file)
subsuite_mapping = {
('browser-chrome', 'clipboard'):
'browser-chrome-clipboard',
('chrome', 'clipboard'):
'chrome-clipboard',
('plain', 'clipboard'):
'plain-clipboard',
('browser-chrome', 'devtools'):
'mochitest-devtools-chrome',
('browser-chrome', 'screenshots'):
'browser-chrome-screenshots',
('plain', 'media'):
'mochitest-media',
# below should be on test-verify-gpu job
('browser-chrome', 'gpu'):
'browser-chrome-gpu',
('chrome', 'gpu'):
'chrome-gpu',
('plain', 'gpu'):
'plain-gpu',
('plain', 'webgl'):
'mochitest-gl',
}
if entry in subsuite_mapping:
suite = subsuite_mapping[entry]
else:
suite = entry[0]
suite_files = self.suites.get(suite)
if not suite_files:
suite_files = []
suite_files.append(file)
self.suites[suite] = suite_files
def _find_misc_tests(self, dirs, changed_files, gpu=False):
manifests = [
(
os.path.join(dirs["abs_mochitest_dir"], "tests",
"mochitest.ini"),
"mochitest-plain",
),
(
os.path.join(dirs["abs_mochitest_dir"], "chrome",
"chrome.ini"),
"mochitest-chrome",
),
(
os.path.join(dirs["abs_mochitest_dir"], "browser",
"browser-chrome.ini"),
"mochitest-browser-chrome",
),
(
os.path.join(dirs["abs_mochitest_dir"], "a11y", "a11y.ini"),
"mochitest-a11y",
),
(
os.path.join(dirs["abs_xpcshell_dir"], "tests",
"xpcshell.ini"),
"xpcshell",
),
]
tests_by_path = {}
all_disabled = []
for (path, suite) in manifests:
if os.path.exists(path):
man = TestManifest([path], strict=False)
active = man.active_tests(exists=False,
disabled=True,
filters=[],
**mozinfo.info)
# Remove disabled tests. Also, remove tests with the same path as
# disabled tests, even if they are not disabled, since per-test mode
# specifies tests by path (it cannot distinguish between two or more
# tests with the same path specified in multiple manifests).
disabled = [t["relpath"] for t in active if "disabled" in t]
all_disabled += disabled
new_by_path = {
t["relpath"]: (suite, t.get("subsuite"), None)
for t in active
if "disabled" not in t and t["relpath"] not in disabled
}
tests_by_path.update(new_by_path)
self.info(
"Per-test run updated with manifest %s (%d active, %d skipped)"
% (path, len(new_by_path), len(disabled)))
ref_manifests = [
(
os.path.join(
dirs["abs_reftest_dir"],
"tests",
"layout",
"reftests",
"reftest.list",
),
"reftest",
"gpu",
), # gpu
(
os.path.join(
dirs["abs_reftest_dir"],
"tests",
"testing",
"crashtest",
"crashtests.list",
),
"crashtest",
None,
),
]
sys.path.append(dirs["abs_reftest_dir"])
import manifest
self.reftest_test_dir = os.path.join(dirs["abs_reftest_dir"], "tests")
for (path, suite, subsuite) in ref_manifests:
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
for t in man.tests:
relpath = os.path.relpath(t["path"], self.reftest_test_dir)
referenced = (t["referenced-test"]
if "referenced-test" in t else None)
tests_by_path[relpath] = (suite, subsuite, referenced)
self._map_test_path_to_source(t["path"], relpath)
self.info("Per-test run updated with manifest %s (%d tests)" %
(path, len(man.tests)))
suite = "jsreftest"
self.jsreftest_test_dir = os.path.join(dirs["abs_test_install_dir"],
"jsreftest", "tests")
path = os.path.join(self.jsreftest_test_dir, "jstests.list")
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
for t in man.files:
# expect manifest test to look like:
# ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js"
# while the test is in mercurial at:
# js/src/tests/test262/.../some_test.js
epos = t.find("=")
if epos > 0:
relpath = t[epos + 1:]
test_path = os.path.join(self.jsreftest_test_dir, relpath)
relpath = os.path.join("js", "src", "tests", relpath)
self._map_test_path_to_source(test_path, relpath)
tests_by_path.update({relpath: (suite, None, None)})
else:
self.warning("unexpected jsreftest test format: %s" %
str(t))
self.info("Per-test run updated with manifest %s (%d tests)" %
(path, len(man.files)))
# for each changed file, determine if it is a test file, and what suite it is in
for file in changed_files:
# manifest paths use os.sep (like backslash on Windows) but
# automation-relevance uses posixpath.sep
file = file.replace(posixpath.sep, os.sep)
entry = tests_by_path.get(file)
if not entry:
if file in all_disabled:
self.info("'%s' has been skipped on this platform." % file)
if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None:
self.fatal(
"Per-test run could not find requested test '%s'" %
file)
continue
if gpu and not self._is_gpu_suite(entry[1]):
self.info("Per-test run (gpu) discarded non-gpu test %s (%s)" %
(file, entry[1]))
continue
elif not gpu and self._is_gpu_suite(entry[1]):
self.info("Per-test run (non-gpu) discarded gpu test %s (%s)" %
(file, entry[1]))
continue
if entry[2] is not None:
# Test name substitution, for reftest reference file handling:
# - if both test and reference modified, run the test file
# - if only reference modified, run the test file
test_file = os.path.join(os.path.dirname(file),
os.path.basename(entry[2]))
self.info("Per-test run substituting %s for %s" %
(test_file, file))
file = test_file
self.info("Per-test run found test %s (%s/%s)" %
(file, entry[0], entry[1]))
subsuite_mapping = {
# Map (<suite>, <subsuite>): <full-suite>
# <suite> is associated with a manifest, explicitly in code above
# <subsuite> comes from "subsuite" tags in some manifest entries
# <full-suite> is a unique id for the suite, matching desktop mozharness configs
(
"mochitest-browser-chrome",
"devtools",
None,
):
"mochitest-devtools-chrome",
("mochitest-browser-chrome", "remote", None):
"mochitest-remote",
(
"mochitest-browser-chrome",
"screenshots",
None,
):
"mochitest-browser-chrome-screenshots", # noqa
("mochitest-plain", "media", None):
"mochitest-media",
# below should be on test-verify-gpu job
("mochitest-chrome", "gpu", None):
"mochitest-chrome-gpu",
("mochitest-plain", "gpu", None):
"mochitest-plain-gpu",
("mochitest-plain", "webgl1-core", None):
"mochitest-webgl1-core",
("mochitest-plain", "webgl1-ext", None):
"mochitest-webgl1-ext",
("mochitest-plain", "webgl2-core", None):
"mochitest-webgl2-core",
("mochitest-plain", "webgl2-ext", None):
"mochitest-webgl2-ext",
("mochitest-plain", "webgl2-deqp", None):
"mochitest-webgl2-deqp",
("mochitest-plain", "webgpu", None):
"mochitest-webgpu",
}
if entry in subsuite_mapping:
suite = subsuite_mapping[entry]
else:
suite = entry[0]
suite_files = self.suites.get(suite)
if not suite_files:
suite_files = []
if file not in suite_files:
suite_files.append(file)
self.suites[suite] = suite_files
def _find_misc_tests(self, dirs, changed_files):
manifests = [
(os.path.join(dirs['abs_mochitest_dir'], 'tests',
'mochitest.ini'), 'plain'),
(os.path.join(dirs['abs_mochitest_dir'], 'chrome',
'chrome.ini'), 'chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'browser',
'browser-chrome.ini'), 'browser-chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'a11y',
'a11y.ini'), 'a11y'),
(os.path.join(dirs['abs_xpcshell_dir'], 'tests',
'xpcshell.ini'), 'xpcshell'),
]
tests_by_path = {}
for (path, suite) in manifests:
if os.path.exists(path):
man = TestManifest([path], strict=False)
active = man.active_tests(exists=False,
disabled=False,
filters=[],
**mozinfo.info)
tests_by_path.update(
{t['relpath']: (suite, t.get('subsuite'))
for t in active})
self.info("Verification updated with manifest %s" % path)
ref_manifests = [
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout',
'reftests', 'reftest.list'), 'reftest'),
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing',
'crashtest', 'crashtests.list'), 'crashtest'),
]
sys.path.append(dirs['abs_reftest_dir'])
import manifest
self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests')
for (path, suite) in ref_manifests:
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
tests_by_path.update({
os.path.relpath(t, self.reftest_test_dir): (suite, None)
for t in man.files
})
self.info("Verification updated with manifest %s" % path)
suite = 'jsreftest'
self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'],
'jsreftest', 'tests')
path = os.path.join(self.jsreftest_test_dir, 'jstests.list')
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
for t in man.files:
# expect manifest test to look like:
# ".../tests/jsreftest/tests/jsreftest.html?test=test262/.../some_test.js"
# while the test is in mercurial at:
# js/src/tests/test262/.../some_test.js
epos = t.find('=')
if epos > 0:
relpath = t[epos + 1:]
relpath = os.path.join('js', 'src', 'tests', relpath)
tests_by_path.update({relpath: (suite, None)})
else:
self.warning("unexpected jsreftest test format: %s" %
str(t))
self.info("Verification updated with manifest %s" % path)
# for each changed file, determine if it is a test file, and what suite it is in
for file in changed_files:
# manifest paths use os.sep (like backslash on Windows) but
# automation-relevance uses posixpath.sep
file = file.replace(posixpath.sep, os.sep)
entry = tests_by_path.get(file)
if entry:
self.info("Verification found test %s" % file)
subsuite_mapping = {
('browser-chrome', 'clipboard'):
'browser-chrome-clipboard',
('chrome', 'clipboard'): 'chrome-clipboard',
('plain', 'clipboard'): 'plain-clipboard',
('browser-chrome', 'devtools'):
'mochitest-devtools-chrome',
('browser-chrome', 'gpu'): 'browser-chrome-gpu',
('browser-chrome', 'screenshots'):
'browser-chrome-screenshots',
('chrome', 'gpu'): 'chrome-gpu',
('plain', 'gpu'): 'plain-gpu',
('plain', 'media'): 'mochitest-media',
('plain', 'webgl'): 'mochitest-gl',
}
if entry in subsuite_mapping:
suite = subsuite_mapping[entry]
else:
suite = entry[0]
suite_files = self.verify_suites.get(suite)
if not suite_files:
suite_files = []
suite_files.append(file)
self.verify_suites[suite] = suite_files
def find_tests_for_verification(self, action, success=None):
"""
For each file modified on this push, determine if the modified file
is a test, by searching test manifests. Populate self.verify_suites
with test files, organized by suite.
This depends on test manifests, so can only run after test zips have
been downloaded and extracted.
"""
if self.config.get('verify') != True:
return
repository = os.environ.get("GECKO_HEAD_REPOSITORY")
revision = os.environ.get("GECKO_HEAD_REV")
if not repository or not revision:
self.warning("unable to verify tests: no repo or revision!")
return []
def get_automationrelevance():
response = self.load_json_url(url)
return response
dirs = self.query_abs_dirs()
mozinfo.find_and_update_from_json(dirs['abs_test_install_dir'])
if self.config.get('e10s') == True:
mozinfo.update({"e10s": True})
# Additional mozinfo properties like "headless" and "coverage" are
# also normally updated dynamically in the harness, but neither of
# these apply to the test-verify task.
manifests = [
(os.path.join(dirs['abs_mochitest_dir'], 'tests',
'mochitest.ini'), 'plain'),
(os.path.join(dirs['abs_mochitest_dir'], 'chrome',
'chrome.ini'), 'chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'browser',
'browser-chrome.ini'), 'browser-chrome'),
(os.path.join(dirs['abs_mochitest_dir'], 'a11y',
'a11y.ini'), 'a11y'),
(os.path.join(dirs['abs_xpcshell_dir'], 'tests',
'xpcshell.ini'), 'xpcshell'),
]
tests_by_path = {}
for (path, suite) in manifests:
if os.path.exists(path):
man = TestManifest([path], strict=False)
active = man.active_tests(exists=False,
disabled=False,
filters=[],
**mozinfo.info)
tests_by_path.update(
{t['relpath']: (suite, t.get('subsuite'))
for t in active})
self.info("Verification updated with manifest %s" % path)
ref_manifests = [
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout',
'reftests', 'reftest.list'), 'reftest'),
(os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing',
'crashtest', 'crashtests.list'), 'crashtest'),
# TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', 'jstests.list'), 'jstestbrowser'),
]
sys.path.append(dirs['abs_reftest_dir'])
import manifest
self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests')
for (path, suite) in ref_manifests:
if os.path.exists(path):
man = manifest.ReftestManifest()
man.load(path)
tests_by_path.update({
os.path.relpath(t, self.reftest_test_dir): (suite, None)
for t in man.files
})
self.info("Verification updated with manifest %s" % path)
# determine which files were changed on this push
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
revision)
contents = self.retry(get_automationrelevance,
attempts=2,
sleeptime=10)
changed_files = set()
for c in contents['changesets']:
self.info(" {cset} {desc}".format(
cset=c['node'][0:12],
desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
changed_files |= set(c['files'])
# for each changed file, determine if it is a test file, and what suite it is in
for file in changed_files:
# manifest paths use os.sep (like backslash on Windows) but
# automation-relevance uses posixpath.sep
file = file.replace(posixpath.sep, os.sep)
entry = tests_by_path.get(file)
if entry:
self.info("Verification found test %s" % file)
subsuite_mapping = {
('browser-chrome', 'clipboard'):
'browser-chrome-clipboard',
('chrome', 'clipboard'): 'chrome-clipboard',
('plain', 'clipboard'): 'plain-clipboard',
('browser-chrome', 'devtools'):
'mochitest-devtools-chrome',
('browser-chrome', 'gpu'): 'browser-chrome-gpu',
('chrome', 'gpu'): 'chrome-gpu',
('plain', 'gpu'): 'plain-gpu',
('plain', 'media'): 'mochitest-media',
('plain', 'webgl'): 'mochitest-gl',
}
if entry in subsuite_mapping:
suite = subsuite_mapping[entry]
else:
suite = entry[0]
suite_files = self.verify_suites.get(suite)
if not suite_files:
suite_files = []
suite_files.append(file)
self.verify_suites[suite] = suite_files
self.verify_downloaded = True
