def getActiveTests(self, manifests, options, testDumpFile=None):
# These prefs will cause reftest.jsm to parse the manifests,
# dump the resulting tests to a file, and exit.
prefs = {
'reftest.manifests': json.dumps(manifests),
'reftest.manifests.dumpTests': testDumpFile or self.testDumpFile,
}
cmdargs = []
self.runApp(options, cmdargs=cmdargs, prefs=prefs)
with open(self.testDumpFile, 'r') as fh:
tests = json.load(fh)
if os.path.isfile(self.testDumpFile):
mozfile.remove(self.testDumpFile)
for test in tests:
# Name and path are expected by manifestparser, but not used in reftest.
test['name'] = test['path'] = test['url1']
mp = TestManifest(strict=False)
mp.tests = tests
filters = []
if options.totalChunks:
filters.append(
mpf.chunk_by_manifest(options.thisChunk, options.totalChunks))
tests = mp.active_tests(exists=False, filters=filters)
return tests
def test_all_js(tests, options):
print "Running JS Tests"
# We run each test in its own instance since these are harness tests.
# That just seems safer, no opportunity for cross-talk since
# we are sorta using the framework to test itself
results = JSResults()
for t in tests:
# write a temporary manifest
manifest = TestManifest()
manifest.tests = [t]
fd, filename = tempfile.mkstemp(suffix=".ini")
os.close(fd)
fp = file(filename, "w")
manifest.write(fp=fp)
fp.close()
# get CLI arguments to mozmill
args = ["-b", options.binary]
args.append("--console-level=DEBUG")
args.append("-m")
args.append(filename)
# run the test
proc = ProcessHandler("mozmill", args=args)
proc.run()
status = proc.waitForFinish(timeout=300)
command = proc.commandline
results.acquire(t["name"], proc.output, status, command)
# remove the temporary manifest
os.remove(filename)
return results
def get_manifests(self, suite, mozinfo):
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = self.get_tests(suite)
if "web-platform-tests" in suite:
manifests = set()
for t in tests:
manifests.add(t["manifest"])
return {"active": list(manifests), "skipped": []}
manifests = set(chunk_by_runtime.get_manifest(t) for t in tests)
# Compute the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active = set(chunk_by_runtime.get_manifest(t) for t in tests)
skipped = manifests - active
return {"active": list(active), "skipped": list(skipped)}
def get_chunked_manifests(flavor, subsuite, chunks, mozinfo):
"""Compute which manifests should run in which chunks with the given category
of tests.
Args:
flavor (str): The suite to run. Values are defined by the 'build_flavor' key
in `moztest.resolve.TEST_SUITES`.
subsuite (str): The subsuite to run or 'undefined' to denote no subsuite.
chunks (int): Number of chunks to split manifests across.
mozinfo (frozenset): Set of data in the form of (<key>, <value>) used
for filtering.
Returns:
A list of manifests where each item contains the manifest that should
run in the corresponding chunk.
"""
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = get_tests(flavor, subsuite)
all_manifests = set(t['manifest_relpath'] for t in tests)
# Compute only the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active_manifests = set(t['manifest_relpath'] for t in tests)
# Run the chunking algorithm.
chunked_manifests = [
c[1] for c in chunk_by_runtime(
None,
chunks,
get_runtimes(mozinfo['os'])
).get_chunked_manifests(tests)
]
# Add all skipped manifests to the first chunk so they still show up in the
# logs. They won't impact runtime much.
skipped_manifests = all_manifests - active_manifests
chunked_manifests[0].extend(skipped_manifests)
return chunked_manifests
def get_manifests(self, flavor, subsuite, mozinfo):
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = self.get_tests(flavor, subsuite)
if flavor == "web-platform-tests":
manifests = set()
for t in tests:
group = self.get_wpt_group(t)
wpt_group_translation[t['manifest']].add(group)
manifests.add(t['manifest'])
return {"active": list(manifests), "skipped": []}
manifests = set(chunk_by_runtime.get_manifest(t) for t in tests)
# Compute the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active = set(chunk_by_runtime.get_manifest(t) for t in tests)
skipped = manifests - active
return {"active": list(active), "skipped": list(skipped)}
def test_all_js(tests, options):
print "Running JS Tests"
# We run each test in its own instance since these are harness tests.
# That just seems safer, no opportunity for cross-talk since
# we are sorta using the framework to test itself
results = JSResults()
for t in tests:
# write a temporary manifest
manifest = TestManifest()
manifest.tests = [t]
fd, filename = tempfile.mkstemp(suffix='.ini')
os.close(fd)
fp = file(filename, 'w')
manifest.write(fp=fp)
fp.close()
# get CLI arguments to mozmill
args = []
if options.binary:
args.extend(['-b', options.binary])
args.append('--console-level=DEBUG')
args.append('-m')
args.append(filename)
# run the test
proc = ProcessHandler("mozmill", args=args)
proc.run()
status = proc.waitForFinish(timeout=300)
command = proc.commandline
results.acquire(t['name'], proc.output, status, command)
# remove the temporary manifest
os.remove(filename)
return results
def get_chunked_manifests(flavor, subsuite, chunks, mozinfo):
"""Compute which manifests should run in which chunks with the given category
of tests.
Args:
flavor (str): The suite to run. Values are defined by the 'build_flavor' key
in `moztest.resolve.TEST_SUITES`.
subsuite (str): The subsuite to run or 'undefined' to denote no subsuite.
chunks (int): Number of chunks to split manifests across.
mozinfo (frozenset): Set of data in the form of (<key>, <value>) used
for filtering.
Returns:
A list of manifests where each item contains the manifest that should
run in the corresponding chunk.
"""
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = get_tests(flavor, subsuite)
if flavor == 'web-platform-tests':
paths = tests_by_top_directory(tests, 3)
# Filter out non-web-platform-test runtime information.
runtimes = get_runtimes(mozinfo['os'])
runtimes = [(k, v) for k, v in runtimes.items()
if k.startswith('/') and not os.path.splitext(k)[-1]]
# Keep track of test paths for each chunk, and the runtime information.
chunked_manifests = [[[], 0] for _ in range(chunks)]
# First, chunk tests that have runtime information available.
for key, rt in sorted(runtimes, key=lambda x: x[1], reverse=True):
# Sort the chunks from fastest to slowest, based on runtimme info
# at x[1], then the number of test paths.
chunked_manifests.sort(key=lambda x: (x[1], len(x[0])))
test_paths = set(paths[key])
if test_paths:
# The full set of paths that live under the key must be added
# to the chunks.
# The smart scheduling algorithm uses paths of up to depth of 3
# as generated by the WPT harness and adding the set of paths will
# enable the scheduling algorithm to evaluate if the given path
# should be scheduled or not.
chunked_manifests[0][0].extend(test_paths)
chunked_manifests[0][1] += rt
# Remove from the list of paths that need scheduling.
paths.pop(key)
# Chunk remaining test paths that were not chunked in the previous step.
# These are test paths that did not have runtime information available.
for test_paths in paths.values():
chunked_manifests.sort(key=lambda x: (x[1], len(x[0])))
chunked_manifests[0][0].extend(set(test_paths))
# One last sort by the runtime, then number of test paths.
chunked_manifests.sort(key=lambda x: (x[1], len(x[0])))
# Reassign variable to contain just the chunked test paths.
chunked_manifests = [c[0] for c in chunked_manifests]
else:
chunker = chunk_by_runtime(None, chunks, get_runtimes(mozinfo['os']))
all_manifests = set(chunker.get_manifest(t) for t in tests)
# Compute only the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active_manifests = set(chunker.get_manifest(t) for t in tests)
# Run the chunking algorithm.
chunked_manifests = [
c[1] for c in chunker.get_chunked_manifests(active_manifests)
]
# Add all skipped manifests to the first chunk so they still show up in the
# logs. They won't impact runtime much.
skipped_manifests = all_manifests - active_manifests
chunked_manifests[0].extend(skipped_manifests)
return chunked_manifests
