def run_mochitest_test(self,
suite=None,
test_file=None,
debugger=None,
shuffle=False,
keep_open=False,
rerun_failures=False,
no_autorun=False,
repeat=0,
slow=False):
"""Runs a mochitest.
test_file is a path to a test file. It can be a relative path from the
top source directory, an absolute filename, or a directory containing
test files.
suite is the type of mochitest to run. It can be one of ('plain',
'chrome', 'browser', 'metro', 'a11y').
debugger is a program name or path to a binary (presumably a debugger)
to run the test in. e.g. 'gdb'
shuffle is whether test order should be shuffled (defaults to false).
keep_open denotes whether to keep the browser open after tests
complete.
"""
# TODO Bug 794506 remove once mach integrates with virtualenv.
build_path = os.path.join(self.topobjdir, 'build')
if build_path not in sys.path:
sys.path.append(build_path)
if rerun_failures and test_file:
print('Cannot specify both --rerun-failures and a test path.')
return 1
# Need to call relpath before os.chdir() below.
test_path = ''
if test_file:
test_path = self._wrap_path_argument(test_file).relpath()
tests_dir = os.path.join(self.topobjdir, '_tests')
mochitest_dir = os.path.join(tests_dir, 'testing', 'mochitest')
failure_file_path = os.path.join(self.statedir,
'mochitest_failures.json')
if rerun_failures and not os.path.exists(failure_file_path):
print('No failure file present. Did you run mochitests before?')
return 1
from StringIO import StringIO
from automation import Automation
# runtests.py is ambiguous, so we load the file/module manually.
if 'mochitest' not in sys.modules:
import imp
path = os.path.join(mochitest_dir, 'runtests.py')
with open(path, 'r') as fh:
imp.load_module('mochitest', fh, path,
('.py', 'r', imp.PY_SOURCE))
import mochitest
# This is required to make other components happy. Sad, isn't it?
os.chdir(self.topobjdir)
automation = Automation()
# Automation installs its own stream handler to stdout. Since we want
# all logging to go through us, we just remove their handler.
remove_handlers = [
l for l in logging.getLogger().handlers
if isinstance(l, logging.StreamHandler)
]
for handler in remove_handlers:
logging.getLogger().removeHandler(handler)
runner = mochitest.Mochitest(automation)
opts = mochitest.MochitestOptions(automation, tests_dir)
options, args = opts.parse_args([])
# Need to set the suite options before verifyOptions below.
if suite == 'plain':
# Don't need additional options for plain.
pass
elif suite == 'chrome':
options.chrome = True
elif suite == 'browser':
options.browserChrome = True
elif suite == 'metro':
options.immersiveMode = True
options.browserChrome = True
elif suite == 'a11y':
options.a11y = True
else:
raise Exception('None or unrecognized mochitest suite type.')
options = opts.verifyOptions(options, runner)
if options is None:
raise Exception('mochitest option validator failed.')
options.autorun = not no_autorun
options.closeWhenDone = not keep_open
options.shuffle = shuffle
options.consoleLevel = 'INFO'
options.repeat = repeat
options.runSlower = slow
options.testingModulesDir = os.path.join(tests_dir, 'modules')
options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins'))
options.symbolsPath = os.path.join(self.distdir,
'crashreporter-symbols')
options.failureFile = failure_file_path
automation.setServerInfo(options.webServer, options.httpPort,
options.sslPort, options.webSocketPort)
if test_path:
test_root = runner.getTestRoot(options)
test_root_file = mozpack.path.join(mochitest_dir, test_root,
test_path)
if not os.path.exists(test_root_file):
print('Specified test path does not exist: %s' %
test_root_file)
print(
'You may need to run |mach build| to build the test files.'
)
return 1
options.testPath = test_path
env = {'TEST_PATH': test_path}
if rerun_failures:
options.testManifest = failure_file_path
if debugger:
options.debugger = debugger
# We need this to enable colorization of output.
self.log_manager.enable_unstructured()
# Output processing is a little funky here. The old make targets
# grepped the log output from TEST-UNEXPECTED-* and printed these lines
# after test execution. Ideally the test runner would expose a Python
# API for obtaining test results and we could just format failures
# appropriately. Unfortunately, it doesn't yet do that. So, we capture
# all output to a buffer then "grep" the buffer after test execution.
# Bug 858197 tracks a Python API that would facilitate this.
test_output = StringIO()
handler = logging.StreamHandler(test_output)
handler.addFilter(UnexpectedFilter())
handler.setFormatter(StructuredHumanFormatter(0, write_times=False))
logging.getLogger().addHandler(handler)
result = runner.runTests(options)
# Need to remove our buffering handler before we echo failures or else
# it will catch them again!
logging.getLogger().removeHandler(handler)
self.log_manager.disable_unstructured()
if test_output.getvalue():
for line in test_output.getvalue().splitlines():
self.log(logging.INFO, 'unexpected', {'msg': line}, '{msg}')
return result
def run_desktop_test(self,
context,
suite=None,
test_paths=None,
debugger=None,
debugger_args=None,
slowscript=False,
shuffle=False,
keep_open=False,
rerun_failures=False,
no_autorun=False,
repeat=0,
run_until_failure=False,
slow=False,
chunk_by_dir=0,
total_chunks=None,
this_chunk=None,
jsdebugger=False,
debug_on_failure=False,
start_at=None,
end_at=None,
e10s=False,
dmd=False,
dump_output_directory=None,
dump_about_memory_after_test=False,
dump_dmd_after_test=False,
install_extension=None,
quiet=False,
**kwargs):
"""Runs a mochitest.
test_paths are path to tests. They can be a relative path from the
top source directory, an absolute filename, or a directory containing
test files.
suite is the type of mochitest to run. It can be one of ('plain',
'chrome', 'browser', 'metro', 'a11y').
debugger is a program name or path to a binary (presumably a debugger)
to run the test in. e.g. 'gdb'
debugger_args are the arguments passed to the debugger.
slowscript is true if the user has requested the SIGSEGV mechanism of
invoking the slow script dialog.
shuffle is whether test order should be shuffled (defaults to false).
keep_open denotes whether to keep the browser open after tests
complete.
"""
if rerun_failures and test_paths:
print('Cannot specify both --rerun-failures and a test path.')
return 1
# Need to call relpath before os.chdir() below.
if test_paths:
test_paths = [
self._wrap_path_argument(p).relpath() for p in test_paths
]
failure_file_path = os.path.join(self.statedir,
'mochitest_failures.json')
if rerun_failures and not os.path.exists(failure_file_path):
print('No failure file present. Did you run mochitests before?')
return 1
from StringIO import StringIO
# runtests.py is ambiguous, so we load the file/module manually.
if 'mochitest' not in sys.modules:
import imp
path = os.path.join(self.mochitest_dir, 'runtests.py')
with open(path, 'r') as fh:
imp.load_module('mochitest', fh, path,
('.py', 'r', imp.PY_SOURCE))
import mozinfo
import mochitest
from manifestparser import TestManifest
from mozbuild.testing import TestResolver
# This is required to make other components happy. Sad, isn't it?
os.chdir(self.topobjdir)
# Automation installs its own stream handler to stdout. Since we want
# all logging to go through us, we just remove their handler.
remove_handlers = [
l for l in logging.getLogger().handlers
if isinstance(l, logging.StreamHandler)
]
for handler in remove_handlers:
logging.getLogger().removeHandler(handler)
runner = mochitest.Mochitest()
opts = mochitest.MochitestOptions()
options, args = opts.parse_args([])
flavor = suite
# Need to set the suite options before verifyOptions below.
if suite == 'plain':
# Don't need additional options for plain.
flavor = 'mochitest'
elif suite == 'chrome':
options.chrome = True
elif suite == 'browser':
options.browserChrome = True
flavor = 'browser-chrome'
elif suite == 'metro':
options.immersiveMode = True
options.browserChrome = True
elif suite == 'a11y':
options.a11y = True
elif suite == 'webapprt-content':
options.webapprtContent = True
options.app = self.get_webapp_runtime_path()
elif suite == 'webapprt-chrome':
options.webapprtChrome = True
options.app = self.get_webapp_runtime_path()
options.browserArgs.append("-test-mode")
else:
raise Exception('None or unrecognized mochitest suite type.')
if dmd:
options.dmdPath = self.bin_dir
options.autorun = not no_autorun
options.closeWhenDone = not keep_open
options.slowscript = slowscript
options.shuffle = shuffle
options.consoleLevel = 'INFO'
options.repeat = repeat
options.runUntilFailure = run_until_failure
options.runSlower = slow
options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins'))
options.symbolsPath = os.path.join(self.distdir,
'crashreporter-symbols')
options.chunkByDir = chunk_by_dir
options.totalChunks = total_chunks
options.thisChunk = this_chunk
options.jsdebugger = jsdebugger
options.debugOnFailure = debug_on_failure
options.startAt = start_at
options.endAt = end_at
options.e10s = e10s
options.dumpAboutMemoryAfterTest = dump_about_memory_after_test
options.dumpDMDAfterTest = dump_dmd_after_test
options.dumpOutputDirectory = dump_output_directory
options.quiet = quiet
options.failureFile = failure_file_path
if install_extension != None:
options.extensionsToInstall = [
os.path.join(self.topsrcdir, install_extension)
]
for k, v in kwargs.iteritems():
setattr(options, k, v)
if test_paths:
resolver = self._spawn(TestResolver)
tests = list(
resolver.resolve_tests(paths=test_paths,
flavor=flavor,
cwd=context.cwd))
if not tests:
print('No tests could be found in the path specified. Please '
'specify a path that is a test file or is a directory '
'containing tests.')
return 1
manifest = TestManifest()
manifest.tests.extend(tests)
options.manifestFile = manifest
if rerun_failures:
options.testManifest = failure_file_path
if debugger:
options.debugger = debugger
if debugger_args:
if options.debugger == None:
print("--debugger-args passed, but no debugger specified.")
return 1
options.debuggerArgs = debugger_args
options = opts.verifyOptions(options, runner)
if options is None:
raise Exception('mochitest option validator failed.')
# We need this to enable colorization of output.
self.log_manager.enable_unstructured()
# Output processing is a little funky here. The old make targets
# grepped the log output from TEST-UNEXPECTED-* and printed these lines
# after test execution. Ideally the test runner would expose a Python
# API for obtaining test results and we could just format failures
# appropriately. Unfortunately, it doesn't yet do that. So, we capture
# all output to a buffer then "grep" the buffer after test execution.
# Bug 858197 tracks a Python API that would facilitate this.
test_output = StringIO()
handler = logging.StreamHandler(test_output)
handler.addFilter(UnexpectedFilter())
handler.setFormatter(StructuredHumanFormatter(0, write_times=False))
logging.getLogger().addHandler(handler)
result = runner.runTests(options)
# Need to remove our buffering handler before we echo failures or else
# it will catch them again!
logging.getLogger().removeHandler(handler)
self.log_manager.disable_unstructured()
if test_output.getvalue():
result = 1
for line in test_output.getvalue().splitlines():
self.log(logging.INFO, 'unexpected', {'msg': line}, '{msg}')
return result
def run_desktop_test(
self,
context,
suite=None,
test_paths=None,
debugger=None,
debugger_args=None,
slowscript=False,
screenshot_on_fail=False,
shuffle=False,
closure_behaviour='auto',
rerun_failures=False,
no_autorun=False,
repeat=0,
run_until_failure=False,
slow=False,
chunk_by_dir=0,
total_chunks=None,
this_chunk=None,
extraPrefs=[],
jsdebugger=False,
debug_on_failure=False,
start_at=None,
end_at=None,
e10s=False,
strict_content_sandbox=False,
nested_oop=False,
dmd=False,
dump_output_directory=None,
dump_about_memory_after_test=False,
dump_dmd_after_test=False,
install_extension=None,
quiet=False,
environment=[],
app_override=None,
bisectChunk=None,
runByDir=False,
useTestMediaDevices=False,
timeout=None,
max_timeouts=None,
**kwargs):
"""Runs a mochitest.
test_paths are path to tests. They can be a relative path from the
top source directory, an absolute filename, or a directory containing
test files.
suite is the type of mochitest to run. It can be one of ('plain',
'chrome', 'browser', 'metro', 'a11y', 'jetpack-package', 'jetpack-addon').
debugger is a program name or path to a binary (presumably a debugger)
to run the test in. e.g. 'gdb'
debugger_args are the arguments passed to the debugger.
slowscript is true if the user has requested the SIGSEGV mechanism of
invoking the slow script dialog.
shuffle is whether test order should be shuffled (defaults to false).
closure_behaviour denotes whether to keep the browser open after tests
complete.
"""
if rerun_failures and test_paths:
print('Cannot specify both --rerun-failures and a test path.')
return 1
# Make absolute paths relative before calling os.chdir() below.
if test_paths:
test_paths = [self._wrap_path_argument(
p).relpath() if os.path.isabs(p) else p for p in test_paths]
failure_file_path = os.path.join(
self.statedir,
'mochitest_failures.json')
if rerun_failures and not os.path.exists(failure_file_path):
print('No failure file present. Did you run mochitests before?')
return 1
# runtests.py is ambiguous, so we load the file/module manually.
if 'mochitest' not in sys.modules:
import imp
path = os.path.join(self.mochitest_dir, 'runtests.py')
with open(path, 'r') as fh:
imp.load_module('mochitest', fh, path,
('.py', 'r', imp.PY_SOURCE))
import mochitest
from manifestparser import TestManifest
from mozbuild.testing import TestResolver
# This is required to make other components happy. Sad, isn't it?
os.chdir(self.topobjdir)
# Automation installs its own stream handler to stdout. Since we want
# all logging to go through us, we just remove their handler.
remove_handlers = [l for l in logging.getLogger().handlers
if isinstance(l, logging.StreamHandler)]
for handler in remove_handlers:
logging.getLogger().removeHandler(handler)
opts = mochitest.MochitestOptions()
options, args = opts.parse_args([])
options.subsuite = ''
flavor = suite
# Need to set the suite options before verifyOptions below.
if suite == 'plain':
# Don't need additional options for plain.
flavor = 'mochitest'
elif suite == 'chrome':
options.chrome = True
elif suite == 'browser':
options.browserChrome = True
flavor = 'browser-chrome'
elif suite == 'devtools':
options.browserChrome = True
options.subsuite = 'devtools'
elif suite == 'jetpack-package':
options.jetpackPackage = True
elif suite == 'jetpack-addon':
options.jetpackAddon = True
elif suite == 'metro':
options.immersiveMode = True
options.browserChrome = True
elif suite == 'a11y':
options.a11y = True
elif suite == 'webapprt-content':
options.webapprtContent = True
options.app = self.get_webapp_runtime_path()
elif suite == 'webapprt-chrome':
options.webapprtChrome = True
options.app = self.get_webapp_runtime_path()
options.browserArgs.append("-test-mode")
else:
raise Exception('None or unrecognized mochitest suite type.')
if dmd:
options.dmdPath = self.bin_dir
options.autorun = not no_autorun
options.closeWhenDone = closure_behaviour != 'open'
options.slowscript = slowscript
options.screenshotOnFail = screenshot_on_fail
options.shuffle = shuffle
options.consoleLevel = 'INFO'
options.repeat = repeat
options.runUntilFailure = run_until_failure
options.runSlower = slow
options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins'))
options.symbolsPath = os.path.join(
self.distdir,
'crashreporter-symbols')
options.chunkByDir = chunk_by_dir
options.totalChunks = total_chunks
options.thisChunk = this_chunk
options.jsdebugger = jsdebugger
options.debugOnFailure = debug_on_failure
options.startAt = start_at
options.endAt = end_at
options.e10s = e10s
options.strictContentSandbox = strict_content_sandbox
options.nested_oop = nested_oop
options.dumpAboutMemoryAfterTest = dump_about_memory_after_test
options.dumpDMDAfterTest = dump_dmd_after_test
options.dumpOutputDirectory = dump_output_directory
options.quiet = quiet
options.environment = environment
options.extraPrefs = extraPrefs
options.bisectChunk = bisectChunk
options.runByDir = runByDir
options.useTestMediaDevices = useTestMediaDevices
if timeout:
options.timeout = int(timeout)
if max_timeouts:
options.maxTimeouts = int(max_timeouts)
options.failureFile = failure_file_path
if install_extension is not None:
options.extensionsToInstall = [
os.path.join(
self.topsrcdir,
install_extension)]
for k, v in kwargs.iteritems():
setattr(options, k, v)
if test_paths:
resolver = self._spawn(TestResolver)
tests = list(
resolver.resolve_tests(
paths=test_paths,
flavor=flavor))
if not tests:
print('No tests could be found in the path specified. Please '
'specify a path that is a test file or is a directory '
'containing tests.')
return 1
manifest = TestManifest()
manifest.tests.extend(tests)
if len(
tests) == 1 and closure_behaviour == 'auto' and suite == 'plain':
options.closeWhenDone = False
options.manifestFile = manifest
if rerun_failures:
options.testManifest = failure_file_path
if debugger:
options.debugger = debugger
if debugger_args:
if options.debugger is None:
print("--debugger-args passed, but no debugger specified.")
return 1
options.debuggerArgs = debugger_args
if app_override:
if app_override == "dist":
options.app = self.get_binary_path(where='staged-package')
elif app_override:
options.app = app_override
if options.gmp_path is None:
# Need to fix the location of gmp_fake which might not be
# shipped in the binary
bin_path = self.get_binary_path()
options.gmp_path = os.path.join(
os.path.dirname(bin_path),
'gmp-fake',
'1.0')
options.gmp_path += os.pathsep
options.gmp_path += os.path.join(
os.path.dirname(bin_path),
'gmp-clearkey',
'0.1')
logger_options = {
key: value for key,
value in vars(options).iteritems() if key.startswith('log')}
runner = mochitest.Mochitest(logger_options)
options = opts.verifyOptions(options, runner)
if options is None:
raise Exception('mochitest option validator failed.')
# We need this to enable colorization of output.
self.log_manager.enable_unstructured()
result = runner.runTests(options)
self.log_manager.disable_unstructured()
if runner.message_logger.errors:
result = 1
runner.message_logger.logger.warning("The following tests failed:")
for error in runner.message_logger.errors:
runner.message_logger.logger.log_raw(error)
runner.message_logger.finish()
return result
