def main():
logger = StructuredLogger('mozregression-gui')
set_default_logger(logger)
# Create a Qt application
log_model = LogModel()
logger.add_handler(log_model)
argv = [sys.argv[0].replace("main.py", "mozregression")] + sys.argv[1:]
app = QApplication(argv)
crash_reporter = CrashReporter(app)
crash_reporter.install()
app.setOrganizationName('mozilla')
app.setOrganizationDomain('mozilla.org')
app.setApplicationName('mozregression-gui')
set_default_prefs()
# Create the main window and show it
win = MainWindow()
app.aboutToQuit.connect(win.bisect_runner.stop)
app.aboutToQuit.connect(win.single_runner.stop)
app.aboutToQuit.connect(win.clear)
release_checker = CheckRelease(win)
release_checker.check()
log_model.log.connect(win.ui.log_view.on_log_received)
win.show()
# Enter Qt application main loop
sys.exit(app.exec_())
def test_check_stability_iterations():
logger = StructuredLogger("test-stability")
logger.add_handler(StreamHandler(sys.stdout, TbplFormatter()))
kwargs = {"verify_log_full": False}
def mock_run_tests(**kwargs):
repeats = kwargs.get("repeat", 1)
for _ in range(repeats):
logger.suite_start(tests=[], name="test")
for _ in range(kwargs.get("rerun", 1)):
logger.test_start("/example/test.html")
logger.test_status("/example/test.html",
subtest="test1",
status="PASS")
logger.test_end("/example/test.html", status="OK")
logger.suite_end()
status = wptrunner.TestStatus()
status.total_tests = 1
status.repeated_runs = repeats
status.expected_repeated_runs = repeats
return (None, status)
# Don't actually load wptrunner, because that will end up starting a browser
# which we don't want to do in this test.
with mock.patch("wptrunner.stability.wptrunner.run_tests") as mock_run:
mock_run.side_effect = mock_run_tests
assert stability.check_stability(logger,
repeat_loop=10,
repeat_restart=5,
chaos_mode=False,
output_results=False,
**kwargs) is None
def logger():
logger = StructuredLogger("mochitest_message_logger")
buf = six.StringIO()
handler = StreamHandler(buf, JSONFormatter())
logger.add_handler(handler)
return logger
def main():
logger = StructuredLogger("mozregression-gui")
set_default_logger(logger)
# Create a Qt application
log_model = LogModel()
logger.add_handler(log_model)
argv = [sys.argv[0].replace("mozregression-gui.py", "mozregression")
] + sys.argv[1:]
# enable hi-dpi scaling
QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
app = QApplication(argv)
crash_reporter = CrashReporter(app)
crash_reporter.install()
app.setOrganizationName("mozilla")
app.setOrganizationDomain("mozilla.org")
app.setApplicationName("mozregression-gui")
set_default_prefs()
# Create the main window and show it
win = MainWindow()
app.aboutToQuit.connect(win.bisect_runner.stop)
app.aboutToQuit.connect(win.single_runner.stop)
app.aboutToQuit.connect(win.clear)
release_checker = CheckRelease(win)
release_checker.check()
log_model.log.connect(win.ui.log_view.on_log_received)
win.show()
# Enter Qt application main loop
sys.exit(app.exec_())
def init_logger(debug=True, allow_color=ALLOW_COLOR, output=None):
"""
Initialize the mozlog logger. Must be called once before using logs.
"""
# late binding of sys.stdout is required for windows color to work
output = output or sys.stdout
start = time.time() * 1000
level_color = {
'WARNING': Fore.MAGENTA + Style.BRIGHT,
'CRITICAL': Fore.RED + Style.BRIGHT,
'ERROR': Fore.RED + Style.BRIGHT,
'DEBUG': Fore.CYAN + Style.BRIGHT,
'INFO': Style.BRIGHT,
}
time_color = Fore.BLUE
if mozinfo.os == "win":
time_color += Style.BRIGHT # this is unreadable on windows without it
def format_log(data):
level = data['level']
elapsed = _format_seconds((data['time'] - start) / 1000)
if allow_color:
elapsed = time_color + elapsed + Style.RESET_ALL
if level in level_color:
level = level_color[level] + level + Style.RESET_ALL
return "%s %s: %s\n" % (elapsed, level, data['message'])
logger = StructuredLogger("mozregression")
handler = LogLevelFilter(StreamHandler(output, format_log),
'debug' if debug else 'info')
logger.add_handler(handler)
set_default_logger(logger)
return logger
class FormatterTest(unittest.TestCase):
def setUp(self):
self.position = 0
self.logger = StructuredLogger("test_%s" % type(self).__name__)
self.output_file = BytesIO()
self.handler = StreamHandler(self.output_file, self.get_formatter())
self.logger.add_handler(self.handler)
def set_position(self, pos=None):
if pos is None:
pos = self.output_file.tell()
self.position = pos
def get_formatter(self):
raise NotImplementedError(
"FormatterTest subclasses must implement get_formatter")
@property
def loglines(self):
self.output_file.seek(self.position)
return [line.rstrip() for line in self.output_file.readlines()]
def init_logger(debug=True, allow_color=ALLOW_COLOR, output=None):
"""
Initialize the mozlog logger. Must be called once before using logs.
"""
# late binding of sys.stdout is required for windows color to work
output = output or sys.stdout
start = time.time() * 1000
level_color = {
"WARNING": Fore.MAGENTA + Style.BRIGHT,
"CRITICAL": Fore.RED + Style.BRIGHT,
"ERROR": Fore.RED + Style.BRIGHT,
"DEBUG": Fore.CYAN + Style.BRIGHT,
"INFO": Style.BRIGHT,
}
time_color = Fore.BLUE
if mozinfo.os == "win":
time_color += Style.BRIGHT # this is unreadable on windows without it
def format_log(data):
level = data["level"]
elapsed = _format_seconds((data["time"] - start) / 1000)
if allow_color:
elapsed = time_color + elapsed + Style.RESET_ALL
if level in level_color:
level = level_color[level] + level + Style.RESET_ALL
msg = data["message"]
if "stack" in data:
msg += "\n%s" % data["stack"]
return "%s %s: %s\n" % (elapsed, level, msg)
logger = StructuredLogger("mozregression")
handler = LogLevelFilter(StreamHandler(output, format_log),
"debug" if debug else "info")
logger.add_handler(handler)
init_python_redirect_logger(logger)
set_default_logger(logger)
return logger
def test(self, what, extra_args):
"""Run tests from names or paths.
mach test accepts arguments specifying which tests to run. Each argument
can be:
* The path to a test file
* A directory containing tests
* A test suite name
* An alias to a test suite name (codes used on TreeHerder)
If no input is provided, tests will be run based on files changed in
the local tree. Relevant tests, tags, or flavors are determined by
IMPACTED_TESTS annotations in moz.build files relevant to the
changed files.
When paths or directories are given, they are first resolved to test
files known to the build system.
If resolved tests belong to more than one test type/flavor/harness,
the harness for each relevant type/flavor will be invoked. e.g. if
you specify a directory with xpcshell and browser chrome mochitests,
both harnesses will be invoked.
"""
from mozlog.commandline import log_formatters
from mozlog.handlers import StreamHandler, LogLevelFilter
from mozlog.structuredlog import StructuredLogger
from moztest.resolve import TestResolver, TEST_FLAVORS, TEST_SUITES
resolver = self._spawn(TestResolver)
run_suites, run_tests = resolver.resolve_metadata(what)
if not run_suites and not run_tests:
print(UNKNOWN_TEST)
return 1
# Create shared logger
formatter = log_formatters[self._mach_context.settings['test']
['format']][0]()
formatter.summary_on_shutdown = True
level = self._mach_context.settings['test']['level']
log = StructuredLogger('mach-test')
log.add_handler(
StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
status = None
for suite_name in run_suites:
suite = TEST_SUITES[suite_name]
kwargs = suite['kwargs']
kwargs['log'] = log
if 'mach_command' in suite:
res = self._mach_context.commands.dispatch(
suite['mach_command'],
self._mach_context,
argv=extra_args,
**kwargs)
if res:
status = res
buckets = {}
for test in run_tests:
key = (test['flavor'], test.get('subsuite', ''))
buckets.setdefault(key, []).append(test)
for (flavor, subsuite), tests in sorted(buckets.items()):
if flavor not in TEST_FLAVORS:
print(UNKNOWN_FLAVOR % flavor)
status = 1
continue
m = TEST_FLAVORS[flavor]
if 'mach_command' not in m:
print(UNKNOWN_FLAVOR % flavor)
status = 1
continue
kwargs = dict(m['kwargs'])
kwargs['log'] = log
kwargs['subsuite'] = subsuite
res = self._mach_context.commands.dispatch(m['mach_command'],
self._mach_context,
argv=extra_args,
test_objects=tests,
**kwargs)
if res:
status = res
log.shutdown()
return status
def test_fail(name, opts, expected):
stack = """
SimpleTest.is@SimpleTest/SimpleTest.js:312:5
@caps/tests/mochitest/test_bug246699.html:53:1
""".strip('\n')
buf = BytesIO()
fmt = formatters[name](**opts)
logger = StructuredLogger('test_logger')
logger.add_handler(StreamHandler(buf, fmt))
logger.suite_start(['test_foo', 'test_bar', 'test_baz'])
logger.test_start('test_foo')
logger.test_end('test_foo', 'FAIL', 'PASS', 'expected 0 got 1')
logger.test_start('test_bar')
logger.test_status('test_bar', 'a subtest', 'FAIL', 'PASS',
'expected 0 got 1', stack)
logger.test_status('test_bar', 'another subtest', 'TIMEOUT')
logger.test_end('test_bar', 'OK')
logger.test_start('test_baz')
logger.test_end('test_baz', 'PASS', 'FAIL')
logger.suite_end()
result = buf.getvalue()
print("Dumping result for copy/paste:")
print(result)
assert result == expected
def test_known_intermittent(name, opts, expected):
buf = BytesIO()
fmt = formatters[name](**opts)
logger = StructuredLogger('test_logger')
logger.add_handler(StreamHandler(buf, fmt))
logger.suite_start(['test_foo', 'test_bar', 'test_baz'])
logger.test_start('test_foo')
logger.test_end('test_foo', 'FAIL', 'PASS', known_intermittent=['FAIL'])
logger.test_start('test_bar')
logger.test_status('test_bar',
'a subtest',
'PASS',
'FAIL',
known_intermittent=['PASS'])
logger.test_end('test_bar', 'OK')
logger.test_start('test_baz')
logger.test_end('test_baz',
'FAIL',
'FAIL',
'expected 0 got 1',
known_intermittent=['PASS'])
logger.suite_end()
result = buf.getvalue()
print("Dumping result for copy/paste:")
print(result)
assert result == expected
def inner(name, **fmt_args):
buf = StringIO()
fmt = formatters[name](**fmt_args)
logger = StructuredLogger("test_logger")
logger.add_handler(StreamHandler(buf, fmt))
return logger
from __future__ import absolute_import, unicode_literals
import os
import mozinfo
import mozunit
from mozlog.structuredlog import set_default_logger, StructuredLogger
set_default_logger(StructuredLogger('test_playback'))
from raptor.playback import get_playback, MitmproxyDesktop
config = {}
run_local = True
if os.environ.get('TOOLTOOLCACHE', None) is None:
run_local = False
def test_get_playback(get_binary):
config['platform'] = mozinfo.os
if 'win' in config['platform']:
# this test is not yet supported on windows
assert True
return
config['obj_path'] = os.path.dirname(get_binary('firefox'))
config['playback_tool'] = 'mitmproxy'
config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
config['playback_pageset_manifest'] = 'mitmproxy-playback-set.manifest'
config['playback_recordings'] = 'mitmproxy-recording-amazon.mp'
def test_precondition_failed(name, opts, expected):
buf = BytesIO()
fmt = formatters[name](**opts)
logger = StructuredLogger('test_logger')
logger.add_handler(StreamHandler(buf, fmt))
logger.suite_start(['test_foo', 'test_bar'])
logger.test_start('test_foo')
logger.test_end('test_foo', 'PRECONDITION_FAILED')
logger.test_start('test_bar')
logger.test_status('test_bar', 'a subtest', 'PASS')
logger.test_status('test_bar', 'another subtest', 'PRECONDITION_FAILED')
logger.test_end('test_bar', 'OK')
logger.suite_end()
result = buf.getvalue()
print("Dumping result for copy/paste:")
print(result)
assert result == expected
try:
from http.server import HTTPServer # py3
except ImportError:
from BaseHTTPServer import HTTPServer # py2
from mozlog.structuredlog import set_default_logger, StructuredLogger
from raptor.control_server import RaptorControlServer
# need this so the raptor unit tests can find output & filter classes
here = os.path.abspath(os.path.dirname(__file__))
raptor_dir = os.path.join(os.path.dirname(here), 'raptor')
sys.path.insert(0, raptor_dir)
from raptor.results import RaptorResultsHandler
set_default_logger(StructuredLogger('test_control_server'))
def test_start_and_stop():
results_handler = RaptorResultsHandler()
control = RaptorControlServer(results_handler)
assert control.server is None
control.start()
assert isinstance(control.server, HTTPServer)
assert control.server.fileno()
assert control._server_thread.is_alive()
control.stop()
assert not control._server_thread.is_alive()
def structured_logger():
return StructuredLogger("logger")
def run_mochitest_general(self,
flavor=None,
test_objects=None,
resolve_tests=True,
**kwargs):
from mochitest_options import ALL_FLAVORS
from mozlog.commandline import log_formatters
from mozlog.handlers import StreamHandler, LogLevelFilter
from mozlog.structuredlog import StructuredLogger
buildapp = None
for app in SUPPORTED_APPS:
if is_buildapp_in(app)(self):
buildapp = app
break
flavors = None
if flavor:
for fname, fobj in ALL_FLAVORS.iteritems():
if flavor in fobj['aliases']:
if buildapp not in fobj['enabled_apps']:
continue
flavors = [fname]
break
else:
flavors = [
f for f, v in ALL_FLAVORS.iteritems()
if buildapp in v['enabled_apps']
]
if not kwargs.get('log'):
# Create shared logger
formatter = log_formatters[self._mach_context.settings['test']
['format']][0]()
formatter.summary_on_shutdown = True
level = self._mach_context.settings['test']['level']
kwargs['log'] = StructuredLogger('mach-mochitest')
kwargs['log'].add_handler(
StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
from mozbuild.controller.building import BuildDriver
self._ensure_state_subdir_exists('.')
test_paths = kwargs['test_paths']
kwargs['test_paths'] = []
mochitest = self._spawn(MochitestRunner)
tests = []
if resolve_tests:
tests = mochitest.resolve_tests(test_paths,
test_objects,
cwd=self._mach_context.cwd)
driver = self._spawn(BuildDriver)
driver.install_tests(tests)
subsuite = kwargs.get('subsuite')
if subsuite == 'default':
kwargs['subsuite'] = None
suites = defaultdict(list)
unsupported = set()
for test in tests:
# Filter out non-mochitests and unsupported flavors.
if test['flavor'] not in ALL_FLAVORS:
continue
key = (test['flavor'], test.get('subsuite', ''))
if test['flavor'] not in flavors:
unsupported.add(key)
continue
if subsuite == 'default':
# "--subsuite default" means only run tests that don't have a subsuite
if test.get('subsuite'):
unsupported.add(key)
continue
elif subsuite and test.get('subsuite', '') != subsuite:
unsupported.add(key)
continue
suites[key].append(test)
if ('mochitest', 'media') in suites:
req = os.path.join('testing', 'tools', 'websocketprocessbridge',
'websocketprocessbridge_requirements.txt')
self.virtualenv_manager.activate()
self.virtualenv_manager.install_pip_requirements(
req, require_hashes=False)
# sys.executable is used to start the websocketprocessbridge, though for some
# reason it doesn't get set when calling `activate_this.py` in the virtualenv.
sys.executable = self.virtualenv_manager.python_path
# This is a hack to introduce an option in mach to not send
# filtered tests to the mochitest harness. Mochitest harness will read
# the master manifest in that case.
if not resolve_tests:
for flavor in flavors:
key = (flavor, kwargs.get('subsuite'))
suites[key] = []
if not suites:
# Make it very clear why no tests were found
if not unsupported:
print(
TESTS_NOT_FOUND.format('\n'.join(
sorted(list(test_paths or test_objects)))))
return 1
msg = []
for f, s in unsupported:
fobj = ALL_FLAVORS[f]
apps = fobj['enabled_apps']
name = fobj['aliases'][0]
if s:
name = '{} --subsuite {}'.format(name, s)
if buildapp not in apps:
reason = 'requires {}'.format(' or '.join(apps))
else:
reason = 'excluded by the command line'
msg.append(' mochitest -f {} ({})'.format(name, reason))
print(
SUPPORTED_TESTS_NOT_FOUND.format(buildapp,
'\n'.join(sorted(msg))))
return 1
if buildapp == 'android':
from mozrunner.devices.android_device import grant_runtime_permissions
grant_runtime_permissions(self)
run_mochitest = mochitest.run_android_test
else:
run_mochitest = mochitest.run_desktop_test
overall = None
for (flavor, subsuite), tests in sorted(suites.items()):
fobj = ALL_FLAVORS[flavor]
msg = fobj['aliases'][0]
if subsuite:
msg = '{} with subsuite {}'.format(msg, subsuite)
print(NOW_RUNNING.format(msg))
harness_args = kwargs.copy()
harness_args['subsuite'] = subsuite
harness_args.update(fobj.get('extra_args', {}))
result = run_mochitest(self._mach_context,
tests=tests,
suite=fobj['suite'],
**harness_args)
if result:
overall = result
# Halt tests on keyboard interrupt
if result == -1:
break
# Only shutdown the logger if we created it
if kwargs['log'].name == 'mach-mochitest':
kwargs['log'].shutdown()
return overall
from http.server import HTTPServer # py3
except ImportError:
from BaseHTTPServer import HTTPServer # py2
from mozlog.structuredlog import set_default_logger, StructuredLogger
from raptor.control_server import RaptorControlServer
# need this so the raptor unit tests can find output & filter classes
here = os.path.abspath(os.path.dirname(__file__))
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
sys.path.insert(0, raptor_dir)
from raptor.results import RaptorResultsHandler
set_default_logger(StructuredLogger("test_control_server"))
def clear_cache():
# remove the condprof download cache
from condprof.client import CONDPROF_CACHE # noqa
if os.path.exists(CONDPROF_CACHE):
shutil.rmtree(CONDPROF_CACHE)
clear_cache()
def test_start_and_stop():
from __future__ import absolute_import, unicode_literals
import os
import time
import mozinfo
import mozunit
from mozlog.structuredlog import set_default_logger, StructuredLogger
# need this so raptor imports work both from /raptor and via mach
here = os.path.abspath(os.path.dirname(__file__))
set_default_logger(StructuredLogger("test_playback"))
from mozproxy import get_playback
from mozproxy.backends.mitm.desktop import MitmproxyDesktop
config = {}
run_local = True
if os.environ.get("TOOLTOOLCACHE") is None:
run_local = False
def test_get_playback(get_binary):
config["platform"] = mozinfo.os
if "win" in config["platform"]:
# this test is not yet supported on windows
assert True
return
def setUp(self):
self.position = 0
self.logger = StructuredLogger("test_%s" % type(self).__name__)
self.output_file = BytesIO()
self.handler = StreamHandler(self.output_file, self.get_formatter())
self.logger.add_handler(self.handler)
