def init_logger(debug=True, allow_color=ALLOW_COLOR, output=None):
"""
Initialize the mozlog logger. Must be called once before using logs.
"""
# late binding of sys.stdout is required for windows color to work
output = output or sys.stdout
start = time.time() * 1000
level_color = {
'WARNING': Fore.MAGENTA + Style.BRIGHT,
'CRITICAL': Fore.RED + Style.BRIGHT,
'ERROR': Fore.RED + Style.BRIGHT,
'DEBUG': Fore.CYAN + Style.BRIGHT,
'INFO': Style.BRIGHT,
}
time_color = Fore.BLUE
if mozinfo.os == "win":
time_color += Style.BRIGHT # this is unreadable on windows without it
def format_log(data):
level = data['level']
elapsed = _format_seconds((data['time'] - start) / 1000)
if allow_color:
elapsed = time_color + elapsed + Style.RESET_ALL
if level in level_color:
level = level_color[level] + level + Style.RESET_ALL
return "%s %s: %s\n" % (elapsed, level, data['message'])
logger = StructuredLogger("mozregression")
handler = LogLevelFilter(StreamHandler(output, format_log),
'debug' if debug else 'info')
logger.add_handler(handler)
set_default_logger(logger)
return logger
def run_step(logger, iterations, restart_after_iteration, kwargs_extras,
**kwargs):
import wptrunner
kwargs = copy.deepcopy(kwargs)
if restart_after_iteration:
kwargs["repeat"] = iterations
else:
kwargs["rerun"] = iterations
kwargs["pause_after_test"] = False
kwargs.update(kwargs_extras)
handler = LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"),
["log", "process_output"])
# There is a public API for this in the next mozlog
initial_handlers = logger._state.handlers
logger._state.handlers = []
with open("raw.log", "wb") as log:
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
logger.add_handler(handler)
logger.add_handler(StreamHandler(log, JSONFormatter()))
wptrunner.run_tests(**kwargs)
logger._state.handlers = initial_handlers
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, iterations)
return results, inconsistent, iterations
def run(venv, logger, **kwargs):
kwargs["pause_after_test"] = False
if kwargs["repeat"] == 1:
kwargs["repeat"] = 10
handler = LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()), "WARNING"),
["log", "process_output"])
# There is a public API for this in the next mozlog
initial_handlers = logger._state.handlers
logger._state.handlers = []
with open("raw.log", "wb") as log:
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
logger.add_handler(handler)
logger.add_handler(StreamHandler(log, JSONFormatter()))
wptrunner.run_tests(**kwargs)
logger._state.handlers = initial_handlers
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, kwargs["repeat"])
return kwargs["repeat"], results, inconsistent
def init_logger(debug=True, allow_color=ALLOW_COLOR, output=None):
"""
Initialize the mozlog logger. Must be called once before using logs.
"""
# late binding of sys.stdout is required for windows color to work
output = output or sys.stdout
start = time.time() * 1000
level_color = {
"WARNING": Fore.MAGENTA + Style.BRIGHT,
"CRITICAL": Fore.RED + Style.BRIGHT,
"ERROR": Fore.RED + Style.BRIGHT,
"DEBUG": Fore.CYAN + Style.BRIGHT,
"INFO": Style.BRIGHT,
}
time_color = Fore.BLUE
if mozinfo.os == "win":
time_color += Style.BRIGHT # this is unreadable on windows without it
def format_log(data):
level = data["level"]
elapsed = _format_seconds((data["time"] - start) / 1000)
if allow_color:
elapsed = time_color + elapsed + Style.RESET_ALL
if level in level_color:
level = level_color[level] + level + Style.RESET_ALL
msg = data["message"]
if "stack" in data:
msg += "\n%s" % data["stack"]
return "%s %s: %s\n" % (elapsed, level, msg)
logger = StructuredLogger("mozregression")
handler = LogLevelFilter(StreamHandler(output, format_log),
"debug" if debug else "info")
logger.add_handler(handler)
init_python_redirect_logger(logger)
set_default_logger(logger)
return logger
def main():
retcode = 0
parser = get_parser()
args = parser.parse_args()
if not os.path.exists(args.root):
logger.critical("Root directory %s does not exist" % args.root)
return 1
os.chdir(args.root)
if args.gh_token:
gh_handler = setup_github_logging(args)
else:
logger.warning("Can't log to GitHub")
gh_handler = None
print >> sys.stderr, "travis_fold:start:browser_setup"
logger.info("# %s #" % args.browser.title())
browser_cls = {"firefox": Firefox, "chrome": Chrome}.get(args.browser)
if browser_cls is None:
logger.critical("Unrecognised browser %s" % args.browser)
return 1
fetch_wpt_master()
head_sha1 = get_sha1()
logger.info("Testing revision %s" % head_sha1)
# For now just pass the whole list of changed files to wptrunner and
# assume that it will run everything that's actually a test
files_changed = get_files_changed()
if not files_changed:
logger.info("No files changed")
return 0
build_manifest()
install_wptrunner()
do_delayed_imports()
logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item
for item in files_changed))
browser = browser_cls(args.gh_token)
browser.install()
browser.install_webdriver()
kwargs = wptrunner_args(args.root, files_changed, args.iterations, browser)
print >> sys.stderr, "travis_fold:end:browser_setup"
print >> sys.stderr, "travis_fold:start:running_tests"
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
print >> sys.stderr, "travis_fold:end:running_tests"
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
print >> sys.stderr, "travis_fold:start:full_results"
write_results(results, args.iterations)
print >> sys.stderr, "travis_fold:end:full_results"
else:
logger.info("No tests run.")
try:
if gh_handler:
gh_handler.send()
except Exception:
logger.error(traceback.format_exc())
return retcode
def test(self, what, extra_args):
"""Run tests from names or paths.
mach test accepts arguments specifying which tests to run. Each argument
can be:
* The path to a test file
* A directory containing tests
* A test suite name
* An alias to a test suite name (codes used on TreeHerder)
If no input is provided, tests will be run based on files changed in
the local tree. Relevant tests, tags, or flavors are determined by
IMPACTED_TESTS annotations in moz.build files relevant to the
changed files.
When paths or directories are given, they are first resolved to test
files known to the build system.
If resolved tests belong to more than one test type/flavor/harness,
the harness for each relevant type/flavor will be invoked. e.g. if
you specify a directory with xpcshell and browser chrome mochitests,
both harnesses will be invoked.
"""
from mozlog.commandline import log_formatters
from mozlog.handlers import StreamHandler, LogLevelFilter
from mozlog.structuredlog import StructuredLogger
from moztest.resolve import TestResolver, TEST_FLAVORS, TEST_SUITES
resolver = self._spawn(TestResolver)
run_suites, run_tests = resolver.resolve_metadata(what)
if not run_suites and not run_tests:
print(UNKNOWN_TEST)
return 1
# Create shared logger
formatter = log_formatters[self._mach_context.settings['test']
['format']][0]()
formatter.summary_on_shutdown = True
level = self._mach_context.settings['test']['level']
log = StructuredLogger('mach-test')
log.add_handler(
StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
status = None
for suite_name in run_suites:
suite = TEST_SUITES[suite_name]
kwargs = suite['kwargs']
kwargs['log'] = log
if 'mach_command' in suite:
res = self._mach_context.commands.dispatch(
suite['mach_command'],
self._mach_context,
argv=extra_args,
**kwargs)
if res:
status = res
buckets = {}
for test in run_tests:
key = (test['flavor'], test.get('subsuite', ''))
buckets.setdefault(key, []).append(test)
for (flavor, subsuite), tests in sorted(buckets.items()):
if flavor not in TEST_FLAVORS:
print(UNKNOWN_FLAVOR % flavor)
status = 1
continue
m = TEST_FLAVORS[flavor]
if 'mach_command' not in m:
print(UNKNOWN_FLAVOR % flavor)
status = 1
continue
kwargs = dict(m['kwargs'])
kwargs['log'] = log
kwargs['subsuite'] = subsuite
res = self._mach_context.commands.dispatch(m['mach_command'],
self._mach_context,
argv=extra_args,
test_objects=tests,
**kwargs)
if res:
status = res
log.shutdown()
return status
files_changed.extend(affected_testfiles)
kwargs = wptrunner_args(args.root, files_changed, args.iterations,
browser)
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs, {"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(StreamHandler(sys.stdout, TbplFormatter()),
"WARNING"), ["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(results, args.iterations, args.comment_pr)
else:
def wrap_handler(x):
if not kwargs["verify_log_full"]:
x = LogLevelFilter(x, "WARNING")
x = LogActionFilter(x, ["log", "process_output"])
return x
def run_mochitest_general(self,
flavor=None,
test_objects=None,
resolve_tests=True,
**kwargs):
from mochitest_options import ALL_FLAVORS
from mozlog.commandline import log_formatters
from mozlog.handlers import StreamHandler, LogLevelFilter
from mozlog.structuredlog import StructuredLogger
buildapp = None
for app in SUPPORTED_APPS:
if is_buildapp_in(app)(self):
buildapp = app
break
flavors = None
if flavor:
for fname, fobj in ALL_FLAVORS.iteritems():
if flavor in fobj['aliases']:
if buildapp not in fobj['enabled_apps']:
continue
flavors = [fname]
break
else:
flavors = [
f for f, v in ALL_FLAVORS.iteritems()
if buildapp in v['enabled_apps']
]
if not kwargs.get('log'):
# Create shared logger
formatter = log_formatters[self._mach_context.settings['test']
['format']][0]()
formatter.summary_on_shutdown = True
level = self._mach_context.settings['test']['level']
kwargs['log'] = StructuredLogger('mach-mochitest')
kwargs['log'].add_handler(
StreamHandler(sys.stdout, LogLevelFilter(formatter, level)))
from mozbuild.controller.building import BuildDriver
self._ensure_state_subdir_exists('.')
test_paths = kwargs['test_paths']
kwargs['test_paths'] = []
mochitest = self._spawn(MochitestRunner)
tests = []
if resolve_tests:
tests = mochitest.resolve_tests(test_paths,
test_objects,
cwd=self._mach_context.cwd)
driver = self._spawn(BuildDriver)
driver.install_tests(tests)
subsuite = kwargs.get('subsuite')
if subsuite == 'default':
kwargs['subsuite'] = None
suites = defaultdict(list)
unsupported = set()
for test in tests:
# Filter out non-mochitests and unsupported flavors.
if test['flavor'] not in ALL_FLAVORS:
continue
key = (test['flavor'], test.get('subsuite', ''))
if test['flavor'] not in flavors:
unsupported.add(key)
continue
if subsuite == 'default':
# "--subsuite default" means only run tests that don't have a subsuite
if test.get('subsuite'):
unsupported.add(key)
continue
elif subsuite and test.get('subsuite', '') != subsuite:
unsupported.add(key)
continue
suites[key].append(test)
if ('mochitest', 'media') in suites:
req = os.path.join('testing', 'tools', 'websocketprocessbridge',
'websocketprocessbridge_requirements.txt')
self.virtualenv_manager.activate()
self.virtualenv_manager.install_pip_requirements(
req, require_hashes=False)
# sys.executable is used to start the websocketprocessbridge, though for some
# reason it doesn't get set when calling `activate_this.py` in the virtualenv.
sys.executable = self.virtualenv_manager.python_path
# This is a hack to introduce an option in mach to not send
# filtered tests to the mochitest harness. Mochitest harness will read
# the master manifest in that case.
if not resolve_tests:
for flavor in flavors:
key = (flavor, kwargs.get('subsuite'))
suites[key] = []
if not suites:
# Make it very clear why no tests were found
if not unsupported:
print(
TESTS_NOT_FOUND.format('\n'.join(
sorted(list(test_paths or test_objects)))))
return 1
msg = []
for f, s in unsupported:
fobj = ALL_FLAVORS[f]
apps = fobj['enabled_apps']
name = fobj['aliases'][0]
if s:
name = '{} --subsuite {}'.format(name, s)
if buildapp not in apps:
reason = 'requires {}'.format(' or '.join(apps))
else:
reason = 'excluded by the command line'
msg.append(' mochitest -f {} ({})'.format(name, reason))
print(
SUPPORTED_TESTS_NOT_FOUND.format(buildapp,
'\n'.join(sorted(msg))))
return 1
if buildapp == 'android':
from mozrunner.devices.android_device import grant_runtime_permissions
grant_runtime_permissions(self)
run_mochitest = mochitest.run_android_test
else:
run_mochitest = mochitest.run_desktop_test
overall = None
for (flavor, subsuite), tests in sorted(suites.items()):
fobj = ALL_FLAVORS[flavor]
msg = fobj['aliases'][0]
if subsuite:
msg = '{} with subsuite {}'.format(msg, subsuite)
print(NOW_RUNNING.format(msg))
harness_args = kwargs.copy()
harness_args['subsuite'] = subsuite
harness_args.update(fobj.get('extra_args', {}))
result = run_mochitest(self._mach_context,
tests=tests,
suite=fobj['suite'],
**harness_args)
if result:
overall = result
# Halt tests on keyboard interrupt
if result == -1:
break
# Only shutdown the logger if we created it
if kwargs['log'].name == 'mach-mochitest':
kwargs['log'].shutdown()
return overall
browser)
browser.prepare_environment()
with TravisFold("running_tests"):
logger.info("Starting %i test iterations" % args.iterations)
with open("raw.log", "wb") as log:
wptrunner.setup_logging(kwargs,
{"raw": log})
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
wptrunner.logger.add_handler(
LogActionFilter(
LogLevelFilter(
StreamHandler(
sys.stdout,
TbplFormatter()
),
"WARNING"),
["log", "process_output"]))
wptrunner.run_tests(**kwargs)
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, args.iterations)
if results:
if inconsistent:
write_inconsistent(inconsistent, args.iterations)
retcode = 2
else:
logger.info("All results were stable\n")