def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, 'tps'))
self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = testfiles['tests']
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result['productversion']
if not self.addonversion:
self.addonversion = result['addonversion']
self.results.append({
'state': result['state'],
'name': result['name'],
'message': result['message'],
'logdata': result['logdata']
})
if result['state'] == 'TEST-PASS':
self.numpassed += 1
else:
self.numfailed += 1
if self.stop_on_error:
print '\nTest failed with --stop-on-error specified; not running any more tests.\n'
break
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
'tests': self.results,
'os': '%s %sbit' % (mozinfo.version, mozinfo.bits),
'testtype': 'crossweave',
'productversion': self.productversion,
'addonversion': self.addonversion,
'synctype': self.synctype,
}
def start_httpd(self):
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www'))
self.httpd.start()
self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.baseurl)
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def runServer(self):
"""
Start a basic HTML server to host
test related files.
"""
if not self.options.serverPath:
self.logger.warning('Can\'t start HTTP server, --server-path not specified')
return
self.logger.debug('Starting server on port ' + str(self.options.serverPort))
self.server = MozHttpd(port=self.options.serverPort,
docroot=self.options.serverPath,
proxy_host_dirs=self.options.proxyHostDirs)
self.server.start(block=False)
def start(self, block=False):
if self.alive:
return
self._server = MozHttpd(host=self.host,
port=self.port,
docroot=self.root,
urlhandlers=[{
"method": "POST",
"path": "/file_upload",
"function": upload_handler
}])
self._server.start(block=block)
self.port = self._server.httpd.server_port
self.base_url = self.get_url()
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, "tps"))
self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = testfiles["tests"]
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result["productversion"]
if not self.addonversion:
self.addonversion = result["addonversion"]
self.results.append(
{
"state": result["state"],
"name": result["name"],
"message": result["message"],
"logdata": result["logdata"],
}
)
if result["state"] == "TEST-PASS":
self.numpassed += 1
else:
self.numfailed += 1
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
"tests": self.results,
"os": "%s %sbit" % (mozinfo.version, mozinfo.bits),
"testtype": "crossweave",
"productversion": self.productversion,
"addonversion": self.addonversion,
"synctype": self.synctype,
}
def start_httpd(self):
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.baseurl)
def start(self, block=False):
if self.alive:
return
self._server = MozHttpd(host=self.host, port=self.port, docroot=self.root, urlhandlers=[
{"method": "POST", "path": "/file_upload", "function": upload_handler}])
self._server.start(block=block)
self.port = self._server.httpd.server_port
self.base_url = self.get_url()
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, 'tps'))
self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = testfiles['tests']
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result['productversion']
if not self.addonversion:
self.addonversion = result['addonversion']
self.results.append({'state': result['state'],
'name': result['name'],
'message': result['message'],
'logdata': result['logdata']})
if result['state'] == 'TEST-PASS':
self.numpassed += 1
else:
self.numfailed += 1
if self.stop_on_error:
print '\nTest failed with --stop-on-error specified; not running any more tests.\n'
break
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = { 'tests': self.results,
'os': '%s %sbit' % (mozinfo.version, mozinfo.bits),
'testtype': 'crossweave',
'productversion': self.productversion,
'addonversion': self.addonversion,
'synctype': self.synctype,
}
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = "http://%s:%d/" % (host, port)
self.logger.info("running webserver on %s" % self.baseurl)
self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www"))
self.httpd.start()
def start_httpd(self, need_external_ip):
host = "127.0.0.1"
if need_external_ip:
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www'))
self.httpd.start()
self.marionette.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.marionette.baseurl)
class FixtureServer(object):
def __init__(self, root, host="127.0.0.1", port=0):
if not os.path.isdir(root):
raise Exception("Server root is not a valid path: %s" % root)
self.root = root
self.host = host
self.port = port
self._server = None
def start(self, block=False):
if self.alive:
return
self._server = MozHttpd(host=self.host,
port=self.port,
docroot=self.root,
urlhandlers=[{
"method": "POST",
"path": "/file_upload",
"function": upload_handler
}])
self._server.start(block=block)
self.port = self._server.httpd.server_port
self.base_url = self.get_url()
def stop(self):
if not self.alive:
return
self._server.stop()
self._server = None
@property
def alive(self):
return self._server is not None
def get_url(self, path="/"):
if not self.alive:
raise "Server not started"
return self._server.get_url(path)
@property
def urlhandlers(self):
return self._server.urlhandlers
def runServer(self):
"""
Start a basic HTML server to host
test related files.
"""
if not self.options.serverPath:
self.logger.warning("Can't start HTTP server, --server-path not specified")
return
self.logger.debug("Starting server on port " + str(self.options.serverPort))
self.server = MozHttpd(port=self.options.serverPort, docroot=self.options.serverPath)
self.server.start(block=False)
def start_httpd(self):
host = iface.get_lan_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def runServer(self):
"""
Start a basic HTML server to host
test related files.
"""
if not self.options.serverPath:
self.logger.warning('Can\'t start HTTP server, --server-path not specified')
return
self.logger.debug('Starting server on port ' + str(self.options.serverPort))
self.server = MozHttpd(port=self.options.serverPort,
docroot=self.options.serverPath,
proxy_host_dirs=self.options.proxyHostDirs)
self.server.start(block=False)
class FixtureServer(object):
def __init__(self, root, host="127.0.0.1", port=0):
if not os.path.isdir(root):
raise Exception("Server root is not a valid path: %s" % root)
self.root = root
self.host = host
self.port = port
self._server = None
def start(self, block=False):
if self.alive:
return
self._server = MozHttpd(host=self.host, port=self.port, docroot=self.root, urlhandlers=[
{"method": "POST", "path": "/file_upload", "function": upload_handler}])
self._server.start(block=block)
self.port = self._server.httpd.server_port
self.base_url = self.get_url()
def stop(self):
if not self.alive:
return
self._server.stop()
self._server = None
@property
def alive(self):
return self._server is not None
def get_url(self, path="/"):
if not self.alive:
raise "Server not started"
return self._server.get_url(path)
@property
def urlhandlers(self):
return self._server.urlhandlers
import os
import sys
import shutil
import tempfile
from datetime import datetime
from mozbuild.base import MozbuildObject
from buildconfig import substs
PORT = 8888
if __name__ == '__main__':
cli = CLI()
debug_args, interactive = cli.debugger_arguments()
build = MozbuildObject.from_environment()
httpd = MozHttpd(port=PORT,
docroot=os.path.join(build.topsrcdir, "build", "pgo"))
httpd.start(block=False)
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=PORT,
options='primary,privileged')
#TODO: mozfile.TemporaryDirectory
profilePath = tempfile.mkdtemp()
try:
#TODO: refactor this into mozprofile
prefpath = os.path.join(build.topsrcdir, "testing", "profiles", "prefs_general.js")
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = { "server": "%s:%d" % httpd.httpd.server_address,
class MarionetteTestRunner(object):
def __init__(self, address=None, emulator=None, homedir=None,
b2gbin=None, autolog=False, revision=None, es_server=None,
rest_server=None, logger=None, testgroup="marionette",
noWindow=False, logcat_dir=None):
self.address = address
self.emulator = emulator
self.homedir = homedir
self.b2gbin = b2gbin
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = iface.get_lan_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def start_marionette(self):
assert(self.baseurl is not None)
if self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette(host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir)
if self.b2gbin:
self.marionette = Marionette(host=host,
port=int(port),
b2gbin=self.b2gbin,
baseurl=self.baseurl)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette(emulator=self.emulator,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir)
else:
raise Exception("must specify address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup = self.testgroup,
os = 'android',
platform = 'emulator',
harness = 'marionette',
server = self.es_server,
restserver = self.rest_server,
machine = socket.gethostname())
testgroup.set_primary_product(
tree = 'b2g',
buildtype = 'opt',
revision = self.revision)
testgroup.add_test_suite(
testsuite = 'b2g emulator testsuite',
elapsedtime = elapsedtime.seconds,
cmdline = '',
passed = self.passed,
failed = self.failed,
todo = self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests, testtype=None):
self.reset_test_stats()
starttime = datetime.utcnow()
for test in tests:
self.run_test(test, testtype)
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(elapsedtime)
if self.marionette.emulator:
self.marionette.emulator.close()
self.marionette.emulator = None
self.marionette = None
def run_test(self, test, testtype):
if not self.httpd:
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
if testtype is not None:
testargs = {}
testtypes = testtype.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
if testtype is None:
manifest_tests = manifest.get()
else:
manifest_tests = manifest.get(**testargs)
for i in manifest_tests:
self.run_test(i["path"], testtype)
return
self.logger.info('TEST-START %s' % os.path.basename(test))
if file_ext == '.py':
test_mod = imp.load_source(mod_name, filepath)
for name in dir(test_mod):
obj = getattr(test_mod, name)
if (isinstance(obj, (type, types.ClassType)) and
issubclass(obj, unittest.TestCase)):
testnames = testloader.getTestCaseNames(obj)
for testname in testnames:
suite.addTest(obj(self.marionette, methodName=testname))
elif file_ext == '.js':
suite.addTest(MarionetteJSTestCase(self.marionette, jsFile=filepath))
if suite.countTestCases():
results = MarionetteTextTestRunner(verbosity=3).run(suite)
self.failed += len(results.failures) + len(results.errors)
self.todo = 0
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def cleanup(self):
if self.httpd:
self.httpd.stop()
__del__ = cleanup
class Peptest():
"""
Peptest
Runs and logs tests designed to test responsiveness
"""
profile_class = Profile
runner_class = Runner
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger('PEP')
# create the profile
enable_proxy = False
locations = ServerLocations()
if self.options.proxyLocations:
if not self.options.serverPath:
self.logger.warning('Can\'t set up proxy without server path')
else:
enable_proxy = True
for proxyLocation in self.options.proxyLocations:
locations.read(proxyLocation, False)
locations.add_host(host='127.0.0.1',
port=self.options.serverPort,
options='primary,privileged')
self.profile = self.profile_class(
profile=self.options.profilePath,
addons=[os.path.join(here, 'extension')],
locations=locations,
proxy=enable_proxy)
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith('.js'):
# a single test file was passed in
testObj = {}
testObj['path'] = os.path.realpath(self.options.testPath)
testObj['name'] = os.path.basename(self.options.testPath)
testObj['here'] = os.path.dirname(testObj['path'])
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj['tests'] = tests
manifestObj['options'] = options.__dict__
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, 'manifest.json'), 'w')
jsonManifest.write(json.dumps(manifestObj))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(
options.tracerThreshold)
env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval)
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(profile=self.profile,
binary=self.options.binary,
cmdargs=cmdargs,
env=env,
process_class=PepProcess)
def start(self):
self.logger.debug('Starting Peptest')
# start firefox
self.runner.start(outputTimeout=self.options.timeout)
self.runner.wait()
crashed = self.checkForCrashes(results.currentTest)
self.stop()
if crashed or results.has_fails():
return 1
return 0
def runServer(self):
"""
Start a basic HTML server to host
test related files.
"""
if not self.options.serverPath:
self.logger.warning(
'Can\'t start HTTP server, --server-path not specified')
return
self.logger.debug('Starting server on port ' +
str(self.options.serverPort))
self.server = MozHttpd(port=self.options.serverPort,
docroot=self.options.serverPath,
proxy_host_dirs=self.options.proxyHostDirs)
self.server.start(block=False)
def stop(self):
"""Kill the app"""
# stop the runner
if self.runner is not None:
self.runner.stop()
# kill the server process
if self.server:
self.server.stop()
# remove harness related files
files = ['manifest.json']
for f in files:
if os.path.exists(os.path.join(here, f)):
os.remove(os.path.join(here, f))
# delete any minidumps that may have been created
dumpDir = os.path.join(self.profile.profile, 'minidumps')
if self.options.profilePath and os.path.exists(dumpDir):
shutil.rmtree(dumpDir)
def checkForCrashes(self, testName=None):
"""
Detects when a crash occurs and prints the output from
MINIDUMP_STACKWALK. Returns true if crash detected,
otherwise false.
"""
stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
# try to get the caller's filename if no test name is given
if testName is None:
try:
testName = os.path.basename(
sys._getframe(1).f_code.co_filename)
except:
testName = "unknown"
foundCrash = False
dumpDir = os.path.join(self.profile.profile, 'minidumps')
dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
symbolsPath = self.options.symbolsPath
for d in dumps:
import subprocess
foundCrash = True
self.logger.info(
"PROCESS-CRASH | %s | application crashed (minidump found)",
testName)
print "Crash dump filename: " + d
# only proceed if a symbols path and stackwalk path were specified
if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
# if symbolsPath is a url, download and extract the zipfile
if utils.isURL(symbolsPath):
bundle = utils.download(symbolsPath, here)
symbolsPath = os.path.join(os.path.dirname(bundle),
'symbols')
utils.extract(bundle, symbolsPath, delete=True)
# run minidump_stackwalk
p = subprocess.Popen([stackwalkPath, d, symbolsPath],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = p.communicate()
if len(out) > 3:
# minidump_stackwalk is chatty, so ignore stderr when it succeeds.
print out
else:
print "stderr from minidump_stackwalk:"
print err
if p.returncode != 0:
print "minidump_stackwalk exited with return code %d" % p.returncode
else:
self.logger.warning(
'No symbols_path or stackwalk path specified, can\'t process dump'
)
break
# if the symbols path was downloaded, cleanup after ourselves
if utils.isURL(self.options.symbolsPath):
if os.path.exists(symbolsPath):
shutil.rmtree(symbolsPath)
return foundCrash
class MarionetteTestRunner(object):
def __init__(self, address=None, emulator=None, emulatorBinary=None,
emulatorImg=None, emulator_res='480x800', homedir=None,
bin=None, profile=None, autolog=False, revision=None,
es_server=None, rest_server=None, logger=None,
testgroup="marionette", noWindow=False, logcat_dir=None,
xml_output=None, repeat=0, perf=False, perfserv=None):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.perfrequest = None
self.xml_output = xml_output
self.repeat = repeat
self.perf = perf
self.perfserv = perfserv
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
self.perfrequest = None
def start_httpd(self):
host = iface.get_lan_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def start_marionette(self):
assert(self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host, port=int(port),
bin=self.bin, profile=self.profile,
baseurl=self.baseurl)
elif self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette(host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette(emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup = self.testgroup,
os = 'android',
platform = 'emulator',
harness = 'marionette',
server = self.es_server,
restserver = self.rest_server,
machine = socket.gethostname(),
logfile = logfile)
testgroup.set_primary_product(
tree = 'b2g',
buildtype = 'opt',
revision = self.revision)
testgroup.add_test_suite(
testsuite = 'b2g emulator testsuite',
elapsedtime = elapsedtime.seconds,
cmdline = '',
passed = self.passed,
failed = self.failed,
todo = self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests, testtype=None):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >=0:
for test in tests:
self.run_test(test, testtype)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.perfrequest and options.perf:
try:
self.perfrequest.submit()
except Exception, e:
print "Could not submit to datazilla"
print e
if self.xml_output:
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def valgrind_test(self, suppressions):
import json
import re
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
build_dir = os.path.join(self.topsrcdir, "build")
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo"))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, "testing", "profiles", "prefs_general.js")
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = {"server": "%s:%d" % httpd.httpd.server_address, "OOP": "false"}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.distdir, "xpi-stage", "quitter")
locations = ServerLocations()
locations.add_host(host="127.0.0.1", port=httpd.httpd.server_port, options="primary")
profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env["G_SLICE"] = "always-malloc"
env["XPCOM_CC_RUN_DURING_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
class OutputHandler(object):
def __init__(self):
self.found_errors = False
def __call__(self, line):
print(line)
m = re.match(r".*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts", line)
if m:
self.found_errors = True
outputHandler = OutputHandler()
kp_kwargs = {"processOutputLine": [outputHandler]}
valgrind = "valgrind"
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
"--smc-check=all-non-file",
"--vex-iropt-register-updates=allregs-at-mem-access",
"--gen-suppressions=all",
"--num-callers=20",
"--leak-check=full",
"--show-possibly-lost=no",
"--track-origins=yes",
]
for s in suppressions:
valgrind_args.append("--suppressions=" + s)
supps_dir = os.path.join(build_dir, "valgrind")
supps_file1 = os.path.join(supps_dir, "cross-architecture.sup")
valgrind_args.append("--suppressions=" + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(["bash", "-c", "echo $MACHTYPE"]).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + ".sup")
if os.path.isfile(supps_file2):
valgrind_args.append("--suppressions=" + supps_file2)
exitcode = None
try:
runner = FirefoxRunner(
profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, kp_kwargs=kp_kwargs
)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait()
finally:
if not outputHandler.found_errors:
status = 0
print("TEST-PASS | valgrind-test | valgrind found no errors")
else:
status = 1 # turns the TBPL job orange
print("TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors")
if exitcode != 0:
status = 2 # turns the TBPL job red
print("TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind")
httpd.stop()
return status
def run_tests(self):
"""
Generate the PGO profile data
"""
from mozhttpd import MozHttpd
from mozprofile import Preferences
from mozdevice import ADBDevice, ADBProcessError, ADBTimeoutError
from six import string_types
from marionette_driver.marionette import Marionette
app = self.query_package_name()
IP = '10.0.2.2'
PORT = 8888
PATH_MAPPINGS = {
'/js-input/webkit/PerformanceTests':
'third_party/webkit/PerformanceTests',
}
dirs = self.query_abs_dirs()
topsrcdir = os.path.join(dirs['abs_work_dir'], 'src')
adb = self.query_exe('adb')
path_mappings = {
k: os.path.join(topsrcdir, v)
for k, v in PATH_MAPPINGS.items()
}
httpd = MozHttpd(port=PORT,
docroot=os.path.join(topsrcdir, "build", "pgo"),
path_mappings=path_mappings)
httpd.start(block=False)
profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['profileserver']
prefpaths = [
os.path.join(profile_data_dir, profile, 'user.js')
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
"OOP": "false"
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
outputdir = self.config.get('output_directory', '/sdcard')
jarlog = posixpath.join(outputdir, 'en-US.log')
profdata = posixpath.join(outputdir, 'default.profraw')
env = {}
env["XPCOM_DEBUG_BREAK"] = "warn"
env["MOZ_IN_AUTOMATION"] = "1"
env["MOZ_JAR_LOG_FILE"] = jarlog
env["LLVM_PROFILE_FILE"] = profdata
adbdevice = ADBDevice(adb=adb, device='emulator-5554')
try:
# Run Fennec a first time to initialize its profile
driver = Marionette(
app='fennec',
package_name=app,
adb_path=adb,
bin="target.apk",
prefs=prefs,
connect_to_running_emulator=True,
startup_timeout=1000,
env=env,
)
driver.start_session()
# Now generate the profile and wait for it to complete
for page in PAGES:
driver.navigate("http://%s:%d/%s" % (IP, PORT, page))
timeout = 2
if 'Speedometer/index.html' in page:
# The Speedometer test actually runs many tests internally in
# javascript, so it needs extra time to run through them. The
# emulator doesn't get very far through the whole suite, but
# this extra time at least lets some of them process.
timeout = 360
time.sleep(timeout)
driver.set_context("chrome")
driver.execute_script("""
Components.utils.import("resource://gre/modules/Services.jsm");
let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"]
.createInstance(Components.interfaces.nsISupportsPRBool);
Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null);
return cancelQuit.data;
""")
driver.execute_script("""
Components.utils.import("resource://gre/modules/Services.jsm");
Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit)
""")
# There is a delay between execute_script() returning and the profile data
# actually getting written out, so poll the device until we get a profile.
for i in range(50):
try:
localprof = '/builds/worker/workspace/default.profraw'
adbdevice.pull(profdata, localprof)
stats = os.stat(localprof)
if stats.st_size == 0:
# The file may not have been fully written yet, so retry until we
# get actual data.
time.sleep(2)
else:
break
except ADBProcessError:
# The file may not exist at all yet, which would raise an
# ADBProcessError, so retry.
time.sleep(2)
else:
raise Exception("Unable to pull default.profraw")
adbdevice.pull(jarlog, '/builds/worker/workspace/en-US.log')
except ADBTimeoutError:
self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError',
EXIT_STATUS_DICT[TBPL_RETRY])
# We normally merge as part of a GENERATED_FILES step in the profile-use
# build, but Android runs sometimes result in a truncated profile. We do
# a merge here to make sure the data isn't corrupt so we can retry the
# 'run' task if necessary.
merge_cmd = [
'/builds/worker/workspace/build/clang/bin/llvm-profdata',
'merge',
'/builds/worker/workspace/default.profraw',
'-o',
'/builds/worker/workspace/merged.profraw',
]
rc = subprocess.call(merge_cmd)
if rc != 0:
self.fatal(
'INFRA-ERROR: Failed to merge profile data. Corrupt profile?',
EXIT_STATUS_DICT[TBPL_RETRY])
# tarfile doesn't support xz in this version of Python
tar_cmd = [
'tar',
'-acvf',
'/builds/worker/artifacts/profdata.tar.xz',
'-C',
'/builds/worker/workspace',
'merged.profraw',
'en-US.log',
]
subprocess.check_call(tar_cmd)
httpd.stop()
class MarionetteTestRunner(object):
def __init__(self,
address=None,
emulator=False,
homedir=None,
autolog=False,
revision=None,
es_server=None,
rest_server=None,
logger=None,
testgroup="marionette",
noWindow=False):
self.address = address
self.emulator = emulator
self.homedir = homedir
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = iface.get_lan_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__),
'www'))
self.httpd.start()
def start_marionette(self):
assert (self.baseurl is not None)
if self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette(host=host,
port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette(emulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow)
else:
raise Exception("must specify address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(testgroup=self.testgroup,
os='android',
platform='emulator',
harness='marionette',
server=self.es_server,
restserver=self.rest_server,
machine=socket.gethostname())
testgroup.set_primary_product(tree='b2g',
buildtype='opt',
revision=self.revision)
testgroup.add_test_suite(testsuite='b2g emulator testsuite',
elapsedtime=elapsedtime.seconds,
cmdline='',
passed=self.passed,
failed=self.failed,
todo=self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests, testtype=None):
self.reset_test_stats()
starttime = datetime.utcnow()
for test in tests:
self.run_test(test, testtype)
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(elapsedtime)
if self.marionette.emulator:
self.marionette.emulator.close()
self.marionette.emulator = None
self.marionette = None
def run_test(self, test, testtype):
if not self.httpd:
self.start_httpd()
if not self.marionette:
self.start_marionette()
if not os.path.isabs(test):
filepath = os.path.join(os.path.dirname(__file__), test)
else:
filepath = test
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_')
or filename.startswith('browser_'))
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
if testtype is not None:
testargs = {}
testtypes = testtype.replace('+', ' +').replace('-',
' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({atype[1:]: 'true'})
elif atype.startswith('-'):
testargs.update({atype[1:]: 'false'})
else:
testargs.update({atype: 'true'})
manifest = TestManifest()
manifest.read(filepath)
if testtype is None:
manifest_tests = manifest.get()
else:
manifest_tests = manifest.get(**testargs)
for i in manifest_tests:
self.run_test(i["path"], testtype)
return
self.logger.info('TEST-START %s' % os.path.basename(test))
if file_ext == '.py':
test_mod = imp.load_source(mod_name, filepath)
for name in dir(test_mod):
obj = getattr(test_mod, name)
if (isinstance(obj, (type, types.ClassType))
and issubclass(obj, unittest.TestCase)):
testnames = testloader.getTestCaseNames(obj)
for testname in testnames:
suite.addTest(obj(self.marionette,
methodName=testname))
elif file_ext == '.js':
suite.addTest(
MarionetteJSTestCase(self.marionette, jsFile=filepath))
if suite.countTestCases():
results = MarionetteTextTestRunner(verbosity=3).run(suite)
self.failed += len(results.failures) + len(results.errors)
self.todo = 0
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(
results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1],
'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]),
failure[1], 'TEST-UNEXPECTED-PASS'))
def cleanup(self):
if self.httpd:
self.httpd.stop()
__del__ = cleanup
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, "tps"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = []
for (filename, meta) in testfiles["tests"].items():
skip_reason = meta.get("disabled")
if skip_reason:
print("Skipping test {} - {}".format(
filename, skip_reason))
else:
testlist.append(filename)
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result["productversion"]
if not self.addonversion:
self.addonversion = result["addonversion"]
self.results.append({
"state": result["state"],
"name": result["name"],
"message": result["message"],
"logdata": result["logdata"],
})
if result["state"] == "TEST-PASS":
self.numpassed += 1
else:
self.numfailed += 1
if self.stop_on_error:
print("\nTest failed with --stop-on-error specified; "
"not running any more tests.\n")
break
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
"tests": self.results,
"os": "%s %sbit" % (mozinfo.version, mozinfo.bits),
"testtype": "crossweave",
"productversion": self.productversion,
"addonversion": self.addonversion,
"synctype": self.synctype,
}
def run_tests(self):
"""
Generate the PGO profile data
"""
from mozhttpd import MozHttpd
from mozprofile import Preferences
from mozdevice import ADBDeviceFactory, ADBTimeoutError
from six import string_types
from marionette_driver.marionette import Marionette
app = self.query_package_name()
IP = "10.0.2.2"
PORT = 8888
PATH_MAPPINGS = {
"/js-input/webkit/PerformanceTests":
"third_party/webkit/PerformanceTests",
}
dirs = self.query_abs_dirs()
topsrcdir = dirs["abs_src_dir"]
adb = self.query_exe("adb")
path_mappings = {
k: os.path.join(topsrcdir, v)
for k, v in PATH_MAPPINGS.items()
}
httpd = MozHttpd(
port=PORT,
docroot=os.path.join(topsrcdir, "build", "pgo"),
path_mappings=path_mappings,
)
httpd.start(block=False)
profile_data_dir = os.path.join(topsrcdir, "testing", "profiles")
with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
base_profiles = json.load(fh)["profileserver"]
prefpaths = [
os.path.join(profile_data_dir, profile, "user.js")
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
"OOP": "false"
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
outputdir = self.config.get("output_directory", "/sdcard/pgo_profile")
jarlog = posixpath.join(outputdir, "en-US.log")
profdata = posixpath.join(outputdir, "default_%p_random_%m.profraw")
env = {}
env["XPCOM_DEBUG_BREAK"] = "warn"
env["MOZ_IN_AUTOMATION"] = "1"
env["MOZ_JAR_LOG_FILE"] = jarlog
env["LLVM_PROFILE_FILE"] = profdata
if self.query_minidump_stackwalk():
os.environ["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path
os.environ["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs(
)["abs_blob_upload_dir"]
if not self.symbols_path:
self.symbols_path = os.environ.get("MOZ_FETCHES_DIR")
# Force test_root to be on the sdcard for android pgo
# builds which fail for Android 4.3 when profiles are located
# in /data/local/tmp/test_root with
# E AndroidRuntime: FATAL EXCEPTION: Gecko
# E AndroidRuntime: java.lang.IllegalArgumentException: \
# Profile directory must be writable if specified: /data/local/tmp/test_root/profile
# This occurs when .can-write-sentinel is written to
# the profile in
# mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoProfile.java.
# This is not a problem on later versions of Android. This
# over-ride of test_root should be removed when Android 4.3 is no
# longer supported.
sdcard_test_root = "/sdcard/test_root"
adbdevice = ADBDeviceFactory(adb=adb,
device="emulator-5554",
test_root=sdcard_test_root)
if adbdevice.test_root != sdcard_test_root:
# If the test_root was previously set and shared
# the initializer will not have updated the shared
# value. Force it to match the sdcard_test_root.
adbdevice.test_root = sdcard_test_root
adbdevice.mkdir(outputdir, parents=True)
try:
# Run Fennec a first time to initialize its profile
driver = Marionette(
app="fennec",
package_name=app,
adb_path=adb,
bin="geckoview-androidTest.apk",
prefs=prefs,
connect_to_running_emulator=True,
startup_timeout=1000,
env=env,
symbols_path=self.symbols_path,
)
driver.start_session()
# Now generate the profile and wait for it to complete
for page in PAGES:
driver.navigate("http://%s:%d/%s" % (IP, PORT, page))
timeout = 2
if "Speedometer/index.html" in page:
# The Speedometer test actually runs many tests internally in
# javascript, so it needs extra time to run through them. The
# emulator doesn't get very far through the whole suite, but
# this extra time at least lets some of them process.
timeout = 360
time.sleep(timeout)
driver.set_context("chrome")
driver.execute_script("""
Components.utils.import("resource://gre/modules/Services.jsm");
let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"]
.createInstance(Components.interfaces.nsISupportsPRBool);
Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null);
return cancelQuit.data;
""")
driver.execute_script("""
Components.utils.import("resource://gre/modules/Services.jsm");
Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit)
""")
# There is a delay between execute_script() returning and the profile data
# actually getting written out, so poll the device until we get a profile.
for i in range(50):
if not adbdevice.process_exist(app):
break
time.sleep(2)
else:
raise Exception("Android App (%s) never quit" % app)
# Pull all the profraw files and en-US.log
adbdevice.pull(outputdir, "/builds/worker/workspace/")
except ADBTimeoutError:
self.fatal(
"INFRA-ERROR: Failed with an ADBTimeoutError",
EXIT_STATUS_DICT[TBPL_RETRY],
)
profraw_files = glob.glob("/builds/worker/workspace/*.profraw")
if not profraw_files:
self.fatal(
"Could not find any profraw files in /builds/worker/workspace")
merge_cmd = [
os.path.join(os.environ["MOZ_FETCHES_DIR"],
"clang/bin/llvm-profdata"),
"merge",
"-o",
"/builds/worker/workspace/merged.profdata",
] + profraw_files
rc = subprocess.call(merge_cmd)
if rc != 0:
self.fatal(
"INFRA-ERROR: Failed to merge profile data. Corrupt profile?",
EXIT_STATUS_DICT[TBPL_RETRY],
)
# tarfile doesn't support xz in this version of Python
tar_cmd = [
"tar",
"-acvf",
"/builds/worker/artifacts/profdata.tar.xz",
"-C",
"/builds/worker/workspace",
"merged.profdata",
"en-US.log",
]
subprocess.check_call(tar_cmd)
httpd.stop()
def run_tests(self):
"""
Generate the PGO profile data
"""
from mozhttpd import MozHttpd
from mozprofile import Preferences
from mozdevice import ADBDevice, ADBTimeoutError
from six import string_types
from marionette_driver.marionette import Marionette
app = self.query_package_name()
IP = '10.0.2.2'
PORT = 8888
PATH_MAPPINGS = {
'/js-input/webkit/PerformanceTests':
'third_party/webkit/PerformanceTests',
}
dirs = self.query_abs_dirs()
topsrcdir = os.path.join(dirs['abs_work_dir'], 'src')
adb = self.query_exe('adb')
path_mappings = {
k: os.path.join(topsrcdir, v)
for k, v in PATH_MAPPINGS.items()
}
httpd = MozHttpd(port=PORT,
docroot=os.path.join(topsrcdir, "build", "pgo"),
path_mappings=path_mappings)
httpd.start(block=False)
profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['profileserver']
prefpaths = [
os.path.join(profile_data_dir, profile, 'user.js')
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
"OOP": "false"
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
# Enforce 1proc. This isn't in one of the user.js files because those
# are shared with desktop, which wants e10s. We can't interpolate
# because the formatting code only works for strings, and this is a
# bool pref.
prefs["browser.tabs.remote.autostart"] = False
outputdir = self.config.get('output_directory', '/sdcard/pgo_profile')
jarlog = posixpath.join(outputdir, 'en-US.log')
profdata = posixpath.join(outputdir, 'default_%p_random_%m.profraw')
env = {}
env["XPCOM_DEBUG_BREAK"] = "warn"
env["MOZ_IN_AUTOMATION"] = "1"
env["MOZ_JAR_LOG_FILE"] = jarlog
env["LLVM_PROFILE_FILE"] = profdata
adbdevice = ADBDevice(adb=adb, device='emulator-5554')
adbdevice.mkdir(outputdir)
try:
# Run Fennec a first time to initialize its profile
driver = Marionette(
app='fennec',
package_name=app,
adb_path=adb,
bin="target.apk",
prefs=prefs,
connect_to_running_emulator=True,
startup_timeout=1000,
env=env,
)
driver.start_session()
# Now generate the profile and wait for it to complete
for page in PAGES:
driver.navigate("http://%s:%d/%s" % (IP, PORT, page))
timeout = 2
if 'Speedometer/index.html' in page:
# The Speedometer test actually runs many tests internally in
# javascript, so it needs extra time to run through them. The
# emulator doesn't get very far through the whole suite, but
# this extra time at least lets some of them process.
timeout = 360
time.sleep(timeout)
driver.set_context("chrome")
driver.execute_script("""
Components.utils.import("resource://gre/modules/Services.jsm");
let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"]
.createInstance(Components.interfaces.nsISupportsPRBool);
Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null);
return cancelQuit.data;
""")
driver.execute_script("""
Components.utils.import("resource://gre/modules/Services.jsm");
Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit)
""")
# There is a delay between execute_script() returning and the profile data
# actually getting written out, so poll the device until we get a profile.
for i in range(50):
if not adbdevice.process_exist(app):
break
time.sleep(2)
else:
raise Exception("Android App (%s) never quit" % app)
# Pull all the profraw files and en-US.log
adbdevice.pull(outputdir, '/builds/worker/workspace/')
except ADBTimeoutError:
self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError',
EXIT_STATUS_DICT[TBPL_RETRY])
profraw_files = glob.glob('/builds/worker/workspace/*.profraw')
if not profraw_files:
self.fatal(
'Could not find any profraw files in /builds/worker/workspace')
merge_cmd = [
os.path.join(os.environ['MOZ_FETCHES_DIR'],
'clang/bin/llvm-profdata'),
'merge',
'-o',
'/builds/worker/workspace/merged.profdata',
] + profraw_files
rc = subprocess.call(merge_cmd)
if rc != 0:
self.fatal(
'INFRA-ERROR: Failed to merge profile data. Corrupt profile?',
EXIT_STATUS_DICT[TBPL_RETRY])
# tarfile doesn't support xz in this version of Python
tar_cmd = [
'tar',
'-acvf',
'/builds/worker/artifacts/profdata.tar.xz',
'-C',
'/builds/worker/workspace',
'merged.profdata',
'en-US.log',
]
subprocess.check_call(tar_cmd)
httpd.stop()
class MarionetteTestRunner(object):
def __init__(self,
address=None,
emulator=None,
emulatorBinary=None,
emulatorImg=None,
emulator_res='480x800',
homedir=None,
app=None,
bin=None,
profile=None,
autolog=False,
revision=None,
es_server=None,
rest_server=None,
logger=None,
testgroup="marionette",
noWindow=False,
logcat_dir=None,
xml_output=None,
repeat=0,
gecko_path=None,
testvars=None,
tree=None,
type=None,
device=None,
symbols_path=None,
**kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.xml_output = xml_output
self.repeat = repeat
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device = device
self.symbols_path = symbols_path
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__),
'www'))
self.httpd.start()
def start_marionette(self):
assert (self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl)
elif self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host,
port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(
os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(testgroup=self.testgroup,
os='android',
platform='emulator',
harness='marionette',
server=self.es_server,
restserver=self.rest_server,
machine=socket.gethostname(),
logfile=logfile)
testgroup.set_primary_product(tree=self.tree,
buildtype='opt',
revision=self.revision)
testgroup.add_test_suite(testsuite='b2g emulator testsuite',
elapsedtime=elapsedtime.seconds,
cmdline='',
passed=self.passed,
failed=self.failed,
todo=self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >= 0:
for test in tests:
self.run_test(test)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_')
or filename.startswith('browser_'))
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-',
' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({atype[1:]: 'true'})
elif atype.startswith('-'):
testargs.update({atype[1:]: 'false'})
else:
testargs.update({atype: 'true'})
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(disabled=False)
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite,
testloader, self.marionette,
self.testvars, **self.test_kwargs)
break
if suite.countTestCases():
runner = MarionetteTextTestRunner(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(
results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1],
'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]),
failure[1], 'TEST-UNEXPECTED-PASS'))
def register_handlers(self):
self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase])
def cleanup(self):
if self.httpd:
self.httpd.stop()
__del__ = cleanup
def generate_xml(self, results_list):
def _extract_xml(test, text='', result='passed'):
cls_name = test.__class__.__name__
testcase = doc.createElement('testcase')
testcase.setAttribute('classname', cls_name)
testcase.setAttribute('name', unicode(test).split()[0])
testcase.setAttribute('time', str(test.duration))
testsuite.appendChild(testcase)
if result in ['failure', 'error', 'skipped']:
f = doc.createElement(result)
f.setAttribute('message', 'test %s' % result)
f.appendChild(doc.createTextNode(text))
testcase.appendChild(f)
doc = dom.Document()
testsuite = doc.createElement('testsuite')
testsuite.setAttribute('name', 'Marionette')
testsuite.setAttribute('time', str(self.elapsedtime.total_seconds()))
testsuite.setAttribute(
'tests', str(sum([results.testsRun for results in results_list])))
def failed_count(results):
count = len(results.failures)
if hasattr(results, 'unexpectedSuccesses'):
count += len(results.unexpectedSuccesses)
return count
testsuite.setAttribute(
'failures',
str(sum([failed_count(results) for results in results_list])))
testsuite.setAttribute(
'errors',
str(sum([len(results.errors) for results in results_list])))
if hasattr(results, 'skipped'):
testsuite.setAttribute(
'skips',
str(
sum([
len(results.skipped) + len(results.expectedFailures)
for results in results_list
])))
for results in results_list:
for tup in results.errors:
_extract_xml(*tup, result='error')
for tup in results.failures:
_extract_xml(*tup, result='failure')
if hasattr(results, 'unexpectedSuccesses'):
for test in results.unexpectedSuccesses:
# unexpectedSuccesses is a list of Testcases only, no tuples
_extract_xml(test,
text='TEST-UNEXPECTED-PASS',
result='failure')
if hasattr(results, 'skipped'):
for tup in results.skipped:
_extract_xml(*tup, result='skipped')
if hasattr(results, 'expectedFailures'):
for tup in results.expectedFailures:
_extract_xml(*tup, result='skipped')
for test in results.tests_passed:
_extract_xml(test)
doc.appendChild(testsuite)
return doc.toprettyxml(encoding='utf-8')
def valgrind_test(self, suppressions):
import json
import re
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
#TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles',
'prefs_general.js')
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = {
'server': '%s:%d' % httpd.httpd.server_address,
'OOP': 'false'
}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
class OutputHandler(object):
def __init__(self):
self.found_errors = False
def __call__(self, line):
print(line)
m = re.match(
r'.*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts',
line)
if m:
self.found_errors = True
outputHandler = OutputHandler()
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind, '--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all', '--num-callers=20',
'--leak-check=full', '--show-possibly-lost=no',
'--track-origins=yes'
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(
['bash', '-c', 'echo $MACHTYPE']).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
kp_kwargs=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait()
finally:
if not outputHandler.found_errors:
status = 0
print(
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
print(
'TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors'
)
if exitcode != 0:
status = 2 # turns the TBPL job red
print(
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind'
)
httpd.stop()
return status
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(
self.topsrcdir, 'testing', 'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['valgrind']
prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
for profile in base_profiles]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
'server': '%s:%d' % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(
self.topsrcdir, 'tools', 'quitter', '*****@*****.**')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
'-v', # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
'--fair-sched=yes',
# Keep debuginfo after library unmap. See bug 1382280.
'--keep-debuginfo=yes',
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
'--expensive-definedness-checks=yes',
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
if mozinfo.os == 'linux':
machtype = {
'x86_64': 'x86_64-pc-linux-gnu',
'x86': 'i386-pc-linux-gnu',
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(logging.ERROR, 'valgrind-fail-parsing',
{'errs': errs, 'supps': supps},
'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors '
'seen, but {supps} generated suppressions seen')
elif errs == 0:
status = 0
self.log(logging.INFO, 'valgrind-pass', {},
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode is None:
status = 2 # turns the TBPL job red
self.log(logging.ERROR, 'valgrind-fail-timeout',
{'timeout': timeout},
'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out '
'(reached {timeout} second limit)')
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(logging.ERROR, 'valgrind-fail-errors', {},
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code'
'from Valgrind')
httpd.stop()
return status
class TPSTestRunner(object):
default_env = {
'MOZ_CRASHREPORTER_DISABLE': '1',
'GNOME_DISABLE_CRASH_DIALOG': '1',
'XRE_NO_WINDOWS_CRASH_DIALOG': '1',
'MOZ_NO_REMOTE': '1',
'XPCOM_DEBUG_BREAK': 'warn',
}
default_preferences = {
'app.update.enabled': False,
'browser.dom.window.dump.enabled': True,
'browser.sessionstore.resume_from_crash': False,
'browser.shell.checkDefaultBrowser': False,
'browser.tabs.warnOnClose': False,
'browser.warnOnQuit': False,
# Allow installing extensions dropped into the profile folder
'extensions.autoDisableScopes': 10,
'extensions.getAddons.get.url':
'http://127.0.0.1:4567/en-US/firefox/api/%API_VERSION%/search/guid:%IDS%',
'extensions.update.enabled': False,
# Don't open a dialog to show available add-on updates
'extensions.update.notifyUser': False,
'services.sync.addons.ignoreRepositoryChecking': True,
'services.sync.firstSync': 'notReady',
'services.sync.lastversion': '1.0',
'services.sync.log.rootLogger': 'Trace',
'services.sync.log.logger.engine.addons': 'Trace',
'services.sync.log.logger.service.main': 'Trace',
'services.sync.log.logger.engine.bookmarks': 'Trace',
'services.sync.log.appender.console': 'Trace',
'services.sync.log.appender.debugLog.enabled': True,
'toolkit.startup.max_resumed_crashes': -1,
}
syncVerRe = re.compile(r'Sync version: (?P<syncversion>.*)\n')
ffVerRe = re.compile(r'Firefox version: (?P<ffver>.*)\n')
ffDateRe = re.compile(r'Firefox builddate: (?P<ffdate>.*)\n')
def __init__(self,
extensionDir,
testfile='sync.test',
binary=None,
config=None,
rlock=None,
mobile=False,
logfile='tps.log',
resultfile='tps_result.json',
ignore_unused_engines=False):
self.extensions = []
self.testfile = testfile
self.logfile = os.path.abspath(logfile)
self.resultfile = resultfile
self.binary = binary
self.ignore_unused_engines = ignore_unused_engines
self.config = config if config else {}
self.repo = None
self.changeset = None
self.branch = None
self.numfailed = 0
self.numpassed = 0
self.nightly = False
self.rlock = rlock
self.mobile = mobile
self.tpsxpi = None
self.firefoxRunner = None
self.extensionDir = extensionDir
self.productversion = None
self.addonversion = None
self.postdata = {}
self.errorlogs = {}
@property
def mobile(self):
return self._mobile
@mobile.setter
def mobile(self, value):
self._mobile = value
self.synctype = 'desktop' if not self._mobile else 'mobile'
def log(self, msg, printToConsole=False):
"""Appends a string to the logfile"""
f = open(self.logfile, 'a')
f.write(msg)
f.close()
if printToConsole:
print msg
def writeToResultFile(self,
postdata,
body=None,
sendTo=['*****@*****.**']):
"""Writes results to test file"""
results = {'results': []}
if os.access(self.resultfile, os.F_OK):
f = open(self.resultfile, 'r')
results = json.loads(f.read())
f.close()
f = open(self.resultfile, 'w')
if body is not None:
postdata['body'] = body
if self.numpassed is not None:
postdata['numpassed'] = self.numpassed
if self.numfailed is not None:
postdata['numfailed'] = self.numfailed
if self.firefoxRunner and self.firefoxRunner.url:
postdata['firefoxrunnerurl'] = self.firefoxRunner.url
postdata['sendTo'] = sendTo
results['results'].append(postdata)
f.write(json.dumps(results, indent=2))
f.close()
def _zip_add_file(self, zip, file, rootDir):
zip.write(os.path.join(rootDir, file), file)
def _zip_add_dir(self, zip, dir, rootDir):
try:
zip.write(os.path.join(rootDir, dir), dir)
except:
# on some OS's, adding directory entries doesn't seem to work
pass
for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
for f in files:
zip.write(os.path.join(root, f), os.path.join(dir, f))
def run_single_test(self, testdir, testname):
testpath = os.path.join(testdir, testname)
self.log("Running test %s\n" % testname, True)
# Read and parse the test file, merge it with the contents of the config
# file, and write the combined output to a temporary file.
f = open(testpath, 'r')
testcontent = f.read()
f.close()
try:
test = json.loads(testcontent)
except:
test = json.loads(
testcontent[testcontent.find('{'):testcontent.find('}') + 1])
testcontent += 'var config = %s;\n' % json.dumps(self.config, indent=2)
testcontent += 'var seconds_since_epoch = %d;\n' % int(time.time())
tmpfile = TempFile(prefix='tps_test_')
tmpfile.write(testcontent)
tmpfile.close()
# generate the profiles defined in the test, and a list of test phases
profiles = {}
phaselist = []
for phase in test:
profilename = test[phase]
# create the profile if necessary
if not profilename in profiles:
profiles[profilename] = Profile(preferences=self.preferences,
addons=self.extensions)
# create the test phase
phaselist.append(
TPSTestPhase(phase,
profiles[profilename],
testname,
tmpfile.filename,
self.logfile,
self.env,
self.firefoxRunner,
self.log,
ignore_unused_engines=self.ignore_unused_engines))
# sort the phase list by name
phaselist = sorted(phaselist, key=lambda phase: phase.phase)
# run each phase in sequence, aborting at the first failure
for phase in phaselist:
phase.run()
# if a failure occurred, dump the entire sync log into the test log
if phase.status != 'PASS':
for profile in profiles:
self.log('\nDumping sync log for profile %s\n' %
profiles[profile].profile)
for root, dirs, files in os.walk(
os.path.join(profiles[profile].profile, 'weave',
'logs')):
for f in files:
weavelog = os.path.join(profiles[profile].profile,
'weave', 'logs', f)
if os.access(weavelog, os.F_OK):
with open(weavelog, 'r') as fh:
for line in fh:
possible_time = line[0:13]
if len(
possible_time
) == 13 and possible_time.isdigit():
time_ms = int(possible_time)
formatted = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time_ms / 1000))
self.log('%s.%03d %s' %
(formatted, time_ms %
1000, line[14:]))
else:
self.log(line)
break
# grep the log for FF and sync versions
f = open(self.logfile)
logdata = f.read()
match = self.syncVerRe.search(logdata)
sync_version = match.group('syncversion') if match else 'unknown'
match = self.ffVerRe.search(logdata)
firefox_version = match.group('ffver') if match else 'unknown'
match = self.ffDateRe.search(logdata)
firefox_builddate = match.group('ffdate') if match else 'unknown'
f.close()
if phase.status == 'PASS':
logdata = ''
else:
# we only care about the log data for this specific test
logdata = logdata[logdata.find('Running test %s' %
(str(testname))):]
result = {
'PASS': lambda x: ('TEST-PASS', ''),
'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()),
'unknown': lambda x:
('TEST-UNEXPECTED-FAIL', 'test did not complete')
}[phase.status](phase.errline)
logstr = "\n%s | %s%s\n" % (result[0], testname,
(' | %s' % result[1] if result[1] else ''))
try:
repoinfo = self.firefoxRunner.runner.get_repositoryInfo()
except:
repoinfo = {}
apprepo = repoinfo.get('application_repository', '')
appchangeset = repoinfo.get('application_changeset', '')
# save logdata to a temporary file for posting to the db
tmplogfile = None
if logdata:
tmplogfile = TempFile(prefix='tps_log_')
tmplogfile.write(logdata)
tmplogfile.close()
self.errorlogs[testname] = tmplogfile
resultdata = ({
'productversion': {
'version': firefox_version,
'buildid': firefox_builddate,
'builddate': firefox_builddate[0:8],
'product': 'Firefox',
'repository': apprepo,
'changeset': appchangeset,
},
'addonversion': {
'version': sync_version,
'product': 'Firefox Sync'
},
'name': testname,
'message': result[1],
'state': result[0],
'logdata': logdata
})
self.log(logstr, True)
for phase in phaselist:
print "\t%s: %s" % (phase.phase, phase.status)
if phase.status == 'FAIL':
break
return resultdata
def run_tests(self):
# delete the logfile if it already exists
if os.access(self.logfile, os.F_OK):
os.remove(self.logfile)
# Make a copy of the default env variables and preferences, and update
# them for mobile settings if needed.
self.env = self.default_env.copy()
self.preferences = self.default_preferences.copy()
if self.mobile:
self.preferences.update({'services.sync.client.type': 'mobile'})
# Set a dummy username to force the correct authentication type. For the
# old sync, the username is not allowed to contain a '@'.
dummy = {'fx_account': 'dummy@somewhere', 'sync_account': 'dummy'}
auth_type = self.config.get('auth_type', 'fx_account')
self.preferences.update({'services.sync.username': dummy[auth_type]})
# Acquire a lock to make sure no other threads are running tests
# at the same time.
if self.rlock:
self.rlock.acquire()
try:
# Create the Firefox runner, which will download and install the
# build, as needed.
if not self.firefoxRunner:
self.firefoxRunner = TPSFirefoxRunner(self.binary)
# now, run the test group
self.run_test_group()
except:
traceback.print_exc()
self.numpassed = 0
self.numfailed = 1
try:
self.writeToResultFile(
self.postdata, '<pre>%s</pre>' % traceback.format_exc())
except:
traceback.print_exc()
else:
try:
if self.numfailed > 0 or self.numpassed == 0:
To = self.config['email'].get('notificationlist')
else:
To = self.config['email'].get('passednotificationlist')
self.writeToResultFile(self.postdata, sendTo=To)
except:
traceback.print_exc()
try:
self.writeToResultFile(
self.postdata,
'<pre>%s</pre>' % traceback.format_exc())
except:
traceback.print_exc()
# release our lock
if self.rlock:
self.rlock.release()
# dump out a summary of test results
print 'Test Summary\n'
for test in self.postdata.get('tests', {}):
print '%s | %s | %s' % (test['state'], test['name'],
test['message'])
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, 'tps'))
self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = testfiles['tests']
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result['productversion']
if not self.addonversion:
self.addonversion = result['addonversion']
self.results.append({
'state': result['state'],
'name': result['name'],
'message': result['message'],
'logdata': result['logdata']
})
if result['state'] == 'TEST-PASS':
self.numpassed += 1
else:
self.numfailed += 1
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
'tests': self.results,
'os': '%s %sbit' % (mozinfo.version, mozinfo.bits),
'testtype': 'crossweave',
'productversion': self.productversion,
'addonversion': self.addonversion,
'synctype': self.synctype,
}
class TPSTestRunner(object):
extra_env = {
"MOZ_CRASHREPORTER_DISABLE": "1",
"GNOME_DISABLE_CRASH_DIALOG": "1",
"XRE_NO_WINDOWS_CRASH_DIALOG": "1",
"MOZ_NO_REMOTE": "1",
"XPCOM_DEBUG_BREAK": "warn",
}
default_preferences = {
"app.update.checkInstallTime":
False,
"app.update.disabledForTesting":
True,
"security.turn_off_all_security_so_that_viruses_can_take_over_this_computer":
True,
"browser.dom.window.dump.enabled":
True,
"devtools.console.stdout.chrome":
True,
"browser.sessionstore.resume_from_crash":
False,
"browser.shell.checkDefaultBrowser":
False,
"browser.tabs.warnOnClose":
False,
"browser.warnOnQuit":
False,
# Allow installing extensions dropped into the profile folder
"extensions.autoDisableScopes":
10,
"extensions.getAddons.get.url":
"http://127.0.0.1:4567/addons/api/%IDS%.json",
# Our pretend addons server doesn't support metadata...
"extensions.getAddons.cache.enabled":
False,
"extensions.install.requireSecureOrigin":
False,
"extensions.update.enabled":
False,
# Don't open a dialog to show available add-on updates
"extensions.update.notifyUser":
False,
"services.sync.firstSync":
"notReady",
"services.sync.lastversion":
"1.0",
"services.sync.autoconnectDelay":
60 * 60 * 10,
"toolkit.startup.max_resumed_crashes":
-1,
# hrm - not sure what the release/beta channels will do?
"xpinstall.signatures.required":
False,
"services.sync.testing.tps":
True,
"engine.bookmarks.repair.enabled":
False,
"extensions.experiments.enabled":
True,
"webextensions.storage.sync.kinto":
False,
}
debug_preferences = {
"services.sync.log.appender.console": "Trace",
"services.sync.log.appender.dump": "Trace",
"services.sync.log.appender.file.level": "Trace",
"services.sync.log.appender.file.logOnSuccess": True,
"services.sync.log.logger": "Trace",
"services.sync.log.logger.engine": "Trace",
}
syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n")
ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n")
ffBuildIDRe = re.compile(r"Firefox buildid: (?P<ffbuildid>.*)\n")
def __init__(
self,
extensionDir,
binary=None,
config=None,
debug=False,
ignore_unused_engines=False,
logfile="tps.log",
mobile=False,
rlock=None,
resultfile="tps_result.json",
testfile=None,
stop_on_error=False,
):
self.binary = binary
self.config = config if config else {}
self.debug = debug
self.extensions = []
self.ignore_unused_engines = ignore_unused_engines
self.logfile = os.path.abspath(logfile)
self.mobile = mobile
self.rlock = rlock
self.resultfile = resultfile
self.testfile = testfile
self.stop_on_error = stop_on_error
self.addonversion = None
self.branch = None
self.changeset = None
self.errorlogs = {}
self.extensionDir = extensionDir
self.firefoxRunner = None
self.nightly = False
self.numfailed = 0
self.numpassed = 0
self.postdata = {}
self.productversion = None
self.repo = None
self.tpsxpi = None
@property
def mobile(self):
return self._mobile
@mobile.setter
def mobile(self, value):
self._mobile = value
self.synctype = "desktop" if not self._mobile else "mobile"
def log(self, msg, printToConsole=False):
"""Appends a string to the logfile"""
f = open(self.logfile, "a")
f.write(msg)
f.close()
if printToConsole:
print(msg)
def writeToResultFile(self,
postdata,
body=None,
sendTo=["*****@*****.**"]):
"""Writes results to test file"""
results = {"results": []}
if os.access(self.resultfile, os.F_OK):
f = open(self.resultfile, "r")
results = json.loads(f.read())
f.close()
f = open(self.resultfile, "w")
if body is not None:
postdata["body"] = body
if self.numpassed is not None:
postdata["numpassed"] = self.numpassed
if self.numfailed is not None:
postdata["numfailed"] = self.numfailed
if self.firefoxRunner and self.firefoxRunner.url:
postdata["firefoxrunnerurl"] = self.firefoxRunner.url
postdata["sendTo"] = sendTo
results["results"].append(postdata)
f.write(json.dumps(results, indent=2))
f.close()
def _zip_add_file(self, zip, file, rootDir):
zip.write(os.path.join(rootDir, file), file)
def _zip_add_dir(self, zip, dir, rootDir):
try:
zip.write(os.path.join(rootDir, dir), dir)
except Exception:
# on some OS's, adding directory entries doesn't seem to work
pass
for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
for f in files:
zip.write(os.path.join(root, f), os.path.join(dir, f))
def handle_phase_failure(self, profiles):
for profile in profiles:
self.log("\nDumping sync log for profile %s\n" %
profiles[profile].profile)
for root, dirs, files in os.walk(
os.path.join(profiles[profile].profile, "weave", "logs")):
for f in files:
weavelog = os.path.join(profiles[profile].profile, "weave",
"logs", f)
if os.access(weavelog, os.F_OK):
with open(weavelog, "r") as fh:
for line in fh:
possible_time = line[0:13]
if len(possible_time
) == 13 and possible_time.isdigit():
time_ms = int(possible_time)
formatted = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(time_ms / 1000),
)
self.log(
"%s.%03d %s" %
(formatted, time_ms % 1000, line[14:]))
else:
self.log(line)
def run_single_test(self, testdir, testname):
testpath = os.path.join(testdir, testname)
self.log("Running test %s\n" % testname, True)
# Read and parse the test file, merge it with the contents of the config
# file, and write the combined output to a temporary file.
f = open(testpath, "r")
testcontent = f.read()
f.close()
# We use yaml to parse the tests because it is a superset of json
# but tolerates things like property names not being quoted, trailing
# commas, etc.
try:
test = yaml.safe_load(testcontent)
except Exception:
test = yaml.safe_load(
testcontent[testcontent.find("{"):testcontent.find("}") + 1])
self.preferences["tps.seconds_since_epoch"] = int(time.time())
# generate the profiles defined in the test, and a list of test phases
profiles = {}
phaselist = []
for phase in test:
profilename = test[phase]
# create the profile if necessary
if profilename not in profiles:
profiles[profilename] = Profile(
preferences=self.preferences.copy(),
addons=self.extensions)
# create the test phase
phaselist.append(
TPSTestPhase(
phase,
profiles[profilename],
testname,
testpath,
self.logfile,
self.env,
self.firefoxRunner,
self.log,
ignore_unused_engines=self.ignore_unused_engines,
))
# sort the phase list by name
phaselist = sorted(phaselist, key=lambda phase: phase.phase)
# run each phase in sequence, aborting at the first failure
failed = False
for phase in phaselist:
phase.run()
if phase.status != "PASS":
failed = True
break
for profilename in profiles:
print("### Cleanup Profile ", profilename)
cleanup_phase = TPSTestPhase(
"cleanup-" + profilename,
profiles[profilename],
testname,
testpath,
self.logfile,
self.env,
self.firefoxRunner,
self.log,
)
cleanup_phase.run()
if cleanup_phase.status != "PASS":
failed = True
# Keep going to run the remaining cleanup phases.
if failed:
self.handle_phase_failure(profiles)
# grep the log for FF and sync versions
f = open(self.logfile)
logdata = f.read()
match = self.syncVerRe.search(logdata)
sync_version = match.group("syncversion") if match else "unknown"
match = self.ffVerRe.search(logdata)
firefox_version = match.group("ffver") if match else "unknown"
match = self.ffBuildIDRe.search(logdata)
firefox_buildid = match.group("ffbuildid") if match else "unknown"
f.close()
if phase.status == "PASS":
logdata = ""
else:
# we only care about the log data for this specific test
logdata = logdata[logdata.find("Running test %s" %
(str(testname))):]
result = {
"PASS": lambda x: ("TEST-PASS", ""),
"FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()),
"unknown": lambda x:
("TEST-UNEXPECTED-FAIL", "test did not complete"),
}[phase.status](phase.errline)
logstr = "\n%s | %s%s\n" % (
result[0],
testname,
(" | %s" % result[1] if result[1] else ""),
)
try:
repoinfo = mozversion.get_version(self.binary)
except Exception:
repoinfo = {}
apprepo = repoinfo.get("application_repository", "")
appchangeset = repoinfo.get("application_changeset", "")
# save logdata to a temporary file for posting to the db
tmplogfile = None
if logdata:
tmplogfile = TempFile(prefix="tps_log_")
tmplogfile.write(logdata)
tmplogfile.close()
self.errorlogs[testname] = tmplogfile
resultdata = {
"productversion": {
"version": firefox_version,
"buildid": firefox_buildid,
"builddate": firefox_buildid[0:8],
"product": "Firefox",
"repository": apprepo,
"changeset": appchangeset,
},
"addonversion": {
"version": sync_version,
"product": "Firefox Sync"
},
"name": testname,
"message": result[1],
"state": result[0],
"logdata": logdata,
}
self.log(logstr, True)
for phase in phaselist:
print("\t{}: {}".format(phase.phase, phase.status))
return resultdata
def update_preferences(self):
self.preferences = self.default_preferences.copy()
if self.mobile:
self.preferences.update({"services.sync.client.type": "mobile"})
# If we are using legacy Sync, then set a dummy username to force the
# correct authentication type. Without this pref set to a value
# without an '@' character, Sync will initialize for FxA.
if self.config.get("auth_type", "fx_account") != "fx_account":
self.preferences.update({"services.sync.username": "******"})
if self.debug:
self.preferences.update(self.debug_preferences)
if "preferences" in self.config:
self.preferences.update(self.config["preferences"])
self.preferences["tps.config"] = json.dumps(self.config)
def run_tests(self):
# delete the logfile if it already exists
if os.access(self.logfile, os.F_OK):
os.remove(self.logfile)
# Copy the system env variables, and update them for custom settings
self.env = os.environ.copy()
self.env.update(self.extra_env)
# Update preferences for custom settings
self.update_preferences()
# Acquire a lock to make sure no other threads are running tests
# at the same time.
if self.rlock:
self.rlock.acquire()
try:
# Create the Firefox runner, which will download and install the
# build, as needed.
if not self.firefoxRunner:
self.firefoxRunner = TPSFirefoxRunner(self.binary)
# now, run the test group
self.run_test_group()
except Exception:
traceback.print_exc()
self.numpassed = 0
self.numfailed = 1
try:
self.writeToResultFile(
self.postdata, "<pre>%s</pre>" % traceback.format_exc())
except Exception:
traceback.print_exc()
else:
try:
if self.numfailed > 0 or self.numpassed == 0:
To = self.config["email"].get("notificationlist")
else:
To = self.config["email"].get("passednotificationlist")
self.writeToResultFile(self.postdata, sendTo=To)
except Exception:
traceback.print_exc()
try:
self.writeToResultFile(
self.postdata,
"<pre>%s</pre>" % traceback.format_exc())
except Exception:
traceback.print_exc()
# release our lock
if self.rlock:
self.rlock.release()
# dump out a summary of test results
print("Test Summary\n")
for test in self.postdata.get("tests", {}):
print("{} | {} | {}".format(test["state"], test["name"],
test["message"]))
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, "tps"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = []
for (filename, meta) in testfiles["tests"].items():
skip_reason = meta.get("disabled")
if skip_reason:
print("Skipping test {} - {}".format(
filename, skip_reason))
else:
testlist.append(filename)
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result["productversion"]
if not self.addonversion:
self.addonversion = result["addonversion"]
self.results.append({
"state": result["state"],
"name": result["name"],
"message": result["message"],
"logdata": result["logdata"],
})
if result["state"] == "TEST-PASS":
self.numpassed += 1
else:
self.numfailed += 1
if self.stop_on_error:
print("\nTest failed with --stop-on-error specified; "
"not running any more tests.\n")
break
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
"tests": self.results,
"os": "%s %sbit" % (mozinfo.version, mozinfo.bits),
"testtype": "crossweave",
"productversion": self.productversion,
"addonversion": self.addonversion,
"synctype": self.synctype,
}
class Peptest():
"""
Peptest
Runs and logs tests designed to test responsiveness
"""
profile_class = Profile
runner_class = Runner
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger('PEP')
# create the profile
enable_proxy = False
locations = ServerLocations()
if self.options.proxyLocations:
if not self.options.serverPath:
self.logger.warning('Can\'t set up proxy without server path')
else:
enable_proxy = True
for proxyLocation in self.options.proxyLocations:
locations.read(proxyLocation, False)
locations.add_host(host='127.0.0.1',
port=self.options.serverPort,
options='primary,privileged')
self.profile = self.profile_class(profile=self.options.profilePath,
addons=[os.path.join(here, 'extension')],
locations=locations,
proxy=enable_proxy)
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith('.js'):
# a single test file was passed in
testObj = {}
testObj['path'] = os.path.realpath(self.options.testPath)
testObj['name'] = os.path.basename(self.options.testPath)
testObj['here'] = os.path.dirname(testObj['path'])
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj['tests'] = tests
manifestObj['options'] = options.__dict__
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, 'manifest.json'), 'w')
jsonManifest.write(json.dumps(manifestObj))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(options.tracerThreshold)
env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval)
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(profile=self.profile,
binary=self.options.binary,
cmdargs=cmdargs,
env=env,
process_class=PepProcess)
def start(self):
self.logger.debug('Starting Peptest')
# start firefox
self.runner.start(outputTimeout=self.options.timeout)
self.runner.wait()
crashed = self.checkForCrashes(results.currentTest)
self.stop()
if crashed or results.has_fails():
return 1
return 0
def runServer(self):
"""
Start a basic HTML server to host
test related files.
"""
if not self.options.serverPath:
self.logger.warning('Can\'t start HTTP server, --server-path not specified')
return
self.logger.debug('Starting server on port ' + str(self.options.serverPort))
self.server = MozHttpd(port=self.options.serverPort,
docroot=self.options.serverPath,
proxy_host_dirs=self.options.proxyHostDirs)
self.server.start(block=False)
def stop(self):
"""Kill the app"""
# stop the runner
if self.runner is not None:
self.runner.stop()
# kill the server process
if self.server:
self.server.stop()
# remove harness related files
files = ['manifest.json']
for f in files:
if os.path.exists(os.path.join(here, f)):
os.remove(os.path.join(here, f))
# delete any minidumps that may have been created
dumpDir = os.path.join(self.profile.profile, 'minidumps')
if self.options.profilePath and os.path.exists(dumpDir):
shutil.rmtree(dumpDir)
def checkForCrashes(self, testName=None):
"""
Detects when a crash occurs and prints the output from
MINIDUMP_STACKWALK. Returns true if crash detected,
otherwise false.
"""
stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
# try to get the caller's filename if no test name is given
if testName is None:
try:
testName = os.path.basename(sys._getframe(1).f_code.co_filename)
except:
testName = "unknown"
foundCrash = False
dumpDir = os.path.join(self.profile.profile, 'minidumps')
dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
symbolsPath = self.options.symbolsPath
for d in dumps:
import subprocess
foundCrash = True
self.logger.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
print "Crash dump filename: " + d
# only proceed if a symbols path and stackwalk path were specified
if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
# if symbolsPath is a url, download and extract the zipfile
if utils.isURL(symbolsPath):
bundle = utils.download(symbolsPath, here)
symbolsPath = os.path.join(os.path.dirname(bundle), 'symbols')
utils.extract(bundle, symbolsPath, delete=True)
# run minidump_stackwalk
p = subprocess.Popen([stackwalkPath, d, symbolsPath],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = p.communicate()
if len(out) > 3:
# minidump_stackwalk is chatty, so ignore stderr when it succeeds.
print out
else:
print "stderr from minidump_stackwalk:"
print err
if p.returncode != 0:
print "minidump_stackwalk exited with return code %d" % p.returncode
else:
self.logger.warning('No symbols_path or stackwalk path specified, can\'t process dump')
break
# if the symbols path was downloaded, cleanup after ourselves
if utils.isURL(self.options.symbolsPath):
if os.path.exists(symbolsPath):
shutil.rmtree(symbolsPath)
return foundCrash
class MarionetteTestRunner(object):
textrunnerclass = MarionetteTextTestRunner
def __init__(self, address=None, emulator=None, emulatorBinary=None,
emulatorImg=None, emulator_res='480x800', homedir=None,
app=None, bin=None, profile=None, autolog=False, revision=None,
logger=None, testgroup="marionette", noWindow=False,
logcat_dir=None, xml_output=None, repeat=0, gecko_path=None,
testvars=None, tree=None, type=None, device_serial=None,
symbols_path=None, timeout=None, es_servers=None, **kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.xml_output = xml_output
self.repeat = repeat
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device_serial = device_serial
self.symbols_path = symbols_path
self.timeout = timeout
self._device = None
self._capabilities = None
self._appName = None
self.es_servers = es_servers
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
@property
def capabilities(self):
if self._capabilities:
return self._capabilities
self.marionette.start_session()
self._capabilities = self.marionette.session_capabilities
self.marionette.delete_session()
return self._capabilities
@property
def device(self):
if self._device:
return self._device
self._device = self.capabilities.get('device')
return self._device
@property
def appName(self):
if self._appName:
return self._appName
self._appName = self.capabilities.get('browserName')
return self._appName
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def start_marionette(self):
assert(self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl,
timeout=self.timeout)
elif self.address:
host, port = self.address.split(':')
try:
#establish a telnet connection so we can vertify the data come back
tlconnection = Telnet(host, port)
except:
raise Exception("could not connect to given marionette host/port")
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl,
timeout=self.timeout)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
for es_server in self.es_servers:
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup=self.testgroup,
os='android',
platform='emulator',
harness='marionette',
server=es_server,
restserver=None,
machine=socket.gethostname(),
logfile=logfile)
testgroup.set_primary_product(
tree=self.tree,
buildtype='opt',
revision=self.revision)
testgroup.add_test_suite(
testsuite='b2g emulator testsuite',
elapsedtime=elapsedtime.seconds,
cmdline='',
passed=self.passed,
failed=self.failed,
todo=self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >=0:
for test in tests:
self.run_test(test)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
if self.failed > 0:
self.logger.info('\nFAILED TESTS\n-------')
for failed_test in self.failures:
self.logger.info('%s' % failed_test[0])
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
all_tests = manifest.active_tests(disabled=False)
manifest_tests = manifest.active_tests(disabled=False,
device=self.device,
app=self.appName)
skip_tests = list(set([x['path'] for x in all_tests]) -
set([x['path'] for x in manifest_tests]))
for skipped in skip_tests:
self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
(os.path.basename(skipped),
self.device,
self.appName))
self.todo += 1
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name,
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
def register_handlers(self):
self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase])
def cleanup(self):
if self.httpd:
self.httpd.stop()
__del__ = cleanup
def generate_xml(self, results_list):
def _extract_xml(test, text='', result='passed'):
cls_name = test.__class__.__name__
testcase = doc.createElement('testcase')
testcase.setAttribute('classname', cls_name)
testcase.setAttribute('name', unicode(test).split()[0])
testcase.setAttribute('time', str(test.duration))
testsuite.appendChild(testcase)
if result in ['failure', 'error', 'skipped']:
f = doc.createElement(result)
f.setAttribute('message', 'test %s' % result)
f.appendChild(doc.createTextNode(text))
testcase.appendChild(f)
doc = dom.Document()
testsuite = doc.createElement('testsuite')
testsuite.setAttribute('name', 'Marionette')
testsuite.setAttribute('time', str(self.elapsedtime.total_seconds()))
testsuite.setAttribute('tests', str(sum([results.testsRun for
results in results_list])))
def failed_count(results):
count = len(results.failures)
if hasattr(results, 'unexpectedSuccesses'):
count += len(results.unexpectedSuccesses)
return count
testsuite.setAttribute('failures', str(sum([failed_count(results)
for results in results_list])))
testsuite.setAttribute('errors', str(sum([len(results.errors)
for results in results_list])))
if hasattr(results, 'skipped'):
testsuite.setAttribute('skips', str(sum([len(results.skipped) +
len(results.expectedFailures)
for results in results_list])))
for results in results_list:
for tup in results.errors:
_extract_xml(tup[0], text=tup[1], result='error')
for tup in results.failures:
_extract_xml(tup[0], text=tup[1], result='failure')
if hasattr(results, 'unexpectedSuccesses'):
for test in results.unexpectedSuccesses:
# unexpectedSuccesses is a list of Testcases only, no tuples
_extract_xml(test, text='TEST-UNEXPECTED-PASS', result='failure')
if hasattr(results, 'skipped'):
for tup in results.skipped:
_extract_xml(tup[0], text=tup[1], result='skipped')
if hasattr(results, 'expectedFailures'):
for tup in results.expectedFailures:
_extract_xml(tup[0], text=tup[1], result='skipped')
for test in results.tests_passed:
_extract_xml(test)
doc.appendChild(testsuite)
return doc.toprettyxml(encoding='utf-8')
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, "build")
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo"))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(self.topsrcdir, "testing",
"profiles")
with open(os.path.join(profile_data_dir, "profiles.json"),
"r") as fh:
base_profiles = json.load(fh)["valgrind"]
prefpaths = [
os.path.join(profile_data_dir, profile, "user.js")
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(self.topsrcdir, "tools", "quitter",
"*****@*****.**")
locations = ServerLocations()
locations.add_host(host="127.0.0.1",
port=httpd.httpd.server_port,
options="primary")
profile = FirefoxProfile(
profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations,
)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env["G_SLICE"] = "always-malloc"
env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
outputHandler = OutputHandler(self.log)
kp_kwargs = {
"processOutputLine": [outputHandler],
"universal_newlines": True,
}
valgrind = "valgrind"
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
"--sym-offsets=yes",
"--smc-check=all-non-file",
"--vex-iropt-register-updates=allregs-at-mem-access",
"--gen-suppressions=all",
"--num-callers=36",
"--leak-check=full",
"--show-possibly-lost=no",
"--track-origins=yes",
"--trace-children=yes",
"-v", # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
"--fair-sched=yes",
# Keep debuginfo after library unmap. See bug 1382280.
"--keep-debuginfo=yes",
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
"--expensive-definedness-checks=yes",
# Compensate for the compiler inlining `new` but not `delete`
# or vice versa.
"--show-mismatched-frees=no",
]
for s in suppressions:
valgrind_args.append("--suppressions=" + s)
supps_dir = os.path.join(build_dir, "valgrind")
supps_file1 = os.path.join(supps_dir, "cross-architecture.sup")
valgrind_args.append("--suppressions=" + supps_file1)
if mozinfo.os == "linux":
machtype = {
"x86_64": "x86_64-pc-linux-gnu",
"x86": "i386-pc-linux-gnu",
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + ".sup")
if os.path.isfile(supps_file2):
valgrind_args.append("--suppressions=" + supps_file2)
exitcode = None
timeout = 1800
binary_not_found_exception = None
try:
runner = FirefoxRunner(
profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs,
)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
except BinaryNotFoundException as e:
binary_not_found_exception = e
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR,
"valgrind-fail-parsing",
{
"errs": errs,
"supps": supps
},
"TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors "
"seen, but {supps} generated suppressions seen",
)
elif errs == 0:
status = 0
self.log(
logging.INFO,
"valgrind-pass",
{},
"TEST-PASS | valgrind-test | valgrind found no errors",
)
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if binary_not_found_exception:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{"error": str(binary_not_found_exception)},
"TEST-UNEXPECTED-FAIL | valgrind-test | {error}",
)
self.log(
logging.INFO,
"valgrind-fail-errors",
{"help": binary_not_found_exception.help()},
"{help}",
)
elif exitcode is None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-timeout",
{"timeout": timeout},
"TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out "
"(reached {timeout} second limit)",
)
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{"exitcode": exitcode},
"TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code "
"from Valgrind: {exitcode}",
)
httpd.stop()
return status
def valgrind_test(self, suppressions):
import json
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
#TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js')
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = { 'server': '%s:%d' % httpd.httpd.server_address,
'OOP': 'false'}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', '*****@*****.**')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
'-v', # Enable verbosity to get the list of used suppressions
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(logging.ERROR, 'valgrind-fail-parsing',
{'errs': errs, 'supps': supps},
'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen')
elif errs == 0:
status = 0
self.log(logging.INFO, 'valgrind-pass', {},
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode == None:
status = 2 # turns the TBPL job red
self.log(logging.ERROR, 'valgrind-fail-timeout',
{'timeout': timeout},
'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)')
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(logging.ERROR, 'valgrind-fail-errors', {},
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind')
httpd.stop()
return status
class TPSTestRunner(object):
extra_env = {
'MOZ_CRASHREPORTER_DISABLE': '1',
'GNOME_DISABLE_CRASH_DIALOG': '1',
'XRE_NO_WINDOWS_CRASH_DIALOG': '1',
'MOZ_NO_REMOTE': '1',
'XPCOM_DEBUG_BREAK': 'warn',
}
default_preferences = {
'app.update.checkInstallTime':
False,
'app.update.disabledForTesting':
True,
'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer':
True,
'browser.dom.window.dump.enabled':
True,
'devtools.console.stdout.chrome':
True,
'browser.sessionstore.resume_from_crash':
False,
'browser.shell.checkDefaultBrowser':
False,
'browser.tabs.warnOnClose':
False,
'browser.warnOnQuit':
False,
# Allow installing extensions dropped into the profile folder
'extensions.autoDisableScopes':
10,
'extensions.getAddons.get.url':
'http://127.0.0.1:4567/addons/api/%IDS%.json',
# Our pretend addons server doesn't support metadata...
'extensions.getAddons.cache.enabled':
False,
'extensions.install.requireSecureOrigin':
False,
'extensions.update.enabled':
False,
# Don't open a dialog to show available add-on updates
'extensions.update.notifyUser':
False,
'services.sync.firstSync':
'notReady',
'services.sync.lastversion':
'1.0',
'services.sync.autoconnectDelay':
60 * 60 * 10,
'toolkit.startup.max_resumed_crashes':
-1,
# hrm - not sure what the release/beta channels will do?
'xpinstall.signatures.required':
False,
'services.sync.testing.tps':
True,
'engine.bookmarks.repair.enabled':
False,
'extensions.experiments.enabled':
True,
'webextensions.storage.sync.kinto':
False,
}
debug_preferences = {
'services.sync.log.appender.console': 'Trace',
'services.sync.log.appender.dump': 'Trace',
'services.sync.log.appender.file.level': 'Trace',
'services.sync.log.appender.file.logOnSuccess': True,
'services.sync.log.logger': 'Trace',
'services.sync.log.logger.engine': 'Trace',
}
syncVerRe = re.compile(r'Sync version: (?P<syncversion>.*)\n')
ffVerRe = re.compile(r'Firefox version: (?P<ffver>.*)\n')
ffBuildIDRe = re.compile(r'Firefox buildid: (?P<ffbuildid>.*)\n')
def __init__(self,
extensionDir,
binary=None,
config=None,
debug=False,
ignore_unused_engines=False,
logfile='tps.log',
mobile=False,
rlock=None,
resultfile='tps_result.json',
testfile=None,
stop_on_error=False):
self.binary = binary
self.config = config if config else {}
self.debug = debug
self.extensions = []
self.ignore_unused_engines = ignore_unused_engines
self.logfile = os.path.abspath(logfile)
self.mobile = mobile
self.rlock = rlock
self.resultfile = resultfile
self.testfile = testfile
self.stop_on_error = stop_on_error
self.addonversion = None
self.branch = None
self.changeset = None
self.errorlogs = {}
self.extensionDir = extensionDir
self.firefoxRunner = None
self.nightly = False
self.numfailed = 0
self.numpassed = 0
self.postdata = {}
self.productversion = None
self.repo = None
self.tpsxpi = None
@property
def mobile(self):
return self._mobile
@mobile.setter
def mobile(self, value):
self._mobile = value
self.synctype = 'desktop' if not self._mobile else 'mobile'
def log(self, msg, printToConsole=False):
"""Appends a string to the logfile"""
f = open(self.logfile, 'a')
f.write(msg)
f.close()
if printToConsole:
print(msg)
def writeToResultFile(self,
postdata,
body=None,
sendTo=['*****@*****.**']):
"""Writes results to test file"""
results = {'results': []}
if os.access(self.resultfile, os.F_OK):
f = open(self.resultfile, 'r')
results = json.loads(f.read())
f.close()
f = open(self.resultfile, 'w')
if body is not None:
postdata['body'] = body
if self.numpassed is not None:
postdata['numpassed'] = self.numpassed
if self.numfailed is not None:
postdata['numfailed'] = self.numfailed
if self.firefoxRunner and self.firefoxRunner.url:
postdata['firefoxrunnerurl'] = self.firefoxRunner.url
postdata['sendTo'] = sendTo
results['results'].append(postdata)
f.write(json.dumps(results, indent=2))
f.close()
def _zip_add_file(self, zip, file, rootDir):
zip.write(os.path.join(rootDir, file), file)
def _zip_add_dir(self, zip, dir, rootDir):
try:
zip.write(os.path.join(rootDir, dir), dir)
except Exception:
# on some OS's, adding directory entries doesn't seem to work
pass
for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
for f in files:
zip.write(os.path.join(root, f), os.path.join(dir, f))
def handle_phase_failure(self, profiles):
for profile in profiles:
self.log('\nDumping sync log for profile %s\n' %
profiles[profile].profile)
for root, dirs, files in os.walk(
os.path.join(profiles[profile].profile, 'weave', 'logs')):
for f in files:
weavelog = os.path.join(profiles[profile].profile, 'weave',
'logs', f)
if os.access(weavelog, os.F_OK):
with open(weavelog, 'r') as fh:
for line in fh:
possible_time = line[0:13]
if len(possible_time
) == 13 and possible_time.isdigit():
time_ms = int(possible_time)
formatted = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time_ms / 1000))
self.log(
'%s.%03d %s' %
(formatted, time_ms % 1000, line[14:]))
else:
self.log(line)
def run_single_test(self, testdir, testname):
testpath = os.path.join(testdir, testname)
self.log("Running test %s\n" % testname, True)
# Read and parse the test file, merge it with the contents of the config
# file, and write the combined output to a temporary file.
f = open(testpath, 'r')
testcontent = f.read()
f.close()
# We use yaml to parse the tests because it is a superset of json
# but tolerates things like property names not being quoted, trailing
# commas, etc.
try:
test = yaml.safe_load(testcontent)
except Exception:
test = yaml.safe_load(
testcontent[testcontent.find('{'):testcontent.find('}') + 1])
self.preferences['tps.seconds_since_epoch'] = int(time.time())
# generate the profiles defined in the test, and a list of test phases
profiles = {}
phaselist = []
for phase in test:
profilename = test[phase]
# create the profile if necessary
if profilename not in profiles:
profiles[profilename] = Profile(
preferences=self.preferences.copy(),
addons=self.extensions)
# create the test phase
phaselist.append(
TPSTestPhase(phase,
profiles[profilename],
testname,
testpath,
self.logfile,
self.env,
self.firefoxRunner,
self.log,
ignore_unused_engines=self.ignore_unused_engines))
# sort the phase list by name
phaselist = sorted(phaselist, key=lambda phase: phase.phase)
# run each phase in sequence, aborting at the first failure
failed = False
for phase in phaselist:
phase.run()
if phase.status != 'PASS':
failed = True
break
for profilename in profiles:
print("### Cleanup Profile ", profilename)
cleanup_phase = TPSTestPhase('cleanup-' + profilename,
profiles[profilename], testname,
testpath, self.logfile, self.env,
self.firefoxRunner, self.log)
cleanup_phase.run()
if cleanup_phase.status != 'PASS':
failed = True
# Keep going to run the remaining cleanup phases.
if failed:
self.handle_phase_failure(profiles)
# grep the log for FF and sync versions
f = open(self.logfile)
logdata = f.read()
match = self.syncVerRe.search(logdata)
sync_version = match.group('syncversion') if match else 'unknown'
match = self.ffVerRe.search(logdata)
firefox_version = match.group('ffver') if match else 'unknown'
match = self.ffBuildIDRe.search(logdata)
firefox_buildid = match.group('ffbuildid') if match else 'unknown'
f.close()
if phase.status == 'PASS':
logdata = ''
else:
# we only care about the log data for this specific test
logdata = logdata[logdata.find('Running test %s' %
(str(testname))):]
result = {
'PASS': lambda x: ('TEST-PASS', ''),
'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()),
'unknown': lambda x:
('TEST-UNEXPECTED-FAIL', 'test did not complete')
}[phase.status](phase.errline)
logstr = "\n%s | %s%s\n" % (result[0], testname,
(' | %s' % result[1] if result[1] else ''))
try:
repoinfo = mozversion.get_version(self.binary)
except Exception:
repoinfo = {}
apprepo = repoinfo.get('application_repository', '')
appchangeset = repoinfo.get('application_changeset', '')
# save logdata to a temporary file for posting to the db
tmplogfile = None
if logdata:
tmplogfile = TempFile(prefix='tps_log_')
tmplogfile.write(logdata)
tmplogfile.close()
self.errorlogs[testname] = tmplogfile
resultdata = ({
'productversion': {
'version': firefox_version,
'buildid': firefox_buildid,
'builddate': firefox_buildid[0:8],
'product': 'Firefox',
'repository': apprepo,
'changeset': appchangeset,
},
'addonversion': {
'version': sync_version,
'product': 'Firefox Sync'
},
'name': testname,
'message': result[1],
'state': result[0],
'logdata': logdata
})
self.log(logstr, True)
for phase in phaselist:
print("\t{}: {}".format(phase.phase, phase.status))
return resultdata
def update_preferences(self):
self.preferences = self.default_preferences.copy()
if self.mobile:
self.preferences.update({'services.sync.client.type': 'mobile'})
# If we are using legacy Sync, then set a dummy username to force the
# correct authentication type. Without this pref set to a value
# without an '@' character, Sync will initialize for FxA.
if self.config.get('auth_type', 'fx_account') != "fx_account":
self.preferences.update({'services.sync.username': "******"})
if self.debug:
self.preferences.update(self.debug_preferences)
if 'preferences' in self.config:
self.preferences.update(self.config['preferences'])
self.preferences['tps.config'] = json.dumps(self.config)
def run_tests(self):
# delete the logfile if it already exists
if os.access(self.logfile, os.F_OK):
os.remove(self.logfile)
# Copy the system env variables, and update them for custom settings
self.env = os.environ.copy()
self.env.update(self.extra_env)
# Update preferences for custom settings
self.update_preferences()
# Acquire a lock to make sure no other threads are running tests
# at the same time.
if self.rlock:
self.rlock.acquire()
try:
# Create the Firefox runner, which will download and install the
# build, as needed.
if not self.firefoxRunner:
self.firefoxRunner = TPSFirefoxRunner(self.binary)
# now, run the test group
self.run_test_group()
except Exception:
traceback.print_exc()
self.numpassed = 0
self.numfailed = 1
try:
self.writeToResultFile(
self.postdata, '<pre>%s</pre>' % traceback.format_exc())
except Exception:
traceback.print_exc()
else:
try:
if self.numfailed > 0 or self.numpassed == 0:
To = self.config['email'].get('notificationlist')
else:
To = self.config['email'].get('passednotificationlist')
self.writeToResultFile(self.postdata, sendTo=To)
except Exception:
traceback.print_exc()
try:
self.writeToResultFile(
self.postdata,
'<pre>%s</pre>' % traceback.format_exc())
except Exception:
traceback.print_exc()
# release our lock
if self.rlock:
self.rlock.release()
# dump out a summary of test results
print('Test Summary\n')
for test in self.postdata.get('tests', {}):
print('{} | {} | {}'.format(test['state'], test['name'],
test['message']))
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, 'tps'))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = []
for (filename, meta) in testfiles['tests'].items():
skip_reason = meta.get("disabled")
if skip_reason:
print('Skipping test {} - {}'.format(
filename, skip_reason))
else:
testlist.append(filename)
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result['productversion']
if not self.addonversion:
self.addonversion = result['addonversion']
self.results.append({
'state': result['state'],
'name': result['name'],
'message': result['message'],
'logdata': result['logdata']
})
if result['state'] == 'TEST-PASS':
self.numpassed += 1
else:
self.numfailed += 1
if self.stop_on_error:
print('\nTest failed with --stop-on-error specified; '
'not running any more tests.\n')
break
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
'tests': self.results,
'os': '%s %sbit' % (mozinfo.version, mozinfo.bits),
'testtype': 'crossweave',
'productversion': self.productversion,
'addonversion': self.addonversion,
'synctype': self.synctype,
}
def main(args):
parser = Options()
options, args = parser.parse_args()
kill_port = 20703
if not options.html_manifest or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server:
parser.print_usage()
return 2
package_options = get_package_options(parser, options)
if not package_options:
parser.print_usage()
return 2
if not os.path.isdir(options.specialpowers):
parser.error("SpecialPowers directory %s does not exist" %
options.specialpowers)
return 2
if options.prefs and not os.path.isfile(options.prefs):
parser.error("Prefs file %s does not exist" % options.prefs)
return 2
if options.log_dest and not os.path.isdir(options.log_dest):
parser.error("Log directory %s does not exist" % options.log_dest)
return 2
log = mozlog.unstructured.getLogger('steeplechase')
log.setLevel(logging.DEBUG)
if ':' in options.host1:
host1, port = options.host1.split(':')
dm1 = DeviceManagerSUT(host1, port)
else:
dm1 = DeviceManagerSUT(options.host1)
if ':' in options.host2:
host2, port = options.host2.split(':')
dm2 = DeviceManagerSUT(host2, port)
else:
dm2 = DeviceManagerSUT(options.host2)
if (options.killall is not None) and (options.killall == 1):
kill_dm1 = DeviceManagerSUT(host1, kill_port)
kill_dm2 = DeviceManagerSUT(host2, kill_port)
os_type = GetOStypes(package_options)
print("OS type of host1 is " + os_type[0] + " and host2 is " +
os_type[1])
KillFirefoxesCommand(kill_dm1, os_type[0])
KillFirefoxesCommand(kill_dm2, os_type[1])
remote_info = [{
'dm': dm1,
'binary': package_options.binary,
'package': package_options.package,
'is_initiator': True,
'name': 'Client1'
}, {
'dm': dm2,
'binary': package_options.binary2,
'package': package_options.package2,
'is_initiator': False,
'name': 'Client2'
}]
# first, push app
for info in remote_info:
dm = info['dm']
if info['binary']:
asset = Binary(path=info['binary'],
log=log,
dm=info['dm'],
name=info['name'])
else:
asset = generate_package_asset(path=info['package'],
log=log,
dm=info['dm'],
name=info['name'])
if options.setup:
asset.setup_test_root()
info['test_root'] = asset.test_root()
if options.setup:
log.info("Pushing app to %s...", info["name"])
asset.setup_client()
info['remote_app_path'] = asset.path_to_launch()
if not options.setup and not dm.fileExists(info['remote_app_path']):
log.error("App does not exist on %s, don't use --noSetup",
info['name'])
return 2
pass_count, fail_count = 0, 0
if options.html_manifest:
manifest = TestManifest(strict=False)
manifest.read(options.html_manifest)
manifest_data = {
"tests": [{
"path": t["relpath"]
} for t in manifest.active_tests(disabled=False, **mozinfo.info)]
}
remote_port = 0
if options.remote_webserver:
result = re.search(':(\d+)', options.remote_webserver)
if result:
remote_port = int(result.groups()[0])
@json_response
def get_manifest(req):
return (200, manifest_data)
handlers = [{
'method': 'GET',
'path': '/manifest.json',
'function': get_manifest
}]
httpd = MozHttpd(
host=moznetwork.get_ip(),
port=remote_port,
log_requests=True,
docroot=os.path.join(os.path.dirname(__file__), "..",
"webharness"),
urlhandlers=handlers,
path_mappings={"/tests": os.path.dirname(options.html_manifest)})
httpd.start(block=False)
test = HTMLTests(httpd, remote_info, log, options)
html_pass_count, html_fail_count = test.run()
pass_count += html_pass_count
fail_count += html_fail_count
httpd.stop()
log.info("Result summary:")
log.info("Passed: %d" % pass_count)
log.info("Failed: %d" % fail_count)
return pass_count > 0 and fail_count == 0
class MarionetteTestRunner(object):
def __init__(self, address=None, emulator=None, emulatorBinary=None,
emulatorImg=None, emulator_res='480x800', homedir=None,
bin=None, profile=None, autolog=False, revision=None,
es_server=None, rest_server=None, logger=None,
testgroup="marionette", noWindow=False, logcat_dir=None,
xml_output=None, repeat=0, perf=False, perfserv=None,
gecko_path=None, testvars=None, tree=None, device=None):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.perfrequest = None
self.xml_output = xml_output
self.repeat = repeat
self.perf = perf
self.perfserv = perfserv
self.gecko_path = gecko_path
self.testvars = {}
self.tree = tree
self.device = device
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
self.perfrequest = None
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def start_marionette(self):
assert(self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host, port=int(port),
bin=self.bin, profile=self.profile,
baseurl=self.baseurl)
elif self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup = self.testgroup,
os = 'android',
platform = 'emulator',
harness = 'marionette',
server = self.es_server,
restserver = self.rest_server,
machine = socket.gethostname(),
logfile = logfile)
testgroup.set_primary_product(
tree = self.tree,
buildtype = 'opt',
revision = self.revision)
testgroup.add_test_suite(
testsuite = 'b2g emulator testsuite',
elapsedtime = elapsedtime.seconds,
cmdline = '',
passed = self.passed,
failed = self.failed,
todo = self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests, testtype=None):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >=0:
for test in tests:
self.run_test(test, testtype)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.perfrequest and options.perf:
try:
self.perfrequest.submit()
except Exception, e:
print "Could not submit to datazilla"
print e
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(self.topsrcdir, 'testing',
'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['valgrind']
prefpaths = [
os.path.join(profile_data_dir, profile, 'user.js')
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
'server': '%s:%d' % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(self.topsrcdir, 'tools', 'quitter',
'*****@*****.**')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--sym-offsets=yes',
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
'-v', # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
'--fair-sched=yes',
# Keep debuginfo after library unmap. See bug 1382280.
'--keep-debuginfo=yes',
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
'--expensive-definedness-checks=yes',
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
if mozinfo.os == 'linux':
machtype = {
'x86_64': 'x86_64-pc-linux-gnu',
'x86': 'i386-pc-linux-gnu',
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR, 'valgrind-fail-parsing', {
'errs': errs,
'supps': supps
},
'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors '
'seen, but {supps} generated suppressions seen')
elif errs == 0:
status = 0
self.log(
logging.INFO, 'valgrind-pass', {},
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode is None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR, 'valgrind-fail-timeout',
{'timeout': timeout},
'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out '
'(reached {timeout} second limit)')
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR, 'valgrind-fail-errors',
{'exitcode': exitcode},
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code '
'from Valgrind: {exitcode}')
httpd.stop()
return status
class TPSTestRunner(object):
default_env = { 'MOZ_CRASHREPORTER_DISABLE': '1',
'GNOME_DISABLE_CRASH_DIALOG': '1',
'XRE_NO_WINDOWS_CRASH_DIALOG': '1',
'MOZ_NO_REMOTE': '1',
'XPCOM_DEBUG_BREAK': 'warn',
}
default_preferences = { 'app.update.enabled' : False,
'browser.dom.window.dump.enabled': True,
'browser.sessionstore.resume_from_crash': False,
'browser.shell.checkDefaultBrowser' : False,
'browser.tabs.warnOnClose' : False,
'browser.warnOnQuit': False,
# Allow installing extensions dropped into the profile folder
'extensions.autoDisableScopes': 10,
'extensions.getAddons.get.url': 'http://127.0.0.1:4567/addons/api/%IDS%.xml',
'extensions.update.enabled' : False,
# Don't open a dialog to show available add-on updates
'extensions.update.notifyUser' : False,
'services.sync.addons.ignoreRepositoryChecking': True,
'services.sync.firstSync': 'notReady',
'services.sync.lastversion': '1.0',
'services.sync.log.rootLogger': 'Trace',
'services.sync.log.logger.engine.addons': 'Trace',
'services.sync.log.logger.service.main': 'Trace',
'services.sync.log.logger.engine.bookmarks': 'Trace',
'services.sync.log.appender.console': 'Trace',
'services.sync.log.appender.debugLog.enabled': True,
'toolkit.startup.max_resumed_crashes': -1,
}
syncVerRe = re.compile(
r'Sync version: (?P<syncversion>.*)\n')
ffVerRe = re.compile(
r'Firefox version: (?P<ffver>.*)\n')
ffDateRe = re.compile(
r'Firefox builddate: (?P<ffdate>.*)\n')
def __init__(self, extensionDir,
testfile='sync.test',
binary=None, config=None, rlock=None, mobile=False,
logfile='tps.log', resultfile='tps_result.json',
ignore_unused_engines=False):
self.extensions = []
self.testfile = testfile
self.logfile = os.path.abspath(logfile)
self.resultfile = resultfile
self.binary = binary
self.ignore_unused_engines = ignore_unused_engines
self.config = config if config else {}
self.repo = None
self.changeset = None
self.branch = None
self.numfailed = 0
self.numpassed = 0
self.nightly = False
self.rlock = rlock
self.mobile = mobile
self.tpsxpi = None
self.firefoxRunner = None
self.extensionDir = extensionDir
self.productversion = None
self.addonversion = None
self.postdata = {}
self.errorlogs = {}
@property
def mobile(self):
return self._mobile
@mobile.setter
def mobile(self, value):
self._mobile = value
self.synctype = 'desktop' if not self._mobile else 'mobile'
def log(self, msg, printToConsole=False):
"""Appends a string to the logfile"""
f = open(self.logfile, 'a')
f.write(msg)
f.close()
if printToConsole:
print msg
def writeToResultFile(self, postdata, body=None,
sendTo=['*****@*****.**']):
"""Writes results to test file"""
results = {'results': []}
if os.access(self.resultfile, os.F_OK):
f = open(self.resultfile, 'r')
results = json.loads(f.read())
f.close()
f = open(self.resultfile, 'w')
if body is not None:
postdata['body'] = body
if self.numpassed is not None:
postdata['numpassed'] = self.numpassed
if self.numfailed is not None:
postdata['numfailed'] = self.numfailed
if self.firefoxRunner and self.firefoxRunner.url:
postdata['firefoxrunnerurl'] = self.firefoxRunner.url
postdata['sendTo'] = sendTo
results['results'].append(postdata)
f.write(json.dumps(results, indent=2))
f.close()
def _zip_add_file(self, zip, file, rootDir):
zip.write(os.path.join(rootDir, file), file)
def _zip_add_dir(self, zip, dir, rootDir):
try:
zip.write(os.path.join(rootDir, dir), dir)
except:
# on some OS's, adding directory entries doesn't seem to work
pass
for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
for f in files:
zip.write(os.path.join(root, f), os.path.join(dir, f))
def run_single_test(self, testdir, testname):
testpath = os.path.join(testdir, testname)
self.log("Running test %s\n" % testname, True)
# Read and parse the test file, merge it with the contents of the config
# file, and write the combined output to a temporary file.
f = open(testpath, 'r')
testcontent = f.read()
f.close()
try:
test = json.loads(testcontent)
except:
test = json.loads(testcontent[testcontent.find('{'):testcontent.find('}') + 1])
testcontent += 'var config = %s;\n' % json.dumps(self.config, indent=2)
testcontent += 'var seconds_since_epoch = %d;\n' % int(time.time())
tmpfile = TempFile(prefix='tps_test_')
tmpfile.write(testcontent)
tmpfile.close()
# generate the profiles defined in the test, and a list of test phases
profiles = {}
phaselist = []
for phase in test:
profilename = test[phase]
# create the profile if necessary
if not profilename in profiles:
profiles[profilename] = Profile(preferences = self.preferences,
addons = self.extensions)
# create the test phase
phaselist.append(TPSTestPhase(
phase,
profiles[profilename],
testname,
tmpfile.filename,
self.logfile,
self.env,
self.firefoxRunner,
self.log,
ignore_unused_engines=self.ignore_unused_engines))
# sort the phase list by name
phaselist = sorted(phaselist, key=lambda phase: phase.phase)
# run each phase in sequence, aborting at the first failure
for phase in phaselist:
phase.run()
# if a failure occurred, dump the entire sync log into the test log
if phase.status != 'PASS':
for profile in profiles:
self.log('\nDumping sync log for profile %s\n' % profiles[profile].profile)
for root, dirs, files in os.walk(os.path.join(profiles[profile].profile, 'weave', 'logs')):
for f in files:
weavelog = os.path.join(profiles[profile].profile, 'weave', 'logs', f)
if os.access(weavelog, os.F_OK):
with open(weavelog, 'r') as fh:
for line in fh:
possible_time = line[0:13]
if len(possible_time) == 13 and possible_time.isdigit():
time_ms = int(possible_time)
formatted = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time_ms / 1000))
self.log('%s.%03d %s' % (
formatted, time_ms % 1000, line[14:] ))
else:
self.log(line)
break;
# grep the log for FF and sync versions
f = open(self.logfile)
logdata = f.read()
match = self.syncVerRe.search(logdata)
sync_version = match.group('syncversion') if match else 'unknown'
match = self.ffVerRe.search(logdata)
firefox_version = match.group('ffver') if match else 'unknown'
match = self.ffDateRe.search(logdata)
firefox_builddate = match.group('ffdate') if match else 'unknown'
f.close()
if phase.status == 'PASS':
logdata = ''
else:
# we only care about the log data for this specific test
logdata = logdata[logdata.find('Running test %s' % (str(testname))):]
result = {
'PASS': lambda x: ('TEST-PASS', ''),
'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()),
'unknown': lambda x: ('TEST-UNEXPECTED-FAIL', 'test did not complete')
} [phase.status](phase.errline)
logstr = "\n%s | %s%s\n" % (result[0], testname, (' | %s' % result[1] if result[1] else ''))
try:
repoinfo = self.firefoxRunner.runner.get_repositoryInfo()
except:
repoinfo = {}
apprepo = repoinfo.get('application_repository', '')
appchangeset = repoinfo.get('application_changeset', '')
# save logdata to a temporary file for posting to the db
tmplogfile = None
if logdata:
tmplogfile = TempFile(prefix='tps_log_')
tmplogfile.write(logdata)
tmplogfile.close()
self.errorlogs[testname] = tmplogfile
resultdata = ({ 'productversion': { 'version': firefox_version,
'buildid': firefox_builddate,
'builddate': firefox_builddate[0:8],
'product': 'Firefox',
'repository': apprepo,
'changeset': appchangeset,
},
'addonversion': { 'version': sync_version,
'product': 'Firefox Sync' },
'name': testname,
'message': result[1],
'state': result[0],
'logdata': logdata
})
self.log(logstr, True)
for phase in phaselist:
print "\t%s: %s" % (phase.phase, phase.status)
if phase.status == 'FAIL':
break
return resultdata
def run_tests(self):
# delete the logfile if it already exists
if os.access(self.logfile, os.F_OK):
os.remove(self.logfile)
# Make a copy of the default env variables and preferences, and update
# them for mobile settings if needed.
self.env = self.default_env.copy()
self.preferences = self.default_preferences.copy()
if self.mobile:
self.preferences.update({'services.sync.client.type' : 'mobile'})
# Set a dummy username to force the correct authentication type. For the
# old sync, the username is not allowed to contain a '@'.
dummy = {'fx_account': 'dummy@somewhere', 'sync_account': 'dummy'}
auth_type = self.config.get('auth_type', 'fx_account')
self.preferences.update({'services.sync.username': dummy[auth_type]})
# Acquire a lock to make sure no other threads are running tests
# at the same time.
if self.rlock:
self.rlock.acquire()
try:
# Create the Firefox runner, which will download and install the
# build, as needed.
if not self.firefoxRunner:
self.firefoxRunner = TPSFirefoxRunner(self.binary)
# now, run the test group
self.run_test_group()
except:
traceback.print_exc()
self.numpassed = 0
self.numfailed = 1
try:
self.writeToResultFile(self.postdata,
'<pre>%s</pre>' % traceback.format_exc())
except:
traceback.print_exc()
else:
try:
if self.numfailed > 0 or self.numpassed == 0:
To = self.config['email'].get('notificationlist')
else:
To = self.config['email'].get('passednotificationlist')
self.writeToResultFile(self.postdata,
sendTo=To)
except:
traceback.print_exc()
try:
self.writeToResultFile(self.postdata,
'<pre>%s</pre>' % traceback.format_exc())
except:
traceback.print_exc()
# release our lock
if self.rlock:
self.rlock.release()
# dump out a summary of test results
print 'Test Summary\n'
for test in self.postdata.get('tests', {}):
print '%s | %s | %s' % (test['state'], test['name'], test['message'])
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, 'tps'))
self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = testfiles['tests']
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result['productversion']
if not self.addonversion:
self.addonversion = result['addonversion']
self.results.append({'state': result['state'],
'name': result['name'],
'message': result['message'],
'logdata': result['logdata']})
if result['state'] == 'TEST-PASS':
self.numpassed += 1
else:
self.numfailed += 1
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = { 'tests': self.results,
'os': '%s %sbit' % (mozinfo.version, mozinfo.bits),
'testtype': 'crossweave',
'productversion': self.productversion,
'addonversion': self.addonversion,
'synctype': self.synctype,
}
def main(args):
parser = Options()
options, args = parser.parse_args()
if not options.html_manifest or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server:
parser.print_usage()
return 2
package_options = get_package_options(parser, options)
if not package_options:
parser.print_usage()
return 2
if not os.path.isdir(options.specialpowers):
parser.error("SpecialPowers directory %s does not exist" % options.specialpowers)
return 2
if options.prefs and not os.path.isfile(options.prefs):
parser.error("Prefs file %s does not exist" % options.prefs)
return 2
if options.log_dest and not os.path.isdir(options.log_dest):
parser.error("Log directory %s does not exist" % options.log_dest)
return 2
log = mozlog.getLogger('steeplechase')
log.setLevel(mozlog.DEBUG)
if ':' in options.host1:
host, port = options.host1.split(':')
dm1 = DeviceManagerSUT(host, port)
else:
dm1 = DeviceManagerSUT(options.host1)
if ':' in options.host2:
host, port = options.host2.split(':')
dm2 = DeviceManagerSUT(host, port)
else:
dm2 = DeviceManagerSUT(options.host2)
remote_info = [{'dm': dm1,
'binary': package_options.binary,
'package': package_options.package,
'is_initiator': True,
'name': 'Client1'},
{'dm': dm2,
'binary': package_options.binary2,
'package': package_options.package2,
'is_initiator': False,
'name': 'Client2'}]
# first, push app
for info in remote_info:
dm = info['dm']
if info['binary']:
asset = Binary(path=info['binary'], log=log, dm=info['dm'], name=info['name'])
else:
asset = generate_package_asset(path=info['package'], log=log, dm=info['dm'], name=info['name'])
if options.setup:
asset.setup_test_root()
info['test_root'] = asset.test_root()
if options.setup:
log.info("Pushing app to %s...", info["name"])
asset.setup_client()
info['remote_app_path'] = asset.path_to_launch()
if not options.setup and not dm.fileExists(info['remote_app_path']):
log.error("App does not exist on %s, don't use --noSetup", info['name'])
return 2
pass_count, fail_count = 0, 0
if options.html_manifest:
manifest = TestManifest(strict=False)
manifest.read(options.html_manifest)
manifest_data = {"tests": [{"path": t["relpath"]} for t in manifest.active_tests(disabled=False, **mozinfo.info)]}
remote_port = 0
if options.remote_webserver:
result = re.search(':(\d+)', options.remote_webserver)
if result:
remote_port = int(result.groups()[0])
@json_response
def get_manifest(req):
return (200, manifest_data)
handlers = [{
'method': 'GET',
'path': '/manifest.json',
'function': get_manifest
}]
httpd = MozHttpd(host=moznetwork.get_ip(), port=remote_port, log_requests=True,
docroot=os.path.join(os.path.dirname(__file__), "..", "webharness"),
urlhandlers=handlers,
path_mappings={"/tests": os.path.dirname(options.html_manifest)})
httpd.start(block=False)
test = HTMLTests(httpd, remote_info, log, options)
html_pass_count, html_fail_count = test.run()
pass_count += html_pass_count
fail_count += html_fail_count
httpd.stop()
log.info("Result summary:")
log.info("Passed: %d" % pass_count)
log.info("Failed: %d" % fail_count)
return pass_count > 0 and fail_count == 0
cli = CLI()
debug_args, interactive = cli.debugger_arguments()
runner_args = cli.runner_args()
build = MozbuildObject.from_environment()
binary = runner_args.get('binary')
if not binary:
binary = build.get_binary_path(where="staged-package")
path_mappings = {
k: os.path.join(build.topsrcdir, v)
for k, v in PATH_MAPPINGS.items()
}
httpd = MozHttpd(port=PORT,
docroot=os.path.join(build.topsrcdir, "build", "pgo"),
path_mappings=path_mappings)
httpd.start(block=False)
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=PORT,
options='primary,privileged')
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['profileserver']
prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
class TPSTestRunner(object):
default_env = {
"MOZ_CRASHREPORTER_DISABLE": "1",
"GNOME_DISABLE_CRASH_DIALOG": "1",
"XRE_NO_WINDOWS_CRASH_DIALOG": "1",
"MOZ_NO_REMOTE": "1",
"XPCOM_DEBUG_BREAK": "warn",
}
default_preferences = {
"app.update.enabled": False,
"browser.dom.window.dump.enabled": True,
"browser.sessionstore.resume_from_crash": False,
"browser.shell.checkDefaultBrowser": False,
"browser.tabs.warnOnClose": False,
"browser.warnOnQuit": False,
# Allow installing extensions dropped into the profile folder
"extensions.autoDisableScopes": 10,
"extensions.getAddons.get.url": "http://127.0.0.1:4567/en-US/firefox/api/%API_VERSION%/search/guid:%IDS%",
"extensions.update.enabled": False,
# Don't open a dialog to show available add-on updates
"extensions.update.notifyUser": False,
"services.sync.addons.ignoreRepositoryChecking": True,
"services.sync.firstSync": "notReady",
"services.sync.lastversion": "1.0",
"services.sync.log.rootLogger": "Trace",
"services.sync.log.logger.engine.addons": "Trace",
"services.sync.log.logger.service.main": "Trace",
"services.sync.log.logger.engine.bookmarks": "Trace",
"services.sync.log.appender.console": "Trace",
"services.sync.log.appender.debugLog.enabled": True,
"toolkit.startup.max_resumed_crashes": -1,
}
syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n")
ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n")
ffDateRe = re.compile(r"Firefox builddate: (?P<ffdate>.*)\n")
def __init__(
self,
extensionDir,
testfile="sync.test",
binary=None,
config=None,
rlock=None,
mobile=False,
logfile="tps.log",
resultfile="tps_result.json",
ignore_unused_engines=False,
):
self.extensions = []
self.testfile = testfile
self.logfile = os.path.abspath(logfile)
self.resultfile = resultfile
self.binary = binary
self.ignore_unused_engines = ignore_unused_engines
self.config = config if config else {}
self.repo = None
self.changeset = None
self.branch = None
self.numfailed = 0
self.numpassed = 0
self.nightly = False
self.rlock = rlock
self.mobile = mobile
self.tpsxpi = None
self.firefoxRunner = None
self.extensionDir = extensionDir
self.productversion = None
self.addonversion = None
self.postdata = {}
self.errorlogs = {}
@property
def mobile(self):
return self._mobile
@mobile.setter
def mobile(self, value):
self._mobile = value
self.synctype = "desktop" if not self._mobile else "mobile"
def log(self, msg, printToConsole=False):
"""Appends a string to the logfile"""
f = open(self.logfile, "a")
f.write(msg)
f.close()
if printToConsole:
print msg
def writeToResultFile(self, postdata, body=None, sendTo=["*****@*****.**"]):
"""Writes results to test file"""
results = {"results": []}
if os.access(self.resultfile, os.F_OK):
f = open(self.resultfile, "r")
results = json.loads(f.read())
f.close()
f = open(self.resultfile, "w")
if body is not None:
postdata["body"] = body
if self.numpassed is not None:
postdata["numpassed"] = self.numpassed
if self.numfailed is not None:
postdata["numfailed"] = self.numfailed
if self.firefoxRunner and self.firefoxRunner.url:
postdata["firefoxrunnerurl"] = self.firefoxRunner.url
postdata["sendTo"] = sendTo
results["results"].append(postdata)
f.write(json.dumps(results, indent=2))
f.close()
def _zip_add_file(self, zip, file, rootDir):
zip.write(os.path.join(rootDir, file), file)
def _zip_add_dir(self, zip, dir, rootDir):
try:
zip.write(os.path.join(rootDir, dir), dir)
except:
# on some OS's, adding directory entries doesn't seem to work
pass
for root, dirs, files in os.walk(os.path.join(rootDir, dir)):
for f in files:
zip.write(os.path.join(root, f), os.path.join(dir, f))
def run_single_test(self, testdir, testname):
testpath = os.path.join(testdir, testname)
self.log("Running test %s\n" % testname, True)
# Read and parse the test file, merge it with the contents of the config
# file, and write the combined output to a temporary file.
f = open(testpath, "r")
testcontent = f.read()
f.close()
try:
test = json.loads(testcontent)
except:
test = json.loads(testcontent[testcontent.find("{") : testcontent.find("}") + 1])
testcontent += "var config = %s;\n" % json.dumps(self.config, indent=2)
testcontent += "var seconds_since_epoch = %d;\n" % int(time.time())
tmpfile = TempFile(prefix="tps_test_")
tmpfile.write(testcontent)
tmpfile.close()
# generate the profiles defined in the test, and a list of test phases
profiles = {}
phaselist = []
for phase in test:
profilename = test[phase]
# create the profile if necessary
if not profilename in profiles:
profiles[profilename] = Profile(preferences=self.preferences, addons=self.extensions)
# create the test phase
phaselist.append(
TPSTestPhase(
phase,
profiles[profilename],
testname,
tmpfile.filename,
self.logfile,
self.env,
self.firefoxRunner,
self.log,
ignore_unused_engines=self.ignore_unused_engines,
)
)
# sort the phase list by name
phaselist = sorted(phaselist, key=lambda phase: phase.phase)
# run each phase in sequence, aborting at the first failure
for phase in phaselist:
phase.run()
# if a failure occurred, dump the entire sync log into the test log
if phase.status != "PASS":
for profile in profiles:
self.log("\nDumping sync log for profile %s\n" % profiles[profile].profile)
for root, dirs, files in os.walk(os.path.join(profiles[profile].profile, "weave", "logs")):
for f in files:
weavelog = os.path.join(profiles[profile].profile, "weave", "logs", f)
if os.access(weavelog, os.F_OK):
with open(weavelog, "r") as fh:
for line in fh:
possible_time = line[0:13]
if len(possible_time) == 13 and possible_time.isdigit():
time_ms = int(possible_time)
formatted = time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime(time_ms / 1000)
)
self.log("%s.%03d %s" % (formatted, time_ms % 1000, line[14:]))
else:
self.log(line)
break
# grep the log for FF and sync versions
f = open(self.logfile)
logdata = f.read()
match = self.syncVerRe.search(logdata)
sync_version = match.group("syncversion") if match else "unknown"
match = self.ffVerRe.search(logdata)
firefox_version = match.group("ffver") if match else "unknown"
match = self.ffDateRe.search(logdata)
firefox_builddate = match.group("ffdate") if match else "unknown"
f.close()
if phase.status == "PASS":
logdata = ""
else:
# we only care about the log data for this specific test
logdata = logdata[logdata.find("Running test %s" % (str(testname))) :]
result = {
"PASS": lambda x: ("TEST-PASS", ""),
"FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()),
"unknown": lambda x: ("TEST-UNEXPECTED-FAIL", "test did not complete"),
}[phase.status](phase.errline)
logstr = "\n%s | %s%s\n" % (result[0], testname, (" | %s" % result[1] if result[1] else ""))
try:
repoinfo = self.firefoxRunner.runner.get_repositoryInfo()
except:
repoinfo = {}
apprepo = repoinfo.get("application_repository", "")
appchangeset = repoinfo.get("application_changeset", "")
# save logdata to a temporary file for posting to the db
tmplogfile = None
if logdata:
tmplogfile = TempFile(prefix="tps_log_")
tmplogfile.write(logdata)
tmplogfile.close()
self.errorlogs[testname] = tmplogfile
resultdata = {
"productversion": {
"version": firefox_version,
"buildid": firefox_builddate,
"builddate": firefox_builddate[0:8],
"product": "Firefox",
"repository": apprepo,
"changeset": appchangeset,
},
"addonversion": {"version": sync_version, "product": "Firefox Sync"},
"name": testname,
"message": result[1],
"state": result[0],
"logdata": logdata,
}
self.log(logstr, True)
for phase in phaselist:
print "\t%s: %s" % (phase.phase, phase.status)
if phase.status == "FAIL":
break
return resultdata
def run_tests(self):
# delete the logfile if it already exists
if os.access(self.logfile, os.F_OK):
os.remove(self.logfile)
# Make a copy of the default env variables and preferences, and update
# them for mobile settings if needed.
self.env = self.default_env.copy()
self.preferences = self.default_preferences.copy()
if self.mobile:
self.preferences.update({"services.sync.client.type": "mobile"})
# If sync accounts have been chosen, disable Firefox Accounts
if self.config.get("auth_type", "fx_account") != "fx_account":
self.preferences.update({"services.sync.fxaccounts.enabled": False})
# Acquire a lock to make sure no other threads are running tests
# at the same time.
if self.rlock:
self.rlock.acquire()
try:
# Create the Firefox runner, which will download and install the
# build, as needed.
if not self.firefoxRunner:
self.firefoxRunner = TPSFirefoxRunner(self.binary)
# now, run the test group
self.run_test_group()
except:
traceback.print_exc()
self.numpassed = 0
self.numfailed = 1
try:
self.writeToResultFile(self.postdata, "<pre>%s</pre>" % traceback.format_exc())
except:
traceback.print_exc()
else:
try:
if self.numfailed > 0 or self.numpassed == 0:
To = self.config["email"].get("notificationlist")
else:
To = self.config["email"].get("passednotificationlist")
self.writeToResultFile(self.postdata, sendTo=To)
except:
traceback.print_exc()
try:
self.writeToResultFile(self.postdata, "<pre>%s</pre>" % traceback.format_exc())
except:
traceback.print_exc()
# release our lock
if self.rlock:
self.rlock.release()
# dump out a summary of test results
print "Test Summary\n"
for test in self.postdata.get("tests", {}):
print "%s | %s | %s" % (test["state"], test["name"], test["message"])
def run_test_group(self):
self.results = []
# reset number of passed/failed tests
self.numpassed = 0
self.numfailed = 0
# build our tps.xpi extension
self.extensions = []
self.extensions.append(os.path.join(self.extensionDir, "tps"))
self.extensions.append(os.path.join(self.extensionDir, "mozmill"))
# build the test list
try:
f = open(self.testfile)
jsondata = f.read()
f.close()
testfiles = json.loads(jsondata)
testlist = testfiles["tests"]
except ValueError:
testlist = [os.path.basename(self.testfile)]
testdir = os.path.dirname(self.testfile)
self.mozhttpd = MozHttpd(port=4567, docroot=testdir)
self.mozhttpd.start()
# run each test, and save the results
for test in testlist:
result = self.run_single_test(testdir, test)
if not self.productversion:
self.productversion = result["productversion"]
if not self.addonversion:
self.addonversion = result["addonversion"]
self.results.append(
{
"state": result["state"],
"name": result["name"],
"message": result["message"],
"logdata": result["logdata"],
}
)
if result["state"] == "TEST-PASS":
self.numpassed += 1
else:
self.numfailed += 1
self.mozhttpd.stop()
# generate the postdata we'll use to post the results to the db
self.postdata = {
"tests": self.results,
"os": "%s %sbit" % (mozinfo.version, mozinfo.bits),
"testtype": "crossweave",
"productversion": self.productversion,
"addonversion": self.addonversion,
"synctype": self.synctype,
}
class BaseMarionetteTestRunner(object):
textrunnerclass = MarionetteTextTestRunner
def __init__(self,
address=None,
emulator=None,
emulatorBinary=None,
emulatorImg=None,
emulator_res='480x800',
homedir=None,
app=None,
app_args=None,
bin=None,
profile=None,
autolog=False,
revision=None,
logger=None,
testgroup="marionette",
noWindow=False,
logcat_dir=None,
xml_output=None,
repeat=0,
gecko_path=None,
testvars=None,
tree=None,
type=None,
device_serial=None,
symbols_path=None,
timeout=None,
es_servers=None,
shuffle=False,
sdcard=None,
**kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.app_args = app_args or []
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.xml_output = xml_output
self.repeat = repeat
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device_serial = device_serial
self.symbols_path = symbols_path
self.timeout = timeout
self._device = None
self._capabilities = None
self._appName = None
self.es_servers = es_servers
self.shuffle = shuffle
self.sdcard = sdcard
self.mixin_run_tests = []
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
@property
def capabilities(self):
if self._capabilities:
return self._capabilities
self.marionette.start_session()
self._capabilities = self.marionette.session_capabilities
self.marionette.delete_session()
return self._capabilities
@property
def device(self):
if self._device:
return self._device
self._device = self.capabilities.get('device')
return self._device
@property
def appName(self):
if self._appName:
return self._appName
self._appName = self.capabilities.get('browserName')
return self._appName
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'www'))
self.httpd.start()
self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.baseurl)
def start_marionette(self):
assert (self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
app_args=self.app_args,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl,
timeout=self.timeout,
device_serial=self.device_serial)
elif self.address:
host, port = self.address.split(':')
try:
#establish a socket connection so we can vertify the data come back
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect((host, int(port)))
connection.close()
except Exception, e:
raise Exception(
"Could not connect to given marionette host:port: %s" % e)
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host,
port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl,
timeout=self.timeout)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout,
sdcard=self.sdcard)
class MarionetteTestRunner(object):
def __init__(
self,
address=None,
emulator=None,
emulatorBinary=None,
emulatorImg=None,
emulator_res="480x800",
homedir=None,
app=None,
bin=None,
profile=None,
autolog=False,
revision=None,
es_server=None,
rest_server=None,
logger=None,
testgroup="marionette",
noWindow=False,
logcat_dir=None,
xml_output=None,
repeat=0,
perf=False,
perfserv=None,
gecko_path=None,
testvars=None,
tree=None,
device=None,
symbols_path=None,
):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.perfrequest = None
self.xml_output = xml_output
self.repeat = repeat
self.perf = perf
self.perfserv = perfserv
self.gecko_path = gecko_path
self.testvars = {}
self.tree = tree
self.device = device
self.symbols_path = symbols_path
if testvars:
if not os.path.exists(testvars):
raise Exception("--testvars file does not exist")
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger("Marionette")
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars["xml_output"] = self.xml_output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
self.perfrequest = None
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = "http://%s:%d/" % (host, port)
self.logger.info("running webserver on %s" % self.baseurl)
self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www"))
self.httpd.start()
def start_marionette(self):
assert self.baseurl is not None
if self.bin:
if self.address:
host, port = self.address.split(":")
else:
host = "localhost"
port = 2828
self.marionette = Marionette(
host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl
)
elif self.address:
host, port = self.address.split(":")
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host,
port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
)
else:
self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info("posting results to autolog")
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup=self.testgroup,
os="android",
platform="emulator",
harness="marionette",
server=self.es_server,
restserver=self.rest_server,
machine=socket.gethostname(),
logfile=logfile,
)
testgroup.set_primary_product(tree=self.tree, buildtype="opt", revision=self.revision)
testgroup.add_test_suite(
testsuite="b2g emulator testsuite",
elapsedtime=elapsedtime.seconds,
cmdline="",
passed=self.passed,
failed=self.failed,
todo=self.todo,
)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests, testtype=None):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >= 0:
for test in tests:
self.run_test(test, testtype)
self.repeat -= 1
self.logger.info("\nSUMMARY\n-------")
self.logger.info("passed: %d" % self.passed)
self.logger.info("failed: %d" % self.failed)
self.logger.info("todo: %d" % self.todo)
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.perfrequest and options.perf:
try:
self.perfrequest.submit()
except Exception, e:
print "Could not submit to datazilla"
print e
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, "w") as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def valgrind_test(self, suppressions):
import json
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
#TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js')
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = { 'server': '%s:%d' % httpd.httpd.server_address,
'OOP': 'false'}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler()
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
# The gstreamer plugin scanner can run as part of executing
# firefox, but is an external program. In some weird cases,
# valgrind finds errors while executing __libc_freeres when
# it runs, but those are not relevant, as it's related to
# executing third party code. So don't trace
# gst-plugin-scanner.
'--trace-children-skip=*/gst-plugin-scanner',
'-v', # Enable verbosity to get the list of used suppressions
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1100
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
# This timeout is slightly less than the no-output timeout on
# TBPL, so we'll timeout here first and give an informative
# message.
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
print('TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {} errors seen, but {} generated suppressions seen'.format(errs, supps))
elif errs == 0:
status = 0
print('TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode == None:
status = 2 # turns the TBPL job red
print('TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {} second limit)'.format(timeout))
elif exitcode != 0:
status = 2 # turns the TBPL job red
print('TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind')
httpd.stop()
return status
def valgrind_test(self, suppressions):
import json
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
#TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js')
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = { 'server': '%s:%d' % httpd.httpd.server_address,
'OOP': 'false'}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
outputHandler = OutputHandler()
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=20',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes'
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
kp_kwargs=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait()
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
print('TEST-UNEXPECTED-FAILURE | valgrind-test | error parsing:', errs, "errors seen, but", supps, "generated suppressions seen")
elif errs == 0:
status = 0
print('TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode != 0:
status = 2 # turns the TBPL job red
print('TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind')
httpd.stop()
return status
class MarionetteTestRunner(object):
textrunnerclass = MarionetteTextTestRunner
def __init__(self, address=None, emulator=None, emulatorBinary=None,
emulatorImg=None, emulator_res='480x800', homedir=None,
app=None, app_args=None, bin=None, profile=None, autolog=False,
revision=None, logger=None, testgroup="marionette", noWindow=False,
logcat_dir=None, xml_output=None, repeat=0, gecko_path=None,
testvars=None, tree=None, type=None, device_serial=None,
symbols_path=None, timeout=None, es_servers=None, shuffle=False, **kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.app_args = app_args or []
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.xml_output = xml_output
self.repeat = repeat
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device_serial = device_serial
self.symbols_path = symbols_path
self.timeout = timeout
self._device = None
self._capabilities = None
self._appName = None
self.es_servers = es_servers
self.shuffle = shuffle
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
@property
def capabilities(self):
if self._capabilities:
return self._capabilities
self.marionette.start_session()
self._capabilities = self.marionette.session_capabilities
self.marionette.delete_session()
return self._capabilities
@property
def device(self):
if self._device:
return self._device
self._device = self.capabilities.get('device')
return self._device
@property
def appName(self):
if self._appName:
return self._appName
self._appName = self.capabilities.get('browserName')
return self._appName
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.baseurl)
def start_marionette(self):
assert(self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
app_args=self.app_args,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl,
timeout=self.timeout)
elif self.address:
host, port = self.address.split(':')
try:
#establish a socket connection so we can vertify the data come back
connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
connection.connect((host,int(port)))
connection.close()
except Exception, e:
raise Exception("Could not connect to given marionette host:port: %s" % e)
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl,
timeout=self.timeout)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
def valgrind_test(self, suppressions):
import json
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, "build")
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo"))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, "testing", "profiles", "prefs_general.js")
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = {"server": "%s:%d" % httpd.httpd.server_address, "OOP": "false"}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.topsrcdir, "tools", "quitter", "*****@*****.**")
locations = ServerLocations()
locations.add_host(host="127.0.0.1", port=httpd.httpd.server_port, options="primary")
profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env["G_SLICE"] = "always-malloc"
env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {"processOutputLine": [outputHandler]}
valgrind = "valgrind"
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
"--smc-check=all-non-file",
"--vex-iropt-register-updates=allregs-at-mem-access",
"--gen-suppressions=all",
"--num-callers=36",
"--leak-check=full",
"--show-possibly-lost=no",
"--track-origins=yes",
"--trace-children=yes",
"-v", # Enable verbosity to get the list of used suppressions
]
for s in suppressions:
valgrind_args.append("--suppressions=" + s)
supps_dir = os.path.join(build_dir, "valgrind")
supps_file1 = os.path.join(supps_dir, "cross-architecture.sup")
valgrind_args.append("--suppressions=" + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(["bash", "-c", "echo $MACHTYPE"]).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + ".sup")
if os.path.isfile(supps_file2):
valgrind_args.append("--suppressions=" + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(
profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs,
)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR,
"valgrind-fail-parsing",
{"errs": errs, "supps": supps},
"TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen",
)
elif errs == 0:
status = 0
self.log(logging.INFO, "valgrind-pass", {}, "TEST-PASS | valgrind-test | valgrind found no errors")
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode == None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-timeout",
{"timeout": timeout},
"TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)",
)
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{},
"TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind",
)
httpd.stop()
return status
