def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, 'tps')) self.extensions.append(os.path.join(self.extensionDir, "mozmill")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = testfiles['tests'] except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result['productversion'] if not self.addonversion: self.addonversion = result['addonversion'] self.results.append({ 'state': result['state'], 'name': result['name'], 'message': result['message'], 'logdata': result['logdata'] }) if result['state'] == 'TEST-PASS': self.numpassed += 1 else: self.numfailed += 1 if self.stop_on_error: print '\nTest failed with --stop-on-error specified; not running any more tests.\n' break self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { 'tests': self.results, 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits), 'testtype': 'crossweave', 'productversion': self.productversion, 'addonversion': self.addonversion, 'synctype': self.synctype, }
def start_httpd(self): host = moznetwork.get_ip() self.httpd = MozHttpd(host=host, port=0, docroot=os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www')) self.httpd.start() self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port) self.logger.info('running webserver on %s' % self.baseurl)
def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start()
def runServer(self): """ Start a basic HTML server to host test related files. """ if not self.options.serverPath: self.logger.warning('Can\'t start HTTP server, --server-path not specified') return self.logger.debug('Starting server on port ' + str(self.options.serverPort)) self.server = MozHttpd(port=self.options.serverPort, docroot=self.options.serverPath, proxy_host_dirs=self.options.proxyHostDirs) self.server.start(block=False)
def start(self, block=False): if self.alive: return self._server = MozHttpd(host=self.host, port=self.port, docroot=self.root, urlhandlers=[{ "method": "POST", "path": "/file_upload", "function": upload_handler }]) self._server.start(block=block) self.port = self._server.httpd.server_port self.base_url = self.get_url()
def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, "tps")) self.extensions.append(os.path.join(self.extensionDir, "mozmill")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = testfiles["tests"] except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result["productversion"] if not self.addonversion: self.addonversion = result["addonversion"] self.results.append( { "state": result["state"], "name": result["name"], "message": result["message"], "logdata": result["logdata"], } ) if result["state"] == "TEST-PASS": self.numpassed += 1 else: self.numfailed += 1 self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { "tests": self.results, "os": "%s %sbit" % (mozinfo.version, mozinfo.bits), "testtype": "crossweave", "productversion": self.productversion, "addonversion": self.addonversion, "synctype": self.synctype, }
def start_httpd(self): host = moznetwork.get_ip() self.httpd = MozHttpd(host=host, port=0, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port) self.logger.info('running webserver on %s' % self.baseurl)
def start(self, block=False): if self.alive: return self._server = MozHttpd(host=self.host, port=self.port, docroot=self.root, urlhandlers=[ {"method": "POST", "path": "/file_upload", "function": upload_handler}]) self._server.start(block=block) self.port = self._server.httpd.server_port self.base_url = self.get_url()
def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, 'tps')) self.extensions.append(os.path.join(self.extensionDir, "mozmill")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = testfiles['tests'] except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result['productversion'] if not self.addonversion: self.addonversion = result['addonversion'] self.results.append({'state': result['state'], 'name': result['name'], 'message': result['message'], 'logdata': result['logdata']}) if result['state'] == 'TEST-PASS': self.numpassed += 1 else: self.numfailed += 1 if self.stop_on_error: print '\nTest failed with --stop-on-error specified; not running any more tests.\n' break self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { 'tests': self.results, 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits), 'testtype': 'crossweave', 'productversion': self.productversion, 'addonversion': self.addonversion, 'synctype': self.synctype, }
def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] s.close() self.baseurl = "http://%s:%d/" % (host, port) self.logger.info("running webserver on %s" % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www")) self.httpd.start()
def start_httpd(self, need_external_ip): host = "127.0.0.1" if need_external_ip: host = moznetwork.get_ip() self.httpd = MozHttpd(host=host, port=0, docroot=os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www')) self.httpd.start() self.marionette.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port) self.logger.info('running webserver on %s' % self.marionette.baseurl)
class FixtureServer(object): def __init__(self, root, host="127.0.0.1", port=0): if not os.path.isdir(root): raise Exception("Server root is not a valid path: %s" % root) self.root = root self.host = host self.port = port self._server = None def start(self, block=False): if self.alive: return self._server = MozHttpd(host=self.host, port=self.port, docroot=self.root, urlhandlers=[{ "method": "POST", "path": "/file_upload", "function": upload_handler }]) self._server.start(block=block) self.port = self._server.httpd.server_port self.base_url = self.get_url() def stop(self): if not self.alive: return self._server.stop() self._server = None @property def alive(self): return self._server is not None def get_url(self, path="/"): if not self.alive: raise "Server not started" return self._server.get_url(path) @property def urlhandlers(self): return self._server.urlhandlers
def runServer(self): """ Start a basic HTML server to host test related files. """ if not self.options.serverPath: self.logger.warning("Can't start HTTP server, --server-path not specified") return self.logger.debug("Starting server on port " + str(self.options.serverPort)) self.server = MozHttpd(port=self.options.serverPort, docroot=self.options.serverPath) self.server.start(block=False)
def start_httpd(self): host = iface.get_lan_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start()
class FixtureServer(object): def __init__(self, root, host="127.0.0.1", port=0): if not os.path.isdir(root): raise Exception("Server root is not a valid path: %s" % root) self.root = root self.host = host self.port = port self._server = None def start(self, block=False): if self.alive: return self._server = MozHttpd(host=self.host, port=self.port, docroot=self.root, urlhandlers=[ {"method": "POST", "path": "/file_upload", "function": upload_handler}]) self._server.start(block=block) self.port = self._server.httpd.server_port self.base_url = self.get_url() def stop(self): if not self.alive: return self._server.stop() self._server = None @property def alive(self): return self._server is not None def get_url(self, path="/"): if not self.alive: raise "Server not started" return self._server.get_url(path) @property def urlhandlers(self): return self._server.urlhandlers
import os import sys import shutil import tempfile from datetime import datetime from mozbuild.base import MozbuildObject from buildconfig import substs PORT = 8888 if __name__ == '__main__': cli = CLI() debug_args, interactive = cli.debugger_arguments() build = MozbuildObject.from_environment() httpd = MozHttpd(port=PORT, docroot=os.path.join(build.topsrcdir, "build", "pgo")) httpd.start(block=False) locations = ServerLocations() locations.add_host(host='127.0.0.1', port=PORT, options='primary,privileged') #TODO: mozfile.TemporaryDirectory profilePath = tempfile.mkdtemp() try: #TODO: refactor this into mozprofile prefpath = os.path.join(build.topsrcdir, "testing", "profiles", "prefs_general.js") prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address,
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=None, homedir=None, b2gbin=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None): self.address = address self.emulator = emulator self.homedir = homedir self.b2gbin = b2gbin self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = iface.get_lan_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert(self.baseurl is not None) if self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette(host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir) if self.b2gbin: self.marionette = Marionette(host=host, port=int(port), b2gbin=self.b2gbin, baseurl=self.baseurl) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette(emulator=self.emulator, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir) else: raise Exception("must specify address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup = self.testgroup, os = 'android', platform = 'emulator', harness = 'marionette', server = self.es_server, restserver = self.rest_server, machine = socket.gethostname()) testgroup.set_primary_product( tree = 'b2g', buildtype = 'opt', revision = self.revision) testgroup.add_test_suite( testsuite = 'b2g emulator testsuite', elapsedtime = elapsedtime.seconds, cmdline = '', passed = self.passed, failed = self.failed, todo = self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests, testtype=None): self.reset_test_stats() starttime = datetime.utcnow() for test in tests: self.run_test(test, testtype) self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(elapsedtime) if self.marionette.emulator: self.marionette.emulator.close() self.marionette.emulator = None self.marionette = None def run_test(self, test, testtype): if not self.httpd: self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': if testtype is not None: testargs = {} testtypes = testtype.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) if testtype is None: manifest_tests = manifest.get() else: manifest_tests = manifest.get(**testargs) for i in manifest_tests: self.run_test(i["path"], testtype) return self.logger.info('TEST-START %s' % os.path.basename(test)) if file_ext == '.py': test_mod = imp.load_source(mod_name, filepath) for name in dir(test_mod): obj = getattr(test_mod, name) if (isinstance(obj, (type, types.ClassType)) and issubclass(obj, unittest.TestCase)): testnames = testloader.getTestCaseNames(obj) for testname in testnames: suite.addTest(obj(self.marionette, methodName=testname)) elif file_ext == '.js': suite.addTest(MarionetteJSTestCase(self.marionette, jsFile=filepath)) if suite.countTestCases(): results = MarionetteTextTestRunner(verbosity=3).run(suite) self.failed += len(results.failures) + len(results.errors) self.todo = 0 if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len(results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS')) def cleanup(self): if self.httpd: self.httpd.stop() __del__ = cleanup
class Peptest(): """ Peptest Runs and logs tests designed to test responsiveness """ profile_class = Profile runner_class = Runner def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger('PEP') # create the profile enable_proxy = False locations = ServerLocations() if self.options.proxyLocations: if not self.options.serverPath: self.logger.warning('Can\'t set up proxy without server path') else: enable_proxy = True for proxyLocation in self.options.proxyLocations: locations.read(proxyLocation, False) locations.add_host(host='127.0.0.1', port=self.options.serverPort, options='primary,privileged') self.profile = self.profile_class( profile=self.options.profilePath, addons=[os.path.join(here, 'extension')], locations=locations, proxy=enable_proxy) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith('.js'): # a single test file was passed in testObj = {} testObj['path'] = os.path.realpath(self.options.testPath) testObj['name'] = os.path.basename(self.options.testPath) testObj['here'] = os.path.dirname(testObj['path']) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj['tests'] = tests manifestObj['options'] = options.__dict__ # write manifest to a JSON file jsonManifest = open(os.path.join(here, 'manifest.json'), 'w') jsonManifest.write(json.dumps(manifestObj)) jsonManifest.close() # setup environment env = os.environ.copy() env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str( options.tracerThreshold) env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval) env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class(profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess) def start(self): self.logger.debug('Starting Peptest') # start firefox self.runner.start(outputTimeout=self.options.timeout) self.runner.wait() crashed = self.checkForCrashes(results.currentTest) self.stop() if crashed or results.has_fails(): return 1 return 0 def runServer(self): """ Start a basic HTML server to host test related files. """ if not self.options.serverPath: self.logger.warning( 'Can\'t start HTTP server, --server-path not specified') return self.logger.debug('Starting server on port ' + str(self.options.serverPort)) self.server = MozHttpd(port=self.options.serverPort, docroot=self.options.serverPath, proxy_host_dirs=self.options.proxyHostDirs) self.server.start(block=False) def stop(self): """Kill the app""" # stop the runner if self.runner is not None: self.runner.stop() # kill the server process if self.server: self.server.stop() # remove harness related files files = ['manifest.json'] for f in files: if os.path.exists(os.path.join(here, f)): os.remove(os.path.join(here, f)) # delete any minidumps that may have been created dumpDir = os.path.join(self.profile.profile, 'minidumps') if self.options.profilePath and os.path.exists(dumpDir): shutil.rmtree(dumpDir) def checkForCrashes(self, testName=None): """ Detects when a crash occurs and prints the output from MINIDUMP_STACKWALK. Returns true if crash detected, otherwise false. """ stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None) # try to get the caller's filename if no test name is given if testName is None: try: testName = os.path.basename( sys._getframe(1).f_code.co_filename) except: testName = "unknown" foundCrash = False dumpDir = os.path.join(self.profile.profile, 'minidumps') dumps = glob.glob(os.path.join(dumpDir, '*.dmp')) symbolsPath = self.options.symbolsPath for d in dumps: import subprocess foundCrash = True self.logger.info( "PROCESS-CRASH | %s | application crashed (minidump found)", testName) print "Crash dump filename: " + d # only proceed if a symbols path and stackwalk path were specified if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath): # if symbolsPath is a url, download and extract the zipfile if utils.isURL(symbolsPath): bundle = utils.download(symbolsPath, here) symbolsPath = os.path.join(os.path.dirname(bundle), 'symbols') utils.extract(bundle, symbolsPath, delete=True) # run minidump_stackwalk p = subprocess.Popen([stackwalkPath, d, symbolsPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() if len(out) > 3: # minidump_stackwalk is chatty, so ignore stderr when it succeeds. print out else: print "stderr from minidump_stackwalk:" print err if p.returncode != 0: print "minidump_stackwalk exited with return code %d" % p.returncode else: self.logger.warning( 'No symbols_path or stackwalk path specified, can\'t process dump' ) break # if the symbols path was downloaded, cleanup after ourselves if utils.isURL(self.options.symbolsPath): if os.path.exists(symbolsPath): shutil.rmtree(symbolsPath) return foundCrash
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, perf=False, perfserv=None): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.perfrequest = None self.xml_output = xml_output self.repeat = repeat self.perf = perf self.perfserv = perfserv # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] self.perfrequest = None def start_httpd(self): host = iface.get_lan_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert(self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), bin=self.bin, profile=self.profile, baseurl=self.baseurl) elif self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette(host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette(emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup = self.testgroup, os = 'android', platform = 'emulator', harness = 'marionette', server = self.es_server, restserver = self.rest_server, machine = socket.gethostname(), logfile = logfile) testgroup.set_primary_product( tree = 'b2g', buildtype = 'opt', revision = self.revision) testgroup.add_test_suite( testsuite = 'b2g emulator testsuite', elapsedtime = elapsedtime.seconds, cmdline = '', passed = self.passed, failed = self.failed, todo = self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests, testtype=None): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >=0: for test in tests: self.run_test(test, testtype) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.perfrequest and options.perf: try: self.perfrequest.submit() except Exception, e: print "Could not submit to datazilla" print e if self.xml_output: with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette
def valgrind_test(self, suppressions): import json import re import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath build_dir = os.path.join(self.topsrcdir, "build") # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo")) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, "testing", "profiles", "prefs_general.js") prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = {"server": "%s:%d" % httpd.httpd.server_address, "OOP": "false"} prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.distdir, "xpi-stage", "quitter") locations = ServerLocations() locations.add_host(host="127.0.0.1", port=httpd.httpd.server_port, options="primary") profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env["G_SLICE"] = "always-malloc" env["XPCOM_CC_RUN_DURING_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["XPCOM_DEBUG_BREAK"] = "warn" class OutputHandler(object): def __init__(self): self.found_errors = False def __call__(self, line): print(line) m = re.match(r".*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts", line) if m: self.found_errors = True outputHandler = OutputHandler() kp_kwargs = {"processOutputLine": [outputHandler]} valgrind = "valgrind" if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, "--smc-check=all-non-file", "--vex-iropt-register-updates=allregs-at-mem-access", "--gen-suppressions=all", "--num-callers=20", "--leak-check=full", "--show-possibly-lost=no", "--track-origins=yes", ] for s in suppressions: valgrind_args.append("--suppressions=" + s) supps_dir = os.path.join(build_dir, "valgrind") supps_file1 = os.path.join(supps_dir, "cross-architecture.sup") valgrind_args.append("--suppressions=" + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output(["bash", "-c", "echo $MACHTYPE"]).rstrip() supps_file2 = os.path.join(supps_dir, machtype + ".sup") if os.path.isfile(supps_file2): valgrind_args.append("--suppressions=" + supps_file2) exitcode = None try: runner = FirefoxRunner( profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, kp_kwargs=kp_kwargs ) runner.start(debug_args=valgrind_args) exitcode = runner.wait() finally: if not outputHandler.found_errors: status = 0 print("TEST-PASS | valgrind-test | valgrind found no errors") else: status = 1 # turns the TBPL job orange print("TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors") if exitcode != 0: status = 2 # turns the TBPL job red print("TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind") httpd.stop() return status
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDevice, ADBProcessError, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = '10.0.2.2' PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } dirs = self.query_abs_dirs() topsrcdir = os.path.join(dirs['abs_work_dir'], 'src') adb = self.query_exe('adb') path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) outputdir = self.config.get('output_directory', '/sdcard') jarlog = posixpath.join(outputdir, 'en-US.log') profdata = posixpath.join(outputdir, 'default.profraw') env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata adbdevice = ADBDevice(adb=adb, device='emulator-5554') try: # Run Fennec a first time to initialize its profile driver = Marionette( app='fennec', package_name=app, adb_path=adb, bin="target.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if 'Speedometer/index.html' in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): try: localprof = '/builds/worker/workspace/default.profraw' adbdevice.pull(profdata, localprof) stats = os.stat(localprof) if stats.st_size == 0: # The file may not have been fully written yet, so retry until we # get actual data. time.sleep(2) else: break except ADBProcessError: # The file may not exist at all yet, which would raise an # ADBProcessError, so retry. time.sleep(2) else: raise Exception("Unable to pull default.profraw") adbdevice.pull(jarlog, '/builds/worker/workspace/en-US.log') except ADBTimeoutError: self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError', EXIT_STATUS_DICT[TBPL_RETRY]) # We normally merge as part of a GENERATED_FILES step in the profile-use # build, but Android runs sometimes result in a truncated profile. We do # a merge here to make sure the data isn't corrupt so we can retry the # 'run' task if necessary. merge_cmd = [ '/builds/worker/workspace/build/clang/bin/llvm-profdata', 'merge', '/builds/worker/workspace/default.profraw', '-o', '/builds/worker/workspace/merged.profraw', ] rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( 'INFRA-ERROR: Failed to merge profile data. Corrupt profile?', EXIT_STATUS_DICT[TBPL_RETRY]) # tarfile doesn't support xz in this version of Python tar_cmd = [ 'tar', '-acvf', '/builds/worker/artifacts/profdata.tar.xz', '-C', '/builds/worker/workspace', 'merged.profraw', 'en-US.log', ] subprocess.check_call(tar_cmd) httpd.stop()
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=False, homedir=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False): self.address = address self.emulator = emulator self.homedir = homedir self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = iface.get_lan_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert (self.baseurl is not None) if self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette(host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette(emulator=True, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow) else: raise Exception("must specify address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup(testgroup=self.testgroup, os='android', platform='emulator', harness='marionette', server=self.es_server, restserver=self.rest_server, machine=socket.gethostname()) testgroup.set_primary_product(tree='b2g', buildtype='opt', revision=self.revision) testgroup.add_test_suite(testsuite='b2g emulator testsuite', elapsedtime=elapsedtime.seconds, cmdline='', passed=self.passed, failed=self.failed, todo=self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests, testtype=None): self.reset_test_stats() starttime = datetime.utcnow() for test in tests: self.run_test(test, testtype) self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(elapsedtime) if self.marionette.emulator: self.marionette.emulator.close() self.marionette.emulator = None self.marionette = None def run_test(self, test, testtype): if not self.httpd: self.start_httpd() if not self.marionette: self.start_marionette() if not os.path.isabs(test): filepath = os.path.join(os.path.dirname(__file__), test) else: filepath = test if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': if testtype is not None: testargs = {} testtypes = testtype.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({atype[1:]: 'true'}) elif atype.startswith('-'): testargs.update({atype[1:]: 'false'}) else: testargs.update({atype: 'true'}) manifest = TestManifest() manifest.read(filepath) if testtype is None: manifest_tests = manifest.get() else: manifest_tests = manifest.get(**testargs) for i in manifest_tests: self.run_test(i["path"], testtype) return self.logger.info('TEST-START %s' % os.path.basename(test)) if file_ext == '.py': test_mod = imp.load_source(mod_name, filepath) for name in dir(test_mod): obj = getattr(test_mod, name) if (isinstance(obj, (type, types.ClassType)) and issubclass(obj, unittest.TestCase)): testnames = testloader.getTestCaseNames(obj) for testname in testnames: suite.addTest(obj(self.marionette, methodName=testname)) elif file_ext == '.js': suite.addTest( MarionetteJSTestCase(self.marionette, jsFile=filepath)) if suite.countTestCases(): results = MarionetteTextTestRunner(verbosity=3).run(suite) self.failed += len(results.failures) + len(results.errors) self.todo = 0 if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len( results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS')) def cleanup(self): if self.httpd: self.httpd.stop() __del__ = cleanup
def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, "tps")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = [] for (filename, meta) in testfiles["tests"].items(): skip_reason = meta.get("disabled") if skip_reason: print("Skipping test {} - {}".format( filename, skip_reason)) else: testlist.append(filename) except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result["productversion"] if not self.addonversion: self.addonversion = result["addonversion"] self.results.append({ "state": result["state"], "name": result["name"], "message": result["message"], "logdata": result["logdata"], }) if result["state"] == "TEST-PASS": self.numpassed += 1 else: self.numfailed += 1 if self.stop_on_error: print("\nTest failed with --stop-on-error specified; " "not running any more tests.\n") break self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { "tests": self.results, "os": "%s %sbit" % (mozinfo.version, mozinfo.bits), "testtype": "crossweave", "productversion": self.productversion, "addonversion": self.addonversion, "synctype": self.synctype, }
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDeviceFactory, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = "10.0.2.2" PORT = 8888 PATH_MAPPINGS = { "/js-input/webkit/PerformanceTests": "third_party/webkit/PerformanceTests", } dirs = self.query_abs_dirs() topsrcdir = dirs["abs_src_dir"] adb = self.query_exe("adb") path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd( port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings, ) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, "testing", "profiles") with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: base_profiles = json.load(fh)["profileserver"] prefpaths = [ os.path.join(profile_data_dir, profile, "user.js") for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) outputdir = self.config.get("output_directory", "/sdcard/pgo_profile") jarlog = posixpath.join(outputdir, "en-US.log") profdata = posixpath.join(outputdir, "default_%p_random_%m.profraw") env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata if self.query_minidump_stackwalk(): os.environ["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path os.environ["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs( )["abs_blob_upload_dir"] if not self.symbols_path: self.symbols_path = os.environ.get("MOZ_FETCHES_DIR") # Force test_root to be on the sdcard for android pgo # builds which fail for Android 4.3 when profiles are located # in /data/local/tmp/test_root with # E AndroidRuntime: FATAL EXCEPTION: Gecko # E AndroidRuntime: java.lang.IllegalArgumentException: \ # Profile directory must be writable if specified: /data/local/tmp/test_root/profile # This occurs when .can-write-sentinel is written to # the profile in # mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoProfile.java. # This is not a problem on later versions of Android. This # over-ride of test_root should be removed when Android 4.3 is no # longer supported. sdcard_test_root = "/sdcard/test_root" adbdevice = ADBDeviceFactory(adb=adb, device="emulator-5554", test_root=sdcard_test_root) if adbdevice.test_root != sdcard_test_root: # If the test_root was previously set and shared # the initializer will not have updated the shared # value. Force it to match the sdcard_test_root. adbdevice.test_root = sdcard_test_root adbdevice.mkdir(outputdir, parents=True) try: # Run Fennec a first time to initialize its profile driver = Marionette( app="fennec", package_name=app, adb_path=adb, bin="geckoview-androidTest.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, symbols_path=self.symbols_path, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if "Speedometer/index.html" in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): if not adbdevice.process_exist(app): break time.sleep(2) else: raise Exception("Android App (%s) never quit" % app) # Pull all the profraw files and en-US.log adbdevice.pull(outputdir, "/builds/worker/workspace/") except ADBTimeoutError: self.fatal( "INFRA-ERROR: Failed with an ADBTimeoutError", EXIT_STATUS_DICT[TBPL_RETRY], ) profraw_files = glob.glob("/builds/worker/workspace/*.profraw") if not profraw_files: self.fatal( "Could not find any profraw files in /builds/worker/workspace") merge_cmd = [ os.path.join(os.environ["MOZ_FETCHES_DIR"], "clang/bin/llvm-profdata"), "merge", "-o", "/builds/worker/workspace/merged.profdata", ] + profraw_files rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( "INFRA-ERROR: Failed to merge profile data. Corrupt profile?", EXIT_STATUS_DICT[TBPL_RETRY], ) # tarfile doesn't support xz in this version of Python tar_cmd = [ "tar", "-acvf", "/builds/worker/artifacts/profdata.tar.xz", "-C", "/builds/worker/workspace", "merged.profdata", "en-US.log", ] subprocess.check_call(tar_cmd) httpd.stop()
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDevice, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = '10.0.2.2' PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } dirs = self.query_abs_dirs() topsrcdir = os.path.join(dirs['abs_work_dir'], 'src') adb = self.query_exe('adb') path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) # Enforce 1proc. This isn't in one of the user.js files because those # are shared with desktop, which wants e10s. We can't interpolate # because the formatting code only works for strings, and this is a # bool pref. prefs["browser.tabs.remote.autostart"] = False outputdir = self.config.get('output_directory', '/sdcard/pgo_profile') jarlog = posixpath.join(outputdir, 'en-US.log') profdata = posixpath.join(outputdir, 'default_%p_random_%m.profraw') env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata adbdevice = ADBDevice(adb=adb, device='emulator-5554') adbdevice.mkdir(outputdir) try: # Run Fennec a first time to initialize its profile driver = Marionette( app='fennec', package_name=app, adb_path=adb, bin="target.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if 'Speedometer/index.html' in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): if not adbdevice.process_exist(app): break time.sleep(2) else: raise Exception("Android App (%s) never quit" % app) # Pull all the profraw files and en-US.log adbdevice.pull(outputdir, '/builds/worker/workspace/') except ADBTimeoutError: self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError', EXIT_STATUS_DICT[TBPL_RETRY]) profraw_files = glob.glob('/builds/worker/workspace/*.profraw') if not profraw_files: self.fatal( 'Could not find any profraw files in /builds/worker/workspace') merge_cmd = [ os.path.join(os.environ['MOZ_FETCHES_DIR'], 'clang/bin/llvm-profdata'), 'merge', '-o', '/builds/worker/workspace/merged.profdata', ] + profraw_files rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( 'INFRA-ERROR: Failed to merge profile data. Corrupt profile?', EXIT_STATUS_DICT[TBPL_RETRY]) # tarfile doesn't support xz in this version of Python tar_cmd = [ 'tar', '-acvf', '/builds/worker/artifacts/profdata.tar.xz', '-C', '/builds/worker/workspace', 'merged.profdata', 'en-US.log', ] subprocess.check_call(tar_cmd) httpd.stop()
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, gecko_path=None, testvars=None, tree=None, type=None, device=None, symbols_path=None, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.xml_output = xml_output self.repeat = repeat self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device = device self.symbols_path = symbols_path if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert (self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl) elif self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join( os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup(testgroup=self.testgroup, os='android', platform='emulator', harness='marionette', server=self.es_server, restserver=self.rest_server, machine=socket.gethostname(), logfile=logfile) testgroup.set_primary_product(tree=self.tree, buildtype='opt', revision=self.revision) testgroup.add_test_suite(testsuite='b2g emulator testsuite', elapsedtime=elapsedtime.seconds, cmdline='', passed=self.passed, failed=self.failed, todo=self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >= 0: for test in tests: self.run_test(test) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette def run_test(self, test): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({atype[1:]: 'true'}) elif atype.startswith('-'): testargs.update({atype[1:]: 'false'}) else: testargs.update({atype: 'true'}) manifest = TestManifest() manifest.read(filepath) manifest_tests = manifest.active_tests(disabled=False) for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = MarionetteTextTestRunner(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len( results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS')) def register_handlers(self): self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase]) def cleanup(self): if self.httpd: self.httpd.stop() __del__ = cleanup def generate_xml(self, results_list): def _extract_xml(test, text='', result='passed'): cls_name = test.__class__.__name__ testcase = doc.createElement('testcase') testcase.setAttribute('classname', cls_name) testcase.setAttribute('name', unicode(test).split()[0]) testcase.setAttribute('time', str(test.duration)) testsuite.appendChild(testcase) if result in ['failure', 'error', 'skipped']: f = doc.createElement(result) f.setAttribute('message', 'test %s' % result) f.appendChild(doc.createTextNode(text)) testcase.appendChild(f) doc = dom.Document() testsuite = doc.createElement('testsuite') testsuite.setAttribute('name', 'Marionette') testsuite.setAttribute('time', str(self.elapsedtime.total_seconds())) testsuite.setAttribute( 'tests', str(sum([results.testsRun for results in results_list]))) def failed_count(results): count = len(results.failures) if hasattr(results, 'unexpectedSuccesses'): count += len(results.unexpectedSuccesses) return count testsuite.setAttribute( 'failures', str(sum([failed_count(results) for results in results_list]))) testsuite.setAttribute( 'errors', str(sum([len(results.errors) for results in results_list]))) if hasattr(results, 'skipped'): testsuite.setAttribute( 'skips', str( sum([ len(results.skipped) + len(results.expectedFailures) for results in results_list ]))) for results in results_list: for tup in results.errors: _extract_xml(*tup, result='error') for tup in results.failures: _extract_xml(*tup, result='failure') if hasattr(results, 'unexpectedSuccesses'): for test in results.unexpectedSuccesses: # unexpectedSuccesses is a list of Testcases only, no tuples _extract_xml(test, text='TEST-UNEXPECTED-PASS', result='failure') if hasattr(results, 'skipped'): for tup in results.skipped: _extract_xml(*tup, result='skipped') if hasattr(results, 'expectedFailures'): for tup in results.expectedFailures: _extract_xml(*tup, result='skipped') for test in results.tests_passed: _extract_xml(test) doc.appendChild(testsuite) return doc.toprettyxml(encoding='utf-8')
def valgrind_test(self, suppressions): import json import re import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, 'OOP': 'false' } prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' class OutputHandler(object): def __init__(self): self.found_errors = False def __call__(self, line): print(line) m = re.match( r'.*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts', line) if m: self.found_errors = True outputHandler = OutputHandler() kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=20', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes' ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output( ['bash', '-c', 'echo $MACHTYPE']).rstrip() supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, kp_kwargs=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait() finally: if not outputHandler.found_errors: status = 0 print( 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange print( 'TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors' ) if exitcode != 0: status = 2 # turns the TBPL job red print( 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind' ) httpd.stop() return status
def valgrind_test(self, suppressions): from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from six import string_types from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile profile_data_dir = os.path.join( self.topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['valgrind'] prefpaths = [os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) quitter = os.path.join( self.topsrcdir, 'tools', 'quitter', '*****@*****.**') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' env.update(self.extra_environment_variables) outputHandler = OutputHandler(self.log) kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=36', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes', '--trace-children=yes', '-v', # Enable verbosity to get the list of used suppressions # Avoid excessive delays in the presence of spinlocks. # See bug 1309851. '--fair-sched=yes', # Keep debuginfo after library unmap. See bug 1382280. '--keep-debuginfo=yes', # Reduce noise level on rustc and/or LLVM compiled code. # See bug 1365915 '--expensive-definedness-checks=yes', ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) if mozinfo.os == 'linux': machtype = { 'x86_64': 'x86_64-pc-linux-gnu', 'x86': 'i386-pc-linux-gnu', }.get(mozinfo.processor) if machtype: supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None timeout = 1800 try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log(logging.ERROR, 'valgrind-fail-parsing', {'errs': errs, 'supps': supps}, 'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors ' 'seen, but {supps} generated suppressions seen') elif errs == 0: status = 0 self.log(logging.INFO, 'valgrind-pass', {}, 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode is None: status = 2 # turns the TBPL job red self.log(logging.ERROR, 'valgrind-fail-timeout', {'timeout': timeout}, 'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out ' '(reached {timeout} second limit)') elif exitcode != 0: status = 2 # turns the TBPL job red self.log(logging.ERROR, 'valgrind-fail-errors', {}, 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code' 'from Valgrind') httpd.stop() return status
class TPSTestRunner(object): default_env = { 'MOZ_CRASHREPORTER_DISABLE': '1', 'GNOME_DISABLE_CRASH_DIALOG': '1', 'XRE_NO_WINDOWS_CRASH_DIALOG': '1', 'MOZ_NO_REMOTE': '1', 'XPCOM_DEBUG_BREAK': 'warn', } default_preferences = { 'app.update.enabled': False, 'browser.dom.window.dump.enabled': True, 'browser.sessionstore.resume_from_crash': False, 'browser.shell.checkDefaultBrowser': False, 'browser.tabs.warnOnClose': False, 'browser.warnOnQuit': False, # Allow installing extensions dropped into the profile folder 'extensions.autoDisableScopes': 10, 'extensions.getAddons.get.url': 'http://127.0.0.1:4567/en-US/firefox/api/%API_VERSION%/search/guid:%IDS%', 'extensions.update.enabled': False, # Don't open a dialog to show available add-on updates 'extensions.update.notifyUser': False, 'services.sync.addons.ignoreRepositoryChecking': True, 'services.sync.firstSync': 'notReady', 'services.sync.lastversion': '1.0', 'services.sync.log.rootLogger': 'Trace', 'services.sync.log.logger.engine.addons': 'Trace', 'services.sync.log.logger.service.main': 'Trace', 'services.sync.log.logger.engine.bookmarks': 'Trace', 'services.sync.log.appender.console': 'Trace', 'services.sync.log.appender.debugLog.enabled': True, 'toolkit.startup.max_resumed_crashes': -1, } syncVerRe = re.compile(r'Sync version: (?P<syncversion>.*)\n') ffVerRe = re.compile(r'Firefox version: (?P<ffver>.*)\n') ffDateRe = re.compile(r'Firefox builddate: (?P<ffdate>.*)\n') def __init__(self, extensionDir, testfile='sync.test', binary=None, config=None, rlock=None, mobile=False, logfile='tps.log', resultfile='tps_result.json', ignore_unused_engines=False): self.extensions = [] self.testfile = testfile self.logfile = os.path.abspath(logfile) self.resultfile = resultfile self.binary = binary self.ignore_unused_engines = ignore_unused_engines self.config = config if config else {} self.repo = None self.changeset = None self.branch = None self.numfailed = 0 self.numpassed = 0 self.nightly = False self.rlock = rlock self.mobile = mobile self.tpsxpi = None self.firefoxRunner = None self.extensionDir = extensionDir self.productversion = None self.addonversion = None self.postdata = {} self.errorlogs = {} @property def mobile(self): return self._mobile @mobile.setter def mobile(self, value): self._mobile = value self.synctype = 'desktop' if not self._mobile else 'mobile' def log(self, msg, printToConsole=False): """Appends a string to the logfile""" f = open(self.logfile, 'a') f.write(msg) f.close() if printToConsole: print msg def writeToResultFile(self, postdata, body=None, sendTo=['*****@*****.**']): """Writes results to test file""" results = {'results': []} if os.access(self.resultfile, os.F_OK): f = open(self.resultfile, 'r') results = json.loads(f.read()) f.close() f = open(self.resultfile, 'w') if body is not None: postdata['body'] = body if self.numpassed is not None: postdata['numpassed'] = self.numpassed if self.numfailed is not None: postdata['numfailed'] = self.numfailed if self.firefoxRunner and self.firefoxRunner.url: postdata['firefoxrunnerurl'] = self.firefoxRunner.url postdata['sendTo'] = sendTo results['results'].append(postdata) f.write(json.dumps(results, indent=2)) f.close() def _zip_add_file(self, zip, file, rootDir): zip.write(os.path.join(rootDir, file), file) def _zip_add_dir(self, zip, dir, rootDir): try: zip.write(os.path.join(rootDir, dir), dir) except: # on some OS's, adding directory entries doesn't seem to work pass for root, dirs, files in os.walk(os.path.join(rootDir, dir)): for f in files: zip.write(os.path.join(root, f), os.path.join(dir, f)) def run_single_test(self, testdir, testname): testpath = os.path.join(testdir, testname) self.log("Running test %s\n" % testname, True) # Read and parse the test file, merge it with the contents of the config # file, and write the combined output to a temporary file. f = open(testpath, 'r') testcontent = f.read() f.close() try: test = json.loads(testcontent) except: test = json.loads( testcontent[testcontent.find('{'):testcontent.find('}') + 1]) testcontent += 'var config = %s;\n' % json.dumps(self.config, indent=2) testcontent += 'var seconds_since_epoch = %d;\n' % int(time.time()) tmpfile = TempFile(prefix='tps_test_') tmpfile.write(testcontent) tmpfile.close() # generate the profiles defined in the test, and a list of test phases profiles = {} phaselist = [] for phase in test: profilename = test[phase] # create the profile if necessary if not profilename in profiles: profiles[profilename] = Profile(preferences=self.preferences, addons=self.extensions) # create the test phase phaselist.append( TPSTestPhase(phase, profiles[profilename], testname, tmpfile.filename, self.logfile, self.env, self.firefoxRunner, self.log, ignore_unused_engines=self.ignore_unused_engines)) # sort the phase list by name phaselist = sorted(phaselist, key=lambda phase: phase.phase) # run each phase in sequence, aborting at the first failure for phase in phaselist: phase.run() # if a failure occurred, dump the entire sync log into the test log if phase.status != 'PASS': for profile in profiles: self.log('\nDumping sync log for profile %s\n' % profiles[profile].profile) for root, dirs, files in os.walk( os.path.join(profiles[profile].profile, 'weave', 'logs')): for f in files: weavelog = os.path.join(profiles[profile].profile, 'weave', 'logs', f) if os.access(weavelog, os.F_OK): with open(weavelog, 'r') as fh: for line in fh: possible_time = line[0:13] if len( possible_time ) == 13 and possible_time.isdigit(): time_ms = int(possible_time) formatted = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time_ms / 1000)) self.log('%s.%03d %s' % (formatted, time_ms % 1000, line[14:])) else: self.log(line) break # grep the log for FF and sync versions f = open(self.logfile) logdata = f.read() match = self.syncVerRe.search(logdata) sync_version = match.group('syncversion') if match else 'unknown' match = self.ffVerRe.search(logdata) firefox_version = match.group('ffver') if match else 'unknown' match = self.ffDateRe.search(logdata) firefox_builddate = match.group('ffdate') if match else 'unknown' f.close() if phase.status == 'PASS': logdata = '' else: # we only care about the log data for this specific test logdata = logdata[logdata.find('Running test %s' % (str(testname))):] result = { 'PASS': lambda x: ('TEST-PASS', ''), 'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()), 'unknown': lambda x: ('TEST-UNEXPECTED-FAIL', 'test did not complete') }[phase.status](phase.errline) logstr = "\n%s | %s%s\n" % (result[0], testname, (' | %s' % result[1] if result[1] else '')) try: repoinfo = self.firefoxRunner.runner.get_repositoryInfo() except: repoinfo = {} apprepo = repoinfo.get('application_repository', '') appchangeset = repoinfo.get('application_changeset', '') # save logdata to a temporary file for posting to the db tmplogfile = None if logdata: tmplogfile = TempFile(prefix='tps_log_') tmplogfile.write(logdata) tmplogfile.close() self.errorlogs[testname] = tmplogfile resultdata = ({ 'productversion': { 'version': firefox_version, 'buildid': firefox_builddate, 'builddate': firefox_builddate[0:8], 'product': 'Firefox', 'repository': apprepo, 'changeset': appchangeset, }, 'addonversion': { 'version': sync_version, 'product': 'Firefox Sync' }, 'name': testname, 'message': result[1], 'state': result[0], 'logdata': logdata }) self.log(logstr, True) for phase in phaselist: print "\t%s: %s" % (phase.phase, phase.status) if phase.status == 'FAIL': break return resultdata def run_tests(self): # delete the logfile if it already exists if os.access(self.logfile, os.F_OK): os.remove(self.logfile) # Make a copy of the default env variables and preferences, and update # them for mobile settings if needed. self.env = self.default_env.copy() self.preferences = self.default_preferences.copy() if self.mobile: self.preferences.update({'services.sync.client.type': 'mobile'}) # Set a dummy username to force the correct authentication type. For the # old sync, the username is not allowed to contain a '@'. dummy = {'fx_account': 'dummy@somewhere', 'sync_account': 'dummy'} auth_type = self.config.get('auth_type', 'fx_account') self.preferences.update({'services.sync.username': dummy[auth_type]}) # Acquire a lock to make sure no other threads are running tests # at the same time. if self.rlock: self.rlock.acquire() try: # Create the Firefox runner, which will download and install the # build, as needed. if not self.firefoxRunner: self.firefoxRunner = TPSFirefoxRunner(self.binary) # now, run the test group self.run_test_group() except: traceback.print_exc() self.numpassed = 0 self.numfailed = 1 try: self.writeToResultFile( self.postdata, '<pre>%s</pre>' % traceback.format_exc()) except: traceback.print_exc() else: try: if self.numfailed > 0 or self.numpassed == 0: To = self.config['email'].get('notificationlist') else: To = self.config['email'].get('passednotificationlist') self.writeToResultFile(self.postdata, sendTo=To) except: traceback.print_exc() try: self.writeToResultFile( self.postdata, '<pre>%s</pre>' % traceback.format_exc()) except: traceback.print_exc() # release our lock if self.rlock: self.rlock.release() # dump out a summary of test results print 'Test Summary\n' for test in self.postdata.get('tests', {}): print '%s | %s | %s' % (test['state'], test['name'], test['message']) def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, 'tps')) self.extensions.append(os.path.join(self.extensionDir, "mozmill")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = testfiles['tests'] except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result['productversion'] if not self.addonversion: self.addonversion = result['addonversion'] self.results.append({ 'state': result['state'], 'name': result['name'], 'message': result['message'], 'logdata': result['logdata'] }) if result['state'] == 'TEST-PASS': self.numpassed += 1 else: self.numfailed += 1 self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { 'tests': self.results, 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits), 'testtype': 'crossweave', 'productversion': self.productversion, 'addonversion': self.addonversion, 'synctype': self.synctype, }
class TPSTestRunner(object): extra_env = { "MOZ_CRASHREPORTER_DISABLE": "1", "GNOME_DISABLE_CRASH_DIALOG": "1", "XRE_NO_WINDOWS_CRASH_DIALOG": "1", "MOZ_NO_REMOTE": "1", "XPCOM_DEBUG_BREAK": "warn", } default_preferences = { "app.update.checkInstallTime": False, "app.update.disabledForTesting": True, "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer": True, "browser.dom.window.dump.enabled": True, "devtools.console.stdout.chrome": True, "browser.sessionstore.resume_from_crash": False, "browser.shell.checkDefaultBrowser": False, "browser.tabs.warnOnClose": False, "browser.warnOnQuit": False, # Allow installing extensions dropped into the profile folder "extensions.autoDisableScopes": 10, "extensions.getAddons.get.url": "http://127.0.0.1:4567/addons/api/%IDS%.json", # Our pretend addons server doesn't support metadata... "extensions.getAddons.cache.enabled": False, "extensions.install.requireSecureOrigin": False, "extensions.update.enabled": False, # Don't open a dialog to show available add-on updates "extensions.update.notifyUser": False, "services.sync.firstSync": "notReady", "services.sync.lastversion": "1.0", "services.sync.autoconnectDelay": 60 * 60 * 10, "toolkit.startup.max_resumed_crashes": -1, # hrm - not sure what the release/beta channels will do? "xpinstall.signatures.required": False, "services.sync.testing.tps": True, "engine.bookmarks.repair.enabled": False, "extensions.experiments.enabled": True, "webextensions.storage.sync.kinto": False, } debug_preferences = { "services.sync.log.appender.console": "Trace", "services.sync.log.appender.dump": "Trace", "services.sync.log.appender.file.level": "Trace", "services.sync.log.appender.file.logOnSuccess": True, "services.sync.log.logger": "Trace", "services.sync.log.logger.engine": "Trace", } syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n") ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n") ffBuildIDRe = re.compile(r"Firefox buildid: (?P<ffbuildid>.*)\n") def __init__( self, extensionDir, binary=None, config=None, debug=False, ignore_unused_engines=False, logfile="tps.log", mobile=False, rlock=None, resultfile="tps_result.json", testfile=None, stop_on_error=False, ): self.binary = binary self.config = config if config else {} self.debug = debug self.extensions = [] self.ignore_unused_engines = ignore_unused_engines self.logfile = os.path.abspath(logfile) self.mobile = mobile self.rlock = rlock self.resultfile = resultfile self.testfile = testfile self.stop_on_error = stop_on_error self.addonversion = None self.branch = None self.changeset = None self.errorlogs = {} self.extensionDir = extensionDir self.firefoxRunner = None self.nightly = False self.numfailed = 0 self.numpassed = 0 self.postdata = {} self.productversion = None self.repo = None self.tpsxpi = None @property def mobile(self): return self._mobile @mobile.setter def mobile(self, value): self._mobile = value self.synctype = "desktop" if not self._mobile else "mobile" def log(self, msg, printToConsole=False): """Appends a string to the logfile""" f = open(self.logfile, "a") f.write(msg) f.close() if printToConsole: print(msg) def writeToResultFile(self, postdata, body=None, sendTo=["*****@*****.**"]): """Writes results to test file""" results = {"results": []} if os.access(self.resultfile, os.F_OK): f = open(self.resultfile, "r") results = json.loads(f.read()) f.close() f = open(self.resultfile, "w") if body is not None: postdata["body"] = body if self.numpassed is not None: postdata["numpassed"] = self.numpassed if self.numfailed is not None: postdata["numfailed"] = self.numfailed if self.firefoxRunner and self.firefoxRunner.url: postdata["firefoxrunnerurl"] = self.firefoxRunner.url postdata["sendTo"] = sendTo results["results"].append(postdata) f.write(json.dumps(results, indent=2)) f.close() def _zip_add_file(self, zip, file, rootDir): zip.write(os.path.join(rootDir, file), file) def _zip_add_dir(self, zip, dir, rootDir): try: zip.write(os.path.join(rootDir, dir), dir) except Exception: # on some OS's, adding directory entries doesn't seem to work pass for root, dirs, files in os.walk(os.path.join(rootDir, dir)): for f in files: zip.write(os.path.join(root, f), os.path.join(dir, f)) def handle_phase_failure(self, profiles): for profile in profiles: self.log("\nDumping sync log for profile %s\n" % profiles[profile].profile) for root, dirs, files in os.walk( os.path.join(profiles[profile].profile, "weave", "logs")): for f in files: weavelog = os.path.join(profiles[profile].profile, "weave", "logs", f) if os.access(weavelog, os.F_OK): with open(weavelog, "r") as fh: for line in fh: possible_time = line[0:13] if len(possible_time ) == 13 and possible_time.isdigit(): time_ms = int(possible_time) formatted = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time_ms / 1000), ) self.log( "%s.%03d %s" % (formatted, time_ms % 1000, line[14:])) else: self.log(line) def run_single_test(self, testdir, testname): testpath = os.path.join(testdir, testname) self.log("Running test %s\n" % testname, True) # Read and parse the test file, merge it with the contents of the config # file, and write the combined output to a temporary file. f = open(testpath, "r") testcontent = f.read() f.close() # We use yaml to parse the tests because it is a superset of json # but tolerates things like property names not being quoted, trailing # commas, etc. try: test = yaml.safe_load(testcontent) except Exception: test = yaml.safe_load( testcontent[testcontent.find("{"):testcontent.find("}") + 1]) self.preferences["tps.seconds_since_epoch"] = int(time.time()) # generate the profiles defined in the test, and a list of test phases profiles = {} phaselist = [] for phase in test: profilename = test[phase] # create the profile if necessary if profilename not in profiles: profiles[profilename] = Profile( preferences=self.preferences.copy(), addons=self.extensions) # create the test phase phaselist.append( TPSTestPhase( phase, profiles[profilename], testname, testpath, self.logfile, self.env, self.firefoxRunner, self.log, ignore_unused_engines=self.ignore_unused_engines, )) # sort the phase list by name phaselist = sorted(phaselist, key=lambda phase: phase.phase) # run each phase in sequence, aborting at the first failure failed = False for phase in phaselist: phase.run() if phase.status != "PASS": failed = True break for profilename in profiles: print("### Cleanup Profile ", profilename) cleanup_phase = TPSTestPhase( "cleanup-" + profilename, profiles[profilename], testname, testpath, self.logfile, self.env, self.firefoxRunner, self.log, ) cleanup_phase.run() if cleanup_phase.status != "PASS": failed = True # Keep going to run the remaining cleanup phases. if failed: self.handle_phase_failure(profiles) # grep the log for FF and sync versions f = open(self.logfile) logdata = f.read() match = self.syncVerRe.search(logdata) sync_version = match.group("syncversion") if match else "unknown" match = self.ffVerRe.search(logdata) firefox_version = match.group("ffver") if match else "unknown" match = self.ffBuildIDRe.search(logdata) firefox_buildid = match.group("ffbuildid") if match else "unknown" f.close() if phase.status == "PASS": logdata = "" else: # we only care about the log data for this specific test logdata = logdata[logdata.find("Running test %s" % (str(testname))):] result = { "PASS": lambda x: ("TEST-PASS", ""), "FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()), "unknown": lambda x: ("TEST-UNEXPECTED-FAIL", "test did not complete"), }[phase.status](phase.errline) logstr = "\n%s | %s%s\n" % ( result[0], testname, (" | %s" % result[1] if result[1] else ""), ) try: repoinfo = mozversion.get_version(self.binary) except Exception: repoinfo = {} apprepo = repoinfo.get("application_repository", "") appchangeset = repoinfo.get("application_changeset", "") # save logdata to a temporary file for posting to the db tmplogfile = None if logdata: tmplogfile = TempFile(prefix="tps_log_") tmplogfile.write(logdata) tmplogfile.close() self.errorlogs[testname] = tmplogfile resultdata = { "productversion": { "version": firefox_version, "buildid": firefox_buildid, "builddate": firefox_buildid[0:8], "product": "Firefox", "repository": apprepo, "changeset": appchangeset, }, "addonversion": { "version": sync_version, "product": "Firefox Sync" }, "name": testname, "message": result[1], "state": result[0], "logdata": logdata, } self.log(logstr, True) for phase in phaselist: print("\t{}: {}".format(phase.phase, phase.status)) return resultdata def update_preferences(self): self.preferences = self.default_preferences.copy() if self.mobile: self.preferences.update({"services.sync.client.type": "mobile"}) # If we are using legacy Sync, then set a dummy username to force the # correct authentication type. Without this pref set to a value # without an '@' character, Sync will initialize for FxA. if self.config.get("auth_type", "fx_account") != "fx_account": self.preferences.update({"services.sync.username": "******"}) if self.debug: self.preferences.update(self.debug_preferences) if "preferences" in self.config: self.preferences.update(self.config["preferences"]) self.preferences["tps.config"] = json.dumps(self.config) def run_tests(self): # delete the logfile if it already exists if os.access(self.logfile, os.F_OK): os.remove(self.logfile) # Copy the system env variables, and update them for custom settings self.env = os.environ.copy() self.env.update(self.extra_env) # Update preferences for custom settings self.update_preferences() # Acquire a lock to make sure no other threads are running tests # at the same time. if self.rlock: self.rlock.acquire() try: # Create the Firefox runner, which will download and install the # build, as needed. if not self.firefoxRunner: self.firefoxRunner = TPSFirefoxRunner(self.binary) # now, run the test group self.run_test_group() except Exception: traceback.print_exc() self.numpassed = 0 self.numfailed = 1 try: self.writeToResultFile( self.postdata, "<pre>%s</pre>" % traceback.format_exc()) except Exception: traceback.print_exc() else: try: if self.numfailed > 0 or self.numpassed == 0: To = self.config["email"].get("notificationlist") else: To = self.config["email"].get("passednotificationlist") self.writeToResultFile(self.postdata, sendTo=To) except Exception: traceback.print_exc() try: self.writeToResultFile( self.postdata, "<pre>%s</pre>" % traceback.format_exc()) except Exception: traceback.print_exc() # release our lock if self.rlock: self.rlock.release() # dump out a summary of test results print("Test Summary\n") for test in self.postdata.get("tests", {}): print("{} | {} | {}".format(test["state"], test["name"], test["message"])) def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, "tps")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = [] for (filename, meta) in testfiles["tests"].items(): skip_reason = meta.get("disabled") if skip_reason: print("Skipping test {} - {}".format( filename, skip_reason)) else: testlist.append(filename) except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result["productversion"] if not self.addonversion: self.addonversion = result["addonversion"] self.results.append({ "state": result["state"], "name": result["name"], "message": result["message"], "logdata": result["logdata"], }) if result["state"] == "TEST-PASS": self.numpassed += 1 else: self.numfailed += 1 if self.stop_on_error: print("\nTest failed with --stop-on-error specified; " "not running any more tests.\n") break self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { "tests": self.results, "os": "%s %sbit" % (mozinfo.version, mozinfo.bits), "testtype": "crossweave", "productversion": self.productversion, "addonversion": self.addonversion, "synctype": self.synctype, }
class Peptest(): """ Peptest Runs and logs tests designed to test responsiveness """ profile_class = Profile runner_class = Runner def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger('PEP') # create the profile enable_proxy = False locations = ServerLocations() if self.options.proxyLocations: if not self.options.serverPath: self.logger.warning('Can\'t set up proxy without server path') else: enable_proxy = True for proxyLocation in self.options.proxyLocations: locations.read(proxyLocation, False) locations.add_host(host='127.0.0.1', port=self.options.serverPort, options='primary,privileged') self.profile = self.profile_class(profile=self.options.profilePath, addons=[os.path.join(here, 'extension')], locations=locations, proxy=enable_proxy) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith('.js'): # a single test file was passed in testObj = {} testObj['path'] = os.path.realpath(self.options.testPath) testObj['name'] = os.path.basename(self.options.testPath) testObj['here'] = os.path.dirname(testObj['path']) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj['tests'] = tests manifestObj['options'] = options.__dict__ # write manifest to a JSON file jsonManifest = open(os.path.join(here, 'manifest.json'), 'w') jsonManifest.write(json.dumps(manifestObj)) jsonManifest.close() # setup environment env = os.environ.copy() env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(options.tracerThreshold) env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval) env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class(profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess) def start(self): self.logger.debug('Starting Peptest') # start firefox self.runner.start(outputTimeout=self.options.timeout) self.runner.wait() crashed = self.checkForCrashes(results.currentTest) self.stop() if crashed or results.has_fails(): return 1 return 0 def runServer(self): """ Start a basic HTML server to host test related files. """ if not self.options.serverPath: self.logger.warning('Can\'t start HTTP server, --server-path not specified') return self.logger.debug('Starting server on port ' + str(self.options.serverPort)) self.server = MozHttpd(port=self.options.serverPort, docroot=self.options.serverPath, proxy_host_dirs=self.options.proxyHostDirs) self.server.start(block=False) def stop(self): """Kill the app""" # stop the runner if self.runner is not None: self.runner.stop() # kill the server process if self.server: self.server.stop() # remove harness related files files = ['manifest.json'] for f in files: if os.path.exists(os.path.join(here, f)): os.remove(os.path.join(here, f)) # delete any minidumps that may have been created dumpDir = os.path.join(self.profile.profile, 'minidumps') if self.options.profilePath and os.path.exists(dumpDir): shutil.rmtree(dumpDir) def checkForCrashes(self, testName=None): """ Detects when a crash occurs and prints the output from MINIDUMP_STACKWALK. Returns true if crash detected, otherwise false. """ stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None) # try to get the caller's filename if no test name is given if testName is None: try: testName = os.path.basename(sys._getframe(1).f_code.co_filename) except: testName = "unknown" foundCrash = False dumpDir = os.path.join(self.profile.profile, 'minidumps') dumps = glob.glob(os.path.join(dumpDir, '*.dmp')) symbolsPath = self.options.symbolsPath for d in dumps: import subprocess foundCrash = True self.logger.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName) print "Crash dump filename: " + d # only proceed if a symbols path and stackwalk path were specified if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath): # if symbolsPath is a url, download and extract the zipfile if utils.isURL(symbolsPath): bundle = utils.download(symbolsPath, here) symbolsPath = os.path.join(os.path.dirname(bundle), 'symbols') utils.extract(bundle, symbolsPath, delete=True) # run minidump_stackwalk p = subprocess.Popen([stackwalkPath, d, symbolsPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() if len(out) > 3: # minidump_stackwalk is chatty, so ignore stderr when it succeeds. print out else: print "stderr from minidump_stackwalk:" print err if p.returncode != 0: print "minidump_stackwalk exited with return code %d" % p.returncode else: self.logger.warning('No symbols_path or stackwalk path specified, can\'t process dump') break # if the symbols path was downloaded, cleanup after ourselves if utils.isURL(self.options.symbolsPath): if os.path.exists(symbolsPath): shutil.rmtree(symbolsPath) return foundCrash
class MarionetteTestRunner(object): textrunnerclass = MarionetteTextTestRunner def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, gecko_path=None, testvars=None, tree=None, type=None, device_serial=None, symbols_path=None, timeout=None, es_servers=None, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.xml_output = xml_output self.repeat = repeat self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device_serial = device_serial self.symbols_path = symbols_path self.timeout = timeout self._device = None self._capabilities = None self._appName = None self.es_servers = es_servers if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] @property def capabilities(self): if self._capabilities: return self._capabilities self.marionette.start_session() self._capabilities = self.marionette.session_capabilities self.marionette.delete_session() return self._capabilities @property def device(self): if self._device: return self._device self._device = self.capabilities.get('device') return self._device @property def appName(self): if self._appName: return self._appName self._appName = self.capabilities.get('browserName') return self._appName def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert(self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl, timeout=self.timeout) elif self.address: host, port = self.address.split(':') try: #establish a telnet connection so we can vertify the data come back tlconnection = Telnet(host, port) except: raise Exception("could not connect to given marionette host/port") if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl, timeout=self.timeout) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename for es_server in self.es_servers: # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup=self.testgroup, os='android', platform='emulator', harness='marionette', server=es_server, restserver=None, machine=socket.gethostname(), logfile=logfile) testgroup.set_primary_product( tree=self.tree, buildtype='opt', revision=self.revision) testgroup.add_test_suite( testsuite='b2g emulator testsuite', elapsedtime=elapsedtime.seconds, cmdline='', passed=self.passed, failed=self.failed, todo=self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >=0: for test in tests: self.run_test(test) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) if self.failed > 0: self.logger.info('\nFAILED TESTS\n-------') for failed_test in self.failures: self.logger.info('%s' % failed_test[0]) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette def run_test(self, test): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) all_tests = manifest.active_tests(disabled=False) manifest_tests = manifest.active_tests(disabled=False, device=self.device, app=self.appName) skip_tests = list(set([x['path'] for x in all_tests]) - set([x['path'] for x in manifest_tests])) for skipped in skip_tests: self.logger.info('TEST-SKIP | %s | device=%s, app=%s' % (os.path.basename(skipped), self.device, self.appName)) self.todo += 1 for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = self.textrunnerclass(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccesses'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS')) if hasattr(results, 'expectedFailures'): self.passed += len(results.expectedFailures) def register_handlers(self): self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase]) def cleanup(self): if self.httpd: self.httpd.stop() __del__ = cleanup def generate_xml(self, results_list): def _extract_xml(test, text='', result='passed'): cls_name = test.__class__.__name__ testcase = doc.createElement('testcase') testcase.setAttribute('classname', cls_name) testcase.setAttribute('name', unicode(test).split()[0]) testcase.setAttribute('time', str(test.duration)) testsuite.appendChild(testcase) if result in ['failure', 'error', 'skipped']: f = doc.createElement(result) f.setAttribute('message', 'test %s' % result) f.appendChild(doc.createTextNode(text)) testcase.appendChild(f) doc = dom.Document() testsuite = doc.createElement('testsuite') testsuite.setAttribute('name', 'Marionette') testsuite.setAttribute('time', str(self.elapsedtime.total_seconds())) testsuite.setAttribute('tests', str(sum([results.testsRun for results in results_list]))) def failed_count(results): count = len(results.failures) if hasattr(results, 'unexpectedSuccesses'): count += len(results.unexpectedSuccesses) return count testsuite.setAttribute('failures', str(sum([failed_count(results) for results in results_list]))) testsuite.setAttribute('errors', str(sum([len(results.errors) for results in results_list]))) if hasattr(results, 'skipped'): testsuite.setAttribute('skips', str(sum([len(results.skipped) + len(results.expectedFailures) for results in results_list]))) for results in results_list: for tup in results.errors: _extract_xml(tup[0], text=tup[1], result='error') for tup in results.failures: _extract_xml(tup[0], text=tup[1], result='failure') if hasattr(results, 'unexpectedSuccesses'): for test in results.unexpectedSuccesses: # unexpectedSuccesses is a list of Testcases only, no tuples _extract_xml(test, text='TEST-UNEXPECTED-PASS', result='failure') if hasattr(results, 'skipped'): for tup in results.skipped: _extract_xml(tup[0], text=tup[1], result='skipped') if hasattr(results, 'expectedFailures'): for tup in results.expectedFailures: _extract_xml(tup[0], text=tup[1], result='skipped') for test in results.tests_passed: _extract_xml(test) doc.appendChild(testsuite) return doc.toprettyxml(encoding='utf-8')
def valgrind_test(self, suppressions): from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from six import string_types from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, "build") # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo")) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile profile_data_dir = os.path.join(self.topsrcdir, "testing", "profiles") with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: base_profiles = json.load(fh)["valgrind"] prefpaths = [ os.path.join(profile_data_dir, profile, "user.js") for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) quitter = os.path.join(self.topsrcdir, "tools", "quitter", "*****@*****.**") locations = ServerLocations() locations.add_host(host="127.0.0.1", port=httpd.httpd.server_port, options="primary") profile = FirefoxProfile( profile=profilePath, preferences=prefs, addons=[quitter], locations=locations, ) firefox_args = [httpd.get_url()] env = os.environ.copy() env["G_SLICE"] = "always-malloc" env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["XPCOM_DEBUG_BREAK"] = "warn" outputHandler = OutputHandler(self.log) kp_kwargs = { "processOutputLine": [outputHandler], "universal_newlines": True, } valgrind = "valgrind" if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, "--sym-offsets=yes", "--smc-check=all-non-file", "--vex-iropt-register-updates=allregs-at-mem-access", "--gen-suppressions=all", "--num-callers=36", "--leak-check=full", "--show-possibly-lost=no", "--track-origins=yes", "--trace-children=yes", "-v", # Enable verbosity to get the list of used suppressions # Avoid excessive delays in the presence of spinlocks. # See bug 1309851. "--fair-sched=yes", # Keep debuginfo after library unmap. See bug 1382280. "--keep-debuginfo=yes", # Reduce noise level on rustc and/or LLVM compiled code. # See bug 1365915 "--expensive-definedness-checks=yes", # Compensate for the compiler inlining `new` but not `delete` # or vice versa. "--show-mismatched-frees=no", ] for s in suppressions: valgrind_args.append("--suppressions=" + s) supps_dir = os.path.join(build_dir, "valgrind") supps_file1 = os.path.join(supps_dir, "cross-architecture.sup") valgrind_args.append("--suppressions=" + supps_file1) if mozinfo.os == "linux": machtype = { "x86_64": "x86_64-pc-linux-gnu", "x86": "i386-pc-linux-gnu", }.get(mozinfo.processor) if machtype: supps_file2 = os.path.join(supps_dir, machtype + ".sup") if os.path.isfile(supps_file2): valgrind_args.append("--suppressions=" + supps_file2) exitcode = None timeout = 1800 binary_not_found_exception = None try: runner = FirefoxRunner( profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs, ) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) except BinaryNotFoundException as e: binary_not_found_exception = e finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log( logging.ERROR, "valgrind-fail-parsing", { "errs": errs, "supps": supps }, "TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors " "seen, but {supps} generated suppressions seen", ) elif errs == 0: status = 0 self.log( logging.INFO, "valgrind-pass", {}, "TEST-PASS | valgrind-test | valgrind found no errors", ) else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if binary_not_found_exception: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-errors", {"error": str(binary_not_found_exception)}, "TEST-UNEXPECTED-FAIL | valgrind-test | {error}", ) self.log( logging.INFO, "valgrind-fail-errors", {"help": binary_not_found_exception.help()}, "{help}", ) elif exitcode is None: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-timeout", {"timeout": timeout}, "TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out " "(reached {timeout} second limit)", ) elif exitcode != 0: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-errors", {"exitcode": exitcode}, "TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code " "from Valgrind: {exitcode}", ) httpd.stop() return status
def valgrind_test(self, suppressions): import json import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, 'OOP': 'false'} prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', '*****@*****.**') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' env.update(self.extra_environment_variables) outputHandler = OutputHandler(self.log) kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=36', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes', '--trace-children=yes', '-v', # Enable verbosity to get the list of used suppressions ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip() supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None timeout = 1800 try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log(logging.ERROR, 'valgrind-fail-parsing', {'errs': errs, 'supps': supps}, 'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen') elif errs == 0: status = 0 self.log(logging.INFO, 'valgrind-pass', {}, 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode == None: status = 2 # turns the TBPL job red self.log(logging.ERROR, 'valgrind-fail-timeout', {'timeout': timeout}, 'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)') elif exitcode != 0: status = 2 # turns the TBPL job red self.log(logging.ERROR, 'valgrind-fail-errors', {}, 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind') httpd.stop() return status
class TPSTestRunner(object): extra_env = { 'MOZ_CRASHREPORTER_DISABLE': '1', 'GNOME_DISABLE_CRASH_DIALOG': '1', 'XRE_NO_WINDOWS_CRASH_DIALOG': '1', 'MOZ_NO_REMOTE': '1', 'XPCOM_DEBUG_BREAK': 'warn', } default_preferences = { 'app.update.checkInstallTime': False, 'app.update.disabledForTesting': True, 'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer': True, 'browser.dom.window.dump.enabled': True, 'devtools.console.stdout.chrome': True, 'browser.sessionstore.resume_from_crash': False, 'browser.shell.checkDefaultBrowser': False, 'browser.tabs.warnOnClose': False, 'browser.warnOnQuit': False, # Allow installing extensions dropped into the profile folder 'extensions.autoDisableScopes': 10, 'extensions.getAddons.get.url': 'http://127.0.0.1:4567/addons/api/%IDS%.json', # Our pretend addons server doesn't support metadata... 'extensions.getAddons.cache.enabled': False, 'extensions.install.requireSecureOrigin': False, 'extensions.update.enabled': False, # Don't open a dialog to show available add-on updates 'extensions.update.notifyUser': False, 'services.sync.firstSync': 'notReady', 'services.sync.lastversion': '1.0', 'services.sync.autoconnectDelay': 60 * 60 * 10, 'toolkit.startup.max_resumed_crashes': -1, # hrm - not sure what the release/beta channels will do? 'xpinstall.signatures.required': False, 'services.sync.testing.tps': True, 'engine.bookmarks.repair.enabled': False, 'extensions.experiments.enabled': True, 'webextensions.storage.sync.kinto': False, } debug_preferences = { 'services.sync.log.appender.console': 'Trace', 'services.sync.log.appender.dump': 'Trace', 'services.sync.log.appender.file.level': 'Trace', 'services.sync.log.appender.file.logOnSuccess': True, 'services.sync.log.logger': 'Trace', 'services.sync.log.logger.engine': 'Trace', } syncVerRe = re.compile(r'Sync version: (?P<syncversion>.*)\n') ffVerRe = re.compile(r'Firefox version: (?P<ffver>.*)\n') ffBuildIDRe = re.compile(r'Firefox buildid: (?P<ffbuildid>.*)\n') def __init__(self, extensionDir, binary=None, config=None, debug=False, ignore_unused_engines=False, logfile='tps.log', mobile=False, rlock=None, resultfile='tps_result.json', testfile=None, stop_on_error=False): self.binary = binary self.config = config if config else {} self.debug = debug self.extensions = [] self.ignore_unused_engines = ignore_unused_engines self.logfile = os.path.abspath(logfile) self.mobile = mobile self.rlock = rlock self.resultfile = resultfile self.testfile = testfile self.stop_on_error = stop_on_error self.addonversion = None self.branch = None self.changeset = None self.errorlogs = {} self.extensionDir = extensionDir self.firefoxRunner = None self.nightly = False self.numfailed = 0 self.numpassed = 0 self.postdata = {} self.productversion = None self.repo = None self.tpsxpi = None @property def mobile(self): return self._mobile @mobile.setter def mobile(self, value): self._mobile = value self.synctype = 'desktop' if not self._mobile else 'mobile' def log(self, msg, printToConsole=False): """Appends a string to the logfile""" f = open(self.logfile, 'a') f.write(msg) f.close() if printToConsole: print(msg) def writeToResultFile(self, postdata, body=None, sendTo=['*****@*****.**']): """Writes results to test file""" results = {'results': []} if os.access(self.resultfile, os.F_OK): f = open(self.resultfile, 'r') results = json.loads(f.read()) f.close() f = open(self.resultfile, 'w') if body is not None: postdata['body'] = body if self.numpassed is not None: postdata['numpassed'] = self.numpassed if self.numfailed is not None: postdata['numfailed'] = self.numfailed if self.firefoxRunner and self.firefoxRunner.url: postdata['firefoxrunnerurl'] = self.firefoxRunner.url postdata['sendTo'] = sendTo results['results'].append(postdata) f.write(json.dumps(results, indent=2)) f.close() def _zip_add_file(self, zip, file, rootDir): zip.write(os.path.join(rootDir, file), file) def _zip_add_dir(self, zip, dir, rootDir): try: zip.write(os.path.join(rootDir, dir), dir) except Exception: # on some OS's, adding directory entries doesn't seem to work pass for root, dirs, files in os.walk(os.path.join(rootDir, dir)): for f in files: zip.write(os.path.join(root, f), os.path.join(dir, f)) def handle_phase_failure(self, profiles): for profile in profiles: self.log('\nDumping sync log for profile %s\n' % profiles[profile].profile) for root, dirs, files in os.walk( os.path.join(profiles[profile].profile, 'weave', 'logs')): for f in files: weavelog = os.path.join(profiles[profile].profile, 'weave', 'logs', f) if os.access(weavelog, os.F_OK): with open(weavelog, 'r') as fh: for line in fh: possible_time = line[0:13] if len(possible_time ) == 13 and possible_time.isdigit(): time_ms = int(possible_time) formatted = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time_ms / 1000)) self.log( '%s.%03d %s' % (formatted, time_ms % 1000, line[14:])) else: self.log(line) def run_single_test(self, testdir, testname): testpath = os.path.join(testdir, testname) self.log("Running test %s\n" % testname, True) # Read and parse the test file, merge it with the contents of the config # file, and write the combined output to a temporary file. f = open(testpath, 'r') testcontent = f.read() f.close() # We use yaml to parse the tests because it is a superset of json # but tolerates things like property names not being quoted, trailing # commas, etc. try: test = yaml.safe_load(testcontent) except Exception: test = yaml.safe_load( testcontent[testcontent.find('{'):testcontent.find('}') + 1]) self.preferences['tps.seconds_since_epoch'] = int(time.time()) # generate the profiles defined in the test, and a list of test phases profiles = {} phaselist = [] for phase in test: profilename = test[phase] # create the profile if necessary if profilename not in profiles: profiles[profilename] = Profile( preferences=self.preferences.copy(), addons=self.extensions) # create the test phase phaselist.append( TPSTestPhase(phase, profiles[profilename], testname, testpath, self.logfile, self.env, self.firefoxRunner, self.log, ignore_unused_engines=self.ignore_unused_engines)) # sort the phase list by name phaselist = sorted(phaselist, key=lambda phase: phase.phase) # run each phase in sequence, aborting at the first failure failed = False for phase in phaselist: phase.run() if phase.status != 'PASS': failed = True break for profilename in profiles: print("### Cleanup Profile ", profilename) cleanup_phase = TPSTestPhase('cleanup-' + profilename, profiles[profilename], testname, testpath, self.logfile, self.env, self.firefoxRunner, self.log) cleanup_phase.run() if cleanup_phase.status != 'PASS': failed = True # Keep going to run the remaining cleanup phases. if failed: self.handle_phase_failure(profiles) # grep the log for FF and sync versions f = open(self.logfile) logdata = f.read() match = self.syncVerRe.search(logdata) sync_version = match.group('syncversion') if match else 'unknown' match = self.ffVerRe.search(logdata) firefox_version = match.group('ffver') if match else 'unknown' match = self.ffBuildIDRe.search(logdata) firefox_buildid = match.group('ffbuildid') if match else 'unknown' f.close() if phase.status == 'PASS': logdata = '' else: # we only care about the log data for this specific test logdata = logdata[logdata.find('Running test %s' % (str(testname))):] result = { 'PASS': lambda x: ('TEST-PASS', ''), 'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()), 'unknown': lambda x: ('TEST-UNEXPECTED-FAIL', 'test did not complete') }[phase.status](phase.errline) logstr = "\n%s | %s%s\n" % (result[0], testname, (' | %s' % result[1] if result[1] else '')) try: repoinfo = mozversion.get_version(self.binary) except Exception: repoinfo = {} apprepo = repoinfo.get('application_repository', '') appchangeset = repoinfo.get('application_changeset', '') # save logdata to a temporary file for posting to the db tmplogfile = None if logdata: tmplogfile = TempFile(prefix='tps_log_') tmplogfile.write(logdata) tmplogfile.close() self.errorlogs[testname] = tmplogfile resultdata = ({ 'productversion': { 'version': firefox_version, 'buildid': firefox_buildid, 'builddate': firefox_buildid[0:8], 'product': 'Firefox', 'repository': apprepo, 'changeset': appchangeset, }, 'addonversion': { 'version': sync_version, 'product': 'Firefox Sync' }, 'name': testname, 'message': result[1], 'state': result[0], 'logdata': logdata }) self.log(logstr, True) for phase in phaselist: print("\t{}: {}".format(phase.phase, phase.status)) return resultdata def update_preferences(self): self.preferences = self.default_preferences.copy() if self.mobile: self.preferences.update({'services.sync.client.type': 'mobile'}) # If we are using legacy Sync, then set a dummy username to force the # correct authentication type. Without this pref set to a value # without an '@' character, Sync will initialize for FxA. if self.config.get('auth_type', 'fx_account') != "fx_account": self.preferences.update({'services.sync.username': "******"}) if self.debug: self.preferences.update(self.debug_preferences) if 'preferences' in self.config: self.preferences.update(self.config['preferences']) self.preferences['tps.config'] = json.dumps(self.config) def run_tests(self): # delete the logfile if it already exists if os.access(self.logfile, os.F_OK): os.remove(self.logfile) # Copy the system env variables, and update them for custom settings self.env = os.environ.copy() self.env.update(self.extra_env) # Update preferences for custom settings self.update_preferences() # Acquire a lock to make sure no other threads are running tests # at the same time. if self.rlock: self.rlock.acquire() try: # Create the Firefox runner, which will download and install the # build, as needed. if not self.firefoxRunner: self.firefoxRunner = TPSFirefoxRunner(self.binary) # now, run the test group self.run_test_group() except Exception: traceback.print_exc() self.numpassed = 0 self.numfailed = 1 try: self.writeToResultFile( self.postdata, '<pre>%s</pre>' % traceback.format_exc()) except Exception: traceback.print_exc() else: try: if self.numfailed > 0 or self.numpassed == 0: To = self.config['email'].get('notificationlist') else: To = self.config['email'].get('passednotificationlist') self.writeToResultFile(self.postdata, sendTo=To) except Exception: traceback.print_exc() try: self.writeToResultFile( self.postdata, '<pre>%s</pre>' % traceback.format_exc()) except Exception: traceback.print_exc() # release our lock if self.rlock: self.rlock.release() # dump out a summary of test results print('Test Summary\n') for test in self.postdata.get('tests', {}): print('{} | {} | {}'.format(test['state'], test['name'], test['message'])) def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, 'tps')) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = [] for (filename, meta) in testfiles['tests'].items(): skip_reason = meta.get("disabled") if skip_reason: print('Skipping test {} - {}'.format( filename, skip_reason)) else: testlist.append(filename) except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result['productversion'] if not self.addonversion: self.addonversion = result['addonversion'] self.results.append({ 'state': result['state'], 'name': result['name'], 'message': result['message'], 'logdata': result['logdata'] }) if result['state'] == 'TEST-PASS': self.numpassed += 1 else: self.numfailed += 1 if self.stop_on_error: print('\nTest failed with --stop-on-error specified; ' 'not running any more tests.\n') break self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { 'tests': self.results, 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits), 'testtype': 'crossweave', 'productversion': self.productversion, 'addonversion': self.addonversion, 'synctype': self.synctype, }
def main(args): parser = Options() options, args = parser.parse_args() kill_port = 20703 if not options.html_manifest or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server: parser.print_usage() return 2 package_options = get_package_options(parser, options) if not package_options: parser.print_usage() return 2 if not os.path.isdir(options.specialpowers): parser.error("SpecialPowers directory %s does not exist" % options.specialpowers) return 2 if options.prefs and not os.path.isfile(options.prefs): parser.error("Prefs file %s does not exist" % options.prefs) return 2 if options.log_dest and not os.path.isdir(options.log_dest): parser.error("Log directory %s does not exist" % options.log_dest) return 2 log = mozlog.unstructured.getLogger('steeplechase') log.setLevel(logging.DEBUG) if ':' in options.host1: host1, port = options.host1.split(':') dm1 = DeviceManagerSUT(host1, port) else: dm1 = DeviceManagerSUT(options.host1) if ':' in options.host2: host2, port = options.host2.split(':') dm2 = DeviceManagerSUT(host2, port) else: dm2 = DeviceManagerSUT(options.host2) if (options.killall is not None) and (options.killall == 1): kill_dm1 = DeviceManagerSUT(host1, kill_port) kill_dm2 = DeviceManagerSUT(host2, kill_port) os_type = GetOStypes(package_options) print("OS type of host1 is " + os_type[0] + " and host2 is " + os_type[1]) KillFirefoxesCommand(kill_dm1, os_type[0]) KillFirefoxesCommand(kill_dm2, os_type[1]) remote_info = [{ 'dm': dm1, 'binary': package_options.binary, 'package': package_options.package, 'is_initiator': True, 'name': 'Client1' }, { 'dm': dm2, 'binary': package_options.binary2, 'package': package_options.package2, 'is_initiator': False, 'name': 'Client2' }] # first, push app for info in remote_info: dm = info['dm'] if info['binary']: asset = Binary(path=info['binary'], log=log, dm=info['dm'], name=info['name']) else: asset = generate_package_asset(path=info['package'], log=log, dm=info['dm'], name=info['name']) if options.setup: asset.setup_test_root() info['test_root'] = asset.test_root() if options.setup: log.info("Pushing app to %s...", info["name"]) asset.setup_client() info['remote_app_path'] = asset.path_to_launch() if not options.setup and not dm.fileExists(info['remote_app_path']): log.error("App does not exist on %s, don't use --noSetup", info['name']) return 2 pass_count, fail_count = 0, 0 if options.html_manifest: manifest = TestManifest(strict=False) manifest.read(options.html_manifest) manifest_data = { "tests": [{ "path": t["relpath"] } for t in manifest.active_tests(disabled=False, **mozinfo.info)] } remote_port = 0 if options.remote_webserver: result = re.search(':(\d+)', options.remote_webserver) if result: remote_port = int(result.groups()[0]) @json_response def get_manifest(req): return (200, manifest_data) handlers = [{ 'method': 'GET', 'path': '/manifest.json', 'function': get_manifest }] httpd = MozHttpd( host=moznetwork.get_ip(), port=remote_port, log_requests=True, docroot=os.path.join(os.path.dirname(__file__), "..", "webharness"), urlhandlers=handlers, path_mappings={"/tests": os.path.dirname(options.html_manifest)}) httpd.start(block=False) test = HTMLTests(httpd, remote_info, log, options) html_pass_count, html_fail_count = test.run() pass_count += html_pass_count fail_count += html_fail_count httpd.stop() log.info("Result summary:") log.info("Passed: %d" % pass_count) log.info("Failed: %d" % fail_count) return pass_count > 0 and fail_count == 0
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, perf=False, perfserv=None, gecko_path=None, testvars=None, tree=None, device=None): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.perfrequest = None self.xml_output = xml_output self.repeat = repeat self.perf = perf self.perfserv = perfserv self.gecko_path = gecko_path self.testvars = {} self.tree = tree self.device = device if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] self.perfrequest = None def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert(self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), bin=self.bin, profile=self.profile, baseurl=self.baseurl) elif self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup = self.testgroup, os = 'android', platform = 'emulator', harness = 'marionette', server = self.es_server, restserver = self.rest_server, machine = socket.gethostname(), logfile = logfile) testgroup.set_primary_product( tree = self.tree, buildtype = 'opt', revision = self.revision) testgroup.add_test_suite( testsuite = 'b2g emulator testsuite', elapsedtime = elapsedtime.seconds, cmdline = '', passed = self.passed, failed = self.failed, todo = self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests, testtype=None): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >=0: for test in tests: self.run_test(test, testtype) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.perfrequest and options.perf: try: self.perfrequest.submit() except Exception, e: print "Could not submit to datazilla" print e if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette
def valgrind_test(self, suppressions): from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from six import string_types from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile profile_data_dir = os.path.join(self.topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['valgrind'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', '*****@*****.**') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' env.update(self.extra_environment_variables) outputHandler = OutputHandler(self.log) kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--sym-offsets=yes', '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=36', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes', '--trace-children=yes', '-v', # Enable verbosity to get the list of used suppressions # Avoid excessive delays in the presence of spinlocks. # See bug 1309851. '--fair-sched=yes', # Keep debuginfo after library unmap. See bug 1382280. '--keep-debuginfo=yes', # Reduce noise level on rustc and/or LLVM compiled code. # See bug 1365915 '--expensive-definedness-checks=yes', ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) if mozinfo.os == 'linux': machtype = { 'x86_64': 'x86_64-pc-linux-gnu', 'x86': 'i386-pc-linux-gnu', }.get(mozinfo.processor) if machtype: supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None timeout = 1800 try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log( logging.ERROR, 'valgrind-fail-parsing', { 'errs': errs, 'supps': supps }, 'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors ' 'seen, but {supps} generated suppressions seen') elif errs == 0: status = 0 self.log( logging.INFO, 'valgrind-pass', {}, 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode is None: status = 2 # turns the TBPL job red self.log( logging.ERROR, 'valgrind-fail-timeout', {'timeout': timeout}, 'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out ' '(reached {timeout} second limit)') elif exitcode != 0: status = 2 # turns the TBPL job red self.log( logging.ERROR, 'valgrind-fail-errors', {'exitcode': exitcode}, 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code ' 'from Valgrind: {exitcode}') httpd.stop() return status
class TPSTestRunner(object): default_env = { 'MOZ_CRASHREPORTER_DISABLE': '1', 'GNOME_DISABLE_CRASH_DIALOG': '1', 'XRE_NO_WINDOWS_CRASH_DIALOG': '1', 'MOZ_NO_REMOTE': '1', 'XPCOM_DEBUG_BREAK': 'warn', } default_preferences = { 'app.update.enabled' : False, 'browser.dom.window.dump.enabled': True, 'browser.sessionstore.resume_from_crash': False, 'browser.shell.checkDefaultBrowser' : False, 'browser.tabs.warnOnClose' : False, 'browser.warnOnQuit': False, # Allow installing extensions dropped into the profile folder 'extensions.autoDisableScopes': 10, 'extensions.getAddons.get.url': 'http://127.0.0.1:4567/addons/api/%IDS%.xml', 'extensions.update.enabled' : False, # Don't open a dialog to show available add-on updates 'extensions.update.notifyUser' : False, 'services.sync.addons.ignoreRepositoryChecking': True, 'services.sync.firstSync': 'notReady', 'services.sync.lastversion': '1.0', 'services.sync.log.rootLogger': 'Trace', 'services.sync.log.logger.engine.addons': 'Trace', 'services.sync.log.logger.service.main': 'Trace', 'services.sync.log.logger.engine.bookmarks': 'Trace', 'services.sync.log.appender.console': 'Trace', 'services.sync.log.appender.debugLog.enabled': True, 'toolkit.startup.max_resumed_crashes': -1, } syncVerRe = re.compile( r'Sync version: (?P<syncversion>.*)\n') ffVerRe = re.compile( r'Firefox version: (?P<ffver>.*)\n') ffDateRe = re.compile( r'Firefox builddate: (?P<ffdate>.*)\n') def __init__(self, extensionDir, testfile='sync.test', binary=None, config=None, rlock=None, mobile=False, logfile='tps.log', resultfile='tps_result.json', ignore_unused_engines=False): self.extensions = [] self.testfile = testfile self.logfile = os.path.abspath(logfile) self.resultfile = resultfile self.binary = binary self.ignore_unused_engines = ignore_unused_engines self.config = config if config else {} self.repo = None self.changeset = None self.branch = None self.numfailed = 0 self.numpassed = 0 self.nightly = False self.rlock = rlock self.mobile = mobile self.tpsxpi = None self.firefoxRunner = None self.extensionDir = extensionDir self.productversion = None self.addonversion = None self.postdata = {} self.errorlogs = {} @property def mobile(self): return self._mobile @mobile.setter def mobile(self, value): self._mobile = value self.synctype = 'desktop' if not self._mobile else 'mobile' def log(self, msg, printToConsole=False): """Appends a string to the logfile""" f = open(self.logfile, 'a') f.write(msg) f.close() if printToConsole: print msg def writeToResultFile(self, postdata, body=None, sendTo=['*****@*****.**']): """Writes results to test file""" results = {'results': []} if os.access(self.resultfile, os.F_OK): f = open(self.resultfile, 'r') results = json.loads(f.read()) f.close() f = open(self.resultfile, 'w') if body is not None: postdata['body'] = body if self.numpassed is not None: postdata['numpassed'] = self.numpassed if self.numfailed is not None: postdata['numfailed'] = self.numfailed if self.firefoxRunner and self.firefoxRunner.url: postdata['firefoxrunnerurl'] = self.firefoxRunner.url postdata['sendTo'] = sendTo results['results'].append(postdata) f.write(json.dumps(results, indent=2)) f.close() def _zip_add_file(self, zip, file, rootDir): zip.write(os.path.join(rootDir, file), file) def _zip_add_dir(self, zip, dir, rootDir): try: zip.write(os.path.join(rootDir, dir), dir) except: # on some OS's, adding directory entries doesn't seem to work pass for root, dirs, files in os.walk(os.path.join(rootDir, dir)): for f in files: zip.write(os.path.join(root, f), os.path.join(dir, f)) def run_single_test(self, testdir, testname): testpath = os.path.join(testdir, testname) self.log("Running test %s\n" % testname, True) # Read and parse the test file, merge it with the contents of the config # file, and write the combined output to a temporary file. f = open(testpath, 'r') testcontent = f.read() f.close() try: test = json.loads(testcontent) except: test = json.loads(testcontent[testcontent.find('{'):testcontent.find('}') + 1]) testcontent += 'var config = %s;\n' % json.dumps(self.config, indent=2) testcontent += 'var seconds_since_epoch = %d;\n' % int(time.time()) tmpfile = TempFile(prefix='tps_test_') tmpfile.write(testcontent) tmpfile.close() # generate the profiles defined in the test, and a list of test phases profiles = {} phaselist = [] for phase in test: profilename = test[phase] # create the profile if necessary if not profilename in profiles: profiles[profilename] = Profile(preferences = self.preferences, addons = self.extensions) # create the test phase phaselist.append(TPSTestPhase( phase, profiles[profilename], testname, tmpfile.filename, self.logfile, self.env, self.firefoxRunner, self.log, ignore_unused_engines=self.ignore_unused_engines)) # sort the phase list by name phaselist = sorted(phaselist, key=lambda phase: phase.phase) # run each phase in sequence, aborting at the first failure for phase in phaselist: phase.run() # if a failure occurred, dump the entire sync log into the test log if phase.status != 'PASS': for profile in profiles: self.log('\nDumping sync log for profile %s\n' % profiles[profile].profile) for root, dirs, files in os.walk(os.path.join(profiles[profile].profile, 'weave', 'logs')): for f in files: weavelog = os.path.join(profiles[profile].profile, 'weave', 'logs', f) if os.access(weavelog, os.F_OK): with open(weavelog, 'r') as fh: for line in fh: possible_time = line[0:13] if len(possible_time) == 13 and possible_time.isdigit(): time_ms = int(possible_time) formatted = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time_ms / 1000)) self.log('%s.%03d %s' % ( formatted, time_ms % 1000, line[14:] )) else: self.log(line) break; # grep the log for FF and sync versions f = open(self.logfile) logdata = f.read() match = self.syncVerRe.search(logdata) sync_version = match.group('syncversion') if match else 'unknown' match = self.ffVerRe.search(logdata) firefox_version = match.group('ffver') if match else 'unknown' match = self.ffDateRe.search(logdata) firefox_builddate = match.group('ffdate') if match else 'unknown' f.close() if phase.status == 'PASS': logdata = '' else: # we only care about the log data for this specific test logdata = logdata[logdata.find('Running test %s' % (str(testname))):] result = { 'PASS': lambda x: ('TEST-PASS', ''), 'FAIL': lambda x: ('TEST-UNEXPECTED-FAIL', x.rstrip()), 'unknown': lambda x: ('TEST-UNEXPECTED-FAIL', 'test did not complete') } [phase.status](phase.errline) logstr = "\n%s | %s%s\n" % (result[0], testname, (' | %s' % result[1] if result[1] else '')) try: repoinfo = self.firefoxRunner.runner.get_repositoryInfo() except: repoinfo = {} apprepo = repoinfo.get('application_repository', '') appchangeset = repoinfo.get('application_changeset', '') # save logdata to a temporary file for posting to the db tmplogfile = None if logdata: tmplogfile = TempFile(prefix='tps_log_') tmplogfile.write(logdata) tmplogfile.close() self.errorlogs[testname] = tmplogfile resultdata = ({ 'productversion': { 'version': firefox_version, 'buildid': firefox_builddate, 'builddate': firefox_builddate[0:8], 'product': 'Firefox', 'repository': apprepo, 'changeset': appchangeset, }, 'addonversion': { 'version': sync_version, 'product': 'Firefox Sync' }, 'name': testname, 'message': result[1], 'state': result[0], 'logdata': logdata }) self.log(logstr, True) for phase in phaselist: print "\t%s: %s" % (phase.phase, phase.status) if phase.status == 'FAIL': break return resultdata def run_tests(self): # delete the logfile if it already exists if os.access(self.logfile, os.F_OK): os.remove(self.logfile) # Make a copy of the default env variables and preferences, and update # them for mobile settings if needed. self.env = self.default_env.copy() self.preferences = self.default_preferences.copy() if self.mobile: self.preferences.update({'services.sync.client.type' : 'mobile'}) # Set a dummy username to force the correct authentication type. For the # old sync, the username is not allowed to contain a '@'. dummy = {'fx_account': 'dummy@somewhere', 'sync_account': 'dummy'} auth_type = self.config.get('auth_type', 'fx_account') self.preferences.update({'services.sync.username': dummy[auth_type]}) # Acquire a lock to make sure no other threads are running tests # at the same time. if self.rlock: self.rlock.acquire() try: # Create the Firefox runner, which will download and install the # build, as needed. if not self.firefoxRunner: self.firefoxRunner = TPSFirefoxRunner(self.binary) # now, run the test group self.run_test_group() except: traceback.print_exc() self.numpassed = 0 self.numfailed = 1 try: self.writeToResultFile(self.postdata, '<pre>%s</pre>' % traceback.format_exc()) except: traceback.print_exc() else: try: if self.numfailed > 0 or self.numpassed == 0: To = self.config['email'].get('notificationlist') else: To = self.config['email'].get('passednotificationlist') self.writeToResultFile(self.postdata, sendTo=To) except: traceback.print_exc() try: self.writeToResultFile(self.postdata, '<pre>%s</pre>' % traceback.format_exc()) except: traceback.print_exc() # release our lock if self.rlock: self.rlock.release() # dump out a summary of test results print 'Test Summary\n' for test in self.postdata.get('tests', {}): print '%s | %s | %s' % (test['state'], test['name'], test['message']) def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, 'tps')) self.extensions.append(os.path.join(self.extensionDir, "mozmill")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = testfiles['tests'] except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result['productversion'] if not self.addonversion: self.addonversion = result['addonversion'] self.results.append({'state': result['state'], 'name': result['name'], 'message': result['message'], 'logdata': result['logdata']}) if result['state'] == 'TEST-PASS': self.numpassed += 1 else: self.numfailed += 1 self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { 'tests': self.results, 'os': '%s %sbit' % (mozinfo.version, mozinfo.bits), 'testtype': 'crossweave', 'productversion': self.productversion, 'addonversion': self.addonversion, 'synctype': self.synctype, }
def main(args): parser = Options() options, args = parser.parse_args() if not options.html_manifest or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server: parser.print_usage() return 2 package_options = get_package_options(parser, options) if not package_options: parser.print_usage() return 2 if not os.path.isdir(options.specialpowers): parser.error("SpecialPowers directory %s does not exist" % options.specialpowers) return 2 if options.prefs and not os.path.isfile(options.prefs): parser.error("Prefs file %s does not exist" % options.prefs) return 2 if options.log_dest and not os.path.isdir(options.log_dest): parser.error("Log directory %s does not exist" % options.log_dest) return 2 log = mozlog.getLogger('steeplechase') log.setLevel(mozlog.DEBUG) if ':' in options.host1: host, port = options.host1.split(':') dm1 = DeviceManagerSUT(host, port) else: dm1 = DeviceManagerSUT(options.host1) if ':' in options.host2: host, port = options.host2.split(':') dm2 = DeviceManagerSUT(host, port) else: dm2 = DeviceManagerSUT(options.host2) remote_info = [{'dm': dm1, 'binary': package_options.binary, 'package': package_options.package, 'is_initiator': True, 'name': 'Client1'}, {'dm': dm2, 'binary': package_options.binary2, 'package': package_options.package2, 'is_initiator': False, 'name': 'Client2'}] # first, push app for info in remote_info: dm = info['dm'] if info['binary']: asset = Binary(path=info['binary'], log=log, dm=info['dm'], name=info['name']) else: asset = generate_package_asset(path=info['package'], log=log, dm=info['dm'], name=info['name']) if options.setup: asset.setup_test_root() info['test_root'] = asset.test_root() if options.setup: log.info("Pushing app to %s...", info["name"]) asset.setup_client() info['remote_app_path'] = asset.path_to_launch() if not options.setup and not dm.fileExists(info['remote_app_path']): log.error("App does not exist on %s, don't use --noSetup", info['name']) return 2 pass_count, fail_count = 0, 0 if options.html_manifest: manifest = TestManifest(strict=False) manifest.read(options.html_manifest) manifest_data = {"tests": [{"path": t["relpath"]} for t in manifest.active_tests(disabled=False, **mozinfo.info)]} remote_port = 0 if options.remote_webserver: result = re.search(':(\d+)', options.remote_webserver) if result: remote_port = int(result.groups()[0]) @json_response def get_manifest(req): return (200, manifest_data) handlers = [{ 'method': 'GET', 'path': '/manifest.json', 'function': get_manifest }] httpd = MozHttpd(host=moznetwork.get_ip(), port=remote_port, log_requests=True, docroot=os.path.join(os.path.dirname(__file__), "..", "webharness"), urlhandlers=handlers, path_mappings={"/tests": os.path.dirname(options.html_manifest)}) httpd.start(block=False) test = HTMLTests(httpd, remote_info, log, options) html_pass_count, html_fail_count = test.run() pass_count += html_pass_count fail_count += html_fail_count httpd.stop() log.info("Result summary:") log.info("Passed: %d" % pass_count) log.info("Failed: %d" % fail_count) return pass_count > 0 and fail_count == 0
cli = CLI() debug_args, interactive = cli.debugger_arguments() runner_args = cli.runner_args() build = MozbuildObject.from_environment() binary = runner_args.get('binary') if not binary: binary = build.get_binary_path(where="staged-package") path_mappings = { k: os.path.join(build.topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(build.topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) locations = ServerLocations() locations.add_host(host='127.0.0.1', port=PORT, options='primary,privileged') with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
class TPSTestRunner(object): default_env = { "MOZ_CRASHREPORTER_DISABLE": "1", "GNOME_DISABLE_CRASH_DIALOG": "1", "XRE_NO_WINDOWS_CRASH_DIALOG": "1", "MOZ_NO_REMOTE": "1", "XPCOM_DEBUG_BREAK": "warn", } default_preferences = { "app.update.enabled": False, "browser.dom.window.dump.enabled": True, "browser.sessionstore.resume_from_crash": False, "browser.shell.checkDefaultBrowser": False, "browser.tabs.warnOnClose": False, "browser.warnOnQuit": False, # Allow installing extensions dropped into the profile folder "extensions.autoDisableScopes": 10, "extensions.getAddons.get.url": "http://127.0.0.1:4567/en-US/firefox/api/%API_VERSION%/search/guid:%IDS%", "extensions.update.enabled": False, # Don't open a dialog to show available add-on updates "extensions.update.notifyUser": False, "services.sync.addons.ignoreRepositoryChecking": True, "services.sync.firstSync": "notReady", "services.sync.lastversion": "1.0", "services.sync.log.rootLogger": "Trace", "services.sync.log.logger.engine.addons": "Trace", "services.sync.log.logger.service.main": "Trace", "services.sync.log.logger.engine.bookmarks": "Trace", "services.sync.log.appender.console": "Trace", "services.sync.log.appender.debugLog.enabled": True, "toolkit.startup.max_resumed_crashes": -1, } syncVerRe = re.compile(r"Sync version: (?P<syncversion>.*)\n") ffVerRe = re.compile(r"Firefox version: (?P<ffver>.*)\n") ffDateRe = re.compile(r"Firefox builddate: (?P<ffdate>.*)\n") def __init__( self, extensionDir, testfile="sync.test", binary=None, config=None, rlock=None, mobile=False, logfile="tps.log", resultfile="tps_result.json", ignore_unused_engines=False, ): self.extensions = [] self.testfile = testfile self.logfile = os.path.abspath(logfile) self.resultfile = resultfile self.binary = binary self.ignore_unused_engines = ignore_unused_engines self.config = config if config else {} self.repo = None self.changeset = None self.branch = None self.numfailed = 0 self.numpassed = 0 self.nightly = False self.rlock = rlock self.mobile = mobile self.tpsxpi = None self.firefoxRunner = None self.extensionDir = extensionDir self.productversion = None self.addonversion = None self.postdata = {} self.errorlogs = {} @property def mobile(self): return self._mobile @mobile.setter def mobile(self, value): self._mobile = value self.synctype = "desktop" if not self._mobile else "mobile" def log(self, msg, printToConsole=False): """Appends a string to the logfile""" f = open(self.logfile, "a") f.write(msg) f.close() if printToConsole: print msg def writeToResultFile(self, postdata, body=None, sendTo=["*****@*****.**"]): """Writes results to test file""" results = {"results": []} if os.access(self.resultfile, os.F_OK): f = open(self.resultfile, "r") results = json.loads(f.read()) f.close() f = open(self.resultfile, "w") if body is not None: postdata["body"] = body if self.numpassed is not None: postdata["numpassed"] = self.numpassed if self.numfailed is not None: postdata["numfailed"] = self.numfailed if self.firefoxRunner and self.firefoxRunner.url: postdata["firefoxrunnerurl"] = self.firefoxRunner.url postdata["sendTo"] = sendTo results["results"].append(postdata) f.write(json.dumps(results, indent=2)) f.close() def _zip_add_file(self, zip, file, rootDir): zip.write(os.path.join(rootDir, file), file) def _zip_add_dir(self, zip, dir, rootDir): try: zip.write(os.path.join(rootDir, dir), dir) except: # on some OS's, adding directory entries doesn't seem to work pass for root, dirs, files in os.walk(os.path.join(rootDir, dir)): for f in files: zip.write(os.path.join(root, f), os.path.join(dir, f)) def run_single_test(self, testdir, testname): testpath = os.path.join(testdir, testname) self.log("Running test %s\n" % testname, True) # Read and parse the test file, merge it with the contents of the config # file, and write the combined output to a temporary file. f = open(testpath, "r") testcontent = f.read() f.close() try: test = json.loads(testcontent) except: test = json.loads(testcontent[testcontent.find("{") : testcontent.find("}") + 1]) testcontent += "var config = %s;\n" % json.dumps(self.config, indent=2) testcontent += "var seconds_since_epoch = %d;\n" % int(time.time()) tmpfile = TempFile(prefix="tps_test_") tmpfile.write(testcontent) tmpfile.close() # generate the profiles defined in the test, and a list of test phases profiles = {} phaselist = [] for phase in test: profilename = test[phase] # create the profile if necessary if not profilename in profiles: profiles[profilename] = Profile(preferences=self.preferences, addons=self.extensions) # create the test phase phaselist.append( TPSTestPhase( phase, profiles[profilename], testname, tmpfile.filename, self.logfile, self.env, self.firefoxRunner, self.log, ignore_unused_engines=self.ignore_unused_engines, ) ) # sort the phase list by name phaselist = sorted(phaselist, key=lambda phase: phase.phase) # run each phase in sequence, aborting at the first failure for phase in phaselist: phase.run() # if a failure occurred, dump the entire sync log into the test log if phase.status != "PASS": for profile in profiles: self.log("\nDumping sync log for profile %s\n" % profiles[profile].profile) for root, dirs, files in os.walk(os.path.join(profiles[profile].profile, "weave", "logs")): for f in files: weavelog = os.path.join(profiles[profile].profile, "weave", "logs", f) if os.access(weavelog, os.F_OK): with open(weavelog, "r") as fh: for line in fh: possible_time = line[0:13] if len(possible_time) == 13 and possible_time.isdigit(): time_ms = int(possible_time) formatted = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(time_ms / 1000) ) self.log("%s.%03d %s" % (formatted, time_ms % 1000, line[14:])) else: self.log(line) break # grep the log for FF and sync versions f = open(self.logfile) logdata = f.read() match = self.syncVerRe.search(logdata) sync_version = match.group("syncversion") if match else "unknown" match = self.ffVerRe.search(logdata) firefox_version = match.group("ffver") if match else "unknown" match = self.ffDateRe.search(logdata) firefox_builddate = match.group("ffdate") if match else "unknown" f.close() if phase.status == "PASS": logdata = "" else: # we only care about the log data for this specific test logdata = logdata[logdata.find("Running test %s" % (str(testname))) :] result = { "PASS": lambda x: ("TEST-PASS", ""), "FAIL": lambda x: ("TEST-UNEXPECTED-FAIL", x.rstrip()), "unknown": lambda x: ("TEST-UNEXPECTED-FAIL", "test did not complete"), }[phase.status](phase.errline) logstr = "\n%s | %s%s\n" % (result[0], testname, (" | %s" % result[1] if result[1] else "")) try: repoinfo = self.firefoxRunner.runner.get_repositoryInfo() except: repoinfo = {} apprepo = repoinfo.get("application_repository", "") appchangeset = repoinfo.get("application_changeset", "") # save logdata to a temporary file for posting to the db tmplogfile = None if logdata: tmplogfile = TempFile(prefix="tps_log_") tmplogfile.write(logdata) tmplogfile.close() self.errorlogs[testname] = tmplogfile resultdata = { "productversion": { "version": firefox_version, "buildid": firefox_builddate, "builddate": firefox_builddate[0:8], "product": "Firefox", "repository": apprepo, "changeset": appchangeset, }, "addonversion": {"version": sync_version, "product": "Firefox Sync"}, "name": testname, "message": result[1], "state": result[0], "logdata": logdata, } self.log(logstr, True) for phase in phaselist: print "\t%s: %s" % (phase.phase, phase.status) if phase.status == "FAIL": break return resultdata def run_tests(self): # delete the logfile if it already exists if os.access(self.logfile, os.F_OK): os.remove(self.logfile) # Make a copy of the default env variables and preferences, and update # them for mobile settings if needed. self.env = self.default_env.copy() self.preferences = self.default_preferences.copy() if self.mobile: self.preferences.update({"services.sync.client.type": "mobile"}) # If sync accounts have been chosen, disable Firefox Accounts if self.config.get("auth_type", "fx_account") != "fx_account": self.preferences.update({"services.sync.fxaccounts.enabled": False}) # Acquire a lock to make sure no other threads are running tests # at the same time. if self.rlock: self.rlock.acquire() try: # Create the Firefox runner, which will download and install the # build, as needed. if not self.firefoxRunner: self.firefoxRunner = TPSFirefoxRunner(self.binary) # now, run the test group self.run_test_group() except: traceback.print_exc() self.numpassed = 0 self.numfailed = 1 try: self.writeToResultFile(self.postdata, "<pre>%s</pre>" % traceback.format_exc()) except: traceback.print_exc() else: try: if self.numfailed > 0 or self.numpassed == 0: To = self.config["email"].get("notificationlist") else: To = self.config["email"].get("passednotificationlist") self.writeToResultFile(self.postdata, sendTo=To) except: traceback.print_exc() try: self.writeToResultFile(self.postdata, "<pre>%s</pre>" % traceback.format_exc()) except: traceback.print_exc() # release our lock if self.rlock: self.rlock.release() # dump out a summary of test results print "Test Summary\n" for test in self.postdata.get("tests", {}): print "%s | %s | %s" % (test["state"], test["name"], test["message"]) def run_test_group(self): self.results = [] # reset number of passed/failed tests self.numpassed = 0 self.numfailed = 0 # build our tps.xpi extension self.extensions = [] self.extensions.append(os.path.join(self.extensionDir, "tps")) self.extensions.append(os.path.join(self.extensionDir, "mozmill")) # build the test list try: f = open(self.testfile) jsondata = f.read() f.close() testfiles = json.loads(jsondata) testlist = testfiles["tests"] except ValueError: testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) self.mozhttpd = MozHttpd(port=4567, docroot=testdir) self.mozhttpd.start() # run each test, and save the results for test in testlist: result = self.run_single_test(testdir, test) if not self.productversion: self.productversion = result["productversion"] if not self.addonversion: self.addonversion = result["addonversion"] self.results.append( { "state": result["state"], "name": result["name"], "message": result["message"], "logdata": result["logdata"], } ) if result["state"] == "TEST-PASS": self.numpassed += 1 else: self.numfailed += 1 self.mozhttpd.stop() # generate the postdata we'll use to post the results to the db self.postdata = { "tests": self.results, "os": "%s %sbit" % (mozinfo.version, mozinfo.bits), "testtype": "crossweave", "productversion": self.productversion, "addonversion": self.addonversion, "synctype": self.synctype, }
class BaseMarionetteTestRunner(object): textrunnerclass = MarionetteTextTestRunner def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, app_args=None, bin=None, profile=None, autolog=False, revision=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, gecko_path=None, testvars=None, tree=None, type=None, device_serial=None, symbols_path=None, timeout=None, es_servers=None, shuffle=False, sdcard=None, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.app_args = app_args or [] self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.xml_output = xml_output self.repeat = repeat self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device_serial = device_serial self.symbols_path = symbols_path self.timeout = timeout self._device = None self._capabilities = None self._appName = None self.es_servers = es_servers self.shuffle = shuffle self.sdcard = sdcard self.mixin_run_tests = [] if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] @property def capabilities(self): if self._capabilities: return self._capabilities self.marionette.start_session() self._capabilities = self.marionette.session_capabilities self.marionette.delete_session() return self._capabilities @property def device(self): if self._device: return self._device self._device = self.capabilities.get('device') return self._device @property def appName(self): if self._appName: return self._appName self._appName = self.capabilities.get('browserName') return self._appName def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = moznetwork.get_ip() self.httpd = MozHttpd(host=host, port=0, docroot=os.path.join( os.path.dirname(os.path.dirname(__file__)), 'www')) self.httpd.start() self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port) self.logger.info('running webserver on %s' % self.baseurl) def start_marionette(self): assert (self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, app_args=self.app_args, bin=self.bin, profile=self.profile, baseurl=self.baseurl, timeout=self.timeout, device_serial=self.device_serial) elif self.address: host, port = self.address.split(':') try: #establish a socket connection so we can vertify the data come back connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect((host, int(port))) connection.close() except Exception, e: raise Exception( "Could not connect to given marionette host:port: %s" % e) if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl, timeout=self.timeout) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout, sdcard=self.sdcard)
class MarionetteTestRunner(object): def __init__( self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res="480x800", homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, perf=False, perfserv=None, gecko_path=None, testvars=None, tree=None, device=None, symbols_path=None, ): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.perfrequest = None self.xml_output = xml_output self.repeat = repeat self.perf = perf self.perfserv = perfserv self.gecko_path = gecko_path self.testvars = {} self.tree = tree self.device = device self.symbols_path = symbols_path if testvars: if not os.path.exists(testvars): raise Exception("--testvars file does not exist") import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger("Marionette") self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars["xml_output"] = self.xml_output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] self.perfrequest = None def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] s.close() self.baseurl = "http://%s:%d/" % (host, port) self.logger.info("running webserver on %s" % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www")) self.httpd.start() def start_marionette(self): assert self.baseurl is not None if self.bin: if self.address: host, port = self.address.split(":") else: host = "localhost" port = 2828 self.marionette = Marionette( host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl ) elif self.address: host, port = self.address.split(":") if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, ) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, ) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info("posting results to autolog") logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup=self.testgroup, os="android", platform="emulator", harness="marionette", server=self.es_server, restserver=self.rest_server, machine=socket.gethostname(), logfile=logfile, ) testgroup.set_primary_product(tree=self.tree, buildtype="opt", revision=self.revision) testgroup.add_test_suite( testsuite="b2g emulator testsuite", elapsedtime=elapsedtime.seconds, cmdline="", passed=self.passed, failed=self.failed, todo=self.todo, ) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests, testtype=None): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >= 0: for test in tests: self.run_test(test, testtype) self.repeat -= 1 self.logger.info("\nSUMMARY\n-------") self.logger.info("passed: %d" % self.passed) self.logger.info("failed: %d" % self.failed) self.logger.info("todo: %d" % self.todo) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.perfrequest and options.perf: try: self.perfrequest.submit() except Exception, e: print "Could not submit to datazilla" print e if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, "w") as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette
def valgrind_test(self, suppressions): import json import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, 'OOP': 'false'} prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' env.update(self.extra_environment_variables) outputHandler = OutputHandler() kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=36', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes', '--trace-children=yes', # The gstreamer plugin scanner can run as part of executing # firefox, but is an external program. In some weird cases, # valgrind finds errors while executing __libc_freeres when # it runs, but those are not relevant, as it's related to # executing third party code. So don't trace # gst-plugin-scanner. '--trace-children-skip=*/gst-plugin-scanner', '-v', # Enable verbosity to get the list of used suppressions ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip() supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None timeout = 1100 try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs) runner.start(debug_args=valgrind_args) # This timeout is slightly less than the no-output timeout on # TBPL, so we'll timeout here first and give an informative # message. exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange print('TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {} errors seen, but {} generated suppressions seen'.format(errs, supps)) elif errs == 0: status = 0 print('TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode == None: status = 2 # turns the TBPL job red print('TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {} second limit)'.format(timeout)) elif exitcode != 0: status = 2 # turns the TBPL job red print('TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind') httpd.stop() return status
def valgrind_test(self, suppressions): import json import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, 'OOP': 'false'} prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' outputHandler = OutputHandler() kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=20', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes' ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip() supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, kp_kwargs=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait() finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange print('TEST-UNEXPECTED-FAILURE | valgrind-test | error parsing:', errs, "errors seen, but", supps, "generated suppressions seen") elif errs == 0: status = 0 print('TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode != 0: status = 2 # turns the TBPL job red print('TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind') httpd.stop() return status
class MarionetteTestRunner(object): textrunnerclass = MarionetteTextTestRunner def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, app_args=None, bin=None, profile=None, autolog=False, revision=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, gecko_path=None, testvars=None, tree=None, type=None, device_serial=None, symbols_path=None, timeout=None, es_servers=None, shuffle=False, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.app_args = app_args or [] self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.xml_output = xml_output self.repeat = repeat self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device_serial = device_serial self.symbols_path = symbols_path self.timeout = timeout self._device = None self._capabilities = None self._appName = None self.es_servers = es_servers self.shuffle = shuffle if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] @property def capabilities(self): if self._capabilities: return self._capabilities self.marionette.start_session() self._capabilities = self.marionette.session_capabilities self.marionette.delete_session() return self._capabilities @property def device(self): if self._device: return self._device self._device = self.capabilities.get('device') return self._device @property def appName(self): if self._appName: return self._appName self._appName = self.capabilities.get('browserName') return self._appName def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = moznetwork.get_ip() self.httpd = MozHttpd(host=host, port=0, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port) self.logger.info('running webserver on %s' % self.baseurl) def start_marionette(self): assert(self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, app_args=self.app_args, bin=self.bin, profile=self.profile, baseurl=self.baseurl, timeout=self.timeout) elif self.address: host, port = self.address.split(':') try: #establish a socket connection so we can vertify the data come back connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM) connection.connect((host,int(port))) connection.close() except Exception, e: raise Exception("Could not connect to given marionette host:port: %s" % e) if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl, timeout=self.timeout) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout)
def valgrind_test(self, suppressions): import json import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, "build") # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo")) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, "testing", "profiles", "prefs_general.js") prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = {"server": "%s:%d" % httpd.httpd.server_address, "OOP": "false"} prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.topsrcdir, "tools", "quitter", "*****@*****.**") locations = ServerLocations() locations.add_host(host="127.0.0.1", port=httpd.httpd.server_port, options="primary") profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env["G_SLICE"] = "always-malloc" env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["XPCOM_DEBUG_BREAK"] = "warn" env.update(self.extra_environment_variables) outputHandler = OutputHandler(self.log) kp_kwargs = {"processOutputLine": [outputHandler]} valgrind = "valgrind" if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, "--smc-check=all-non-file", "--vex-iropt-register-updates=allregs-at-mem-access", "--gen-suppressions=all", "--num-callers=36", "--leak-check=full", "--show-possibly-lost=no", "--track-origins=yes", "--trace-children=yes", "-v", # Enable verbosity to get the list of used suppressions ] for s in suppressions: valgrind_args.append("--suppressions=" + s) supps_dir = os.path.join(build_dir, "valgrind") supps_file1 = os.path.join(supps_dir, "cross-architecture.sup") valgrind_args.append("--suppressions=" + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output(["bash", "-c", "echo $MACHTYPE"]).rstrip() supps_file2 = os.path.join(supps_dir, machtype + ".sup") if os.path.isfile(supps_file2): valgrind_args.append("--suppressions=" + supps_file2) exitcode = None timeout = 1800 try: runner = FirefoxRunner( profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs, ) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log( logging.ERROR, "valgrind-fail-parsing", {"errs": errs, "supps": supps}, "TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen", ) elif errs == 0: status = 0 self.log(logging.INFO, "valgrind-pass", {}, "TEST-PASS | valgrind-test | valgrind found no errors") else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode == None: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-timeout", {"timeout": timeout}, "TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)", ) elif exitcode != 0: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-errors", {}, "TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind", ) httpd.stop() return status