def test_update_file(self): """Test that mozinfo.update can load a JSON file.""" j = os.path.join(self.tempdir, "mozinfo.json") with open(j, "w") as f: f.write(json.dumps({"foo": "xyz"})) mozinfo.update(j) self.assertEqual(mozinfo.info["foo"], "xyz")
def validate(self, parser, options, context): """Validate android options.""" if build_obj: options.log_mach = '-' if options.remoteWebServer is None: if os.name != "nt": options.remoteWebServer = moznetwork.get_ip() else: parser.error( "you must specify a --remote-webserver=<ip address>") options.webServer = options.remoteWebServer if options.app is None: if build_obj: options.app = build_obj.substs['ANDROID_PACKAGE_NAME'] else: parser.error("You must specify either appPath or app") if build_obj and 'MOZ_HOST_BIN' in os.environ: options.xrePath = os.environ['MOZ_HOST_BIN'] # Only reset the xrePath if it wasn't provided if options.xrePath is None: options.xrePath = options.utilityPath if build_obj: options.topsrcdir = build_obj.topsrcdir if options.pidFile != "": f = open(options.pidFile, 'w') f.write("%s" % os.getpid()) f.close() if not options.robocopApk and build_obj: apk = build_obj.substs.get('GRADLE_ANDROID_APP_ANDROIDTEST_APK') if apk and os.path.exists(apk): options.robocopApk = apk if options.robocopApk != "": if not os.path.exists(options.robocopApk): parser.error("Unable to find robocop APK '%s'" % options.robocopApk) options.robocopApk = os.path.abspath(options.robocopApk) # Disable e10s by default on Android because we don't run Android # e10s jobs anywhere yet. options.e10s = False mozinfo.update({'e10s': options.e10s}) # allow us to keep original application around for cleanup while # running robocop via 'am' options.remoteappname = options.app return options
def find_tests_for_verification(self, action, success=None): """ For each file modified on this push, determine if the modified file is a test, by searching test manifests. Populate self.verify_suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ if self.config.get('verify') != True: return repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: self.warning("unable to verify tests: no repo or revision!") return [] def get_automationrelevance(): response = self.load_json_url(url) return response dirs = self.query_abs_dirs() mozinfo.find_and_update_from_json(dirs['abs_test_install_dir']) e10s = self.config.get('e10s', False) mozinfo.update({"e10s": e10s}) headless = self.config.get('headless', False) mozinfo.update({"headless": headless}) # FIXME(emilio): Need to update test expectations. mozinfo.update({'stylo': True}) mozinfo.update({'verify': True}) self.info("Verification using mozinfo: %s" % str(mozinfo.info)) # determine which files were changed on this push url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) changed_files = set() for c in contents['changesets']: self.info(" {cset} {desc}".format( cset=c['node'][0:12], desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) if self.config.get('verify_category') == "web-platform": self._find_wpt_tests(dirs, changed_files) else: self._find_misc_tests(dirs, changed_files) self.verify_downloaded = True
def run_desktop_test(self, suite=None, test_file=None, debugger=None, debugger_args=None, shuffle=False, keep_open=False, rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False, slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None, jsdebugger=False, debug_on_failure=False, start_at=None, end_at=None, e10s=False, dmd=False, dump_output_directory=None, dump_about_memory_after_test=False, dump_dmd_after_test=False): """Runs a mochitest. test_file is a path to a test file. It can be a relative path from the top source directory, an absolute filename, or a directory containing test files. suite is the type of mochitest to run. It can be one of ('plain', 'chrome', 'browser', 'metro', 'a11y'). debugger is a program name or path to a binary (presumably a debugger) to run the test in. e.g. 'gdb' debugger_args are the arguments passed to the debugger. shuffle is whether test order should be shuffled (defaults to false). keep_open denotes whether to keep the browser open after tests complete. """ if rerun_failures and test_file: print('Cannot specify both --rerun-failures and a test path.') return 1 # Need to call relpath before os.chdir() below. test_path = '' if test_file: test_path = self._wrap_path_argument(test_file).relpath() failure_file_path = os.path.join(self.statedir, 'mochitest_failures.json') if rerun_failures and not os.path.exists(failure_file_path): print('No failure file present. Did you run mochitests before?') return 1 from StringIO import StringIO # runtests.py is ambiguous, so we load the file/module manually. if 'mochitest' not in sys.modules: import imp path = os.path.join(self.mochitest_dir, 'runtests.py') with open(path, 'r') as fh: imp.load_module('mochitest', fh, path, ('.py', 'r', imp.PY_SOURCE)) import mozinfo import mochitest # This is required to make other components happy. Sad, isn't it? os.chdir(self.topobjdir) # Automation installs its own stream handler to stdout. Since we want # all logging to go through us, we just remove their handler. remove_handlers = [l for l in logging.getLogger().handlers if isinstance(l, logging.StreamHandler)] for handler in remove_handlers: logging.getLogger().removeHandler(handler) runner = mochitest.Mochitest() opts = mochitest.MochitestOptions() options, args = opts.parse_args([]) # Need to set the suite options before verifyOptions below. if suite == 'plain': # Don't need additional options for plain. pass elif suite == 'chrome': options.chrome = True elif suite == 'browser': options.browserChrome = True elif suite == 'metro': options.immersiveMode = True options.browserChrome = True elif suite == 'a11y': options.a11y = True elif suite == 'webapprt-content': options.webapprtContent = True options.app = self.get_webapp_runtime_path() elif suite == 'webapprt-chrome': options.webapprtChrome = True options.app = self.get_webapp_runtime_path() options.browserArgs.append("-test-mode") else: raise Exception('None or unrecognized mochitest suite type.') if dmd: options.dmdPath = self.lib_dir options.autorun = not no_autorun options.closeWhenDone = not keep_open options.shuffle = shuffle options.consoleLevel = 'INFO' options.repeat = repeat options.runUntilFailure = run_until_failure options.runSlower = slow options.testingModulesDir = os.path.join(self.tests_dir, 'modules') options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins')) options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols') options.chunkByDir = chunk_by_dir options.totalChunks = total_chunks options.thisChunk = this_chunk options.jsdebugger = jsdebugger options.debugOnFailure = debug_on_failure options.startAt = start_at options.endAt = end_at options.e10s = e10s options.dumpAboutMemoryAfterTest = dump_about_memory_after_test options.dumpDMDAfterTest = dump_dmd_after_test options.dumpOutputDirectory = dump_output_directory mozinfo.update({"e10s": e10s}) # for test manifest parsing. options.failureFile = failure_file_path if test_path: test_root = runner.getTestRoot(options) test_root_file = mozpack.path.join(self.mochitest_dir, test_root, test_path) if not os.path.exists(test_root_file): print('Specified test path does not exist: %s' % test_root_file) print('You may need to run |mach build| to build the test files.') return 1 # Handle test_path pointing at a manifest file so conditions in # the manifest are processed. This is a temporary solution # pending bug 938019. # The manifest basename is the same as |suite|, except for plain manifest_base = 'mochitest' if suite == 'plain' else suite if os.path.basename(test_root_file) == manifest_base + '.ini': options.manifestFile = test_root_file else: options.testPath = test_path if rerun_failures: options.testManifest = failure_file_path if debugger: options.debugger = debugger if debugger_args: if options.debugger == None: print("--debugger-args passed, but no debugger specified.") return 1 options.debuggerArgs = debugger_args options = opts.verifyOptions(options, runner) if options is None: raise Exception('mochitest option validator failed.') # We need this to enable colorization of output. self.log_manager.enable_unstructured() # Output processing is a little funky here. The old make targets # grepped the log output from TEST-UNEXPECTED-* and printed these lines # after test execution. Ideally the test runner would expose a Python # API for obtaining test results and we could just format failures # appropriately. Unfortunately, it doesn't yet do that. So, we capture # all output to a buffer then "grep" the buffer after test execution. # Bug 858197 tracks a Python API that would facilitate this. test_output = StringIO() handler = logging.StreamHandler(test_output) handler.addFilter(UnexpectedFilter()) handler.setFormatter(StructuredHumanFormatter(0, write_times=False)) logging.getLogger().addHandler(handler) result = runner.runTests(options) # Need to remove our buffering handler before we echo failures or else # it will catch them again! logging.getLogger().removeHandler(handler) self.log_manager.disable_unstructured() if test_output.getvalue(): result = 1 for line in test_output.getvalue().splitlines(): self.log(logging.INFO, 'unexpected', {'msg': line}, '{msg}') return result
def validate(self, parser, options, context): """Validate android options.""" if build_obj: options.log_mach = '-' device_args = {'deviceRoot': options.remoteTestRoot} device_args['adbPath'] = options.adbPath if options.deviceIP: device_args['host'] = options.deviceIP device_args['port'] = options.devicePort elif options.deviceSerial: device_args['deviceSerial'] = options.deviceSerial if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': device_args['logLevel'] = logging.DEBUG options.dm = DroidADB(**device_args) if not options.remoteTestRoot: options.remoteTestRoot = options.dm.deviceRoot if options.remoteWebServer is None: if os.name != "nt": options.remoteWebServer = moznetwork.get_ip() else: parser.error( "you must specify a --remote-webserver=<ip address>") options.webServer = options.remoteWebServer if options.remoteLogFile is None: options.remoteLogFile = options.remoteTestRoot + \ '/logs/mochitest.log' if options.remoteLogFile.count('/') < 1: options.remoteLogFile = options.remoteTestRoot + \ '/' + options.remoteLogFile if options.remoteAppPath and options.app: parser.error( "You cannot specify both the remoteAppPath and the app setting") elif options.remoteAppPath: options.app = options.remoteTestRoot + "/" + options.remoteAppPath elif options.app is None: if build_obj: options.app = build_obj.substs['ANDROID_PACKAGE_NAME'] else: # Neither remoteAppPath nor app are set -- error parser.error("You must specify either appPath or app") if build_obj and 'MOZ_HOST_BIN' in os.environ: options.xrePath = os.environ['MOZ_HOST_BIN'] # Only reset the xrePath if it wasn't provided if options.xrePath is None: options.xrePath = options.utilityPath if build_obj: options.topsrcdir = build_obj.topsrcdir if options.pidFile != "": f = open(options.pidFile, 'w') f.write("%s" % os.getpid()) f.close() # Robocop specific options if options.robocopIni != "": if not os.path.exists(options.robocopIni): parser.error( "Unable to find specified robocop .ini manifest '%s'" % options.robocopIni) options.robocopIni = os.path.abspath(options.robocopIni) if not options.robocopApk and build_obj: if build_obj.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'): options.robocopApk = os.path.join(build_obj.topobjdir, 'gradle', 'build', 'mobile', 'android', 'app', 'outputs', 'apk', 'app-official-australis-debug-androidTest-' 'unaligned.apk') else: options.robocopApk = os.path.join(build_obj.topobjdir, 'mobile', 'android', 'tests', 'browser', 'robocop', 'robocop-debug.apk') if options.robocopApk != "": if not os.path.exists(options.robocopApk): parser.error( "Unable to find robocop APK '%s'" % options.robocopApk) options.robocopApk = os.path.abspath(options.robocopApk) # Disable e10s by default on Android because we don't run Android # e10s jobs anywhere yet. options.e10s = False mozinfo.update({'e10s': options.e10s}) # allow us to keep original application around for cleanup while # running robocop via 'am' options.remoteappname = options.app return options
def find_modified_tests(self): """ For each file modified on this push, determine if the modified file is a test, by searching test manifests. Populate self.suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: self.warning( "unable to run tests in per-test mode: no repo or revision!") return [] def get_automationrelevance(): response = self.load_json_url(url) return response dirs = self.query_abs_dirs() mozinfo.find_and_update_from_json(dirs['abs_test_install_dir']) e10s = self.config.get('e10s', False) mozinfo.update({"e10s": e10s}) headless = self.config.get('headless', False) mozinfo.update({"headless": headless}) if mozinfo.info['buildapp'] == 'mobile/android': # extra android mozinfo normally comes from device queries, but this # code may run before the device is ready, so rely on configuration mozinfo.update( {'android_version': self.config.get('android_version', 18)}) mozinfo.update({'is_fennec': self.config.get('is_fennec', True)}) mozinfo.update( {'is_emulator': self.config.get('is_emulator', True)}) mozinfo.update({'verify': True}) self.info("Per-test run using mozinfo: %s" % str(mozinfo.info)) changed_files = set() if os.environ.get('MOZHARNESS_TEST_PATHS', None) is not None: suite_to_paths = json.loads(os.environ['MOZHARNESS_TEST_PATHS']) specified_files = itertools.chain.from_iterable( suite_to_paths.values()) changed_files.update(specified_files) self.info( "Per-test run found explicit request in MOZHARNESS_TEST_PATHS:" ) self.info(str(changed_files)) else: # determine which files were changed on this push url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) for c in contents['changesets']: self.info(" {cset} {desc}".format( cset=c['node'][0:12], desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) if self.config.get('per_test_category') == "web-platform": self._find_wpt_tests(dirs, changed_files) elif self.config.get('gpu_required', False) is not False: self._find_misc_tests(dirs, changed_files, gpu=True) else: self._find_misc_tests(dirs, changed_files) # per test mode run specific tests from any given test suite # _find_*_tests organizes tests to run into suites so we can # run each suite at a time # chunk files total_tests = sum([len(self.suites[x]) for x in self.suites]) files_per_chunk = total_tests / float( self.config.get('total_chunks', 1)) files_per_chunk = int(math.ceil(files_per_chunk)) chunk_number = int(self.config.get('this_chunk', 1)) suites = {} start = (chunk_number - 1) * files_per_chunk end = (chunk_number * files_per_chunk) current = -1 for suite in self.suites: for test in self.suites[suite]: current += 1 if current >= start and current < end: if suite not in suites: suites[suite] = [] suites[suite].append(test) if current >= end: break self.suites = suites self.tests_downloaded = True
def test_update(self): """Test that mozinfo.update works.""" mozinfo.update({"foo": 123}) self.assertEqual(mozinfo.info["foo"], 123)
def validate(self, parser, options, context): """Validate generic options.""" # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) # and android doesn't use 'app' the same way, so skip validation if parser.app != 'android': if options.app is None: if build_obj: options.app = build_obj.get_binary_path() else: parser.error( "could not find the application path, --appname must be specified") elif options.app == "dist" and build_obj: options.app = build_obj.get_binary_path(where='staged-package') options.app = self.get_full_path(options.app, parser.oldcwd) if not os.path.exists(options.app): parser.error("Error: Path {} doesn't exist. Are you executing " "$objdir/_tests/testing/mochitest/runtests.py?".format( options.app)) if options.flavor is None: options.flavor = 'plain' if options.gmp_path is None and options.app and build_obj: # Need to fix the location of gmp_fake which might not be shipped in the binary gmp_modules = ( ('gmp-fake', '1.0'), ('gmp-clearkey', '0.1'), ('gmp-fakeopenh264', '1.0') ) options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") if options.extra_mozinfo_json: if not os.path.isfile(options.extra_mozinfo_json): parser.error("Error: couldn't find mozinfo.json at '%s'." % options.extra_mozinfo_json) options.extra_mozinfo_json = json.load(open(options.extra_mozinfo_json)) if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname( options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: parser.error( "could not find xre directory, --xre-path must be specified") # allow relative paths if options.xrePath: options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: options.certPath = self.get_full_path(options.certPath, parser.oldcwd) elif build_obj: options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) elif not options.symbolsPath and build_obj: options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] if options.debugOnFailure and not options.jsdebugger: parser.error( "--debug-on-failure requires --jsdebugger.") if options.debuggerArgs and not options.debugger: parser.error( "--debugger-args requires --debugger.") if options.valgrind or options.debugger: # valgrind and some debuggers may cause Gecko to start slowly. Make sure # marionette waits long enough to connect. options.marionette_startup_timeout = 900 options.marionette_socket_timeout = 540 if options.store_chrome_manifest: options.store_chrome_manifest = os.path.abspath(options.store_chrome_manifest) if not os.path.isdir(os.path.dirname(options.store_chrome_manifest)): parser.error( "directory for %s does not exist as a destination to copy a " "chrome manifest." % options.store_chrome_manifest) if options.jscov_dir_prefix: options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix) if not os.path.isdir(options.jscov_dir_prefix): parser.error( "directory %s does not exist as a destination for coverage " "data." % options.jscov_dir_prefix) if options.testingModulesDir is None: # Try to guess the testing modules directory. possible = [os.path.join(here, os.path.pardir, 'modules')] if build_obj: possible.insert(0, os.path.join(build_obj.topobjdir, '_tests', 'modules')) for p in possible: if os.path.isdir(p): options.testingModulesDir = p break if build_obj: plugins_dir = os.path.join(build_obj.distdir, 'plugins') if os.path.isdir(plugins_dir) and plugins_dir not in options.extraProfileFiles: options.extraProfileFiles.append(plugins_dir) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): parser.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: parser.error("immersive is only supported on Windows 8 and up.") options.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") if not os.path.exists(options.immersiveHelperPath): parser.error("%s not found, cannot launch immersive tests." % options.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): parser.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: parser.error( '--use-test-media-devices is only supported on Linux currently') gst01 = spawn.find_executable("gst-launch-0.1") gst10 = spawn.find_executable("gst-launch-1.0") pactl = spawn.find_executable("pactl") if not (gst01 or gst10): parser.error( 'Missing gst-launch-{0.1,1.0}, required for ' '--use-test-media-devices') if not pactl: parser.error( 'Missing binary pactl required for ' '--use-test-media-devices') if options.nested_oop: options.e10s = True options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": options.defaultLeakThreshold, # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, } # See the dependencies of bug 1401764. if mozinfo.isWin: options.leakThresholds["tab"] = 1000 # XXX We can't normalize test_paths in the non build_obj case here, # because testRoot depends on the flavor, which is determined by the # mach command and therefore not finalized yet. Conversely, test paths # need to be normalized here for the mach case. if options.test_paths and build_obj: # Normalize test paths so they are relative to test root options.test_paths = [build_obj._wrap_path_argument(p).relpath() for p in options.test_paths] return options
def validate(self, parser, options, context): """Validate generic options.""" # for test manifest parsing. mozinfo.update({"strictContentSandbox": options.strictContentSandbox}) # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) # b2g and android don't use 'app' the same way, so skip validation if parser.app not in ('b2g', 'android'): if options.app is None: if build_obj: options.app = build_obj.get_binary_path() else: parser.error( "could not find the application path, --appname must be specified") elif options.app == "dist" and build_obj: options.app = build_obj.get_binary_path(where='staged-package') options.app = self.get_full_path(options.app, parser.oldcwd) if not os.path.exists(options.app): parser.error("Error: Path {} doesn't exist. Are you executing " "$objdir/_tests/testing/mochitest/runtests.py?".format( options.app)) if options.gmp_path is None and options.app and build_obj: # Need to fix the location of gmp_fake which might not be shipped in the binary gmp_modules = ( ('gmp-fake', '1.0'), ('gmp-clearkey', '0.1'), ('gmp-fakeopenh264', '1.0') ) options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.ipcplugins: options.test_paths.append('dom/plugins/test/mochitest') if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname( options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: parser.error( "could not find xre directory, --xre-path must be specified") # allow relative paths if options.xrePath: options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) if options.dmdPath: options.dmdPath = self.get_full_path(options.dmdPath, parser.oldcwd) if options.dmd and not options.dmdPath: if build_obj: options.dmdPath = build_obj.bin_dir else: parser.error( "could not find dmd libraries, specify them with --dmd-path") if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: options.certPath = self.get_full_path(options.certPath, parser.oldcwd) elif build_obj: options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) elif not options.symbolsPath and build_obj: options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.vmwareRecording: if not mozinfo.isWin: parser.error( "use-vmware-recording is only supported on Windows.") options.vmwareHelperPath = os.path.join( options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") if not os.path.exists(options.vmwareHelperPath): parser.error("%s not found, cannot automate VMware recording." % options.vmwareHelperPath) if options.webapprtContent and options.webapprtChrome: parser.error( "Only one of --webapprt-content and --webapprt-chrome may be given.") if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] options.autorun = False if options.debugOnFailure and not options.jsdebugger: parser.error( "--debug-on-failure requires --jsdebugger.") if options.debuggerArgs and not options.debugger: parser.error( "--debugger-args requires --debugger.") if options.testingModulesDir is None: if build_obj: options.testingModulesDir = os.path.join( build_obj.topobjdir, '_tests', 'modules') else: # Try to guess the testing modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code should never be executed in local runs because the build system # should always set the flag that populates this variable. If buildbot ever # passes this argument, this code can be deleted. possible = os.path.join(here, os.path.pardir, 'modules') if os.path.isdir(possible): options.testingModulesDir = possible if build_obj: plugins_dir = os.path.join(build_obj.distdir, 'plugins') if plugins_dir not in options.extraProfileFiles: options.extraProfileFiles.append(plugins_dir) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): parser.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: parser.error("immersive is only supported on Windows 8 and up.") options.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") if not os.path.exists(options.immersiveHelperPath): parser.error("%s not found, cannot launch immersive tests." % options.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): parser.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: parser.error( '--use-test-media-devices is only supported on Linux currently') for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): parser.error( 'Missing binary %s required for ' '--use-test-media-devices' % f) if options.nested_oop: if not options.e10s: options.e10s = True mozinfo.update({"e10s": options.e10s}) # for test manifest parsing. options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": 10000, # See dependencies of bug 1051230. # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, } # Bug 1091917 - We exit early in tab processes on Windows, so we don't # get leak logs yet. if mozinfo.isWin: options.ignoreMissingLeaks.append("tab") # XXX We can't normalize test_paths in the non build_obj case here, # because testRoot depends on the flavor, which is determined by the # mach command and therefore not finalized yet. Conversely, test paths # need to be normalized here for the mach case. if options.test_paths and build_obj: # Normalize test paths so they are relative to test root options.test_paths = [build_obj._wrap_path_argument(p).relpath() for p in options.test_paths] return options
def validate(self, parser, options, context): """Validate generic options.""" # for test manifest parsing. mozinfo.update({"strictContentSandbox": options.strictContentSandbox}) # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) # and android doesn't use 'app' the same way, so skip validation if parser.app != 'android': if options.app is None: if build_obj: options.app = build_obj.get_binary_path() else: parser.error( "could not find the application path, --appname must be specified") elif options.app == "dist" and build_obj: options.app = build_obj.get_binary_path(where='staged-package') options.app = self.get_full_path(options.app, parser.oldcwd) if not os.path.exists(options.app): parser.error("Error: Path {} doesn't exist. Are you executing " "$objdir/_tests/testing/mochitest/runtests.py?".format( options.app)) if options.flavor is None: options.flavor = 'plain' if options.gmp_path is None and options.app and build_obj: # Need to fix the location of gmp_fake which might not be shipped in the binary gmp_modules = ( ('gmp-fake', '1.0'), ('gmp-clearkey', '0.1'), ('gmp-fakeopenh264', '1.0') ) options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") if options.extra_mozinfo_json: if not os.path.isfile(options.extra_mozinfo_json): parser.error("Error: couldn't find mozinfo.json at '%s'." % options.extra_mozinfo_json) options.extra_mozinfo_json = json.load(open(options.extra_mozinfo_json)) if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname( options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: parser.error( "could not find xre directory, --xre-path must be specified") # allow relative paths if options.xrePath: options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) if options.dmdPath: options.dmdPath = self.get_full_path(options.dmdPath, parser.oldcwd) if options.dmd and not options.dmdPath: if build_obj: options.dmdPath = build_obj.bindir else: parser.error( "could not find dmd libraries, specify them with --dmd-path") if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: options.certPath = self.get_full_path(options.certPath, parser.oldcwd) elif build_obj: options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) elif not options.symbolsPath and build_obj: options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] options.autorun = False if options.debugOnFailure and not options.jsdebugger: parser.error( "--debug-on-failure requires --jsdebugger.") if options.debuggerArgs and not options.debugger: parser.error( "--debugger-args requires --debugger.") if options.valgrind or options.debugger: # valgrind and some debuggers may cause Gecko to start slowly. Make sure # marionette waits long enough to connect. options.marionette_port_timeout = 900 options.marionette_socket_timeout = 540 if options.store_chrome_manifest: options.store_chrome_manifest = os.path.abspath(options.store_chrome_manifest) if not os.path.isdir(os.path.dirname(options.store_chrome_manifest)): parser.error( "directory for %s does not exist as a destination to copy a " "chrome manifest." % options.store_chrome_manifest) if options.jscov_dir_prefix: options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix) if not os.path.isdir(options.jscov_dir_prefix): parser.error( "directory %s does not exist as a destination for coverage " "data." % options.jscov_dir_prefix) if options.testingModulesDir is None: if build_obj: options.testingModulesDir = os.path.join( build_obj.topobjdir, '_tests', 'modules') else: # Try to guess the testing modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code should never be executed in local runs because the build system # should always set the flag that populates this variable. If buildbot ever # passes this argument, this code can be deleted. possible = os.path.join(here, os.path.pardir, 'modules') if os.path.isdir(possible): options.testingModulesDir = possible if build_obj: plugins_dir = os.path.join(build_obj.distdir, 'plugins') if plugins_dir not in options.extraProfileFiles: options.extraProfileFiles.append(plugins_dir) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): parser.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: parser.error("immersive is only supported on Windows 8 and up.") options.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") if not os.path.exists(options.immersiveHelperPath): parser.error("%s not found, cannot launch immersive tests." % options.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): parser.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: parser.error( '--use-test-media-devices is only supported on Linux currently') for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): parser.error( 'Missing binary %s required for ' '--use-test-media-devices' % f) if options.nested_oop: options.e10s = True options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": 10000, # See dependencies of bug 1051230. # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, } # XXX We can't normalize test_paths in the non build_obj case here, # because testRoot depends on the flavor, which is determined by the # mach command and therefore not finalized yet. Conversely, test paths # need to be normalized here for the mach case. if options.test_paths and build_obj: # Normalize test paths so they are relative to test root options.test_paths = [build_obj._wrap_path_argument(p).relpath() for p in options.test_paths] return options
def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, manifest=None, testdirs=[], testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, profileName=None, mozInfo=None, **otherOptions): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |appPath|, if provided, is the path to an application directory. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of test directories to run. |testdirs|, if provided, is a list of absolute paths of test directories. No-manifest only option. |testPath|, if provided, indicates a single path and/or test to run. |interactive|, if set to True, indicates to provide an xpcshell prompt instead of automatically executing the test. |verbose|, if set to True, will cause stdout/stderr from tests to be printed always |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile directory if running only a subset of tests. |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. |otherOptions| may be present for the convenience of subclasses """ global gotSIGINT self.xpcshell = xpcshell self.xrePath = xrePath self.appPath = appPath self.symbolsPath = symbolsPath self.manifest = manifest self.testdirs = testdirs self.testPath = testPath self.interactive = interactive self.verbose = verbose self.keepGoing = keepGoing self.logfiles = logfiles self.totalChunks = totalChunks self.thisChunk = thisChunk self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) self.profileName = profileName or "xpcshell" self.mozInfo = mozInfo # If we have an interactive debugger, disable ctrl-c. if self.debuggerInfo and self.debuggerInfo["interactive"]: signal.signal(signal.SIGINT, lambda signum, frame: None) if not testdirs and not manifest: # nothing to test! self.log.error("Error: No test dirs or test manifest specified!") return False self.testCount = 0 self.passCount = 0 self.failCount = 0 self.todoCount = 0 self.setAbsPath() self.buildXpcsRunArgs() self.buildEnvironment() # Handle filenames in mozInfo if not isinstance(self.mozInfo, dict): mozInfoFile = self.mozInfo if not os.path.isfile(mozInfoFile): self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) return False self.mozInfo = parse_json(open(mozInfoFile).read()) mozinfo.update(self.mozInfo) pStdout, pStderr = self.getPipes() self.buildTestList() for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue self.testCount += 1 # Check for skipped tests if 'disabled' in test: self.log.info("TEST-INFO | skipping %s | %s" % (name, test['disabled'])) continue # Check for known-fail tests expected = test['expected'] == 'pass' testdir = os.path.dirname(name) self.buildXpcsCmd(testdir) testHeadFiles = self.getHeadFiles(test) testTailFiles = self.getTailFiles(test) cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd) # create a temp dir that the JS harness can stick a profile in self.profileDir = self.setupProfileDir() self.leakLogFile = self.setupLeakLogging() # The test file will have to be loaded after the head files. cmdT = self.buildCmdTestFile(name) args = self.xpcsRunArgs if 'debug' in test: args.insert(0, '-d') try: self.log.info("TEST-INFO | %s | running test ..." % name) startTime = time.time() proc = self.launchProcess(cmdH + cmdT + args, stdout=pStdout, stderr=pStderr, env=self.env, cwd=testdir) # Allow user to kill hung subprocess with SIGINT w/o killing this script # - don't move this line above launchProcess, or child will inherit the SIG_IGN signal.signal(signal.SIGINT, markGotSIGINT) # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|. stdout, stderr = self.communicate(proc) signal.signal(signal.SIGINT, signal.SIG_DFL) if interactive: # Not sure what else to do here... return True def print_stdout(stdout): """Print stdout line-by-line to avoid overflowing buffers.""" self.log.info(">>>>>>>") if (stdout): for line in stdout.splitlines(): self.log.info(line) self.log.info("<<<<<<<") result = not ((self.getReturnCode(proc) != 0) or (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", stdout, re.MULTILINE)) or (stdout and re.search(": SyntaxError:", stdout, re.MULTILINE))) if result != expected: self.log.error("TEST-UNEXPECTED-%s | %s | test failed (with xpcshell return code: %d), see following log:" % ("FAIL" if expected else "PASS", name, self.getReturnCode(proc))) print_stdout(stdout) self.failCount += 1 else: timeTaken = (time.time() - startTime) * 1000 self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken)) if verbose: print_stdout(stdout) if expected: self.passCount += 1 else: self.todoCount += 1 checkForCrashes(testdir, self.symbolsPath, testName=name) # Find child process(es) leak log(s), if any: See InitLog() in # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic leakLogs = [self.leakLogFile] for childLog in glob(os.path.join(self.profileDir, "runxpcshelltests_leaks_*_pid*.log")): if os.path.isfile(childLog): leakLogs += [childLog] for log in leakLogs: dumpLeakLog(log, True) if self.logfiles and stdout: self.createLogFile(name, stdout, leakLogs) finally: # We don't want to delete the profile when running check-interactive # or check-one. if self.profileDir and not self.interactive and not self.singleFile: self.removeDir(self.profileDir) if gotSIGINT: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution") if (keepGoing): gotSIGINT = False else: break if self.testCount == 0: self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?") self.failCount = 1 self.log.info("""INFO | Result summary: INFO | Passed: %d INFO | Failed: %d INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount)) if gotSIGINT and not keepGoing: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ "(Use --keep-going to keep running tests after killing one with SIGINT)") return False return self.failCount == 0
def validate(self, parser, options, context): """Validate android options.""" if build_obj: options.log_mach = '-' if options.dm_trans == "adb": if options.deviceIP: options.dm = DroidADB( options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) elif options.deviceSerial: options.dm = DroidADB( None, None, deviceSerial=options.deviceSerial, deviceRoot=options.remoteTestRoot) else: options.dm = DroidADB(deviceRoot=options.remoteTestRoot) elif options.dm_trans == 'sut': if options.deviceIP is None: parser.error( "If --dm_trans = sut, you must provide a device IP") options.dm = DroidSUT( options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) if not options.remoteTestRoot: options.remoteTestRoot = options.dm.deviceRoot if options.remoteWebServer is None: if os.name != "nt": options.remoteWebServer = moznetwork.get_ip() else: parser.error( "you must specify a --remote-webserver=<ip address>") options.webServer = options.remoteWebServer if options.remoteLogFile is None: options.remoteLogFile = options.remoteTestRoot + \ '/logs/mochitest.log' if options.remoteLogFile.count('/') < 1: options.remoteLogFile = options.remoteTestRoot + \ '/' + options.remoteLogFile if options.remoteAppPath and options.app: parser.error( "You cannot specify both the remoteAppPath and the app setting") elif options.remoteAppPath: options.app = options.remoteTestRoot + "/" + options.remoteAppPath elif options.app is None: if build_obj: options.app = build_obj.substs['ANDROID_PACKAGE_NAME'] else: # Neither remoteAppPath nor app are set -- error parser.error("You must specify either appPath or app") if build_obj and 'MOZ_HOST_BIN' in os.environ: options.xrePath = os.environ['MOZ_HOST_BIN'] # Only reset the xrePath if it wasn't provided if options.xrePath is None: options.xrePath = options.utilityPath if options.pidFile != "": f = open(options.pidFile, 'w') f.write("%s" % os.getpid()) f.close() # Robocop specific options if options.robocopIni != "": if not os.path.exists(options.robocopIni): parser.error( "Unable to find specified robocop .ini manifest '%s'" % options.robocopIni) options.robocopIni = os.path.abspath(options.robocopIni) if not options.robocopApk and build_obj: options.robocopApk = os.path.join(build_obj.topobjdir, 'mobile', 'android', 'tests', 'browser', 'robocop', 'robocop-debug.apk') if options.robocopApk != "": if not os.path.exists(options.robocopApk): parser.error( "Unable to find robocop APK '%s'" % options.robocopApk) options.robocopApk = os.path.abspath(options.robocopApk) # Disable e10s by default on Android because we don't run Android # e10s jobs anywhere yet. options.e10s = False mozinfo.update({'e10s': options.e10s}) # allow us to keep original application around for cleanup while # running robocop via 'am' options.remoteappname = options.app return options
def find_tests_for_verification(self, action, success=None): """ For each file modified on this push, determine if the modified file is a test, by searching test manifests. Populate self.verify_suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ if self.config.get('verify') != True: return repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: self.warning("unable to verify tests: no repo or revision!") return [] def get_automationrelevance(): response = self.load_json_url(url) return response dirs = self.query_abs_dirs() mozinfo.find_and_update_from_json(dirs['abs_test_install_dir']) if self.config.get('e10s') == True: mozinfo.update({"e10s": True}) # Additional mozinfo properties like "headless" and "coverage" are # also normally updated dynamically in the harness, but neither of # these apply to the test-verify task. manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] tests_by_path = {} for (path, suite) in manifests: if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=False, filters=[], **mozinfo.info) tests_by_path.update( {t['relpath']: (suite, t.get('subsuite')) for t in active}) self.info("Verification updated with manifest %s" % path) ref_manifests = [ (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'), (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'), # TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', 'jstests.list'), 'jstestbrowser'), ] sys.path.append(dirs['abs_reftest_dir']) import manifest self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests') for (path, suite) in ref_manifests: if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) tests_by_path.update({ os.path.relpath(t, self.reftest_test_dir): (suite, None) for t in man.files }) self.info("Verification updated with manifest %s" % path) # determine which files were changed on this push url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) changed_files = set() for c in contents['changesets']: self.info(" {cset} {desc}".format( cset=c['node'][0:12], desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) # for each changed file, determine if it is a test file, and what suite it is in for file in changed_files: # manifest paths use os.sep (like backslash on Windows) but # automation-relevance uses posixpath.sep file = file.replace(posixpath.sep, os.sep) entry = tests_by_path.get(file) if entry: self.info("Verification found test %s" % file) subsuite_mapping = { ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', ('chrome', 'clipboard'): 'chrome-clipboard', ('plain', 'clipboard'): 'plain-clipboard', ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', ('browser-chrome', 'gpu'): 'browser-chrome-gpu', ('chrome', 'gpu'): 'chrome-gpu', ('plain', 'gpu'): 'plain-gpu', ('plain', 'media'): 'mochitest-media', ('plain', 'webgl'): 'mochitest-gl', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] else: suite = entry[0] suite_files = self.verify_suites.get(suite) if not suite_files: suite_files = [] suite_files.append(file) self.verify_suites[suite] = suite_files self.verify_downloaded = True
class XPCShellTests(object): log = logging.getLogger() oldcwd = os.getcwd() def __init__(self, log=sys.stdout): """ Init logging and node status """ handler = logging.StreamHandler(log) self.log.setLevel(logging.INFO) self.log.addHandler(handler) self.nodeProc = None def buildTestList(self): """ read the xpcshell.ini manifest and set self.alltests to be an array of test objects. if we are chunking tests, it will be done here as well """ mp = manifestparser.TestManifest(strict=False) if self.manifest is None: for testdir in self.testdirs: if testdir: mp.read(os.path.join(testdir, 'xpcshell.ini')) else: mp.read(self.manifest) self.buildTestPath() self.alltests = mp.active_tests(**mozinfo.info) if self.singleFile is None and self.totalChunks > 1: self.chunkTests() def chunkTests(self): """ Split the list of tests up into [totalChunks] pieces and filter the self.alltests based on thisChunk, so we only run a subset. """ totalTests = 0 for dir in self.alltests: totalTests += len(self.alltests[dir]) testsPerChunk = math.ceil(totalTests / float(self.totalChunks)) start = int(round((self.thisChunk - 1) * testsPerChunk)) end = start + testsPerChunk currentCount = 0 templist = {} for dir in self.alltests: startPosition = 0 dirCount = len(self.alltests[dir]) endPosition = dirCount if currentCount < start and currentCount + dirCount >= start: startPosition = int(start - currentCount) if currentCount + dirCount > end: endPosition = int(end - currentCount) if end - currentCount < 0 or (currentCount + dirCount < start): endPosition = 0 if startPosition is not endPosition: templist[dir] = self.alltests[dir][startPosition:endPosition] currentCount += dirCount self.alltests = templist def setAbsPath(self): """ Set the absolute path for xpcshell, httpdjspath and xrepath. These 3 variables depend on input from the command line and we need to allow for absolute paths. This function is overloaded for a remote solution as os.path* won't work remotely. """ self.testharnessdir = os.path.dirname(os.path.abspath(__file__)) self.headJSPath = self.testharnessdir.replace("\\", "/") + "/head.js" self.xpcshell = os.path.abspath(self.xpcshell) # we assume that httpd.js lives in components/ relative to xpcshell self.httpdJSPath = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.js') self.httpdJSPath = replaceBackSlashes(self.httpdJSPath) self.httpdManifest = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.manifest') self.httpdManifest = replaceBackSlashes(self.httpdManifest) if self.xrePath is None: self.xrePath = os.path.dirname(self.xpcshell) else: self.xrePath = os.path.abspath(self.xrePath) if self.mozInfo is None: self.mozInfo = os.path.join(self.testharnessdir, "mozinfo.json") def buildCoreEnvironment(self): """ Add environment variables likely to be used across all platforms, including remote systems. """ # Make assertions fatal self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" # Don't launch the crash reporter client self.env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" # Capturing backtraces is very slow on some platforms, and it's # disabled by automation.py too self.env["NS_TRACE_MALLOC_DISABLE_STACKS"] = "1" def buildEnvironment(self): """ Create and returns a dictionary of self.env to include all the appropriate env variables and values. On a remote system, we overload this to set different values and are missing things like os.environ and PATH. """ self.env = dict(os.environ) self.buildCoreEnvironment() if sys.platform == 'win32': self.env["PATH"] = self.env["PATH"] + ";" + self.xrePath elif sys.platform in ('os2emx', 'os2knix'): os.environ[ "BEGINLIBPATH"] = self.xrePath + ";" + self.env["BEGINLIBPATH"] os.environ["LIBPATHSTRICT"] = "T" elif sys.platform == 'osx' or sys.platform == "darwin": self.env["DYLD_LIBRARY_PATH"] = self.xrePath else: # unix or linux? if not "LD_LIBRARY_PATH" in self.env or self.env[ "LD_LIBRARY_PATH"] is None: self.env["LD_LIBRARY_PATH"] = self.xrePath else: self.env["LD_LIBRARY_PATH"] = ":".join( [self.xrePath, self.env["LD_LIBRARY_PATH"]]) return self.env def buildXpcsRunArgs(self): """ Add arguments to run the test or make it interactive. """ if self.interactive: self.xpcsRunArgs = [ '-e', 'print("To start the test, type |_execute_test();|.");', '-i' ] else: self.xpcsRunArgs = ['-e', '_execute_test(); quit(0);'] def getPipes(self): """ Determine the value of the stdout and stderr for the test. Return value is a list (pStdout, pStderr). """ if self.interactive: pStdout = None pStderr = None else: if (self.debuggerInfo and self.debuggerInfo["interactive"]): pStdout = None pStderr = None else: if sys.platform == 'os2emx': pStdout = None else: pStdout = PIPE pStderr = STDOUT return pStdout, pStderr def buildXpcsCmd(self, testdir): """ Load the root head.js file as the first file in our test path, before other head, test, and tail files. On a remote system, we overload this to add additional command line arguments, so this gets overloaded. """ # - NOTE: if you rename/add any of the constants set here, update # do_load_child_test_harness() in head.js if not self.appPath: self.appPath = self.xrePath self.xpcsCmd = [ self.xpcshell, '-g', self.xrePath, '-a', self.appPath, '-r', self.httpdManifest, '-m', '-n', '-s', '-e', 'const _HTTPD_JS_PATH = "%s";' % self.httpdJSPath, '-e', 'const _HEAD_JS_PATH = "%s";' % self.headJSPath ] if self.testingModulesDir: # Escape backslashes in string literal. sanitized = self.testingModulesDir.replace('\\', '\\\\') self.xpcsCmd.extend( ['-e', 'const _TESTING_MODULES_DIR = "%s";' % sanitized]) self.xpcsCmd.extend( ['-f', os.path.join(self.testharnessdir, 'head.js')]) if self.debuggerInfo: self.xpcsCmd = [self.debuggerInfo["path"] ] + self.debuggerInfo["args"] + self.xpcsCmd if self.pluginsPath: self.xpcsCmd.extend(['-p', os.path.abspath(self.pluginsPath)]) def buildTestPath(self): """ If we specifiy a testpath, set the self.testPath variable to be the given directory or file. |testPath| will be the optional path only, or |None|. |singleFile| will be the optional test only, or |None|. """ self.singleFile = None if self.testPath is not None: if self.testPath.endswith('.js'): # Split into path and file. if self.testPath.find('/') == -1: # Test only. self.singleFile = self.testPath else: # Both path and test. # Reuse |testPath| temporarily. self.testPath = self.testPath.rsplit('/', 1) self.singleFile = self.testPath[1] self.testPath = self.testPath[0] else: # Path only. # Simply remove optional ending separator. self.testPath = self.testPath.rstrip("/") def getHeadAndTailFiles(self, test): """Obtain the list of head and tail files. Returns a 2-tuple. The first element is a list of head files. The second is a list of tail files. """ def sanitize_list(s, kind): for f in s.strip().split(' '): f = f.strip() if len(f) < 1: continue path = os.path.normpath(os.path.join(test['here'], f)) if not os.path.exists(path): raise Exception('%s file does not exist: %s' % (kind, path)) if not os.path.isfile(path): raise Exception('%s file is not a file: %s' % (kind, path)) yield path return (list(sanitize_list(test['head'], 'head')), list(sanitize_list(test['tail'], 'tail'))) def setupProfileDir(self): """ Create a temporary folder for the profile and set appropriate environment variables. When running check-interactive and check-one, the directory is well-defined and retained for inspection once the tests complete. On a remote system, this may be overloaded to use a remote path structure. """ if self.interactive or self.singleFile: profileDir = os.path.join(gettempdir(), self.profileName, "xpcshellprofile") try: # This could be left over from previous runs self.removeDir(profileDir) except: pass os.makedirs(profileDir) else: profileDir = mkdtemp() self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir if self.interactive or self.singleFile: self.log.info("TEST-INFO | profile dir is %s" % profileDir) return profileDir def setupLeakLogging(self): """ Enable leaks (only) detection to its own log file and set environment variables. On a remote system, this may be overloaded to use a remote filename and path structure """ filename = "runxpcshelltests_leaks.log" leakLogFile = os.path.join(self.profileDir, filename) self.env["XPCOM_MEM_LEAK_LOG"] = leakLogFile return leakLogFile def launchProcess(self, cmd, stdout, stderr, env, cwd): """ Simple wrapper to launch a process. On a remote system, this is more complex and we need to overload this function. """ cmd = wrapCommand(cmd) proc = Popen(cmd, stdout=stdout, stderr=stderr, env=env, cwd=cwd) return proc def communicate(self, proc): """ Simple wrapper to communicate with a process. On a remote system, this is overloaded to handle remote process communication. """ return proc.communicate() def poll(self, proc): """ Simple wrapper to check if a process has terminated. On a remote system, this is overloaded to handle remote process communication. """ return proc.poll() def kill(self, proc): """ Simple wrapper to kill a process. On a remote system, this is overloaded to handle remote process communication. """ return proc.kill() def removeDir(self, dirname): """ Simple wrapper to remove (recursively) a given directory. On a remote system, we need to overload this to work on the remote filesystem. """ shutil.rmtree(dirname) def verifyDirPath(self, dirname): """ Simple wrapper to get the absolute path for a given directory name. On a remote system, we need to overload this to work on the remote filesystem. """ return os.path.abspath(dirname) def getReturnCode(self, proc): """ Simple wrapper to get the return code for a given process. On a remote system we overload this to work with the remote process management. """ return proc.returncode def createLogFile(self, test, stdout, leakLogs): """ For a given test and stdout buffer, create a log file. also log any found leaks. On a remote system we have to fix the test name since it can contain directories. """ try: f = open(test + ".log", "w") f.write(stdout) for leakLog in leakLogs: if os.path.exists(leakLog): leaks = open(leakLog, "r") f.write(leaks.read()) leaks.close() finally: if f: f.close() def buildCmdHead(self, headfiles, tailfiles, xpcscmd): """ Build the command line arguments for the head and tail files, along with the address of the webserver which some tests require. On a remote system, this is overloaded to resolve quoting issues over a secondary command line. """ cmdH = ", ".join( ['"' + replaceBackSlashes(f) + '"' for f in headfiles]) cmdT = ", ".join( ['"' + replaceBackSlashes(f) + '"' for f in tailfiles]) return xpcscmd + \ ['-e', 'const _SERVER_ADDR = "localhost"', '-e', 'const _HEAD_FILES = [%s];' % cmdH, '-e', 'const _TAIL_FILES = [%s];' % cmdT] def buildCmdTestFile(self, name): """ Build the command line arguments for the test file. On a remote system, this may be overloaded to use a remote path structure. """ return ['-e', 'const _TEST_FILE = ["%s"];' % replaceBackSlashes(name)] def trySetupNode(self): """ Run node for SPDY tests, if available, and updates mozinfo as appropriate. """ nodeMozInfo = {'hasNode': False} # Assume the worst nodeBin = None # We try to find the node executable in the path given to us by the user in # the MOZ_NODE_PATH environment variable localPath = os.getenv('MOZ_NODE_PATH', None) if localPath and os.path.exists(localPath) and os.path.isfile( localPath): nodeBin = localPath if nodeBin: self.log.info('Found node at %s' % (nodeBin, )) myDir = os.path.split(os.path.abspath(__file__))[0] mozSpdyJs = os.path.join(myDir, 'moz-spdy', 'moz-spdy.js') if os.path.exists(mozSpdyJs): # OK, we found our SPDY server, let's try to get it running self.log.info('Found moz-spdy at %s' % (mozSpdyJs, )) stdout, stderr = self.getPipes() try: # We pipe stdin to node because the spdy server will exit when its # stdin reaches EOF self.nodeProc = Popen([nodeBin, mozSpdyJs], stdin=PIPE, stdout=PIPE, stderr=STDOUT, env=self.env, cwd=os.getcwd()) # Check to make sure the server starts properly by waiting for it to # tell us it's started msg = self.nodeProc.stdout.readline() if msg.startswith('SPDY server listening'): nodeMozInfo['hasNode'] = True except OSError, e: # This occurs if the subprocess couldn't be started self.log.error('Could not run node SPDY server: %s' % (str(e), )) mozinfo.update(nodeMozInfo)
def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, manifest=None, testdirs=None, testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, profileName=None, mozInfo=None, shuffle=False, testsRootDir=None, xunitFilename=None, xunitName=None, testingModulesDir=None, autolog=False, pluginsPath=None, **otherOptions): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |appPath|, if provided, is the path to an application directory. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of test directories to run. |testdirs|, if provided, is a list of absolute paths of test directories. No-manifest only option. |testPath|, if provided, indicates a single path and/or test to run. |pluginsPath|, if provided, custom plugins directory to be returned from the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST. |interactive|, if set to True, indicates to provide an xpcshell prompt instead of automatically executing the test. |verbose|, if set to True, will cause stdout/stderr from tests to be printed always |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile directory if running only a subset of tests. |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. |shuffle|, if True, execute tests in random order. |testsRootDir|, absolute path to root directory of all tests. This is used by xUnit generation to determine the package name of the tests. |xunitFilename|, if set, specifies the filename to which to write xUnit XML results. |xunitName|, if outputting an xUnit XML file, the str value to use for the testsuite name. |testingModulesDir|, if provided, specifies where JS modules reside. xpcshell will register a resource handler mapping this path. |otherOptions| may be present for the convenience of subclasses """ global gotSIGINT if testdirs is None: testdirs = [] if xunitFilename is not None or xunitName is not None: if not isinstance(testsRootDir, basestring): raise Exception("testsRootDir must be a str when outputting xUnit.") if not os.path.isabs(testsRootDir): testsRootDir = os.path.abspath(testsRootDir) if not os.path.exists(testsRootDir): raise Exception("testsRootDir path does not exists: %s" % testsRootDir) # Try to guess modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code path should never be executed in local runs because the build system # should always set this argument. if not testingModulesDir: ourDir = os.path.dirname(__file__) possible = os.path.join(ourDir, os.path.pardir, 'modules') if os.path.isdir(possible): testingModulesDir = possible if testingModulesDir: # The resource loader expects native paths. Depending on how we were # invoked, a UNIX style path may sneak in on Windows. We try to # normalize that. testingModulesDir = os.path.normpath(testingModulesDir) if not os.path.isabs(testingModulesDir): testingModulesDir = os.path.abspath(testingModulesDir) if not testingModulesDir.endswith(os.path.sep): testingModulesDir += os.path.sep self.xpcshell = xpcshell self.xrePath = xrePath self.appPath = appPath self.symbolsPath = symbolsPath self.manifest = manifest self.testdirs = testdirs self.testPath = testPath self.interactive = interactive self.verbose = verbose self.keepGoing = keepGoing self.logfiles = logfiles self.totalChunks = totalChunks self.thisChunk = thisChunk self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) self.profileName = profileName or "xpcshell" self.mozInfo = mozInfo self.testingModulesDir = testingModulesDir self.pluginsPath = pluginsPath # If we have an interactive debugger, disable ctrl-c. if self.debuggerInfo and self.debuggerInfo["interactive"]: signal.signal(signal.SIGINT, lambda signum, frame: None) if not testdirs and not manifest: # nothing to test! self.log.error("Error: No test dirs or test manifest specified!") return False self.testCount = 0 self.passCount = 0 self.failCount = 0 self.todoCount = 0 self.setAbsPath() self.buildXpcsRunArgs() self.buildEnvironment() # Handle filenames in mozInfo if not isinstance(self.mozInfo, dict): mozInfoFile = self.mozInfo if not os.path.isfile(mozInfoFile): self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) return False self.mozInfo = parse_json(open(mozInfoFile).read()) mozinfo.update(self.mozInfo) # The appDirKey is a optional entry in either the default or individual test # sections that defines a relative application directory for test runs. If # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell # test harness. appDirKey = None if "appname" in self.mozInfo: appDirKey = self.mozInfo["appname"] + "-appdir" # We have to do this before we build the test list so we know whether or # not to run tests that depend on having the node spdy server self.trySetupNode() pStdout, pStderr = self.getPipes() self.buildTestList() if shuffle: random.shuffle(self.alltests) xunitResults = [] for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue self.testCount += 1 keep_going, xunitResult = self.run_test(test, tests_root_dir=testsRootDir, app_dir_key=appDirKey, interactive=interactive, verbose=verbose, pStdout=pStdout, pStderr=pStderr, keep_going=keepGoing) xunitResults.append(xunitResult) if not keep_going: break self.shutdownNode() if self.testCount == 0: self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?") self.failCount = 1 self.log.info("INFO | Result summary:") self.log.info("INFO | Passed: %d" % self.passCount) self.log.info("INFO | Failed: %d" % self.failCount) self.log.info("INFO | Todo: %d" % self.todoCount) if autolog: self.post_to_autolog(xunitResults, xunitName) if xunitFilename is not None: self.writeXunitResults(filename=xunitFilename, results=xunitResults, name=xunitName) if gotSIGINT and not keepGoing: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ "(Use --keep-going to keep running tests after killing one with SIGINT)") return False return self.failCount == 0
def add_test(self, test, expected="pass", group="default"): filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if filename.endswith(".ini"): msg_tmpl = ( "Ignoring manifest '{0}'; running all tests in '{1}'." " See --help for details.") relpath = os.path.relpath(os.path.join(root, filename), filepath) self.logger.warning(msg_tmpl.format(relpath, filepath)) elif self._is_filename_valid(filename): test_file = os.path.join(root, filename) self.add_test(test_file) return file_ext = os.path.splitext(os.path.split(filepath)[-1])[1] if file_ext == ".ini": group = filepath manifest = TestManifest() manifest.read(filepath) json_path = update_mozinfo(filepath) mozinfo.update({ "appname": self.appName, "manage_instance": self.marionette.instance is not None, "headless": self.headless, "webrender": self.enable_webrender, }) self.logger.info("mozinfo updated from: {}".format(json_path)) self.logger.info("mozinfo is: {}".format(mozinfo.info)) filters = [] if self.test_tags: filters.append(tags(self.test_tags)) manifest_tests = manifest.active_tests(exists=False, disabled=True, filters=filters, **mozinfo.info) if len(manifest_tests) == 0: self.logger.error("No tests to run using specified " "combination of filters: {}".format( manifest.fmt_filters())) target_tests = [] for test in manifest_tests: if test.get("disabled"): self.manifest_skipped_tests.append(test) else: target_tests.append(test) for i in target_tests: if not os.path.exists(i["path"]): raise IOError("test file: {} does not exist".format( i["path"])) self.add_test(i["path"], i["expected"], group=group) return self.tests.append({ "filepath": filepath, "expected": expected, "group": group })
def validate(self, parser, options, context): """Validate generic options.""" # for test manifest parsing. mozinfo.update({"strictContentSandbox": options.strictContentSandbox}) # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) # b2g and android don't use 'app' the same way, so skip validation if parser.app not in ('b2g', 'android'): if options.app is None: if build_obj: options.app = build_obj.get_binary_path() else: parser.error( "could not find the application path, --appname must be specified" ) elif options.app == "dist" and build_obj: options.app = build_obj.get_binary_path(where='staged-package') options.app = self.get_full_path(options.app, parser.oldcwd) if not os.path.exists(options.app): parser.error( "Error: Path {} doesn't exist. Are you executing " "$objdir/_tests/testing/mochitest/runtests.py?".format( options.app)) if options.gmp_path is None and options.app and build_obj: # Need to fix the location of gmp_fake which might not be shipped in the binary gmp_modules = (('gmp-fake', '1.0'), ('gmp-clearkey', '0.1'), ('gmp-fakeopenh264', '1.0')) options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname(options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: parser.error( "could not find xre directory, --xre-path must be specified" ) # allow relative paths options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) if options.dmdPath: options.dmdPath = self.get_full_path(options.dmdPath, parser.oldcwd) if options.dmd and not options.dmdPath: if build_obj: options.dmdPath = build_obj.bin_dir else: parser.error( "could not find dmd libraries, specify them with --dmd-path" ) if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: options.certPath = self.get_full_path(options.certPath, parser.oldcwd) elif build_obj: options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if options.symbolsPath and len(urlparse( options.symbolsPath).scheme) < 2: options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) elif not options.symbolsPath and build_obj: options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.vmwareRecording: if not mozinfo.isWin: parser.error( "use-vmware-recording is only supported on Windows.") options.vmwareHelperPath = os.path.join( options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") if not os.path.exists(options.vmwareHelperPath): parser.error( "%s not found, cannot automate VMware recording." % options.vmwareHelperPath) if options.webapprtContent and options.webapprtChrome: parser.error( "Only one of --webapprt-content and --webapprt-chrome may be given." ) if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] options.autorun = False if options.debugOnFailure and not options.jsdebugger: parser.error("--debug-on-failure requires --jsdebugger.") if options.debuggerArgs and not options.debugger: parser.error("--debugger-args requires --debugger.") if options.testingModulesDir is None: if build_obj: options.testingModulesDir = os.path.join( build_obj.topobjdir, '_tests', 'modules') else: # Try to guess the testing modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code should never be executed in local runs because the build system # should always set the flag that populates this variable. If buildbot ever # passes this argument, this code can be deleted. possible = os.path.join(here, os.path.pardir, 'modules') if os.path.isdir(possible): options.testingModulesDir = possible if build_obj: options.extraProfileFiles.append( os.path.join(build_obj.distdir, 'plugins')) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): parser.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: parser.error( "immersive is only supported on Windows 8 and up.") options.immersiveHelperPath = os.path.join(options.utilityPath, "metrotestharness.exe") if not os.path.exists(options.immersiveHelperPath): parser.error("%s not found, cannot launch immersive tests." % options.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): parser.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: parser.error( '--use-test-media-devices is only supported on Linux currently' ) for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): parser.error('Missing binary %s required for ' '--use-test-media-devices' % f) if options.nested_oop: if not options.e10s: options.e10s = True mozinfo.update({"e10s": options.e10s}) # for test manifest parsing. options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": 25000, # See dependencies of bug 1051230. # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, } # Bug 1091917 - We exit early in tab processes on Windows, so we don't # get leak logs yet. if mozinfo.isWin: options.ignoreMissingLeaks.append("tab") # Bug 1121539 - OSX-only intermittent tab process leak in test_ipc.html if mozinfo.isMac: options.leakThresholds["tab"] = 100000 return options
def find_modified_tests(self): """ For each file modified on this push, determine if the modified file is a test, by searching test manifests. Populate self.suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: self.warning( "unable to run tests in per-test mode: no repo or revision!") self.suites = {} self.tests_downloaded = True return def get_automationrelevance(): response = self.load_json_url(url) return response dirs = self.query_abs_dirs() mozinfo.find_and_update_from_json(dirs["abs_test_install_dir"]) e10s = self.config.get("e10s", False) mozinfo.update({"e10s": e10s}) is_fission = "fission.autostart=true" in self.config.get( "extra_prefs", []) mozinfo.update({"fission": is_fission}) headless = self.config.get("headless", False) mozinfo.update({"headless": headless}) if mozinfo.info["buildapp"] == "mobile/android": # extra android mozinfo normally comes from device queries, but this # code may run before the device is ready, so rely on configuration mozinfo.update({ "android_version": str(self.config.get("android_version", 24)) }) mozinfo.update({"is_fennec": self.config.get("is_fennec", False)}) mozinfo.update( {"is_emulator": self.config.get("is_emulator", True)}) mozinfo.update({"verify": True}) self.info("Per-test run using mozinfo: %s" % str(mozinfo.info)) # determine which files were changed on this push changed_files = set() url = "%s/json-automationrelevance/%s" % (repository.rstrip("/"), revision) contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) for c in contents["changesets"]: self.info(" {cset} {desc}".format( cset=c["node"][0:12], desc=c["desc"].splitlines()[0].encode("ascii", "ignore"), )) changed_files |= set(c["files"]) changed_files = list(changed_files) # check specified test paths, as from 'mach try ... <path>' if os.environ.get("MOZHARNESS_TEST_PATHS", None) is not None: suite_to_paths = json.loads(os.environ["MOZHARNESS_TEST_PATHS"]) specified_paths = itertools.chain.from_iterable( suite_to_paths.values()) specified_paths = list(specified_paths) # filter the list of changed files to those found under the # specified path(s) changed_and_specified = set() for changed in changed_files: for specified in specified_paths: if changed.startswith(specified): changed_and_specified.add(changed) break if changed_and_specified: changed_files = changed_and_specified else: # if specified paths do not match changed files, assume the # specified paths are explicitly requested tests changed_files = set() changed_files.update(specified_paths) self.info( "Per-test run found explicit request in MOZHARNESS_TEST_PATHS:" ) self.info(str(changed_files)) if self.config.get("per_test_category") == "web-platform": self._find_wpt_tests(dirs, changed_files) elif self.config.get("gpu_required", False) is not False: self._find_misc_tests(dirs, changed_files, gpu=True) else: self._find_misc_tests(dirs, changed_files) # per test mode run specific tests from any given test suite # _find_*_tests organizes tests to run into suites so we can # run each suite at a time # chunk files total_tests = sum([len(self.suites[x]) for x in self.suites]) if total_tests == 0: self.warning("No tests to verify.") self.suites = {} self.tests_downloaded = True return files_per_chunk = total_tests / float( self.config.get("total_chunks", 1)) files_per_chunk = int(math.ceil(files_per_chunk)) chunk_number = int(self.config.get("this_chunk", 1)) suites = {} start = (chunk_number - 1) * files_per_chunk end = chunk_number * files_per_chunk current = -1 for suite in self.suites: for test in self.suites[suite]: current += 1 if current >= start and current < end: if suite not in suites: suites[suite] = [] suites[suite].append(test) if current >= end: break self.suites = suites self.tests_downloaded = True
def verifyOptions(self, options, mochitest): """ verify correct options and cleanup paths """ # for test manifest parsing. mozinfo.update({"strictContentSandbox": options.strictContentSandbox}) # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) if options.app is None: if build_obj is not None: options.app = build_obj.get_binary_path() else: self.error( "could not find the application path, --appname must be specified") if options.totalChunks is not None and options.thisChunk is None: self.error( "thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: self.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: self.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != self.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname( options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: self.error( "could not find xre directory, --xre-path must be specified") # allow relative paths options.xrePath = mochitest.getFullPath(options.xrePath) if options.profilePath: options.profilePath = mochitest.getFullPath(options.profilePath) options.app = mochitest.getFullPath(options.app) if options.dmdPath is not None: options.dmdPath = mochitest.getFullPath(options.dmdPath) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" self.error(msg % {"app": options.app}) return None if options.utilityPath: options.utilityPath = mochitest.getFullPath(options.utilityPath) if options.certPath: options.certPath = mochitest.getFullPath(options.certPath) if options.symbolsPath and len( urlparse( options.symbolsPath).scheme) < 2: options.symbolsPath = mochitest.getFullPath(options.symbolsPath) # Set server information on the options object options.webServer = '127.0.0.1' options.httpPort = DEFAULT_PORTS['http'] options.sslPort = DEFAULT_PORTS['https'] # options.webSocketPort = DEFAULT_PORTS['ws'] # <- http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l30 options.webSocketPort = str(9988) # The default websocket port is incorrect in mozprofile; it is # set to the SSL proxy setting. See: # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517 if options.vmwareRecording: if not mozinfo.isWin: self.error( "use-vmware-recording is only supported on Windows.") mochitest.vmwareHelperPath = os.path.join( options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") if not os.path.exists(mochitest.vmwareHelperPath): self.error("%s not found, cannot automate VMware recording." % mochitest.vmwareHelperPath) if options.webapprtContent and options.webapprtChrome: self.error( "Only one of --webapprt-content and --webapprt-chrome may be given.") if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] options.autorun = False if options.debugOnFailure and not options.jsdebugger: self.error( "--debug-on-failure should be used together with --jsdebugger.") # Try to guess the testing modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code should never be executed in local runs because the build system # should always set the flag that populates this variable. If buildbot ever # passes this argument, this code can be deleted. if options.testingModulesDir is None: possible = os.path.join(here, os.path.pardir, 'modules') if os.path.isdir(possible): options.testingModulesDir = possible # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): self.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: self.error("immersive is only supported on Windows 8 and up.") mochitest.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") if not os.path.exists(mochitest.immersiveHelperPath): self.error("%s not found, cannot launch immersive tests." % mochitest.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): self.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: self.error( '--use-test-media-devices is only supported on Linux currently') for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): self.error( 'Missing binary %s required for ' '--use-test-media-devices' % f) if options.nested_oop: if not options.e10s: options.e10s = True mozinfo.update({"e10s": options.e10s}) # for test manifest parsing. options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": 25000, # See dependencies of bug 1051230. # GMP rarely gets a log, but when it does, it leaks a little. "geckomediaplugin": 20000, } # Bug 1065098 - The geckomediaplugin process fails to produce a leak # log for some reason. options.ignoreMissingLeaks = ["geckomediaplugin"] # Bug 1091917 - We exit early in tab processes on Windows, so we don't # get leak logs yet. if mozinfo.isWin: options.ignoreMissingLeaks.append("tab") # Bug 1121539 - OSX-only intermittent tab process leak in test_ipc.html if mozinfo.isMac: options.leakThresholds["tab"] = 100000 return options
def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, manifest=None, testdirs=None, testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, profileName=None, mozInfo=None, shuffle=False, testsRootDir=None, xunitFilename=None, xunitName=None, testingModulesDir=None, autolog=False, pluginsPath=None, **otherOptions): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |appPath|, if provided, is the path to an application directory. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of test directories to run. |testdirs|, if provided, is a list of absolute paths of test directories. No-manifest only option. |testPath|, if provided, indicates a single path and/or test to run. |pluginsPath|, if provided, custom plugins directory to be returned from the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST. |interactive|, if set to True, indicates to provide an xpcshell prompt instead of automatically executing the test. |verbose|, if set to True, will cause stdout/stderr from tests to be printed always |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile directory if running only a subset of tests. |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. |shuffle|, if True, execute tests in random order. |testsRootDir|, absolute path to root directory of all tests. This is used by xUnit generation to determine the package name of the tests. |xunitFilename|, if set, specifies the filename to which to write xUnit XML results. |xunitName|, if outputting an xUnit XML file, the str value to use for the testsuite name. |testingModulesDir|, if provided, specifies where JS modules reside. xpcshell will register a resource handler mapping this path. |otherOptions| may be present for the convenience of subclasses """ global gotSIGINT if testdirs is None: testdirs = [] if xunitFilename is not None or xunitName is not None: if not isinstance(testsRootDir, basestring): raise Exception( "testsRootDir must be a str when outputting xUnit.") if not os.path.isabs(testsRootDir): testsRootDir = os.path.abspath(testsRootDir) if not os.path.exists(testsRootDir): raise Exception("testsRootDir path does not exists: %s" % testsRootDir) # Try to guess modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code path should never be executed in local runs because the build system # should always set this argument. if not testingModulesDir: ourDir = os.path.dirname(__file__) possible = os.path.join(ourDir, os.path.pardir, 'modules') if os.path.isdir(possible): testingModulesDir = possible if testingModulesDir: # The resource loader expects native paths. Depending on how we were # invoked, a UNIX style path may sneak in on Windows. We try to # normalize that. testingModulesDir = os.path.normpath(testingModulesDir) if not os.path.isabs(testingModulesDir): testingModulesDir = os.path.abspath(testingModulesDir) if not testingModulesDir.endswith(os.path.sep): testingModulesDir += os.path.sep self.xpcshell = xpcshell self.xrePath = xrePath self.appPath = appPath self.symbolsPath = symbolsPath self.manifest = manifest self.testdirs = testdirs self.testPath = testPath self.interactive = interactive self.verbose = verbose self.keepGoing = keepGoing self.logfiles = logfiles self.totalChunks = totalChunks self.thisChunk = thisChunk self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) self.profileName = profileName or "xpcshell" self.mozInfo = mozInfo self.testingModulesDir = testingModulesDir self.pluginsPath = pluginsPath # If we have an interactive debugger, disable ctrl-c. if self.debuggerInfo and self.debuggerInfo["interactive"]: signal.signal(signal.SIGINT, lambda signum, frame: None) if not testdirs and not manifest: # nothing to test! self.log.error("Error: No test dirs or test manifest specified!") return False self.testCount = 0 self.passCount = 0 self.failCount = 0 self.todoCount = 0 self.setAbsPath() self.buildXpcsRunArgs() self.buildEnvironment() # Handle filenames in mozInfo if not isinstance(self.mozInfo, dict): mozInfoFile = self.mozInfo if not os.path.isfile(mozInfoFile): self.log.error( "Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) return False self.mozInfo = parse_json(open(mozInfoFile).read()) mozinfo.update(self.mozInfo) # The appDirKey is a optional entry in either the default or individual test # sections that defines a relative application directory for test runs. If # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell # test harness. appDirKey = None if "appname" in self.mozInfo: appDirKey = self.mozInfo["appname"] + "-appdir" # We have to do this before we build the test list so we know whether or # not to run tests that depend on having the node spdy server self.trySetupNode() pStdout, pStderr = self.getPipes() self.buildTestList() if shuffle: random.shuffle(self.alltests) xunitResults = [] for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue self.testCount += 1 xunitResult = {"name": test["name"], "classname": "xpcshell"} # The xUnit package is defined as the path component between the root # dir and the test with path characters replaced with '.' (using Java # class notation). if testsRootDir is not None: testsRootDir = os.path.normpath(testsRootDir) if test["here"].find(testsRootDir) != 0: raise Exception("testsRootDir is not a parent path of %s" % test["here"]) relpath = test["here"][len(testsRootDir):].lstrip("/\\") xunitResult["classname"] = relpath.replace("/", ".").replace( "\\", ".") # Check for skipped tests if 'disabled' in test: self.log.info("TEST-INFO | skipping %s | %s" % (name, test['disabled'])) xunitResult["skipped"] = True xunitResults.append(xunitResult) continue # Check for known-fail tests expected = test['expected'] == 'pass' # By default self.appPath will equal the gre dir. If specified in the # xpcshell.ini file, set a different app dir for this test. if appDirKey != None and appDirKey in test: relAppDir = test[appDirKey] relAppDir = os.path.join(self.xrePath, relAppDir) self.appPath = os.path.abspath(relAppDir) else: self.appPath = None testdir = os.path.dirname(name) self.buildXpcsCmd(testdir) testHeadFiles, testTailFiles = self.getHeadAndTailFiles(test) cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd) # create a temp dir that the JS harness can stick a profile in self.profileDir = self.setupProfileDir() self.leakLogFile = self.setupLeakLogging() # The test file will have to be loaded after the head files. cmdT = self.buildCmdTestFile(name) args = self.xpcsRunArgs[:] if 'debug' in test: args.insert(0, '-d') completeCmd = cmdH + cmdT + args proc = None try: self.log.info("TEST-INFO | %s | running test ..." % name) if verbose: self.logCommand(name, completeCmd, testdir) startTime = time.time() proc = self.launchProcess(completeCmd, stdout=pStdout, stderr=pStderr, env=self.env, cwd=testdir) if interactive: self.log.info("TEST-INFO | %s | Process ID: %d" % (name, proc.pid)) # Allow user to kill hung subprocess with SIGINT w/o killing this script # - don't move this line above launchProcess, or child will inherit the SIG_IGN signal.signal(signal.SIGINT, markGotSIGINT) # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|. stdout, stderr = self.communicate(proc) signal.signal(signal.SIGINT, signal.SIG_DFL) if interactive: # Not sure what else to do here... return True def print_stdout(stdout): """Print stdout line-by-line to avoid overflowing buffers.""" self.log.info(">>>>>>>") if (stdout): for line in stdout.splitlines(): self.log.info(line) self.log.info("<<<<<<<") result = not ( (self.getReturnCode(proc) != 0) or # if do_throw or do_check failed (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", stdout, re.MULTILINE)) or # if syntax error in xpcshell file (stdout and re.search(": SyntaxError:", stdout, re.MULTILINE)) or # if e10s test started but never finished (child process crash) (stdout and re.search("^child: CHILD-TEST-STARTED", stdout, re.MULTILINE) and not re.search("^child: CHILD-TEST-COMPLETED", stdout, re.MULTILINE))) if result != expected: failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS") message = "%s | %s | test failed (with xpcshell return code: %d), see following log:" % ( failureType, name, self.getReturnCode(proc)) self.log.error(message) print_stdout(stdout) self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": failureType, "message": message, "text": stdout } else: now = time.time() timeTaken = (now - startTime) * 1000 xunitResult["time"] = now - startTime self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken)) if verbose: print_stdout(stdout) xunitResult["passed"] = True if expected: self.passCount += 1 else: self.todoCount += 1 xunitResult["todo"] = True if checkForCrashes(testdir, self.symbolsPath, testName=name): message = "PROCESS-CRASH | %s | application crashed" % name self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": "PROCESS-CRASH", "message": message, "text": stdout } # Find child process(es) leak log(s), if any: See InitLog() in # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic leakLogs = [self.leakLogFile] for childLog in glob( os.path.join(self.profileDir, "runxpcshelltests_leaks_*_pid*.log")): if os.path.isfile(childLog): leakLogs += [childLog] for log in leakLogs: dumpLeakLog(log, True) if self.logfiles and stdout: self.createLogFile(name, stdout, leakLogs) finally: # We can sometimes get here before the process has terminated, which would # cause removeDir() to fail - so check for the process & kill it it needed. if proc and self.poll(proc) is None: message = "TEST-UNEXPECTED-FAIL | %s | Process still running after test!" % name self.log.error(message) print_stdout(stdout) self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": "TEST-UNEXPECTED-FAIL", "message": message, "text": stdout } self.kill(proc) # We don't want to delete the profile when running check-interactive # or check-one. if self.profileDir and not self.interactive and not self.singleFile: try: self.removeDir(self.profileDir) except Exception: self.log.info( "TEST-INFO | Failed to remove profile directory. Waiting." ) # We suspect the filesystem may still be making changes. Wait a # little bit and try again. time.sleep(5) try: self.removeDir(self.profileDir) except Exception: message = "TEST-UNEXPECTED-FAIL | %s | Failed to clean up the test profile directory: %s" % ( name, sys.exc_info()[1]) self.log.error(message) print_stdout(stdout) print_stdout(traceback.format_exc()) self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": "TEST-UNEXPECTED-FAIL", "message": message, "text": "%s\n%s" % (stdout, traceback.format_exc()) } if gotSIGINT: xunitResult["passed"] = False xunitResult["time"] = "0.0" xunitResult["failure"] = { "type": "SIGINT", "message": "Received SIGINT", "text": "Received SIGINT (control-C) during test execution." } self.log.error( "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution" ) if (keepGoing): gotSIGINT = False else: xunitResults.append(xunitResult) break xunitResults.append(xunitResult) self.shutdownNode() if self.testCount == 0: self.log.error( "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?" ) self.failCount = 1 self.log.info("""INFO | Result summary: INFO | Passed: %d INFO | Failed: %d INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount)) if autolog: self.post_to_autolog(xunitResults, xunitName) if xunitFilename is not None: self.writeXunitResults(filename=xunitFilename, results=xunitResults, name=xunitName) if gotSIGINT and not keepGoing: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ "(Use --keep-going to keep running tests after killing one with SIGINT)") return False return self.failCount == 0
def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, manifest=None, testdirs=[], testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, profileName=None, mozInfo=None, **otherOptions): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |appPath|, if provided, is the path to an application directory. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of test directories to run. |testdirs|, if provided, is a list of absolute paths of test directories. No-manifest only option. |testPath|, if provided, indicates a single path and/or test to run. |interactive|, if set to True, indicates to provide an xpcshell prompt instead of automatically executing the test. |verbose|, if set to True, will cause stdout/stderr from tests to be printed always |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile directory if running only a subset of tests. |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. |otherOptions| may be present for the convenience of subclasses """ global gotSIGINT self.xpcshell = xpcshell self.xrePath = xrePath self.appPath = appPath self.symbolsPath = symbolsPath self.manifest = manifest self.testdirs = testdirs self.testPath = testPath self.interactive = interactive self.verbose = verbose self.keepGoing = keepGoing self.logfiles = logfiles self.totalChunks = totalChunks self.thisChunk = thisChunk self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) self.profileName = profileName or "xpcshell" self.mozInfo = mozInfo # If we have an interactive debugger, disable ctrl-c. if self.debuggerInfo and self.debuggerInfo["interactive"]: signal.signal(signal.SIGINT, lambda signum, frame: None) if not testdirs and not manifest: # nothing to test! self.log.error("Error: No test dirs or test manifest specified!") return False self.testCount = 0 self.passCount = 0 self.failCount = 0 self.todoCount = 0 self.setAbsPath() self.buildXpcsRunArgs() self.buildEnvironment() # Handle filenames in mozInfo if not isinstance(self.mozInfo, dict): mozInfoFile = self.mozInfo if not os.path.isfile(mozInfoFile): self.log.error( "Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) return False self.mozInfo = parse_json(open(mozInfoFile).read()) mozinfo.update(self.mozInfo) pStdout, pStderr = self.getPipes() self.buildTestList() for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue self.testCount += 1 # Check for skipped tests if 'disabled' in test: self.log.info("TEST-INFO | skipping %s | %s" % (name, test['disabled'])) continue # Check for known-fail tests expected = test['expected'] == 'pass' testdir = os.path.dirname(name) self.buildXpcsCmd(testdir) testHeadFiles = self.getHeadFiles(test) testTailFiles = self.getTailFiles(test) cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd) # create a temp dir that the JS harness can stick a profile in self.profileDir = self.setupProfileDir() self.leakLogFile = self.setupLeakLogging() # The test file will have to be loaded after the head files. cmdT = self.buildCmdTestFile(name) args = self.xpcsRunArgs if 'debug' in test: args.insert(0, '-d') try: self.log.info("TEST-INFO | %s | running test ..." % name) startTime = time.time() proc = self.launchProcess(cmdH + cmdT + args, stdout=pStdout, stderr=pStderr, env=self.env, cwd=testdir) # Allow user to kill hung subprocess with SIGINT w/o killing this script # - don't move this line above launchProcess, or child will inherit the SIG_IGN signal.signal(signal.SIGINT, markGotSIGINT) # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|. stdout, stderr = self.communicate(proc) signal.signal(signal.SIGINT, signal.SIG_DFL) if interactive: # Not sure what else to do here... return True def print_stdout(stdout): """Print stdout line-by-line to avoid overflowing buffers.""" self.log.info(">>>>>>>") if (stdout): for line in stdout.splitlines(): self.log.info(line) self.log.info("<<<<<<<") result = not ( (self.getReturnCode(proc) != 0) or (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", stdout, re.MULTILINE)) or (stdout and re.search(": SyntaxError:", stdout, re.MULTILINE))) if result != expected: self.log.error( "TEST-UNEXPECTED-%s | %s | test failed (with xpcshell return code: %d), see following log:" % ("FAIL" if expected else "PASS", name, self.getReturnCode(proc))) print_stdout(stdout) self.failCount += 1 else: timeTaken = (time.time() - startTime) * 1000 self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken)) if verbose: print_stdout(stdout) if expected: self.passCount += 1 else: self.todoCount += 1 checkForCrashes(testdir, self.symbolsPath, testName=name) # Find child process(es) leak log(s), if any: See InitLog() in # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic leakLogs = [self.leakLogFile] for childLog in glob( os.path.join(self.profileDir, "runxpcshelltests_leaks_*_pid*.log")): if os.path.isfile(childLog): leakLogs += [childLog] for log in leakLogs: dumpLeakLog(log, True) if self.logfiles and stdout: self.createLogFile(name, stdout, leakLogs) finally: # We don't want to delete the profile when running check-interactive # or check-one. if self.profileDir and not self.interactive and not self.singleFile: self.removeDir(self.profileDir) if gotSIGINT: self.log.error( "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution" ) if (keepGoing): gotSIGINT = False else: break if self.testCount == 0: self.log.error( "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?" ) self.failCount = 1 self.log.info("""INFO | Result summary: INFO | Passed: %d INFO | Failed: %d INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount)) if gotSIGINT and not keepGoing: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ "(Use --keep-going to keep running tests after killing one with SIGINT)") return False return self.failCount == 0
def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, manifest=None, testdirs=None, testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, profileName=None, mozInfo=None, shuffle=False, testsRootDir=None, xunitFilename=None, xunitName=None, testingModulesDir=None, autolog=False, **otherOptions): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |appPath|, if provided, is the path to an application directory. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of test directories to run. |testdirs|, if provided, is a list of absolute paths of test directories. No-manifest only option. |testPath|, if provided, indicates a single path and/or test to run. |interactive|, if set to True, indicates to provide an xpcshell prompt instead of automatically executing the test. |verbose|, if set to True, will cause stdout/stderr from tests to be printed always |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile directory if running only a subset of tests. |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. |shuffle|, if True, execute tests in random order. |testsRootDir|, absolute path to root directory of all tests. This is used by xUnit generation to determine the package name of the tests. |xunitFilename|, if set, specifies the filename to which to write xUnit XML results. |xunitName|, if outputting an xUnit XML file, the str value to use for the testsuite name. |testingModulesDir|, if provided, specifies where JS modules reside. xpcshell will register a resource handler mapping this path. |otherOptions| may be present for the convenience of subclasses """ global gotSIGINT if testdirs is None: testdirs = [] if xunitFilename is not None or xunitName is not None: if not isinstance(testsRootDir, str): raise Exception("testsRootDir must be a str when outputting xUnit.") if not os.path.isabs(testsRootDir): testsRootDir = os.path.abspath(testsRootDir) if not os.path.exists(testsRootDir): raise Exception("testsRootDir path does not exists: %s" % testsRootDir) # Try to guess modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code path should never be executed in local runs because the build system # should always set this argument. if not testingModulesDir: ourDir = os.path.dirname(__file__) possible = os.path.join(ourDir, os.path.pardir, 'modules') if os.path.isdir(possible): testingModulesDir = possible if testingModulesDir: # The resource loader expects native paths. Depending on how we were # invoked, a UNIX style path may sneak in on Windows. We try to # normalize that. testingModulesDir = os.path.normpath(testingModulesDir) if not os.path.isabs(testingModulesDir): testingModulesDir = os.path.abspath(testingModulesDir) if not testingModulesDir.endswith(os.path.sep): testingModulesDir += os.path.sep self.xpcshell = xpcshell self.xrePath = xrePath self.appPath = appPath self.symbolsPath = symbolsPath self.manifest = manifest self.testdirs = testdirs self.testPath = testPath self.interactive = interactive self.verbose = verbose self.keepGoing = keepGoing self.logfiles = logfiles self.totalChunks = totalChunks self.thisChunk = thisChunk self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) self.profileName = profileName or "xpcshell" self.mozInfo = mozInfo self.testingModulesDir = testingModulesDir # If we have an interactive debugger, disable ctrl-c. if self.debuggerInfo and self.debuggerInfo["interactive"]: signal.signal(signal.SIGINT, lambda signum, frame: None) if not testdirs and not manifest: # nothing to test! self.log.error("Error: No test dirs or test manifest specified!") return False self.testCount = 0 self.passCount = 0 self.failCount = 0 self.todoCount = 0 self.setAbsPath() self.buildXpcsRunArgs() self.buildEnvironment() # Handle filenames in mozInfo if not isinstance(self.mozInfo, dict): mozInfoFile = self.mozInfo if not os.path.isfile(mozInfoFile): self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) return False self.mozInfo = parse_json(open(mozInfoFile).read()) mozinfo.update(self.mozInfo) # We have to do this before we build the test list so we know whether or # not to run tests that depend on having the node spdy server self.trySetupNode() pStdout, pStderr = self.getPipes() self.buildTestList() if shuffle: random.shuffle(self.alltests) xunitResults = [] for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue self.testCount += 1 xunitResult = {"name": test["name"], "classname": "xpcshell"} # The xUnit package is defined as the path component between the root # dir and the test with path characters replaced with '.' (using Java # class notation). if testsRootDir is not None: testsRootDir = os.path.normpath(testsRootDir) if test["here"].find(testsRootDir) != 0: raise Exception("testsRootDir is not a parent path of %s" % test["here"]) relpath = test["here"][len(testsRootDir):].lstrip("/\\") xunitResult["classname"] = relpath.replace("/", ".").replace("\\", ".") # Check for skipped tests if 'disabled' in test: self.log.info("TEST-INFO | skipping %s | %s" % (name, test['disabled'])) xunitResult["skipped"] = True xunitResults.append(xunitResult) continue # Check for known-fail tests expected = test['expected'] == 'pass' testdir = os.path.dirname(name) self.buildXpcsCmd(testdir) testHeadFiles, testTailFiles = self.getHeadAndTailFiles(test) cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd) # create a temp dir that the JS harness can stick a profile in self.profileDir = self.setupProfileDir() self.leakLogFile = self.setupLeakLogging() # The test file will have to be loaded after the head files. cmdT = self.buildCmdTestFile(name) args = self.xpcsRunArgs[:] if 'debug' in test: args.insert(0, '-d') completeCmd = cmdH + cmdT + args try: self.log.info("TEST-INFO | %s | running test ..." % name) if verbose: self.log.info("TEST-INFO | %s | full command: %r" % (name, completeCmd)) self.log.info("TEST-INFO | %s | current directory: %r" % (name, testdir)) # Show only those environment variables that are changed from # the ambient environment. changedEnv = (set("%s=%s" % i for i in self.env.iteritems()) - set("%s=%s" % i for i in os.environ.iteritems())) self.log.info("TEST-INFO | %s | environment: %s" % (name, list(changedEnv))) startTime = time.time() proc = self.launchProcess(completeCmd, stdout=pStdout, stderr=pStderr, env=self.env, cwd=testdir) if interactive: self.log.info("TEST-INFO | %s | Process ID: %d" % (name, proc.pid)) # Allow user to kill hung subprocess with SIGINT w/o killing this script # - don't move this line above launchProcess, or child will inherit the SIG_IGN signal.signal(signal.SIGINT, markGotSIGINT) # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|. stdout, stderr = self.communicate(proc) signal.signal(signal.SIGINT, signal.SIG_DFL) if interactive: # Not sure what else to do here... return True def print_stdout(stdout): """Print stdout line-by-line to avoid overflowing buffers.""" self.log.info(">>>>>>>") if (stdout): for line in stdout.splitlines(): self.log.info(line) self.log.info("<<<<<<<") result = not ((self.getReturnCode(proc) != 0) or # if do_throw or do_check failed (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", stdout, re.MULTILINE)) or # if syntax error in xpcshell file (stdout and re.search(": SyntaxError:", stdout, re.MULTILINE)) or # if e10s test started but never finished (child process crash) (stdout and re.search("^child: CHILD-TEST-STARTED", stdout, re.MULTILINE) and not re.search("^child: CHILD-TEST-COMPLETED", stdout, re.MULTILINE))) if result != expected: failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS") message = "%s | %s | test failed (with xpcshell return code: %d), see following log:" % ( failureType, name, self.getReturnCode(proc)) self.log.error(message) print_stdout(stdout) self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": failureType, "message": message, "text": stdout } else: now = time.time() timeTaken = (now - startTime) * 1000 xunitResult["time"] = now - startTime self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken)) if verbose: print_stdout(stdout) xunitResult["passed"] = True if expected: self.passCount += 1 else: self.todoCount += 1 xunitResult["todo"] = True checkForCrashes(testdir, self.symbolsPath, testName=name) # Find child process(es) leak log(s), if any: See InitLog() in # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic leakLogs = [self.leakLogFile] for childLog in glob(os.path.join(self.profileDir, "runxpcshelltests_leaks_*_pid*.log")): if os.path.isfile(childLog): leakLogs += [childLog] for log in leakLogs: dumpLeakLog(log, True) if self.logfiles and stdout: self.createLogFile(name, stdout, leakLogs) finally: # We can sometimes get here before the process has terminated, which would # cause removeDir() to fail - so check for the process & kill it it needed. if self.poll(proc) is None: message = "TEST-UNEXPECTED-FAIL | %s | Process still running after test!" % name self.log.error(message) print_stdout(stdout) self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": "TEST-UNEXPECTED-FAIL", "message": message, "text": stdout } self.kill(proc) # We don't want to delete the profile when running check-interactive # or check-one. if self.profileDir and not self.interactive and not self.singleFile: try: self.removeDir(self.profileDir) except Exception: message = "TEST-UNEXPECTED-FAIL | %s | Failed to clean up the test profile directory: %s" % (name, sys.exc_info()[1]) self.log.error(message) print_stdout(stdout) print_stdout(traceback.format_exc()) self.failCount += 1 xunitResult["passed"] = False xunitResult["failure"] = { "type": "TEST-UNEXPECTED-FAIL", "message": message, "text": "%s\n%s" % (stdout, traceback.format_exc()) } if gotSIGINT: xunitResult["passed"] = False xunitResult["time"] = "0.0" xunitResult["failure"] = { "type": "SIGINT", "message": "Received SIGINT", "text": "Received SIGINT (control-C) during test execution." } self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution") if (keepGoing): gotSIGINT = False else: xunitResults.append(xunitResult) break xunitResults.append(xunitResult) self.shutdownNode() if self.testCount == 0: self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?") self.failCount = 1 self.log.info("""INFO | Result summary: INFO | Passed: %d INFO | Failed: %d INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount)) if autolog: self.post_to_autolog(xunitResults, xunitName) if xunitFilename is not None: self.writeXunitResults(filename=xunitFilename, results=xunitResults, name=xunitName) if gotSIGINT and not keepGoing: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ "(Use --keep-going to keep running tests after killing one with SIGINT)") return False return self.failCount == 0
def run_desktop_test(self, suite=None, test_file=None, debugger=None, debugger_args=None, shuffle=False, keep_open=False, rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False, slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None, jsdebugger=False, debug_on_failure=False, start_at=None, end_at=None, e10s=False, dmd=False, dump_output_directory=None, dump_about_memory_after_test=False, dump_dmd_after_test=False, **kwargs): """Runs a mochitest. test_file is a path to a test file. It can be a relative path from the top source directory, an absolute filename, or a directory containing test files. suite is the type of mochitest to run. It can be one of ('plain', 'chrome', 'browser', 'metro', 'a11y'). debugger is a program name or path to a binary (presumably a debugger) to run the test in. e.g. 'gdb' debugger_args are the arguments passed to the debugger. shuffle is whether test order should be shuffled (defaults to false). keep_open denotes whether to keep the browser open after tests complete. """ if rerun_failures and test_file: print('Cannot specify both --rerun-failures and a test path.') return 1 # Need to call relpath before os.chdir() below. test_path = '' if test_file: test_path = self._wrap_path_argument(test_file).relpath() failure_file_path = os.path.join(self.statedir, 'mochitest_failures.json') if rerun_failures and not os.path.exists(failure_file_path): print('No failure file present. Did you run mochitests before?') return 1 from StringIO import StringIO # runtests.py is ambiguous, so we load the file/module manually. if 'mochitest' not in sys.modules: import imp path = os.path.join(self.mochitest_dir, 'runtests.py') with open(path, 'r') as fh: imp.load_module('mochitest', fh, path, ('.py', 'r', imp.PY_SOURCE)) import mozinfo import mochitest # This is required to make other components happy. Sad, isn't it? os.chdir(self.topobjdir) # Automation installs its own stream handler to stdout. Since we want # all logging to go through us, we just remove their handler. remove_handlers = [ l for l in logging.getLogger().handlers if isinstance(l, logging.StreamHandler) ] for handler in remove_handlers: logging.getLogger().removeHandler(handler) runner = mochitest.Mochitest() opts = mochitest.MochitestOptions() options, args = opts.parse_args([]) # Need to set the suite options before verifyOptions below. if suite == 'plain': # Don't need additional options for plain. pass elif suite == 'chrome': options.chrome = True elif suite == 'browser': options.browserChrome = True elif suite == 'metro': options.immersiveMode = True options.browserChrome = True elif suite == 'a11y': options.a11y = True elif suite == 'webapprt-content': options.webapprtContent = True options.app = self.get_webapp_runtime_path() elif suite == 'webapprt-chrome': options.webapprtChrome = True options.app = self.get_webapp_runtime_path() options.browserArgs.append("-test-mode") else: raise Exception('None or unrecognized mochitest suite type.') if dmd: options.dmdPath = self.lib_dir options.autorun = not no_autorun options.closeWhenDone = not keep_open options.shuffle = shuffle options.consoleLevel = 'INFO' options.repeat = repeat options.runUntilFailure = run_until_failure options.runSlower = slow options.testingModulesDir = os.path.join(self.tests_dir, 'modules') options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins')) options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols') options.chunkByDir = chunk_by_dir options.totalChunks = total_chunks options.thisChunk = this_chunk options.jsdebugger = jsdebugger options.debugOnFailure = debug_on_failure options.startAt = start_at options.endAt = end_at options.e10s = e10s options.dumpAboutMemoryAfterTest = dump_about_memory_after_test options.dumpDMDAfterTest = dump_dmd_after_test options.dumpOutputDirectory = dump_output_directory mozinfo.update({"e10s": e10s}) # for test manifest parsing. options.failureFile = failure_file_path for k, v in kwargs.iteritems(): setattr(options, k, v) if test_path: test_root = runner.getTestRoot(options) test_root_file = mozpack.path.join(self.mochitest_dir, test_root, test_path) if not os.path.exists(test_root_file): print('Specified test path does not exist: %s' % test_root_file) print( 'You may need to run |mach build| to build the test files.' ) return 1 # Handle test_path pointing at a manifest file so conditions in # the manifest are processed. This is a temporary solution # pending bug 938019. # The manifest basename is the same as |suite|, except for plain manifest_base = 'mochitest' if suite == 'plain' else suite if os.path.basename(test_root_file) == manifest_base + '.ini': options.manifestFile = test_root_file else: options.testPath = test_path if rerun_failures: options.testManifest = failure_file_path if debugger: options.debugger = debugger if debugger_args: if options.debugger == None: print("--debugger-args passed, but no debugger specified.") return 1 options.debuggerArgs = debugger_args options = opts.verifyOptions(options, runner) if options is None: raise Exception('mochitest option validator failed.') # We need this to enable colorization of output. self.log_manager.enable_unstructured() # Output processing is a little funky here. The old make targets # grepped the log output from TEST-UNEXPECTED-* and printed these lines # after test execution. Ideally the test runner would expose a Python # API for obtaining test results and we could just format failures # appropriately. Unfortunately, it doesn't yet do that. So, we capture # all output to a buffer then "grep" the buffer after test execution. # Bug 858197 tracks a Python API that would facilitate this. test_output = StringIO() handler = logging.StreamHandler(test_output) handler.addFilter(UnexpectedFilter()) handler.setFormatter(StructuredHumanFormatter(0, write_times=False)) logging.getLogger().addHandler(handler) result = runner.runTests(options) # Need to remove our buffering handler before we echo failures or else # it will catch them again! logging.getLogger().removeHandler(handler) self.log_manager.disable_unstructured() if test_output.getvalue(): result = 1 for line in test_output.getvalue().splitlines(): self.log(logging.INFO, 'unexpected', {'msg': line}, '{msg}') return result
def verifyOptions(self, options, mochitest): """ verify correct options and cleanup paths """ mozinfo.update({"e10s": options.e10s}) # for test manifest parsing. if options.app is None: if build_obj is not None: options.app = build_obj.get_binary_path() else: self.error( "could not find the application path, --appname must be specified" ) if options.totalChunks is not None and options.thisChunk is None: self.error( "thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: self.error("thisChunk must be between 1 and totalChunks") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != self.defaults['app']: options.xrePath = os.path.dirname(options.app) elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: self.error( "could not find xre directory, --xre-path must be specified" ) # allow relative paths options.xrePath = mochitest.getFullPath(options.xrePath) if options.profilePath: options.profilePath = mochitest.getFullPath(options.profilePath) options.app = mochitest.getFullPath(options.app) if options.dmdPath is not None: options.dmdPath = mochitest.getFullPath(options.dmdPath) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" self.error(msg % {"app": options.app}) return None if options.utilityPath: options.utilityPath = mochitest.getFullPath(options.utilityPath) if options.certPath: options.certPath = mochitest.getFullPath(options.certPath) if options.symbolsPath and not isURL(options.symbolsPath): options.symbolsPath = mochitest.getFullPath(options.symbolsPath) # Set server information on the options object options.webServer = '127.0.0.1' options.httpPort = DEFAULT_PORTS['http'] options.sslPort = DEFAULT_PORTS['https'] # options.webSocketPort = DEFAULT_PORTS['ws'] options.webSocketPort = str( 9988 ) # <- http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l30 # The default websocket port is incorrect in mozprofile; it is # set to the SSL proxy setting. See: # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517 if options.vmwareRecording: if not mozinfo.isWin: self.error( "use-vmware-recording is only supported on Windows.") mochitest.vmwareHelperPath = os.path.join( options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") if not os.path.exists(mochitest.vmwareHelperPath): self.error("%s not found, cannot automate VMware recording." % mochitest.vmwareHelperPath) if options.testManifest and options.runOnlyTests: self.error( "Please use --test-manifest only and not --run-only-tests") if options.runOnlyTests: if not os.path.exists( os.path.abspath(os.path.join(here, options.runOnlyTests))): self.error("unable to find --run-only-tests file '%s'" % options.runOnlyTests) options.runOnly = True options.testManifest = options.runOnlyTests options.runOnlyTests = None if options.manifestFile and options.testManifest: self.error( "Unable to support both --manifest and --test-manifest/--run-only-tests at the same time" ) if options.webapprtContent and options.webapprtChrome: self.error( "Only one of --webapprt-content and --webapprt-chrome may be given." ) if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.debugger.chrome-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] options.autorun = False if options.debugOnFailure and not options.jsdebugger: self.error( "--debug-on-failure should be used together with --jsdebugger." ) # Try to guess the testing modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code should never be executed in local runs because the build system # should always set the flag that populates this variable. If buildbot ever # passes this argument, this code can be deleted. if options.testingModulesDir is None: possible = os.path.join(here, os.path.pardir, 'modules') if os.path.isdir(possible): options.testingModulesDir = possible # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): self.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: self.error("immersive is only supported on Windows 8 and up.") mochitest.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") if not os.path.exists(mochitest.immersiveHelperPath): self.error("%s not found, cannot launch immersive tests." % mochitest.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): self.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: self.error( '--use-test-media-devices is only supported on Linux currently' ) for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): self.error( 'Missing binary %s required for --use-test-media-devices' ) return options
def find_modified_tests(self): """ For each file modified on this push, determine if the modified file is a test, by searching test manifests. Populate self.suites with test files, organized by suite. This depends on test manifests, so can only run after test zips have been downloaded and extracted. """ repository = os.environ.get("GECKO_HEAD_REPOSITORY") revision = os.environ.get("GECKO_HEAD_REV") if not repository or not revision: self.warning( "unable to run tests in per-test mode: no repo or revision!") return [] def get_automationrelevance(): response = self.load_json_url(url) return response dirs = self.query_abs_dirs() mozinfo.find_and_update_from_json(dirs['abs_test_install_dir']) e10s = self.config.get('e10s', False) mozinfo.update({"e10s": e10s}) headless = self.config.get('headless', False) mozinfo.update({"headless": headless}) # FIXME(emilio): Need to update test expectations. mozinfo.update({'stylo': True}) mozinfo.update({'verify': True}) self.info("Per-test run using mozinfo: %s" % str(mozinfo.info)) # determine which files were changed on this push url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10) changed_files = set() for c in contents['changesets']: self.info(" {cset} {desc}".format( cset=c['node'][0:12], desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) if self.config.get('per_test_category') == "web-platform": self._find_wpt_tests(dirs, changed_files) elif self.config.get('gpu_required') == True: self._find_misc_tests(dirs, changed_files, gpu=True) else: self._find_misc_tests(dirs, changed_files) # per test mode run specific tests from any given test suite # _find_*_tests organizes tests to run into suites so we can # run each suite at a time # chunk files total_tests = sum([len(self.suites[x]) for x in self.suites]) files_per_chunk = total_tests / float( self.config.get('total_chunks', 1)) files_per_chunk = int(math.ceil(files_per_chunk)) chunk_number = int(self.config.get('this_chunk', 1)) suites = {} start = (chunk_number - 1) * files_per_chunk end = (chunk_number * files_per_chunk) current = -1 for suite in self.suites: for test in self.suites[suite]: current += 1 if current >= start and current < end: if suite not in suites: suites[suite] = [] suites[suite].append(test) if current >= end: break self.suites = suites self.tests_downloaded = True
def verifyOptions(self, options, mochitest): """ verify correct options and cleanup paths """ mozinfo.update({"e10s": options.e10s}) # for test manifest parsing. if options.app is None: if build_obj is not None: options.app = build_obj.get_binary_path() else: self.error("could not find the application path, --appname must be specified") if options.totalChunks is not None and options.thisChunk is None: self.error("thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: self.error("thisChunk must be between 1 and totalChunks") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != self.defaults['app']: options.xrePath = os.path.dirname(options.app) elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: self.error("could not find xre directory, --xre-path must be specified") # allow relative paths options.xrePath = mochitest.getFullPath(options.xrePath) if options.profilePath: options.profilePath = mochitest.getFullPath(options.profilePath) options.app = mochitest.getFullPath(options.app) if options.dmdPath is not None: options.dmdPath = mochitest.getFullPath(options.dmdPath) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" self.error(msg % {"app": options.app}) return None if options.utilityPath: options.utilityPath = mochitest.getFullPath(options.utilityPath) if options.certPath: options.certPath = mochitest.getFullPath(options.certPath) if options.symbolsPath and not isURL(options.symbolsPath): options.symbolsPath = mochitest.getFullPath(options.symbolsPath) # Set server information on the options object options.webServer = '127.0.0.1' options.httpPort = DEFAULT_PORTS['http'] options.sslPort = DEFAULT_PORTS['https'] # options.webSocketPort = DEFAULT_PORTS['ws'] options.webSocketPort = str(9988) # <- http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l30 # The default websocket port is incorrect in mozprofile; it is # set to the SSL proxy setting. See: # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517 if options.vmwareRecording: if not mozinfo.isWin: self.error("use-vmware-recording is only supported on Windows.") mochitest.vmwareHelperPath = os.path.join( options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") if not os.path.exists(mochitest.vmwareHelperPath): self.error("%s not found, cannot automate VMware recording." % mochitest.vmwareHelperPath) if options.testManifest and options.runOnlyTests: self.error("Please use --test-manifest only and not --run-only-tests") if options.runOnlyTests: if not os.path.exists(os.path.abspath(os.path.join(here, options.runOnlyTests))): self.error("unable to find --run-only-tests file '%s'" % options.runOnlyTests) options.runOnly = True options.testManifest = options.runOnlyTests options.runOnlyTests = None if options.manifestFile and options.testManifest: self.error("Unable to support both --manifest and --test-manifest/--run-only-tests at the same time") if options.webapprtContent and options.webapprtChrome: self.error("Only one of --webapprt-content and --webapprt-chrome may be given.") if options.jsdebugger: options.extraPrefs += [ "devtools.debugger.remote-enabled=true", "devtools.debugger.chrome-enabled=true", "devtools.chrome.enabled=true", "devtools.debugger.prompt-connection=false" ] options.autorun = False if options.debugOnFailure and not options.jsdebugger: self.error("--debug-on-failure should be used together with --jsdebugger.") # Try to guess the testing modules directory. # This somewhat grotesque hack allows the buildbot machines to find the # modules directory without having to configure the buildbot hosts. This # code should never be executed in local runs because the build system # should always set the flag that populates this variable. If buildbot ever # passes this argument, this code can be deleted. if options.testingModulesDir is None: possible = os.path.join(here, os.path.pardir, 'modules') if os.path.isdir(possible): options.testingModulesDir = possible # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath(options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath(options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): self.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace('\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.immersiveMode: if not mozinfo.isWin: self.error("immersive is only supported on Windows 8 and up.") mochitest.immersiveHelperPath = os.path.join( options.utilityPath, "metrotestharness.exe") if not os.path.exists(mochitest.immersiveHelperPath): self.error("%s not found, cannot launch immersive tests." % mochitest.immersiveHelperPath) if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): self.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: self.error('--use-test-media-devices is only supported on Linux currently') for f in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']: if not os.path.isfile(f): self.error('Missing binary %s required for --use-test-media-devices') return options
def validate(self, parser, options, context): """Validate android options.""" if build_obj: options.log_mach = '-' device_args = {'deviceRoot': options.remoteTestRoot} if options.dm_trans == "adb": device_args['adbPath'] = options.adbPath if options.deviceIP: device_args['host'] = options.deviceIP device_args['port'] = options.devicePort elif options.deviceSerial: device_args['deviceSerial'] = options.deviceSerial options.dm = DroidADB(**device_args) elif options.dm_trans == 'sut': if options.deviceIP is None: parser.error( "If --dm_trans = sut, you must provide a device IP") device_args['host'] = options.deviceIP device_args['port'] = options.devicePort options.dm = DroidSUT(**device_args) if not options.remoteTestRoot: options.remoteTestRoot = options.dm.deviceRoot if options.remoteWebServer is None: if os.name != "nt": options.remoteWebServer = moznetwork.get_ip() else: parser.error( "you must specify a --remote-webserver=<ip address>") options.webServer = options.remoteWebServer if options.remoteLogFile is None: options.remoteLogFile = options.remoteTestRoot + \ '/logs/mochitest.log' if options.remoteLogFile.count('/') < 1: options.remoteLogFile = options.remoteTestRoot + \ '/' + options.remoteLogFile if options.remoteAppPath and options.app: parser.error( "You cannot specify both the remoteAppPath and the app setting") elif options.remoteAppPath: options.app = options.remoteTestRoot + "/" + options.remoteAppPath elif options.app is None: if build_obj: options.app = build_obj.substs['ANDROID_PACKAGE_NAME'] else: # Neither remoteAppPath nor app are set -- error parser.error("You must specify either appPath or app") if build_obj and 'MOZ_HOST_BIN' in os.environ: options.xrePath = os.environ['MOZ_HOST_BIN'] # Only reset the xrePath if it wasn't provided if options.xrePath is None: options.xrePath = options.utilityPath if options.pidFile != "": f = open(options.pidFile, 'w') f.write("%s" % os.getpid()) f.close() # Robocop specific options if options.robocopIni != "": if not os.path.exists(options.robocopIni): parser.error( "Unable to find specified robocop .ini manifest '%s'" % options.robocopIni) options.robocopIni = os.path.abspath(options.robocopIni) if not options.robocopApk and build_obj: options.robocopApk = os.path.join(build_obj.topobjdir, 'mobile', 'android', 'tests', 'browser', 'robocop', 'robocop-debug.apk') if options.robocopApk != "": if not os.path.exists(options.robocopApk): parser.error( "Unable to find robocop APK '%s'" % options.robocopApk) options.robocopApk = os.path.abspath(options.robocopApk) # Disable e10s by default on Android because we don't run Android # e10s jobs anywhere yet. options.e10s = False mozinfo.update({'e10s': options.e10s}) # allow us to keep original application around for cleanup while # running robocop via 'am' options.remoteappname = options.app return options
def validate(self, parser, options, context): """Validate generic options.""" # for test manifest parsing. mozinfo.update({"nested_oop": options.nested_oop}) # and android doesn't use 'app' the same way, so skip validation if parser.app != 'android': if options.app is None: if build_obj: options.app = build_obj.get_binary_path() else: parser.error( "could not find the application path, --appname must be specified") elif options.app == "dist" and build_obj: options.app = build_obj.get_binary_path(where='staged-package') options.app = self.get_full_path(options.app, parser.oldcwd) if not os.path.exists(options.app): parser.error("Error: Path {} doesn't exist. Are you executing " "$objdir/_tests/testing/mochitest/runtests.py?".format( options.app)) if options.flavor is None: options.flavor = 'plain' if options.gmp_path is None and options.app and build_obj: # Need to fix the location of gmp_fake which might not be shipped in the binary gmp_modules = ( ('gmp-fake', '1.0'), ('gmp-clearkey', '0.1'), ('gmp-fakeopenh264', '1.0') ) options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") if options.extra_mozinfo_json: if not os.path.isfile(options.extra_mozinfo_json): parser.error("Error: couldn't find mozinfo.json at '%s'." % options.extra_mozinfo_json) options.extra_mozinfo_json = json.load(open(options.extra_mozinfo_json)) if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.chunkByDir and options.chunkByRuntime: parser.error( "can only use one of --chunk-by-dir or --chunk-by-runtime") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.get_default('app'): options.xrePath = os.path.dirname(options.app) if mozinfo.isMac: options.xrePath = os.path.join( os.path.dirname( options.xrePath), "Resources") elif build_obj is not None: # otherwise default to dist/bin options.xrePath = build_obj.bindir else: parser.error( "could not find xre directory, --xre-path must be specified") # allow relative paths if options.xrePath: options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd) if options.profilePath: options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd) if options.utilityPath: options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd) if options.certPath: options.certPath = self.get_full_path(options.certPath, parser.oldcwd) elif build_obj: options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd) elif not options.symbolsPath and build_obj: options.symbolsPath = os.path.join(build_obj.distdir, 'crashreporter-symbols') if options.debugOnFailure and not options.jsdebugger: parser.error( "--debug-on-failure requires --jsdebugger.") if options.debuggerArgs and not options.debugger: parser.error( "--debugger-args requires --debugger.") if options.valgrind or options.debugger: # valgrind and some debuggers may cause Gecko to start slowly. Make sure # marionette waits long enough to connect. options.marionette_startup_timeout = 900 options.marionette_socket_timeout = 540 if options.store_chrome_manifest: options.store_chrome_manifest = os.path.abspath(options.store_chrome_manifest) if not os.path.isdir(os.path.dirname(options.store_chrome_manifest)): parser.error( "directory for %s does not exist as a destination to copy a " "chrome manifest." % options.store_chrome_manifest) if options.jscov_dir_prefix: options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix) if not os.path.isdir(options.jscov_dir_prefix): parser.error( "directory %s does not exist as a destination for coverage " "data." % options.jscov_dir_prefix) if options.testingModulesDir is None: # Try to guess the testing modules directory. possible = [os.path.join(here, os.path.pardir, 'modules')] if build_obj: possible.insert(0, os.path.join(build_obj.topobjdir, '_tests', 'modules')) for p in possible: if os.path.isdir(p): options.testingModulesDir = p break # Paths to specialpowers and mochijar from the tests archive. options.stagedAddons = [ os.path.join(here, 'extensions', 'specialpowers'), os.path.join(here, 'mochijar'), ] if build_obj: objdir_xpi_stage = os.path.join(build_obj.distdir, 'xpi-stage') if os.path.isdir(objdir_xpi_stage): options.stagedAddons = [ os.path.join(objdir_xpi_stage, 'specialpowers'), os.path.join(objdir_xpi_stage, 'mochijar'), ] plugins_dir = os.path.join(build_obj.distdir, 'plugins') if os.path.isdir(plugins_dir) and plugins_dir not in options.extraProfileFiles: options.extraProfileFiles.append(plugins_dir) # Even if buildbot is updated, we still want this, as the path we pass in # to the app must be absolute and have proper slashes. if options.testingModulesDir is not None: options.testingModulesDir = os.path.normpath( options.testingModulesDir) if not os.path.isabs(options.testingModulesDir): options.testingModulesDir = os.path.abspath( options.testingModulesDir) if not os.path.isdir(options.testingModulesDir): parser.error('--testing-modules-dir not a directory: %s' % options.testingModulesDir) options.testingModulesDir = options.testingModulesDir.replace( '\\', '/') if options.testingModulesDir[-1] != '/': options.testingModulesDir += '/' if options.runUntilFailure: if not options.repeat: options.repeat = 29 if options.dumpOutputDirectory is None: options.dumpOutputDirectory = tempfile.gettempdir() if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: if not os.path.isdir(options.dumpOutputDirectory): parser.error('--dump-output-directory not a directory: %s' % options.dumpOutputDirectory) if options.useTestMediaDevices: if not mozinfo.isLinux: parser.error( '--use-test-media-devices is only supported on Linux currently') gst01 = spawn.find_executable("gst-launch-0.1") gst10 = spawn.find_executable("gst-launch-1.0") pactl = spawn.find_executable("pactl") if not (gst01 or gst10): parser.error( 'Missing gst-launch-{0.1,1.0}, required for ' '--use-test-media-devices') if not pactl: parser.error( 'Missing binary pactl required for ' '--use-test-media-devices') if options.nested_oop: options.e10s = True options.leakThresholds = { "default": options.defaultLeakThreshold, "tab": options.defaultLeakThreshold, # GMP rarely gets a log, but when it does, it leaks a little. "gmplugin": 20000, "rdd": 400, } # See the dependencies of bug 1401764. if mozinfo.isWin: options.leakThresholds["tab"] = 1000 # XXX We can't normalize test_paths in the non build_obj case here, # because testRoot depends on the flavor, which is determined by the # mach command and therefore not finalized yet. Conversely, test paths # need to be normalized here for the mach case. if options.test_paths and build_obj: # Normalize test paths so they are relative to test root options.test_paths = [build_obj._wrap_path_argument(p).relpath() for p in options.test_paths] return options
def parseArgs(): home = os.path.expanduser("~") filename = ".obmtoolrc" if os.name == "posix" else "obmtool.ini" defaultconfig = os.path.join(home, filename) # When adding new arguments, DO NOT USE the config dict yet. See config file loading below. parser = argparse.ArgumentParser(description="Start Thunderbird with a preconfigured OBM setup") parser.add_argument('-t', '--thunderbird', type=str, help="The Thunderbird version (17,24,...), or a path to the binary.") # default: defaults.tbversion parser.add_argument('-l', '--lightning', type=str, help="The path to the Lightning XPI") parser.add_argument('-o', '--obm', type=str, help="The path to the OBM XPI") parser.add_argument('-u', '--user', type=str, help="The OBM user to set up") # default: defaults.user parser.add_argument('-s', '--server', type=str, help="The sync services URI") # default: defaults.server parser.add_argument('-e', '--extension', type=str, nargs='+', default=[], help="An additional extension to install, can be specified multiple times") parser.add_argument('-p', '--pref', type=str, nargs='+', default=[], metavar='key=value', help="Additional preferences to set, can be specified multiple times. Value can be a string, integer or true|false.") parser.add_argument('-r', '--reset', action='store_true', help="Reset the currently used profile before starting") # default: defaults.reset parser.add_argument('-c', '--config', default=None, help="Config file to use (default: %s)" % defaultconfig) parser.add_argument('-m', '--mozmill', type=str, nargs='+', default=[], help="Run a specific mozmill test") parser.add_argument('--format', type=str, default='pprint-color', metavar='[pprint|pprint-color|json|xunit]', help="Mozmill output format (default: pprint-color)") parser.add_argument('--logfile', type=str, default=None, help="Log mozmill events to a file in addition to the console") parser.add_argument('-v', '--verbose', action='store_true', help="Show more information about whats going on") # default: defaults.verbose args = parser.parse_args() # Set up logging if args.verbose: logging.basicConfig(level=logging.INFO) # Read user config, this needs to be done fairly early if not args.config: args.config = defaultconfig if not os.path.exists(args.config): print "Config file %s does not exist" % os.path.abspath(args.config) sys.exit(1) mode = os.stat(args.config)[stat.ST_MODE] logging.info("Reading configuration from %s" % os.path.abspath(args.config)) config.readUserFile(args.config) # Protect from footgun if config.get("defaults", "password") and mode & (stat.S_IRGRP | stat.S_IWOTH | stat.S_ISUID | stat.S_ISGID) != 0: print "Attempt to read config file %s that contains a password and has too open permissions. Change mode to 0600 or equivalent." % args.config sys.exit(1) # Set up defaults that are taken from the config file, these need to be # merged after we load the right config file configdefaults = { "thunderbird": config.get("defaults", "tbversion"), "user": config.get("defaults", "user"), "password": config.get("defaults", "password"), "server": config.get("defaults", "server"), "reset": config.get("defaults", "reset"), "verbose": config.get("defaults", "verbose") } for k in configdefaults: if not k in args.__dict__ or args.__dict__[k] is None: args.__dict__[k] = configdefaults[k] # Set up the Thunderbird version and path try: # First check if a version number was passed and get the path from the config args.tbversion = int(args.thunderbird) args.thunderbird = os.path.expanduser(config.require("paths", "thunderbird-%s" % args.tbversion)) args.thunderbird = obmtool.utils.fixBinaryPath(args.thunderbird) except ValueError: # Otherwise it was probably a path. Keep the path in args.thunderbird and # get the version from Thunderbird's application.ini args.thunderbird = obmtool.utils.fixBinaryPath(os.path.expanduser(args.thunderbird)) tbversion = mozversion.get_version(args.thunderbird)['application_version'] args.tbversion = int(tbversion.split(".")[0]) # Set up default lightning xpi based on either passed token (i.e tb3) or # passed thunderbird version if args.lightning and not os.path.exists(args.lightning): args.lightning = config.get("paths", "lightning-%s" % args.lightning) if args.lightning is None: print "Invalid path to Lightning XPI" sys.exit() if args.lightning is None: args.lightning = config.require("paths", "lightning-tb%d" % args.tbversion) # Set up default obm xpi based on either passed token (i.e next-tb17) or # default version in prefs (i.e obmversion=next-tb24) if args.obm is None: args.obm = config.require("defaults", "obmversion") if args.obm and not os.path.exists(args.obm): args.obm = config.require("paths", "obm-%s" % args.obm) logging.info("Using Lighting from %s" % args.lightning) logging.info("Using OBM from %s" % args.obm) # Set up a path for the profile, either from config or using /tmp args.cachePath = config.get("paths", "profileCache", None) if args.cachePath is None: args.cachePath = tempfile.gettempdir() # Expand user path for later use args.obm = os.path.expanduser(args.obm) args.lightning = os.path.expanduser(args.lightning) args.cachePath = os.path.expanduser(args.cachePath) # Add extra addons from prefs and passed options extensions = filter(bool, re.split("[,\n]", config.get("profile", "extensions", ""))) extensions.extend(args.extension) extensions.append(args.obm) extensions.append(args.lightning) if args.mozmill: extensions.extend(mozmill.ADDONS) args.extension = map(os.path.expanduser, extensions) # Add extra preferences specified on commandline extraprefs = {} preferences = config.getAll("preferences") preferences.extend([x.split("=", 2) for x in args.pref]) for k, v in preferences: lv = v.lower() if lv == "true": v = True elif lv == "false": v = False else: try: v = int(v) except: pass extraprefs[k] = v if args.mozmill: # Set up jsbridge port args.jsbridge_port = jsbridge.find_port() # Add testing prefs extraprefs['extensions.jsbridge.port'] = args.jsbridge_port extraprefs['focusmanager.testmode'] = True # TODO main window controller will timeout finding the main window since # the sync takes so long. extraprefs['extensions.obm.syncOnStart'] = False # Set up mozinfo for our current configuration mozinfo.update(obmtool.utils.setupMozinfo(args)) # Set up extra preferences in the profile args.preferences = extraprefs # For the following args we need the runner already runner = createRunner(args) # Add extra certificates from the prefs for cert in filter(bool, re.split("[,\n]", config.get("profile", "certificates", ""))): host,port = cert.split(":") runner.profile.overrides.addEntry(host, int(port)) # Add extra signons from the prefs for signon in filter(bool, re.split("[,\n]", config.get("profile", "signons", ""))): hostname,realm,user,password = signon.split("|") runner.profile.signons.addEntry(hostname, realm, user, password) # Need to flush profile after adding certs/signons runner.profile.flush() return runner, args