class MuletReftest(RefTest): build_type = "mulet" marionette = None def __init__(self, marionette_args): RefTest.__init__(self) self.last_test = os.path.basename(__file__) self.marionette_args = marionette_args self.profile = None self.runner = None self.test_script = os.path.join(here, 'b2g_start_script.js') self.timeout = None def run_marionette_script(self): self.marionette = Marionette(**self.marionette_args) assert (self.marionette.wait_for_port()) self.marionette.start_session() if self.build_type == "mulet": self._wait_for_homescreen(timeout=300) self._unlockScreen() self.marionette.set_context(self.marionette.CONTEXT_CHROME) if os.path.isfile(self.test_script): f = open(self.test_script, 'r') self.test_script = f.read() f.close() self.marionette.execute_script(self.test_script) def run_tests(self, tests, options): manifests = self.resolver.resolveManifests(options, tests) self.profile = self.create_profile(options, manifests, profile_to_clone=options.profile) env = self.buildBrowserEnv(options, self.profile.profile) kp_kwargs = { 'processOutputLine': [self._on_output], 'onTimeout': [self._on_timeout], 'kill_on_timeout': False } if not options.debugger: if not options.timeout: if mozinfo.info['debug']: options.timeout = 420 else: options.timeout = 300 self.timeout = options.timeout + 30.0 log.info("%s | Running tests: start.", os.path.basename(__file__)) cmd, args = self.build_command_line( options.app, ignore_window_size=options.ignoreWindowSize, browser_arg=options.browser_arg) self.runner = FirefoxRunner(profile=self.profile, binary=cmd, cmdargs=args, env=env, process_class=ProcessHandler, process_args=kp_kwargs, symbols_path=options.symbolsPath) status = 0 try: self.runner.start(outputTimeout=self.timeout) log.info("%s | Application pid: %d", os.path.basename(__file__), self.runner.process_handler.pid) # kick starts the reftest harness self.run_marionette_script() status = self.runner.wait() finally: self.runner.check_for_crashes(test_name=self.last_test) self.runner.cleanup() if status > 0: log.testFail("%s | application terminated with exit code %s", self.last_test, status) elif status < 0: log.info("%s | application killed with signal %s", self.last_test, -status) log.info("%s | Running tests: end.", os.path.basename(__file__)) return status def create_profile(self, options, manifests, profile_to_clone=None): profile = RefTest.createReftestProfile( self, options, manifests, profile_to_clone=profile_to_clone) prefs = {} # Turn off the locale picker screen prefs["browser.firstrun.show.localepicker"] = False if not self.build_type == "mulet": # FIXME: With Mulet we can't set this values since Gaia won't launch prefs["b2g.system_startup_url"] = \ "app://test-container.gaiamobile.org/index.html" prefs["b2g.system_manifest_url"] = \ "app://test-container.gaiamobile.org/manifest.webapp" # Make sure we disable system updates prefs["app.update.enabled"] = False prefs["app.update.url"] = "" prefs["app.update.url.override"] = "" # Disable webapp updates prefs["webapps.update.enabled"] = False # Disable tiles also prefs["browser.newtabpage.directory.source"] = "" prefs["browser.newtabpage.directory.ping"] = "" prefs["dom.ipc.tabs.disabled"] = False prefs["dom.mozBrowserFramesEnabled"] = True prefs["font.size.inflation.emPerLine"] = 0 prefs["font.size.inflation.minTwips"] = 0 prefs[ "network.dns.localDomains"] = "app://test-container.gaiamobile.org" prefs["reftest.browser.iframe.enabled"] = False prefs["reftest.remote"] = False # Set a future policy version to avoid the telemetry prompt. prefs["toolkit.telemetry.prompted"] = 999 prefs["toolkit.telemetry.notifiedOptOut"] = 999 # Set the extra prefs. profile.set_preferences(prefs) return profile def build_command_line(self, app, ignore_window_size=False, browser_arg=None): cmd = os.path.abspath(app) args = ['-marionette'] if browser_arg: args += [browser_arg] if not ignore_window_size: args.extend(['--screen', '800x1000']) if self.build_type == "mulet": args += ['-chrome', 'chrome://b2g/content/shell.html'] return cmd, args def _on_output(self, line): sys.stdout.write("%s\n" % line) sys.stdout.flush() # TODO use structured logging if "TEST-START" in line and "|" in line: self.last_test = line.split("|")[1].strip() def _on_timeout(self): msg = "%s | application timed out after %s seconds with no output" log.testFail(msg % (self.last_test, self.timeout)) # kill process to get a stack self.runner.stop(sig=signal.SIGABRT) def _unlockScreen(self): self.marionette.set_context(self.marionette.CONTEXT_CONTENT) self.marionette.import_script( os.path.abspath( os.path.join(__file__, os.path.pardir, "gaia_lock_screen.js"))) self.marionette.switch_to_frame() self.marionette.execute_async_script('GaiaLockScreen.unlock()') def _wait_for_homescreen(self, timeout): log.info("Waiting for home screen to load") Wait(self.marionette, timeout).until( expected.element_present(By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDevice, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = '10.0.2.2' PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } dirs = self.query_abs_dirs() topsrcdir = os.path.join(dirs['abs_work_dir'], 'src') adb = self.query_exe('adb') path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) # Enforce 1proc. This isn't in one of the user.js files because those # are shared with desktop, which wants e10s. We can't interpolate # because the formatting code only works for strings, and this is a # bool pref. prefs["browser.tabs.remote.autostart"] = False outputdir = self.config.get('output_directory', '/sdcard/pgo_profile') jarlog = posixpath.join(outputdir, 'en-US.log') profdata = posixpath.join(outputdir, 'default_%p_random_%m.profraw') env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata adbdevice = ADBDevice(adb=adb, device='emulator-5554') adbdevice.mkdir(outputdir) try: # Run Fennec a first time to initialize its profile driver = Marionette( app='fennec', package_name=app, adb_path=adb, bin="target.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if 'Speedometer/index.html' in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): if not adbdevice.process_exist(app): break time.sleep(2) else: raise Exception("Android App (%s) never quit" % app) # Pull all the profraw files and en-US.log adbdevice.pull(outputdir, '/builds/worker/workspace/') except ADBTimeoutError: self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError', EXIT_STATUS_DICT[TBPL_RETRY]) profraw_files = glob.glob('/builds/worker/workspace/*.profraw') if not profraw_files: self.fatal( 'Could not find any profraw files in /builds/worker/workspace') merge_cmd = [ os.path.join(os.environ['MOZ_FETCHES_DIR'], 'clang/bin/llvm-profdata'), 'merge', '-o', '/builds/worker/workspace/merged.profdata', ] + profraw_files rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( 'INFRA-ERROR: Failed to merge profile data. Corrupt profile?', EXIT_STATUS_DICT[TBPL_RETRY]) # tarfile doesn't support xz in this version of Python tar_cmd = [ 'tar', '-acvf', '/builds/worker/artifacts/profdata.tar.xz', '-C', '/builds/worker/workspace', 'merged.profdata', 'en-US.log', ] subprocess.check_call(tar_cmd) httpd.stop()
class B2GDesktopReftest(RefTest): build_type = "desktop" marionette = None def __init__(self, marionette_args): RefTest.__init__(self) self.last_test = os.path.basename(__file__) self.marionette_args = marionette_args self.profile = None self.runner = None self.test_script = os.path.join(here, 'b2g_start_script.js') self.timeout = None def run_marionette_script(self): self.marionette = Marionette(**self.marionette_args) assert(self.marionette.wait_for_port()) self.marionette.start_session() if self.build_type == "mulet": self._wait_for_homescreen(timeout=300) self._unlockScreen() self.marionette.set_context(self.marionette.CONTEXT_CHROME) if os.path.isfile(self.test_script): f = open(self.test_script, 'r') self.test_script = f.read() f.close() self.marionette.execute_script(self.test_script) def run_tests(self, tests, options): manifests = self.resolver.resolveManifests(options, tests) self.profile = self.create_profile(options, manifests, profile_to_clone=options.profile) env = self.buildBrowserEnv(options, self.profile.profile) kp_kwargs = { 'processOutputLine': [self._on_output], 'onTimeout': [self._on_timeout], 'kill_on_timeout': False } if not options.debugger: if not options.timeout: if mozinfo.info['debug']: options.timeout = 420 else: options.timeout = 300 self.timeout = options.timeout + 30.0 log.info("%s | Running tests: start.", os.path.basename(__file__)) cmd, args = self.build_command_line(options.app, ignore_window_size=options.ignoreWindowSize, browser_arg=options.browser_arg) self.runner = FirefoxRunner(profile=self.profile, binary=cmd, cmdargs=args, env=env, process_class=ProcessHandler, process_args=kp_kwargs, symbols_path=options.symbolsPath) status = 0 try: self.runner.start(outputTimeout=self.timeout) log.info("%s | Application pid: %d", os.path.basename(__file__), self.runner.process_handler.pid) # kick starts the reftest harness self.run_marionette_script() status = self.runner.wait() finally: self.runner.check_for_crashes(test_name=self.last_test) self.runner.cleanup() if status > 0: log.testFail("%s | application terminated with exit code %s", self.last_test, status) elif status < 0: log.info("%s | application killed with signal %s", self.last_test, -status) log.info("%s | Running tests: end.", os.path.basename(__file__)) return status def create_profile(self, options, manifests, profile_to_clone=None): profile = RefTest.createReftestProfile(self, options, manifests, profile_to_clone=profile_to_clone) prefs = {} # Turn off the locale picker screen prefs["browser.firstrun.show.localepicker"] = False if not self.build_type == "mulet": # FIXME: With Mulet we can't set this values since Gaia won't launch prefs["b2g.system_startup_url"] = \ "app://test-container.gaiamobile.org/index.html" prefs["b2g.system_manifest_url"] = \ "app://test-container.gaiamobile.org/manifest.webapp" # Make sure we disable system updates prefs["app.update.enabled"] = False prefs["app.update.url"] = "" prefs["app.update.url.override"] = "" # Disable webapp updates prefs["webapps.update.enabled"] = False # Disable tiles also prefs["browser.newtabpage.directory.source"] = "" prefs["browser.newtabpage.directory.ping"] = "" prefs["dom.ipc.tabs.disabled"] = False prefs["dom.mozBrowserFramesEnabled"] = True prefs["font.size.inflation.emPerLine"] = 0 prefs["font.size.inflation.minTwips"] = 0 prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org" prefs["reftest.browser.iframe.enabled"] = False prefs["reftest.remote"] = False # Set a future policy version to avoid the telemetry prompt. prefs["toolkit.telemetry.prompted"] = 999 prefs["toolkit.telemetry.notifiedOptOut"] = 999 # Disable periodic updates of service workers prefs["dom.serviceWorkers.periodic-updates.enabled"] = False # Set the extra prefs. profile.set_preferences(prefs) return profile def build_command_line(self, app, ignore_window_size=False, browser_arg=None): cmd = os.path.abspath(app) args = ['-marionette'] if browser_arg: args += [browser_arg] if not ignore_window_size: args.extend(['--screen', '800x1000']) if self.build_type == "mulet": args += ['-chrome', 'chrome://b2g/content/shell.html'] return cmd, args def _on_output(self, line): sys.stdout.write("%s\n" % line) sys.stdout.flush() # TODO use structured logging if "TEST-START" in line and "|" in line: self.last_test = line.split("|")[1].strip() def _on_timeout(self): msg = "%s | application timed out after %s seconds with no output" log.testFail(msg % (self.last_test, self.timeout)) # kill process to get a stack self.runner.stop(sig=signal.SIGABRT)
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDeviceFactory, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = "10.0.2.2" PORT = 8888 PATH_MAPPINGS = { "/js-input/webkit/PerformanceTests": "third_party/webkit/PerformanceTests", } dirs = self.query_abs_dirs() topsrcdir = dirs["abs_src_dir"] adb = self.query_exe("adb") path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd( port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings, ) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, "testing", "profiles") with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: base_profiles = json.load(fh)["profileserver"] prefpaths = [ os.path.join(profile_data_dir, profile, "user.js") for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) outputdir = self.config.get("output_directory", "/sdcard/pgo_profile") jarlog = posixpath.join(outputdir, "en-US.log") profdata = posixpath.join(outputdir, "default_%p_random_%m.profraw") env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata if self.query_minidump_stackwalk(): os.environ["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path os.environ["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs( )["abs_blob_upload_dir"] if not self.symbols_path: self.symbols_path = os.environ.get("MOZ_FETCHES_DIR") # Force test_root to be on the sdcard for android pgo # builds which fail for Android 4.3 when profiles are located # in /data/local/tmp/test_root with # E AndroidRuntime: FATAL EXCEPTION: Gecko # E AndroidRuntime: java.lang.IllegalArgumentException: \ # Profile directory must be writable if specified: /data/local/tmp/test_root/profile # This occurs when .can-write-sentinel is written to # the profile in # mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoProfile.java. # This is not a problem on later versions of Android. This # over-ride of test_root should be removed when Android 4.3 is no # longer supported. sdcard_test_root = "/sdcard/test_root" adbdevice = ADBDeviceFactory(adb=adb, device="emulator-5554", test_root=sdcard_test_root) if adbdevice.test_root != sdcard_test_root: # If the test_root was previously set and shared # the initializer will not have updated the shared # value. Force it to match the sdcard_test_root. adbdevice.test_root = sdcard_test_root adbdevice.mkdir(outputdir, parents=True) try: # Run Fennec a first time to initialize its profile driver = Marionette( app="fennec", package_name=app, adb_path=adb, bin="geckoview-androidTest.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, symbols_path=self.symbols_path, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if "Speedometer/index.html" in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): if not adbdevice.process_exist(app): break time.sleep(2) else: raise Exception("Android App (%s) never quit" % app) # Pull all the profraw files and en-US.log adbdevice.pull(outputdir, "/builds/worker/workspace/") except ADBTimeoutError: self.fatal( "INFRA-ERROR: Failed with an ADBTimeoutError", EXIT_STATUS_DICT[TBPL_RETRY], ) profraw_files = glob.glob("/builds/worker/workspace/*.profraw") if not profraw_files: self.fatal( "Could not find any profraw files in /builds/worker/workspace") merge_cmd = [ os.path.join(os.environ["MOZ_FETCHES_DIR"], "clang/bin/llvm-profdata"), "merge", "-o", "/builds/worker/workspace/merged.profdata", ] + profraw_files rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( "INFRA-ERROR: Failed to merge profile data. Corrupt profile?", EXIT_STATUS_DICT[TBPL_RETRY], ) # tarfile doesn't support xz in this version of Python tar_cmd = [ "tar", "-acvf", "/builds/worker/artifacts/profdata.tar.xz", "-C", "/builds/worker/workspace", "merged.profdata", "en-US.log", ] subprocess.check_call(tar_cmd) httpd.stop()
class MuletReftest(RefTest): build_type = "mulet" marionette = None def __init__(self, marionette_args): RefTest.__init__(self) self.last_test = os.path.basename(__file__) self.marionette_args = marionette_args self.profile = None self.runner = None self.test_script = os.path.join(here, 'b2g_start_script.js') self.timeout = None def run_marionette_script(self): self.marionette = Marionette(**self.marionette_args) assert(self.marionette.wait_for_port()) self.marionette.start_session() if self.build_type == "mulet": self._wait_for_homescreen(timeout=300) self._unlockScreen() self.marionette.set_context(self.marionette.CONTEXT_CHROME) if os.path.isfile(self.test_script): f = open(self.test_script, 'r') self.test_script = f.read() f.close() self.marionette.execute_script(self.test_script) def run_tests(self, tests, options): manifests = self.resolver.resolveManifests(options, tests) self.profile = self.create_profile(options, manifests, profile_to_clone=options.profile) env = self.buildBrowserEnv(options, self.profile.profile) self._populate_logger(options) outputHandler = OutputHandler(self.log, options.utilityPath, symbolsPath=options.symbolsPath) kp_kwargs = { 'processOutputLine': [outputHandler], 'onTimeout': [self._on_timeout], 'kill_on_timeout': False } if not options.debugger: if not options.timeout: if mozinfo.info['debug']: options.timeout = 420 else: options.timeout = 300 self.timeout = options.timeout + 30.0 self.log.info("%s | Running tests: start." % os.path.basename(__file__)) cmd, args = self.build_command_line(options.app, ignore_window_size=options.ignoreWindowSize, browser_arg=options.browser_arg) self.runner = FirefoxRunner(profile=self.profile, binary=cmd, cmdargs=args, env=env, process_class=ProcessHandler, process_args=kp_kwargs, symbols_path=options.symbolsPath) status = 0 try: self.runner.start(outputTimeout=self.timeout) self.log.info("%s | Application pid: %d" % ( os.path.basename(__file__), self.runner.process_handler.pid)) # kick starts the reftest harness self.run_marionette_script() status = self.runner.wait() finally: self.runner.check_for_crashes(test_name=self.last_test) self.runner.cleanup() if status > 0: self.log.testFail("%s | application terminated with exit code %s" % ( self.last_test, status)) elif status < 0: self.log.info("%s | application killed with signal %s" % ( self.last_test, -status)) self.log.info("%s | Running tests: end." % os.path.basename(__file__)) return status def create_profile(self, options, manifests, profile_to_clone=None): profile = RefTest.createReftestProfile(self, options, manifests, profile_to_clone=profile_to_clone) prefs = {} # Turn off the locale picker screen prefs["browser.firstrun.show.localepicker"] = False if not self.build_type == "mulet": # FIXME: With Mulet we can't set this values since Gaia won't launch prefs["b2g.system_startup_url"] = \ "app://test-container.gaiamobile.org/index.html" prefs["b2g.system_manifest_url"] = \ "app://test-container.gaiamobile.org/manifest.webapp" # Make sure we disable system updates prefs["app.update.enabled"] = False prefs["app.update.url"] = "" # Disable webapp updates prefs["webapps.update.enabled"] = False # Disable tiles also prefs["browser.newtabpage.directory.source"] = "" prefs["browser.newtabpage.directory.ping"] = "" prefs["dom.ipc.tabs.disabled"] = False prefs["dom.mozBrowserFramesEnabled"] = True prefs["font.size.inflation.emPerLine"] = 0 prefs["font.size.inflation.minTwips"] = 0 prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org" prefs["reftest.browser.iframe.enabled"] = False prefs["reftest.remote"] = False # Set the extra prefs. profile.set_preferences(prefs) return profile def build_command_line(self, app, ignore_window_size=False, browser_arg=None): cmd = os.path.abspath(app) args = ['-marionette'] if browser_arg: args += [browser_arg] if not ignore_window_size: args.extend(['--screen', '800x1000']) if self.build_type == "mulet": args += ['-chrome', 'chrome://b2g/content/shell.html'] return cmd, args def _on_timeout(self): msg = "%s | application timed out after %s seconds with no output" self.log.testFail(msg % (self.last_test, self.timeout)) self.log.error("Force-terminating active process(es)."); # kill process to get a stack self.runner.stop(sig=signal.SIGABRT) def _unlockScreen(self): self.marionette.set_context(self.marionette.CONTEXT_CONTENT) self.marionette.import_script(os.path.abspath( os.path.join(__file__, os.path.pardir, "gaia_lock_screen.js"))) self.marionette.switch_to_frame() self.marionette.execute_async_script('GaiaLockScreen.unlock()') def _wait_for_homescreen(self, timeout): self.log.info("Waiting for home screen to load") Wait(self.marionette, timeout).until(expected.element_present( By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
import os import sys import fnmatch from marionette_driver.marionette import Marionette BASE_DIR = os.path.dirname(os.path.abspath(__file__)) client = Marionette(host='localhost', port=2828) client.start_session() client.set_context(Marionette.CONTEXT_CHROME) with open(os.path.join(BASE_DIR, 'jasmine.js')) as f: client.execute_script(f.read(), sandbox='tests', new_sandbox=False) with open(os.path.join(BASE_DIR, 'boot.js')) as f: client.execute_script(f.read(), sandbox='tests', new_sandbox=False) test_dir = sys.argv[1] for root, dirnames, filenames in os.walk(test_dir): for filename in fnmatch.filter(filenames, 'test*.js'): with open(os.path.join(root, filename)) as f: client.execute_script(f.read(), sandbox='tests', new_sandbox=False) specs = client.execute_script(""" runSpecs(); return jsApiReporter.specs(); """, sandbox='tests', new_sandbox=False)
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDevice, ADBProcessError, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = '10.0.2.2' PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } dirs = self.query_abs_dirs() topsrcdir = os.path.join(dirs['abs_work_dir'], 'src') adb = self.query_exe('adb') path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) outputdir = self.config.get('output_directory', '/sdcard') jarlog = posixpath.join(outputdir, 'en-US.log') profdata = posixpath.join(outputdir, 'default.profraw') env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata adbdevice = ADBDevice(adb=adb, device='emulator-5554') try: # Run Fennec a first time to initialize its profile driver = Marionette( app='fennec', package_name=app, adb_path=adb, bin="target.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if 'Speedometer/index.html' in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): try: localprof = '/builds/worker/workspace/default.profraw' adbdevice.pull(profdata, localprof) stats = os.stat(localprof) if stats.st_size == 0: # The file may not have been fully written yet, so retry until we # get actual data. time.sleep(2) else: break except ADBProcessError: # The file may not exist at all yet, which would raise an # ADBProcessError, so retry. time.sleep(2) else: raise Exception("Unable to pull default.profraw") adbdevice.pull(jarlog, '/builds/worker/workspace/en-US.log') except ADBTimeoutError: self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError', EXIT_STATUS_DICT[TBPL_RETRY]) # We normally merge as part of a GENERATED_FILES step in the profile-use # build, but Android runs sometimes result in a truncated profile. We do # a merge here to make sure the data isn't corrupt so we can retry the # 'run' task if necessary. merge_cmd = [ '/builds/worker/workspace/build/clang/bin/llvm-profdata', 'merge', '/builds/worker/workspace/default.profraw', '-o', '/builds/worker/workspace/merged.profraw', ] rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( 'INFRA-ERROR: Failed to merge profile data. Corrupt profile?', EXIT_STATUS_DICT[TBPL_RETRY]) # tarfile doesn't support xz in this version of Python tar_cmd = [ 'tar', '-acvf', '/builds/worker/artifacts/profdata.tar.xz', '-C', '/builds/worker/workspace', 'merged.profraw', 'en-US.log', ] subprocess.check_call(tar_cmd) httpd.stop()
def run_firefox(args): logger = logging.getLogger('sisyphus') logger.setLevel(logging.DEBUG) streamhandler = logging.StreamHandler(stream=sys.stdout) streamformatter = logging.Formatter( '%(asctime)s %(name)15s %(levelname)-8s %(message)s') streamhandler.setFormatter(streamformatter) logger.addHandler(streamhandler) # On Windows, make sure we point to the same temp directory as cygwin. if platform.system() == 'Windows': tempfile.tempdir = 'C:\\cygwin\\tmp' # Clean up tmpaddons tmpaddons = glob.glob(os.path.join(tempfile.gettempdir(), 'tmpaddon*')) for tmpaddon in tmpaddons: os.unlink(tmpaddon) # Work around Windows issues with shell metacharacters in url. if not args.url: if "URL" in os.environ: args.url = os.environ["URL"] else: logger.error("run_firefox: url is required") return # Load preferences of the form name=value from the # command line arguments. def eval_value(value): """Convert preference string value""" if value == 'true': value = True elif value == 'false': value = False else: try: value = eval(value) except NameError: # Leave string value alone. pass return value mozprofile_preferences = mozprofile.prefs.Preferences() preferences = {} set_preference_args = [] for set_preference_arg in args.set_preferences: (name, value) = set_preference_arg.split('=', 1) set_preference_args.append((name, eval_value(value))) preferences.update(dict(set_preference_args)) # Load preferences of from json files. for preference_json_arg in args.preference_jsons: preferences.update( dict(mozprofile_preferences.read_json(preference_json_arg))) # Load preferences from Firefox prefs.js/user.js files. for preference_file_arg in args.preference_files: preferences.update( dict(mozprofile_preferences.read_prefs(preference_file_arg))) logger.info("preferences: %s", json.dumps(preferences, indent=2, sort_keys=True)) profile = mozprofile.profile.FirefoxProfile(profile=args.profile, preferences=preferences) client = Marionette(host='localhost', port=2828, bin=args.binary, profile=profile, gecko_log=args.gecko_log, symbols_path=args.symbols_path) client.start_session() if args.restart: client.restart(clean=False, in_app=True) client.maximize_window() references = {'time_out_alarm_fired': False} if hasattr(signal, 'SIGALRM'): # Windows doesn't support SIGALRM. marionette # doesn't support cygwin paths... def timeout_handler(signum, frame): logger.warning("navigate: %s timed out" % args.url) references['time_out_alarm_fired'] = True client.quit() default_alarm_handler = signal.getsignal(signal.SIGALRM) signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(args.page_load_timeout + 2 * args.script_timeout) try: client.timeout.page_load = args.page_load_timeout client.timeout.script = args.script_timeout # Register the dialog closer for the browser. If the download # dialog appears, it will be closed and the browser window # will be closed. This forces marionette to return from # navigate and works around Bug 1366035. This version does # not dismiss normal Alerts which can be handled by Marionette's Alert. dialog_closer_script = """ var gDialogCloser; var gDialogCloserObserver; var gDialogCloserSubjects = []; registerDialogCloser = function () { gDialogCloser = Components.classes['@mozilla.org/embedcomp/window-watcher;1'].getService(Components.interfaces.nsIWindowWatcher); gDialogCloserObserver = {observe: dialogCloser_observe}; gDialogCloser.registerNotification(gDialogCloserObserver); } unregisterDialogCloser = function () { if (!gDialogCloserObserver || !gDialogCloser) { return; } gDialogCloser.unregisterNotification(gDialogCloserObserver); gDialogCloserObserver = null; gDialogCloser = null; } dialogCloser_observe = function (subject, topic, data) { if (subject instanceof ChromeWindow && topic == 'domwindowopened' ) { gDialogCloserSubjects.push(subject); subject.setTimeout(closeDialog, 5000) } } closeDialog = function () { var subject; while ( (subject = gDialogCloserSubjects.pop()) != null) { if (subject.document instanceof XULDocument) { var uri = subject.document.documentURI; //if (uri.startsWith('chrome://') && uri.endsWith('ialog.xul')) { // subject.close(); //} else if (uri == 'chrome://mozapps/content/downloads/unknownContentType.xul') { dump('Sisyphus Runner: Closing Window due to download dialog\\n'); subject.close(); window.close(); } } } } registerDialogCloser(); """ client.set_context(client.CONTEXT_CHROME) client.execute_script(dialog_closer_script, new_sandbox=False, script_timeout=client.timeout.script) client.set_context(client.CONTEXT_CONTENT) try: logger.info('New Page: %s' % args.url) client.navigate(args.url) client.maximize_window() except Exception, e: logger.warning('navigate: %s', e) # Do not call client.check_for_crash() as that will invoke # mozcrash which will delete the dump files. Handle the dumps # in the caller. client.set_context(client.CONTEXT_CONTENT) for content_script_url in args.content_scripts: content_script = get_remote_text(content_script_url) if content_script: try: logger.info('<contentscript>\n%s\n</contentscript>', content_script) result = client.execute_script( content_script, script_args=[], script_timeout=client.timeout.script) logger.info('content script result: %s', result) except errors.ScriptTimeoutException, e: logger.warning('content script: %s', e) except Exception, e: logger.error('content script: %s', e)