class AnnounceMessenger: # Create our marionette instance. def __init__(self): self.client = Marionette('localhost', port=2828) self.client.start_session() random.seed() def send_message(self, to_user, message, wait_load=0.5): # Make firefox go to the users page. self.client.navigate("https://www.messenger.com/t/{}".format(to_user)) # Give a little bit of time to load. time.sleep(wait_load) for letter in message: # Make the typing look realistic. keep_down = random.uniform(0.005, 0.010) wait_time = keep_down + random.uniform(0.02, 0.05) # Send off the action action = Actions(self.client) if letter != '\n': action.key_down(letter).wait(keep_down).key_up(letter) else: # Handle multi line messages. action.key_down(Keys.SHIFT).key_down(Keys.ENTER)\ .wait(keep_down).key_up(Keys.SHIFT).key_up(Keys.ENTER) action.perform() # wait for it to complete. time.sleep(wait_time) # Send message. action.key_down('\n').wait(keep_down).key_up('\n') action.perform()
def navigate(url): client = Marionette(host='localhost', port=2828) client.start_session() client.navigate(url) start_page = InformationPage(client) exhibitor_widget = start_page.get_exhibitor_widget() exhibitors = exhibitor_widget.get_exhibitors() return exhibitors
def test_load_vapid_page(): client = Marionette("localhost", port=2828) client.start_session() url = "https://jrconlin.github.io/Webpush_QA/" expected_url = "https://jrconlin.github.io/Webpush_QA/success.html" client.navigate(url) sleep(5) assert expected_url in client.get_url()
def navigate(url): client = Marionette(host='localhost', port=2828) client.start_session() client.navigate(url) start_page = InformationPage(client) exhibitor_page = start_page.navigate_to_page("Exhibitors") exhibitor_widget = exhibitor_page.get_exhibitor_widget() exhibitors = exhibitor_widget.get_exhibitors() for exhibitor in exhibitors: print exhibitor
def test_load_vapid_page(): client = Marionette('localhost', port=2828) client.start_session() url = 'https://jrconlin.github.io/Webpush_QA/' client.navigate(url) sleep(5) # Loop through each expected test element and make sure we have our unicode check mark field_ids = ['pre', 'reg', 'sub', 'gen', 'vap', 'enc', 'snd', 'rcv', 'dec'] command = "return window.getComputedStyle(document.querySelector('li#{0}.done.undone'),':before').getPropertyValue('content')" for field_id in field_ids: resp = client.execute_script(command.format(field_id)) assert resp == u'"\u2611"'
def workerProc(arg): client = Marionette('localhost', port=2828) client.start_session() client.navigate('https://www.mercari.com/sell/') time.sleep(5) #first open the excel file to grab information #photos will be stored in photos file num_rows = arg items = load_excel("Mercari.xlsx", num_rows) for item in items: title = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[3]/div[1]/div[2]/input") title.send_keys(item[0]) textarea = client.find_element(By.TAG_NAME, 'textarea') textarea.send_keys(item[1]) zipcode = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[6]/div[1]/div[2]/input") zipcode.send_keys(item[8]) price = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[7]/div/div[2]/input") price.send_keys(item[10]) brand = client.find_element(By.XPATH, "/html/body/div[1]/main/div[3]/div/div[2]/div[7]/div/div[2]/input") brand.send_keys(item[6]) category(client, item[2], item[3], item[4]) if item[6] >= 0: submit_size(client, item[6]) time.sleep(1) #determineSize(client, 1) selectCondition(client, item[7]) ship_selfpaid(client) for i in range(item[10]): path = os.path.dirname(os.path.realpath("photos\img" + str(i) + ".jpg")) fileUpload(client, getPID(), path + "\img" + str(i) + ".jpg", i) submit_button(client) time.sleep(2) client.navigate('https://www.mercari.com/sell/')
def freeze(self, request, queryset): with open(os.path.join(settings.BASE_DIR, 'build', 'freeze.bundle.js')) as f: freeze_script = f.read() client = None try: client = Marionette(bin=settings.FIREFOX_BIN, headless=True) client.start_session() for webpage in queryset: print('Freezing {}...'.format(webpage.url)) client.navigate(webpage.url) results = client.execute_async_script( freeze_script, script_timeout=1000 * 60 * 5, ) webpage.frozen_html.save(webpage.url, ContentFile(results['html'])) webpage.save() finally: if client: client.cleanup()
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDeviceFactory, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = "10.0.2.2" PORT = 8888 PATH_MAPPINGS = { "/js-input/webkit/PerformanceTests": "third_party/webkit/PerformanceTests", } dirs = self.query_abs_dirs() topsrcdir = dirs["abs_src_dir"] adb = self.query_exe("adb") path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd( port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings, ) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, "testing", "profiles") with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: base_profiles = json.load(fh)["profileserver"] prefpaths = [ os.path.join(profile_data_dir, profile, "user.js") for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) outputdir = self.config.get("output_directory", "/sdcard/pgo_profile") jarlog = posixpath.join(outputdir, "en-US.log") profdata = posixpath.join(outputdir, "default_%p_random_%m.profraw") env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata if self.query_minidump_stackwalk(): os.environ["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path os.environ["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs( )["abs_blob_upload_dir"] if not self.symbols_path: self.symbols_path = os.environ.get("MOZ_FETCHES_DIR") # Force test_root to be on the sdcard for android pgo # builds which fail for Android 4.3 when profiles are located # in /data/local/tmp/test_root with # E AndroidRuntime: FATAL EXCEPTION: Gecko # E AndroidRuntime: java.lang.IllegalArgumentException: \ # Profile directory must be writable if specified: /data/local/tmp/test_root/profile # This occurs when .can-write-sentinel is written to # the profile in # mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoProfile.java. # This is not a problem on later versions of Android. This # over-ride of test_root should be removed when Android 4.3 is no # longer supported. sdcard_test_root = "/sdcard/test_root" adbdevice = ADBDeviceFactory(adb=adb, device="emulator-5554", test_root=sdcard_test_root) if adbdevice.test_root != sdcard_test_root: # If the test_root was previously set and shared # the initializer will not have updated the shared # value. Force it to match the sdcard_test_root. adbdevice.test_root = sdcard_test_root adbdevice.mkdir(outputdir, parents=True) try: # Run Fennec a first time to initialize its profile driver = Marionette( app="fennec", package_name=app, adb_path=adb, bin="geckoview-androidTest.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, symbols_path=self.symbols_path, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if "Speedometer/index.html" in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): if not adbdevice.process_exist(app): break time.sleep(2) else: raise Exception("Android App (%s) never quit" % app) # Pull all the profraw files and en-US.log adbdevice.pull(outputdir, "/builds/worker/workspace/") except ADBTimeoutError: self.fatal( "INFRA-ERROR: Failed with an ADBTimeoutError", EXIT_STATUS_DICT[TBPL_RETRY], ) profraw_files = glob.glob("/builds/worker/workspace/*.profraw") if not profraw_files: self.fatal( "Could not find any profraw files in /builds/worker/workspace") merge_cmd = [ os.path.join(os.environ["MOZ_FETCHES_DIR"], "clang/bin/llvm-profdata"), "merge", "-o", "/builds/worker/workspace/merged.profdata", ] + profraw_files rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( "INFRA-ERROR: Failed to merge profile data. Corrupt profile?", EXIT_STATUS_DICT[TBPL_RETRY], ) # tarfile doesn't support xz in this version of Python tar_cmd = [ "tar", "-acvf", "/builds/worker/artifacts/profdata.tar.xz", "-C", "/builds/worker/workspace", "merged.profdata", "en-US.log", ] subprocess.check_call(tar_cmd) httpd.stop()
class MarionetteHelper: """ Helper for starting firefox and for remote browsing """ def __init__(self, firefox_path, logger=None): self.logger = logger # type: logging.Logger self.client = None # type: Marionette self.ffpopen = None # type: subprocess.Popen self.ffpath = firefox_path # type: str if logger is None: self.logger = logging.getLogger("MarionetteHelper") self.logger.debug("Marionette helper init done") def _open_session(self, host='localhost', port=2828): """ Opens the session for marionette""" caps = {u'acceptInsecureCerts': True, } self.client = Marionette(host, port=port) self.client.start_session(capabilities=caps) def run_firefox(self): """ Start the firefox process""" self.logger.debug("Starting firefox process") self.ffpopen = subprocess.Popen([self.ffpath, "-marionette"]) self.logger.debug("Opening marionette session") self._open_session() def quit_firefox(self): """ Close the firefox process and close marionette session""" #self.logger.debug("Closing firefox") #self.client._send_message("Marionette:Quit") self.client._request_in_app_shutdown("eForceQuit") self.client.delete_session(False) self.client.cleanup() self.client = None # reset client state #try: # self.client.close() # try to close the window anyway #except InvalidSessionIdException: # pass #except socket.error: # pass #finally: # try: # self.logger.debug("Closing marionette session") # self.client.delete_session(False) # close client session # except InvalidSessionIdException: # pass # except socket.error: # pass # self.client = None # reset client state #self.logger.debug("Waiting for firefox to close") #for _ in range(3): # give the process 3 seconds to terminate # time.sleep(1) # self.ffpopen.poll() # if self.ffpopen.returncode is not None: # break #self.ffpopen.poll() #if self.ffpopen.returncode is None: # self.logger.warning("Firefox not closed in time, killing it!") # self.ffpopen.kill() # process was not quit in time, kill it #self.ffpopen = None # reset popen state #self.logger.debug("Firefox is closed") def ___get_client(self): """ Returns the internal marionette client object""" return self.client def navigate_to_url(self, url): """ Open an url in format of http[s]://example.com""" try: if "http" not in url: url = "http://" + url self.client.navigate(url) except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def back(self): """ Go a page backward""" try: self.client.go_back() except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def forward(self): """ Go a page forward""" try: self.client.go_forward() except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def follow_link(self, index=0): """ Click on a link""" try: links = self.client.find_elements(By.TAG_NAME, "a") links[index].click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except IndexError: self.logger.warning("Error: Out of bound") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_class_name(self, html_class_name): """ Click on first element via class name""" try: e = self.client.find_element(By.CLASS_NAME, html_class_name) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_css_selector(self, css_selector): """ Click on first element via css selector""" try: e = self.client.find_element(By.CSS_SELECTOR, css_selector) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_id(self, html_id): """ Click on first element via element id""" try: e = self.client.find_element(By.ID, html_id) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_name(self, html_name): """ Click on first element via element name""" try: e = self.client.find_element(By.NAME, html_name) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_tag_name(self, html_tag_name): """ Click on first element via tag name""" try: e = self.client.find_element(By.TAG_NAME, html_tag_name) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_xpath(self, xpath): """ Click on first element via xpath""" try: e = self.client.find_element(By.XPATH, xpath) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def click_element_by_link_text(self, html_link_text): """ Click on first element via link text""" try: e = self.client.find_element(By.LINK_TEXT, html_link_text) e.click() except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False except InsecureCertificateException: self.logger.warning("Warning: Insecure Certificate") return True except UnknownException as e: if "Reached error page" in str(e): self.logger.warning("Reached error page, ignoring...") else: # reraise, something very unexpected happened here t, v, tb = sys.exc_info() raise t, v, tb return False return True def send_keys_to_element_by_name(self, name, text): """ Sends text to an element via name""" try: e = self.client.find_element(By.NAME, name) e.send_keys(text) except ElementNotVisibleException: self.logger.warning("Error: Element not visible") return False except ElementNotSelectableException: self.logger.warning("Error: Element not selectable") return False except ElementNotAccessibleException: self.logger.warning("Error: Element not accessible") return False except ElementNotInteractableException: self.logger.warning("Error: Element not interactable") return False except NoSuchElementException: self.logger.warning("Error: Element does not exist") return False except TimeoutException: self.logger.warning("Error: Timeout") return False return True def select_window(self, index): """ Switch window via index""" try: self.client.switch_to_window(self.client.window_handles[index]) except NoSuchWindowException: self.logger.warning("Error: Window does not exist") return False return True def close_window(self): """ Close the current window""" self.client.close() # this won't close the last window, call quit_firefox instead def get_current_window_index(self): """ Get current windows index""" return self.client.window_handles.index(self.client.current_window_handle) def new_tab(self): """ Open a new empty tab""" with self.client.using_context("chrome"): self.client.find_element(By.ID, "menu_newNavigatorTab").click() def new_window(self): """ Open a new empty window""" with self.client.using_context("chrome"): self.client.execute_script("window.open();") def close_tab(self): """ Close the current tab""" self.close_window() # basically the same as close windows
def run_firefox(args): logger = logging.getLogger('sisyphus') logger.setLevel(logging.DEBUG) streamhandler = logging.StreamHandler(stream=sys.stdout) streamformatter = logging.Formatter( '%(asctime)s %(name)15s %(levelname)-8s %(message)s') streamhandler.setFormatter(streamformatter) logger.addHandler(streamhandler) # On Windows, make sure we point to the same temp directory as cygwin. if platform.system() == 'Windows': tempfile.tempdir = 'C:\\cygwin\\tmp' # Clean up tmpaddons tmpaddons = glob.glob(os.path.join(tempfile.gettempdir(), 'tmpaddon*')) for tmpaddon in tmpaddons: os.unlink(tmpaddon) # Work around Windows issues with shell metacharacters in url. if not args.url: if "URL" in os.environ: args.url = os.environ["URL"] else: logger.error("run_firefox: url is required") return # Load preferences of the form name=value from the # command line arguments. def eval_value(value): """Convert preference string value""" if value == 'true': value = True elif value == 'false': value = False else: try: value = eval(value) except NameError: # Leave string value alone. pass return value mozprofile_preferences = mozprofile.prefs.Preferences() preferences = {} set_preference_args = [] for set_preference_arg in args.set_preferences: (name, value) = set_preference_arg.split('=', 1) set_preference_args.append((name, eval_value(value))) preferences.update(dict(set_preference_args)) # Load preferences of from json files. for preference_json_arg in args.preference_jsons: preferences.update( dict(mozprofile_preferences.read_json(preference_json_arg))) # Load preferences from Firefox prefs.js/user.js files. for preference_file_arg in args.preference_files: preferences.update( dict(mozprofile_preferences.read_prefs(preference_file_arg))) logger.info("preferences: %s", json.dumps(preferences, indent=2, sort_keys=True)) profile = mozprofile.profile.FirefoxProfile(profile=args.profile, preferences=preferences) client = Marionette(host='localhost', port=2828, bin=args.binary, profile=profile, gecko_log=args.gecko_log, symbols_path=args.symbols_path) client.start_session() if args.restart: client.restart(clean=False, in_app=True) client.maximize_window() references = {'time_out_alarm_fired': False} if hasattr(signal, 'SIGALRM'): # Windows doesn't support SIGALRM. marionette # doesn't support cygwin paths... def timeout_handler(signum, frame): logger.warning("navigate: %s timed out" % args.url) references['time_out_alarm_fired'] = True client.quit() default_alarm_handler = signal.getsignal(signal.SIGALRM) signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(args.page_load_timeout + 2 * args.script_timeout) try: client.timeout.page_load = args.page_load_timeout client.timeout.script = args.script_timeout # Register the dialog closer for the browser. If the download # dialog appears, it will be closed and the browser window # will be closed. This forces marionette to return from # navigate and works around Bug 1366035. This version does # not dismiss normal Alerts which can be handled by Marionette's Alert. dialog_closer_script = """ var gDialogCloser; var gDialogCloserObserver; var gDialogCloserSubjects = []; registerDialogCloser = function () { gDialogCloser = Components.classes['@mozilla.org/embedcomp/window-watcher;1'].getService(Components.interfaces.nsIWindowWatcher); gDialogCloserObserver = {observe: dialogCloser_observe}; gDialogCloser.registerNotification(gDialogCloserObserver); } unregisterDialogCloser = function () { if (!gDialogCloserObserver || !gDialogCloser) { return; } gDialogCloser.unregisterNotification(gDialogCloserObserver); gDialogCloserObserver = null; gDialogCloser = null; } dialogCloser_observe = function (subject, topic, data) { if (subject instanceof ChromeWindow && topic == 'domwindowopened' ) { gDialogCloserSubjects.push(subject); subject.setTimeout(closeDialog, 5000) } } closeDialog = function () { var subject; while ( (subject = gDialogCloserSubjects.pop()) != null) { if (subject.document instanceof XULDocument) { var uri = subject.document.documentURI; //if (uri.startsWith('chrome://') && uri.endsWith('ialog.xul')) { // subject.close(); //} else if (uri == 'chrome://mozapps/content/downloads/unknownContentType.xul') { dump('Sisyphus Runner: Closing Window due to download dialog\\n'); subject.close(); window.close(); } } } } registerDialogCloser(); """ client.set_context(client.CONTEXT_CHROME) client.execute_script(dialog_closer_script, new_sandbox=False, script_timeout=client.timeout.script) client.set_context(client.CONTEXT_CONTENT) try: logger.info('New Page: %s' % args.url) client.navigate(args.url) client.maximize_window() except Exception, e: logger.warning('navigate: %s', e) # Do not call client.check_for_crash() as that will invoke # mozcrash which will delete the dump files. Handle the dumps # in the caller. client.set_context(client.CONTEXT_CONTENT) for content_script_url in args.content_scripts: content_script = get_remote_text(content_script_url) if content_script: try: logger.info('<contentscript>\n%s\n</contentscript>', content_script) result = client.execute_script( content_script, script_args=[], script_timeout=client.timeout.script) logger.info('content script result: %s', result) except errors.ScriptTimeoutException, e: logger.warning('content script: %s', e) except Exception, e: logger.error('content script: %s', e)
from marionette_driver import By from marionette_driver.marionette import Marionette from www.fileserver import FileServer # Start services FILE_SERVER = FileServer() SERVICE_PORT = str(5000) FILE_PORT = str(FILE_SERVER.port) HOST = "http://localhost:" ENDPOINT_PREFIX = HOST + SERVICE_PORT TEST_HTML = HOST + FILE_PORT + "/"+ 'test_IAccessible.html' CLIENT = Marionette(host='localhost', port=2828) CLIENT.start_session() CLIENT.navigate(TEST_HTML) EVENT_PARAMS = urllib.urlencode({'interface': 'IAccessible', 'name': 'MSAA Checkbox', 'type' : 'EVENT_OBJECT_STATECHANGE'}) EVENT_ENDPOINT = ENDPOINT_PREFIX + "/event?%s" CMD_PARAMS = urllib.urlencode({'interface': 'IAccessible', 'name': 'MSAA Checkbox', 'function': 'State'}) CMD_ENPOINT = ENDPOINT_PREFIX + "/cmd?%s" ACCESSSIBLE_PARAMS = urllib.urlencode({'interface': 'IAccessible', 'name': 'MSAA Checkbox', 'depth': -1}) ACCESSIBLE_ENDPOINT = ENDPOINT_PREFIX + "/accessible?%s" print "-----------------ACCESSIBLE------------------" RESPONSE = json.load(urllib.urlopen(ACCESSIBLE_ENDPOINT % ACCESSSIBLE_PARAMS)) pprint(RESPONSE) CHECKBOX = 0x2C assert RESPONSE['IAccessible']['Role'] == CHECKBOX # CHECKBOX = CLIENT.find_element(By.CSS_SELECTOR, "input[type='checkbox']")
class Firefox(DesktopBrowser): """Firefox""" def __init__(self, path, options, job): DesktopBrowser.__init__(self, path, options, job) self.job = job self.task = None self.options = options self.path = path self.event_name = None self.moz_log = None self.marionette = None self.addons = None self.extension_id = None self.nav_error = None self.page_loaded = None self.recording = False self.connected = False self.start_offset = None self.browser_version = None self.log_pos = {} self.page = {} self.requests = {} self.last_activity = monotonic.monotonic() self.script_dir = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'js') self.start_page = 'http://127.0.0.1:8888/orange.html' def prepare(self, job, task): """Prepare the profile/OS for the browser""" self.moz_log = os.path.join(task['dir'], 'moz.log') self.log_pos = {} self.page = {} self.requests = {} os.environ["MOZ_LOG_FILE"] = self.moz_log os.environ["MOZ_LOG"] = 'timestamp,sync,nsHttp:5,nsSocketTransport:5'\ 'nsHostResolver:5,pipnss:5' DesktopBrowser.prepare(self, job, task) profile_template = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'support', 'Firefox', 'profile') if not task['cached'] and os.path.isdir(profile_template): try: if os.path.isdir(task['profile']): shutil.rmtree(task['profile']) shutil.copytree(profile_template, task['profile']) except Exception: pass # delete any unsent crash reports crash_dir = None if platform.system() == 'Windows': if 'APPDATA' in os.environ: crash_dir = os.path.join(os.environ['APPDATA'], 'Mozilla', 'Firefox', 'Crash Reports') else: crash_dir = os.path.join(os.path.expanduser('~'), '.mozilla', 'firefox', 'Crash Reports') if crash_dir and os.path.isdir(crash_dir): logging.debug("Clearing crash reports in %s", crash_dir) try: shutil.rmtree(crash_dir) except Exception: pass # Prepare the config for the extension to query if self.job['message_server'] is not None: config = None names = [ 'block', 'block_domains', 'block_domains_except', 'headers', 'cookies' ] for name in names: if name in task and task[name]: if config is None: config = {} config[name] = task[name] self.job['message_server'].config = config def launch(self, job, task): """Launch the browser""" if self.job['message_server'] is not None: self.job['message_server'].flush_messages() self.connected = False from marionette_driver.marionette import Marionette from marionette_driver.addons import Addons args = [ '-profile', '"{0}"'.format(task['profile']), '-no-remote', '-marionette', 'about:blank' ] if self.path.find(' ') > -1: command_line = '"{0}"'.format(self.path) else: command_line = self.path command_line += ' ' + ' '.join(args) DesktopBrowser.launch_browser(self, command_line) try: self.marionette = Marionette('localhost', port=2828) self.marionette.start_session(timeout=self.task['time_limit']) self.configure_prefs() logging.debug('Installing extension') self.addons = Addons(self.marionette) extension_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'support', 'Firefox', 'extension') self.extension_id = self.addons.install(extension_path, temp=True) logging.debug('Resizing browser to %dx%d', task['width'], task['height']) self.marionette.set_window_position(x=0, y=0) self.marionette.set_window_size(height=task['height'], width=task['width']) if 'browserVersion' in self.marionette.session_capabilities: self.browser_version = self.marionette.session_capabilities[ 'browserVersion'] self.marionette.navigate(self.start_page) time.sleep(0.5) self.wait_for_extension() if self.connected: # Override the UA String if necessary ua_string = self.execute_js('navigator.userAgent;') modified = False if 'uastring' in self.job: ua_string = self.job['uastring'] modified = True if ua_string is not None and 'AppendUA' in task: ua_string += ' ' + task['AppendUA'] modified = True if modified: logging.debug(ua_string) self.marionette.set_pref('general.useragent.override', ua_string) # Figure out the native viewport size size = self.execute_js( "[window.innerWidth, window.innerHeight]") logging.debug(size) if size is not None and len(size) == 2: task['actual_viewport'] = { "width": size[0], "height": size[1] } if 'adjust_viewport' in job and job['adjust_viewport']: delta_x = max(task['width'] - size[0], 0) delta_y = max(task['height'] - size[1], 0) if delta_x or delta_y: width = task['width'] + delta_x height = task['height'] + delta_y logging.debug('Resizing browser to %dx%d', width, height) self.marionette.set_window_position(x=0, y=0) self.marionette.set_window_size(height=height, width=width) # Wait for the browser startup to finish DesktopBrowser.wait_for_idle(self) except Exception as err: task['error'] = 'Error starting Firefox: {0}'.format(err.__str__()) def get_pref_value(self, value): """Convert a json pref value to Python""" str_match = re.match(r'^"(.*)"$', value) if value == 'true': value = True elif value == 'false': value = False elif re.match(r'^[\d]+$', value): value = int(value) elif str_match: value = str_match.group(1) else: value = None return value def configure_prefs(self): """Load the prefs file and configure them through marionette""" prefs = {} prefs_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'support', 'Firefox', 'profile', 'prefs.js') with open(prefs_file) as f_in: for line in f_in: matches = re.search( r'user_pref\("([^"]+)",[\s]*([^\)]*)[\s]*\);', line) if matches: key = matches.group(1).strip() value = self.get_pref_value(matches.group(2).strip()) if value is not None: prefs[key] = value if prefs: try: self.marionette.set_prefs(prefs, True) except Exception: pass def stop(self, job, task): """Kill the browser""" if self.extension_id is not None and self.addons is not None: try: self.addons.uninstall(self.extension_id) except Exception: pass self.extension_id = None self.addons = None if self.marionette is not None: try: self.marionette.close() except Exception: pass self.marionette = None DesktopBrowser.stop(self, job, task) # Make SURE the firefox processes are gone if platform.system() == "Linux": subprocess.call(['killall', '-9', 'firefox']) subprocess.call(['killall', '-9', 'firefox-trunk']) os.environ["MOZ_LOG_FILE"] = '' os.environ["MOZ_LOG"] = '' # delete the raw log files if self.moz_log is not None: files = sorted(glob.glob(self.moz_log + '*')) for path in files: try: os.remove(path) except Exception: pass def run_lighthouse_test(self, task): """Stub for lighthouse test""" pass def run_task(self, task): """Run an individual test""" if self.marionette is not None and self.connected: self.task = task logging.debug("Running test") end_time = monotonic.monotonic() + task['test_time_limit'] task['current_step'] = 1 recording = False while len(task['script']) and task['error'] is None and \ monotonic.monotonic() < end_time: self.prepare_task(task) command = task['script'].pop(0) if not recording and command['record']: recording = True self.on_start_recording(task) try: self.process_command(command) except Exception: logging.exception("Exception running task") if command['record']: self.wait_for_page_load() if not task['combine_steps'] or not len(task['script']): self.on_stop_recording(task) recording = False self.on_start_processing(task) self.wait_for_processing(task) self.step_complete(task) if task['log_data']: # Move on to the next step task['current_step'] += 1 self.event_name = None task['navigated'] = True # Always navigate to about:blank after finishing in case the tab is # remembered across sessions try: self.marionette.navigate('about:blank') except Exception: logging.debug( 'Marionette exception navigating to about:blank after the test' ) self.task = None def wait_for_extension(self): """Wait for the extension to send the started message""" if self.job['message_server'] is not None: end_time = monotonic.monotonic() + 30 while monotonic.monotonic() < end_time: try: self.job['message_server'].get_message(1) logging.debug('Extension started') self.connected = True break except Exception: pass def wait_for_page_load(self): """Wait for the onload event from the extension""" if self.job['message_server'] is not None and self.connected: start_time = monotonic.monotonic() end_time = start_time + self.task['time_limit'] done = False while not done: try: self.process_message( self.job['message_server'].get_message(1)) except Exception: pass now = monotonic.monotonic() elapsed_test = now - start_time if self.nav_error is not None: done = True if self.page_loaded is None: self.task['error'] = self.nav_error elif now >= end_time: done = True # only consider it an error if we didn't get a page load event if self.page_loaded is None: self.task['error'] = "Page Load Timeout" elif 'time' not in self.job or elapsed_test > self.job['time']: elapsed_activity = now - self.last_activity elapsed_page_load = now - self.page_loaded if self.page_loaded else 0 if elapsed_page_load >= 1 and elapsed_activity >= self.task[ 'activity_time']: done = True elif self.task['error'] is not None: done = True def execute_js(self, script): """Run javascipt""" ret = None if self.marionette is not None: try: ret = self.marionette.execute_script('return ' + script, script_timeout=30) except Exception: pass return ret def run_js_file(self, file_name): """Execute one of our js scripts""" ret = None script = None script_file_path = os.path.join(self.script_dir, file_name) if os.path.isfile(script_file_path): with open(script_file_path, 'rb') as script_file: script = script_file.read() if script is not None: try: ret = self.marionette.execute_script('return ' + script, script_timeout=30) except Exception: pass if ret is not None: logging.debug(ret) return ret def collect_browser_metrics(self, task): """Collect all of the in-page browser metrics that we need""" logging.debug("Collecting user timing metrics") user_timing = self.run_js_file('user_timing.js') if user_timing is not None: path = os.path.join(task['dir'], task['prefix'] + '_timed_events.json.gz') with gzip.open(path, 'wb', 7) as outfile: outfile.write(json.dumps(user_timing)) logging.debug("Collecting page-level metrics") page_data = self.run_js_file('page_data.js') if page_data is not None: task['page_data'].update(page_data) if 'customMetrics' in self.job: custom_metrics = {} for name in self.job['customMetrics']: logging.debug("Collecting custom metric %s", name) script = 'var wptCustomMetric = function() {' +\ self.job['customMetrics'][name] +\ '};try{return wptCustomMetric();}catch(e){};' try: custom_metrics[name] = self.marionette.execute_script( script, script_timeout=30) if custom_metrics[name] is not None: logging.debug(custom_metrics[name]) except Exception: pass path = os.path.join(task['dir'], task['prefix'] + '_metrics.json.gz') with gzip.open(path, 'wb', 7) as outfile: outfile.write(json.dumps(custom_metrics)) def process_message(self, message): """Process a message from the extension""" logging.debug(message) if self.recording: self.last_activity = monotonic.monotonic() try: # Make all of the timestamps relative to the test start to match the log events if 'timeStamp' in message['body']: timestamp = message['body']['timeStamp'] seconds = int(timestamp / 1000) milliseconds = timestamp - (seconds * 1000) event_time = datetime.utcfromtimestamp(seconds) event_time += timedelta(milliseconds=milliseconds) elapsed = event_time - self.task['start_time'] message['body']['timeStamp'] = elapsed.total_seconds() cat, msg = message['path'].split('.', 1) if cat == 'webNavigation': self.process_web_navigation(msg, message['body']) elif cat == 'webRequest': self.process_web_request(msg, message['body']) except Exception: pass def process_web_navigation(self, message, evt): """Handle webNavigation.*""" if evt is not None: if message == 'onBeforeNavigate': if 'frameId' in evt and evt['frameId'] == 0: self.page_loaded = None logging.debug("Starting navigation") if 'timeStamp' in evt and 'start' not in self.page: self.page['start'] = evt['timeStamp'] elif message == 'onCommitted': if 'timeStamp' in evt and 'frameId' in evt and evt['frameId'] == 0 \ and 'committed' not in self.page: self.page['committed'] = evt['timeStamp'] if 'injectScript' in self.job and self.marionette is not None: logging.debug("Injecting script: \n%s", self.job['injectScript']) try: self.marionette.execute_script( self.job['injectScript'], script_timeout=30) except Exception: pass elif message == 'onDOMContentLoaded': if 'timeStamp' in evt and 'frameId' in evt and evt[ 'frameId'] == 0: self.page['DOMContentLoaded'] = evt['timeStamp'] elif message == 'onCompleted': if 'frameId' in evt and evt['frameId'] == 0: self.page_loaded = monotonic.monotonic() logging.debug("Page loaded") if 'timeStamp' in evt: self.page['loaded'] = evt['timeStamp'] elif message == 'onErrorOccurred': if 'frameId' in evt and evt['frameId'] == 0: self.page_loaded = monotonic.monotonic() logging.debug("Page load failed") if 'error' in evt: self.nav_error = evt['error'] else: self.nav_error = 'Navigation failed' def process_web_request(self, message, evt): """Handle webNavigation.*""" if evt is not None and 'requestId' in evt and 'timeStamp' in evt: if evt['requestId'] not in self.requests: self.requests[evt['requestId']] = { 'id': evt['requestId'], 'from_net': True } request = self.requests[evt['requestId']] if 'url' in evt and 'url' not in request: request['url'] = evt['url'] if 'method' in evt and 'method' not in request: request['method'] = evt['method'] if 'type' in evt and 'type' not in request: request['type'] = evt['type'] if 'ip' in evt and 'ip' not in request: request['ip'] = evt['ip'] if 'fromCache' in evt and evt['fromCache']: request['from_net'] = False if 'statusLine' in evt: request['status_line'] = evt['statusLine'] if 'statusCode' in evt: request['status'] = evt['statusCode'] if 'requestHeaders' in evt and 'request_headers' not in request: request['request_headers'] = list(evt['requestHeaders']) if 'responseHeaders' in evt and 'response_headers' not in request: request['response_headers'] = list(evt['responseHeaders']) if message == 'onBeforeRequest': request['created'] = evt['timeStamp'] elif message == 'onSendHeaders': request['start'] = evt['timeStamp'] elif message == 'onBeforeRedirect': if 'first_byte' not in request: request['first_byte'] = evt['timeStamp'] if 'end' not in request or evt['timeStamp'] > request['end']: request['end'] = evt['timeStamp'] elif message == 'onHeadersReceived': if 'first_byte' not in request: request['first_byte'] = evt['timeStamp'] if 'end' not in request or evt['timeStamp'] > request['end']: request['end'] = evt['timeStamp'] elif message == 'onResponseStarted': if 'first_byte' not in request: request['first_byte'] = evt['timeStamp'] if 'end' not in request or evt['timeStamp'] > request['end']: request['end'] = evt['timeStamp'] elif message == 'onCompleted': if 'first_byte' not in request: request['first_byte'] = evt['timeStamp'] if 'end' not in request or evt['timeStamp'] > request['end']: request['end'] = evt['timeStamp'] elif message == 'onErrorOccurred': if 'end' not in request or evt['timeStamp'] > request['end']: request['end'] = evt['timeStamp'] if 'error' in evt: request['error'] = evt['error'] if 'status' not in request: request['status'] = 12999 def prepare_task(self, task): """Format the file prefixes for multi-step testing""" if task['current_step'] == 1: task['prefix'] = task['task_prefix'] task['video_subdirectory'] = task['task_video_prefix'] else: task['prefix'] = '{0}_{1:d}'.format(task['task_prefix'], task['current_step']) task['video_subdirectory'] = '{0}_{1:d}'.format( task['task_video_prefix'], task['current_step']) if task['video_subdirectory'] not in task['video_directories']: task['video_directories'].append(task['video_subdirectory']) if self.event_name is not None: task['step_name'] = self.event_name else: task['step_name'] = 'Step_{0:d}'.format(task['current_step']) def on_start_recording(self, task): """Notification that we are about to start an operation that needs to be recorded""" # Clear the state self.page = {} self.requests = {} task['page_data'] = {'date': time.time()} task['page_result'] = None task['run_start_time'] = monotonic.monotonic() if self.browser_version is not None and 'browserVersion' not in task[ 'page_data']: task['page_data']['browserVersion'] = self.browser_version task['page_data']['browser_version'] = self.browser_version # Mark the start point in the various log files self.log_pos = {} if self.moz_log is not None: files = sorted(glob.glob(self.moz_log + '*')) for path in files: self.log_pos[path] = os.path.getsize(path) self.recording = True now = monotonic.monotonic() if not self.task['stop_at_onload']: self.last_activity = now if self.page_loaded is not None: self.page_loaded = now DesktopBrowser.on_start_recording(self, task) logging.debug('Starting measurement') task['start_time'] = datetime.utcnow() def on_stop_recording(self, task): """Notification that we are done with recording""" self.recording = False DesktopBrowser.on_stop_recording(self, task) if self.connected: if self.job['pngScreenShot']: screen_shot = os.path.join(task['dir'], task['prefix'] + '_screen.png') self.grab_screenshot(screen_shot, png=True) else: screen_shot = os.path.join(task['dir'], task['prefix'] + '_screen.jpg') self.grab_screenshot(screen_shot, png=False, resize=600) # Collect end of test data from the browser self.collect_browser_metrics(task) # Collect the interactive periods interactive = self.execute_js( 'window.wrappedJSObject.wptagentGetInteractivePeriods();') if interactive is not None and len(interactive): interactive_file = os.path.join( task['dir'], task['prefix'] + '_interactive.json.gz') with gzip.open(interactive_file, 'wb', 7) as f_out: f_out.write(interactive) # Copy the log files if self.moz_log is not None: task['moz_log'] = os.path.join(task['dir'], task['prefix'] + '_moz.log') files = sorted(glob.glob(self.moz_log + '*')) for path in files: try: base_name = os.path.basename(path) dest = os.path.join( task['dir'], task['prefix'] + '_' + base_name + '.gz') start_pos = self.log_pos[ path] if path in self.log_pos else 0 end_pos = os.path.getsize(path) if end_pos > start_pos: length = end_pos - start_pos logging.debug( 'Preparing moz log %s (%d bytes from %d)', base_name, length, start_pos) with open(path, 'rb') as f_in: f_in.seek(start_pos) with gzip.open(dest, 'wb', 7) as f_out: while length > 0: read_bytes = min(length, 1024 * 1024) buff = f_in.read(read_bytes) read_bytes = len(buff) f_out.write(buff) length -= read_bytes except Exception: pass def on_start_processing(self, task): """Start any processing of the captured data""" DesktopBrowser.on_start_processing(self, task) # Parse the moz log for the accurate request timings request_timings = [] if 'moz_log' in task: from internal.support.firefox_log_parser import FirefoxLogParser parser = FirefoxLogParser() start_time = task['start_time'].strftime('%Y-%m-%d %H:%M:%S.%f') logging.debug('Parsing moz logs relative to %s start time', start_time) request_timings = parser.process_logs(task['moz_log'], start_time) files = sorted(glob.glob(task['moz_log'] + '*')) for path in files: try: os.remove(path) except Exception: pass # Build the request and page data if len(request_timings) and task['current_step'] == 1: self.adjust_timings(request_timings) self.process_requests(request_timings, task) def adjust_timings(self, requests): """Adjust the request timings to start at zero for the earliest timestamp""" timestamps = [ 'dns_start', 'dns_end', 'connect_start', 'connect_end', 'ssl_start', 'ssl_end', 'start', 'first_byte', 'end' ] earliest = None for request in requests: for entry in timestamps: if entry in request and request[entry] >= 0: if earliest is None or request[entry] < earliest: earliest = request[entry] logging.debug("Adjusting request timings by %0.3f seconds", earliest) if earliest is not None and earliest > 0: self.start_offset = earliest for request in requests: for entry in timestamps: if entry in request and request[entry] >= 0: request[entry] -= earliest def wait_for_processing(self, task): """Wait for any background processing threads to finish""" DesktopBrowser.wait_for_processing(self, task) def process_command(self, command): """Process an individual script command""" logging.debug("Processing script command:") logging.debug(command) if command['command'] == 'navigate': self.task['page_data']['URL'] = command['target'] url = str(command['target']).replace('"', '\"') script = 'window.location="{0}";'.format(url) script = self.prepare_script_for_record(script) self.marionette.execute_script(script) elif command['command'] == 'logdata': self.task['combine_steps'] = False if int(re.search(r'\d+', str(command['target'])).group()): logging.debug("Data logging enabled") self.task['log_data'] = True else: logging.debug("Data logging disabled") self.task['log_data'] = False elif command['command'] == 'combinesteps': self.task['log_data'] = True self.task['combine_steps'] = True elif command['command'] == 'seteventname': self.event_name = command['target'] elif command['command'] == 'exec': script = command['target'] if command['record']: script = self.prepare_script_for_record(script) self.marionette.execute_script(script) elif command['command'] == 'sleep': delay = min( 60, max(0, int(re.search(r'\d+', str(command['target'])).group()))) if delay > 0: time.sleep(delay) elif command['command'] == 'setabm': self.task['stop_at_onload'] = \ bool('target' in command and int(re.search(r'\d+', str(command['target'])).group()) == 0) elif command['command'] == 'setactivitytimeout': if 'target' in command: milliseconds = int( re.search(r'\d+', str(command['target'])).group()) self.task['activity_time'] = max( 0, min(30, float(milliseconds) / 1000.0)) elif command['command'] == 'setuseragent': self.task['user_agent_string'] = command['target'] elif command['command'] == 'firefoxpref': if 'target' in command and 'value' in command: self.set_pref(command['target'], command['value']) def navigate(self, url): """Navigate to the given URL""" if self.marionette is not None: try: self.marionette.navigate(url) except Exception as err: logging.debug("Error navigating Firefox: %s", str(err)) def set_pref(self, key, value_str): """Set an individual pref value""" value = self.get_pref_value(value_str.strip()) if value is not None: try: logging.debug('Setting Pref "%s" to %s', key, value_str) self.marionette.set_pref(key, value) except Exception: pass def grab_screenshot(self, path, png=True, resize=0): """Save the screen shot (png or jpeg)""" if self.marionette is not None: try: data = self.marionette.screenshot(format='binary', full=False) if data is not None: resize_string = '' if not resize else '-resize {0:d}x{0:d} '.format( resize) if png: with open(path, 'wb') as image_file: image_file.write(data) if len(resize_string): cmd = '{0} -format png -define png:color-type=2 '\ '-depth 8 {1}"{2}"'.format(self.job['image_magick']['mogrify'], resize_string, path) logging.debug(cmd) subprocess.call(cmd, shell=True) else: tmp_file = path + '.png' with open(tmp_file, 'wb') as image_file: image_file.write(data) command = '{0} "{1}" {2}-quality {3:d} "{4}"'.format( self.job['image_magick']['convert'], tmp_file, resize_string, self.job['imageQuality'], path) logging.debug(command) subprocess.call(command, shell=True) if os.path.isfile(tmp_file): try: os.remove(tmp_file) except Exception: pass except Exception as err: logging.debug('Exception grabbing screen shot: %s', str(err)) def process_requests(self, request_timings, task): """Convert all of the request and page events into the format needed for WPT""" result = {} result['requests'] = self.merge_requests(request_timings) result['pageData'] = self.calculate_page_stats(result['requests']) devtools_file = os.path.join( task['dir'], task['prefix'] + '_devtools_requests.json.gz') with gzip.open(devtools_file, 'wb', 7) as f_out: json.dump(result, f_out) def get_empty_request(self, request_id, url): """Return and empty, initialized request""" parts = urlparse.urlsplit(url) request = { 'type': 3, 'id': request_id, 'request_id': request_id, 'ip_addr': '', 'full_url': url, 'is_secure': 1 if parts.scheme == 'https' else 0, 'method': '', 'host': parts.netloc, 'url': parts.path, 'responseCode': -1, 'load_start': -1, 'load_ms': -1, 'ttfb_ms': -1, 'dns_start': -1, 'dns_end': -1, 'dns_ms': -1, 'connect_start': -1, 'connect_end': -1, 'connect_ms': -1, 'ssl_start': -1, 'ssl_end': -1, 'ssl_ms': -1, 'bytesIn': 0, 'bytesOut': 0, 'objectSize': 0, 'initiator': '', 'initiator_line': '', 'initiator_column': '', 'server_rtt': None, 'headers': { 'request': [], 'response': [] }, 'score_cache': -1, 'score_cdn': -1, 'score_gzip': -1, 'score_cookies': -1, 'score_keep-alive': -1, 'score_minify': -1, 'score_combine': -1, 'score_compress': -1, 'score_etags': -1, 'gzip_total': None, 'gzip_save': None, 'minify_total': None, 'minify_save': None, 'image_total': None, 'image_save': None, 'cache_time': None, 'cdn_provider': None, 'server_count': None, 'socket': -1 } if len(parts.query): request['url'] += '?' + parts.query return request def get_header_value(self, headers, name): """Return the value for the given header""" value = '' name = name.lower() for header in headers: pos = header.find(':') if pos > 0: key = header[0:pos].lower() if key.startswith(name): val = header[pos + 1:].strip() if len(value): value += '; ' value += val return value def merge_requests(self, request_timings): """Merge the requests from the extension and log files""" requests = [] # Start with the requests reported from the extension for req_id in self.requests: try: req = self.requests[req_id] if req['from_net'] and 'start' in req and 'url' in req: request = self.get_empty_request(req['id'], req['url']) if 'ip' in req: request['ip_addr'] = req['ip'] if 'method' in req: request['method'] = req['method'] if 'status' in req: request['responseCode'] = req['status'] if 'type' in req: request['requestType'] = req['type'] if 'request_headers' in req: for header in req['request_headers']: if 'name' in header and 'value' in header: header_text = '{0}: {1}'.format( header['name'], header['value']) request['bytesOut'] += len(header_text) + 2 request['headers']['request'].append( header_text) if 'status_line' in req: request['bytesIn'] += len(req['status_line']) + 2 request['headers']['response'].append( req['status_line']) if 'response_headers' in req: for header in req['response_headers']: if 'name' in header and 'value' in header: try: header_text = '{0}: {1}'.format( header['name'], header['value']) request['bytesIn'] += len(header_text) + 2 request['headers']['response'].append( header_text) except Exception: pass if 'created' in req: request['created'] = req['created'] request['load_start'] = int(round(req['start'] * 1000.0)) if 'first_byte' in req: ttfb = int( round((req['first_byte'] - req['start']) * 1000.0)) request['ttfb_ms'] = max(0, ttfb) if 'end' in req: load_time = int( round((req['end'] - req['start']) * 1000.0)) request['load_ms'] = max(0, load_time) size = self.get_header_value( request['headers']['response'], 'Content-Length') if len(size): request['bytesIn'] += int( re.search(r'\d+', str(size)).group()) requests.append(request) except Exception: pass # Overwrite them with the same requests from the logs for request in requests: for req in request_timings: try: if 'claimed' not in req and 'url' in req and 'full_url' in request \ and 'start' in req and request['full_url'] == req['url']: req['claimed'] = True self.populate_request(request, req) except Exception: pass # Add any events from the logs that weren't reported by the extension for req in request_timings: try: if 'claimed' not in req and 'url' in req and 'start' in req: request = self.get_empty_request(req['id'], req['url']) self.populate_request(request, req) requests.append(request) except Exception: pass # parse values out of the headers for request in requests: try: value = self.get_header_value(request['headers']['response'], 'Expires') if value: request['expires'] = value value = self.get_header_value(request['headers']['response'], 'Cache-Control') if value: request['cacheControl'] = value value = self.get_header_value(request['headers']['response'], 'Content-Type') if value: request['contentType'] = value value = self.get_header_value(request['headers']['response'], 'Content-Encoding') if value: request['contentEncoding'] = value value = self.get_header_value(request['headers']['response'], 'Content-Length') if value: request['objectSize'] = value except Exception: pass requests.sort(key=lambda x: x['load_start']) return requests def populate_request(self, request, log_request): """Populate a request object from the log request values""" request['load_start'] = int(log_request['start'] * 1000) if 'status' in log_request: request['responseCode'] = log_request['status'] if 'dns_start' in log_request and log_request['dns_start'] >= 0: request['dns_start'] = int(log_request['dns_start'] * 1000) if 'dns_end' in log_request and log_request['dns_end'] >= 0: request['dns_end'] = int(round(log_request['dns_end'] * 1000.0)) if 'connect_start' in log_request and log_request['connect_start'] >= 0: request['connect_start'] = int(log_request['connect_start'] * 1000) if 'connect_end' in log_request and log_request['connect_end'] >= 0: request['connect_end'] = int( round(log_request['connect_end'] * 1000.0)) if 'ssl_start' in log_request and log_request['ssl_start'] >= 0: request['ssl_start'] = int(log_request['ssl_start'] * 1000) if 'ssl_end' in log_request and log_request['ssl_end'] >= 0: request['ssl_end'] = int(round(log_request['ssl_end'] * 1000.0)) if 'connection' in log_request: request['socket'] = log_request['connection'] request['load_start'] = int(round(log_request['start'] * 1000.0)) if 'first_byte' in log_request: request['ttfb_ms'] = int(round((log_request['first_byte'] - \ log_request['start']) * 1000.0)) if 'end' in log_request: request['load_ms'] = int(round((log_request['end'] - \ log_request['start']) * 1000.0)) if 'bytes_in' in log_request: request['bytesIn'] = log_request['bytes_in'] if 'request_headers' in log_request: request['headers']['request'] = list( log_request['request_headers']) if 'response_headers' in log_request: request['headers']['response'] = list( log_request['response_headers']) def calculate_page_stats(self, requests): """Calculate the page-level stats""" page = {'loadTime': 0, 'docTime': 0, 'fullyLoaded': 0, 'bytesOut': 0, 'bytesOutDoc': 0, 'bytesIn': 0, 'bytesInDoc': 0, 'requests': len(requests), 'requestsDoc': 0, 'responses_200': 0, 'responses_404': 0, 'responses_other': 0, 'result': 0, 'testStartOffset': 0, 'cached': 1 if self.task['cached'] else 0, 'optimization_checked': 0, 'start_epoch': int((self.task['start_time'] - \ datetime.utcfromtimestamp(0)).total_seconds()) } if 'loaded' in self.page: page['loadTime'] = int(round(self.page['loaded'] * 1000.0)) page['docTime'] = page['loadTime'] page['loadEventStart'] = page['loadTime'] page['loadEventEnd'] = page['loadTime'] if 'DOMContentLoaded' in self.page: page['domContentLoadedEventStart'] = int( round(self.page['DOMContentLoaded'] * 1000.0)) page['domContentLoadedEventEnd'] = page[ 'domContentLoadedEventStart'] main_request = None index = 0 for request in requests: if request['load_ms'] >= 0: end_time = request['load_start'] + request['load_ms'] if end_time > page['fullyLoaded']: page['fullyLoaded'] = end_time if end_time <= page['loadTime']: page['requestsDoc'] += 1 page['bytesInDoc'] += request['bytesIn'] page['bytesOutDoc'] += request['bytesOut'] page['bytesIn'] += request['bytesIn'] page['bytesOut'] += request['bytesOut'] if request['responseCode'] == 200: page['responses_200'] += 1 elif request['responseCode'] == 404: page['responses_404'] += 1 page['result'] = 99999 elif request['responseCode'] > -1: page['responses_other'] += 1 if main_request is None and \ (request['responseCode'] == 200 or request['responseCode'] == 304): main_request = request['id'] request['is_base_page'] = True page['final_base_page_request'] = index page['final_base_page_request_id'] = main_request page['final_url'] = request['full_url'] if 'URL' not in self.task['page_data']: self.task['page_data']['URL'] = page['final_url'] if request['ttfb_ms'] >= 0: page['TTFB'] = request['load_start'] + request['ttfb_ms'] if request['ssl_end'] >= request['ssl_start'] and \ request['ssl_start'] >= 0: page['basePageSSLTime'] = int(round(request['ssl_end'] - \ request['ssl_start'])) if page['responses_200'] == 0 and len(requests): if 'responseCode' in requests[0]: page['result'] = requests[0]['responseCode'] else: page['result'] = 12999 self.task['page_result'] = page['result'] return page
#!/usr/bin/env python # -*- coding: utf-8 -*- from marionette_driver.marionette import Marionette import pdb client = Marionette("localhost", port=2828) client.start_session() client.navigate("http://www.tianyancha.com/company/75507246") element = client.find_element("class name", 'company_info') print element.text
class MarionetteHelper: def __init__(self, logger, success_symbol, failure_symbol): """ Initialise the helper class. """ self.client = None self.logger = logger self.success_symbol = success_symbol self.failure_symbol = failure_symbol def init_ff(self): """ Initialises the connection to Firefox and starts a session. @return: - """ if not check_socket(MARIONETTE_HOST, MARIONETTE_PORT): self.logger.error( u" > [ERROR] Please check if you started Firefox with the '-marionette' " "option or set 'marionette.enabled' to 'true' in 'about:config'. {}" .format(self.failure_symbol)) sys.exit(1) self.client = Marionette(host=MARIONETTE_HOST, port=MARIONETTE_PORT) self.client.start_session() def get_existing_bookmarks(self): """ Get the existing bookmarks from the Google Bookmarks API. We need to do this in Firefox to have the cookie set which authorities us with the API. @return: - """ self.client.navigate( "https://www.google.com/bookmarks/?output=xml&num=10000") # Initialise XML object root = XML(self.client.page_source.encode("utf-8")) # Return set of bookmarks return set([bookmark[1].text for bookmark in root[0]]) def save_button_contains_correct_text_save(self, *args): """ Helper method for Marionette, here: check if fav button contains text "SAVE" @return: Whether or not the fav button contains the text "SAVE" """ save_button = self.client.find_element( By.CLASS_NAME, "section-entity-action-save-button") return save_button.text == "SAVE" def save_button_contains_correct_text_saved(self, *args): """ Helper method for Marionette, here: check if fav button contains text "SAVED" @return: Whether or not the fav button contains the text "SAVED" """ save_button = self.client.find_element( By.CLASS_NAME, "section-entity-action-save-button") return save_button.text == "SAVED" def interactive_add_feature(self, coordinates): """ Navigates to the Google Maps URL for the provided coordinates and waits for input. @return: - """ url = "https://www.google.com/maps/search/?api=1&query={},{}" # This navigates Firefox to the passed URL self.client.navigate(url.format(coordinates[0], coordinates[1])) # Wait for input if sys.version_info[0] < 3: raw_input("Press Enter to continue...") else: input("Press Enter to continue...") def add_feature_2(self, url, list_add): """ Tries to add a feature (bookmark / place) to your Google Maps fav list. @return: - """ self.client.navigate(url) try: saved_button = Wait(self.client, timeout=1).until( expected.element_present(By.CSS_SELECTOR, "[data-value='Saved']")) self.logger.info(" > Feature was already saved") return utils.constants.ADD_FEATURE_ALREADY_ADDED except TimeoutException: pass try: save_button = Wait(self.client, timeout=5).until( expected.element_present(By.CSS_SELECTOR, "[data-value='Save']")) Wait(self.client, timeout=5).until(expected.element_displayed(save_button)) except TimeoutException: self.logger.error(" > Unable to find save button") return utils.constants.ADD_FEATURE_UNKNOWN_ERROR save_button.click() if list_add == utils.constants.LIST_STARRED_PLACES: data_index = 2 elif list_add == utils.constants.LIST_WANT_TO_GO: data_index = 1 else: data_index = -1 css_selector = "#action-menu [data-index='{}']".format(data_index) sub_save_item = Wait(self.client, timeout=5).until( expected.element_present(By.CSS_SELECTOR, css_selector)) Wait(self.client, timeout=5).until(expected.element_displayed(sub_save_item)) sub_save_item.click() def add_feature(self, url): """ Tries to add a feature (bookmark / place) to your Google Maps fav list. @return: - ADD_FEATURE_FAILURE if adding resulted in a known failure - ADD_FEATURE_SUCCESS if everything went fine - ADD_FEATURE_UNKNOWN_ERROR if we don't know what happened """ # This navigates Firefox to the passed URL self.client.navigate(url) # We wait for the fav button to be present... save_button = Wait(self.client, timeout=10).until( expected.element_present(By.CLASS_NAME, "section-entity-action-save-button")) # ... and to be displayed displayed = Wait(self.client, timeout=10).until( expected.element_displayed(save_button)) try: # Now we look for the correct text, it should say "SAVE" Wait(self.client, timeout=6).until(self.save_button_contains_correct_text_save) try: # Click it to add the feature (bookmark / place) to the Google Maps fav list save_button.click() except NoSuchElementException: pass try: # Now the text should be "SAVED" and this indicates it was saved Wait(self.client, timeout=6).until( self.save_button_contains_correct_text_saved) except TimeoutException: # We clicked but the fav button text didn't change, i.e. the click went wrong or timed out self.logger.error(" > [ERROR] Feature: '{}'".format(url)) save_button = self.client.find_element( By.CLASS_NAME, "section-entity-action-save-button") self.logger.error( " > [ERROR] Save button didn't switch to 'SAVED', it contains '{}'" .format(save_button.text)) return ADD_FEATURE_FAILURE return ADD_FEATURE_SUCCESS except TimeoutException: # This is the case if the fave button didn't contain the text "SAVE". # This can happen if it contains "SAVED", but this shouldn't happen in the # first place because we don't try to add features if we know that they're # already added. # So most likely something truly went wrong here. self.logger.error(" > [ERROR] Feature: '{}'".format(url)) save_button = self.client.find_element( By.CLASS_NAME, "section-entity-action-save-button") self.logger.error( " > [ERROR] Save button contained unknown text '{}'".format( save_button.text)) return ADD_FEATURE_UNKNOWN_ERROR
'}') # The location of the Firefox binary, will depend on system. # Be careful to use the actual binary and not a wrapper script. binary = '/usr/lib/firefox/firefox' # Loop through the four different configurations. for mode in ['headless', 'graphical']: # Set up the client with the appropriate settings. if mode == 'headless': os.environ['MOZ_HEADLESS'] = '1' else: os.environ.pop('MOZ_HEADLESS', None) client = Marionette('localhost', bin=binary, port=2828) client.start_session() # Navigate to the test page and inject the JavaScript. client.navigate( 'https://intoli.com/blog/javascript-injection/test-page.html') client.execute_async_script(injected_javascript) # Save the results as an image. filename = os.path.join( 'img', 'marionette-execute-async-scripy-firefox-%s-results.png' % mode) with open(filename, 'wb') as f: f.write(client.screenshot(format='binary')) print 'Saved "%s".' % filename # Cleanup the client before the next test. client.cleanup()
parser.add_argument('--search', dest='search', help='The term with problems', required=True, type=str) parser.add_argument('--remove', dest='remove', help='The wrong translation to remove', required=True, type=str) parser.add_argument('--replace', dest='replace', help='The new translation to submit', required=False, type=str) parser.add_argument('--lang', dest='lang', help='The locale, eg: it', default="it") args = parser.parse_args() # Load configuration config = ConfigParser.RawConfigParser() config.readfp(open('config.ini')) print "Connection in progress to Firefox" client = Marionette(host='127.0.0.1', port=28288) client.start_session() print "Connection to Firefox Done" # Detect if already logged try: client.find_element(By.CSS_SELECTOR, 'body.logged-in') client.navigate("https://translate.wordpress.org/wp-login.php?action=logout&redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F&_wpnonce=583839252e") except: pass # Login client.navigate("https://login.wordpress.org/?redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F") try: #Log In Form usernameLogin = client.find_element(By.ID, 'user_login') usernameLogin.click() usernameLogin.send_keys(config.get('Login', 'user')) passwordLogin = client.find_element(By.ID, 'user_pass') passwordLogin.click() passwordLogin.send_keys(config.get('Login', 'pass')) # Click on the Log in button to connect client.find_element(By.ID, 'wp-submit').click() time.sleep(1)
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDevice, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = '10.0.2.2' PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } dirs = self.query_abs_dirs() topsrcdir = os.path.join(dirs['abs_work_dir'], 'src') adb = self.query_exe('adb') path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) # Enforce 1proc. This isn't in one of the user.js files because those # are shared with desktop, which wants e10s. We can't interpolate # because the formatting code only works for strings, and this is a # bool pref. prefs["browser.tabs.remote.autostart"] = False outputdir = self.config.get('output_directory', '/sdcard/pgo_profile') jarlog = posixpath.join(outputdir, 'en-US.log') profdata = posixpath.join(outputdir, 'default_%p_random_%m.profraw') env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata adbdevice = ADBDevice(adb=adb, device='emulator-5554') adbdevice.mkdir(outputdir) try: # Run Fennec a first time to initialize its profile driver = Marionette( app='fennec', package_name=app, adb_path=adb, bin="target.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if 'Speedometer/index.html' in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): if not adbdevice.process_exist(app): break time.sleep(2) else: raise Exception("Android App (%s) never quit" % app) # Pull all the profraw files and en-US.log adbdevice.pull(outputdir, '/builds/worker/workspace/') except ADBTimeoutError: self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError', EXIT_STATUS_DICT[TBPL_RETRY]) profraw_files = glob.glob('/builds/worker/workspace/*.profraw') if not profraw_files: self.fatal( 'Could not find any profraw files in /builds/worker/workspace') merge_cmd = [ os.path.join(os.environ['MOZ_FETCHES_DIR'], 'clang/bin/llvm-profdata'), 'merge', '-o', '/builds/worker/workspace/merged.profdata', ] + profraw_files rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( 'INFRA-ERROR: Failed to merge profile data. Corrupt profile?', EXIT_STATUS_DICT[TBPL_RETRY]) # tarfile doesn't support xz in this version of Python tar_cmd = [ 'tar', '-acvf', '/builds/worker/artifacts/profdata.tar.xz', '-C', '/builds/worker/workspace', 'merged.profdata', 'en-US.log', ] subprocess.check_call(tar_cmd) httpd.stop()
parser.add_argument('--search', dest='search', help='The term with problems', required=True, type=str) parser.add_argument('--remove', dest='remove', help='The wrong translation to remove', required=True, type=str) parser.add_argument('--replace', dest='replace', help='The new translation to submit', required=False, type=str) parser.add_argument('--lang', dest='lang', help='The locale, eg: it', default="it") args = parser.parse_args() # Load configuration config = ConfigParser.RawConfigParser() config.readfp(open('config.ini')) print "Connection in progress to Firefox" client = Marionette(host='127.0.0.1', port=28288) client.start_session() print "Connection to Firefox Done" # Detect if already logged try: client.find_element(By.CSS_SELECTOR, 'body.logged-in') client.navigate("https://translate.wordpress.org/wp-login.php?action=logout&redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F&_wpnonce=583839252e") except: pass # Login client.navigate("https://login.wordpress.org/?redirect_to=https%3A%2F%2Ftranslate.wordpress.org%2F") try: #Log In Form usernameLogin = client.find_element(By.ID, 'user_login') usernameLogin.click() usernameLogin.send_keys(config.get('Login', 'user')) passwordLogin = client.find_element(By.ID, 'user_pass') passwordLogin.click() passwordLogin.send_keys(config.get('Login', 'pass')) # Click on the Log in button to connect client.find_element(By.ID, 'wp-submit').click() time.sleep(1)
def run_tests(self): """ Generate the PGO profile data """ from mozhttpd import MozHttpd from mozprofile import Preferences from mozdevice import ADBDevice, ADBProcessError, ADBTimeoutError from six import string_types from marionette_driver.marionette import Marionette app = self.query_package_name() IP = '10.0.2.2' PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } dirs = self.query_abs_dirs() topsrcdir = os.path.join(dirs['abs_work_dir'], 'src') adb = self.query_exe('adb') path_mappings = { k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) profile_data_dir = os.path.join(topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['profileserver'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, "OOP": "false" } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) outputdir = self.config.get('output_directory', '/sdcard') jarlog = posixpath.join(outputdir, 'en-US.log') profdata = posixpath.join(outputdir, 'default.profraw') env = {} env["XPCOM_DEBUG_BREAK"] = "warn" env["MOZ_IN_AUTOMATION"] = "1" env["MOZ_JAR_LOG_FILE"] = jarlog env["LLVM_PROFILE_FILE"] = profdata adbdevice = ADBDevice(adb=adb, device='emulator-5554') try: # Run Fennec a first time to initialize its profile driver = Marionette( app='fennec', package_name=app, adb_path=adb, bin="target.apk", prefs=prefs, connect_to_running_emulator=True, startup_timeout=1000, env=env, ) driver.start_session() # Now generate the profile and wait for it to complete for page in PAGES: driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) timeout = 2 if 'Speedometer/index.html' in page: # The Speedometer test actually runs many tests internally in # javascript, so it needs extra time to run through them. The # emulator doesn't get very far through the whole suite, but # this extra time at least lets some of them process. timeout = 360 time.sleep(timeout) driver.set_context("chrome") driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); let cancelQuit = Components.classes["@mozilla.org/supports-PRBool;1"] .createInstance(Components.interfaces.nsISupportsPRBool); Services.obs.notifyObservers(cancelQuit, "quit-application-requested", null); return cancelQuit.data; """) driver.execute_script(""" Components.utils.import("resource://gre/modules/Services.jsm"); Services.startup.quit(Ci.nsIAppStartup.eAttemptQuit) """) # There is a delay between execute_script() returning and the profile data # actually getting written out, so poll the device until we get a profile. for i in range(50): try: localprof = '/builds/worker/workspace/default.profraw' adbdevice.pull(profdata, localprof) stats = os.stat(localprof) if stats.st_size == 0: # The file may not have been fully written yet, so retry until we # get actual data. time.sleep(2) else: break except ADBProcessError: # The file may not exist at all yet, which would raise an # ADBProcessError, so retry. time.sleep(2) else: raise Exception("Unable to pull default.profraw") adbdevice.pull(jarlog, '/builds/worker/workspace/en-US.log') except ADBTimeoutError: self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError', EXIT_STATUS_DICT[TBPL_RETRY]) # We normally merge as part of a GENERATED_FILES step in the profile-use # build, but Android runs sometimes result in a truncated profile. We do # a merge here to make sure the data isn't corrupt so we can retry the # 'run' task if necessary. merge_cmd = [ '/builds/worker/workspace/build/clang/bin/llvm-profdata', 'merge', '/builds/worker/workspace/default.profraw', '-o', '/builds/worker/workspace/merged.profraw', ] rc = subprocess.call(merge_cmd) if rc != 0: self.fatal( 'INFRA-ERROR: Failed to merge profile data. Corrupt profile?', EXIT_STATUS_DICT[TBPL_RETRY]) # tarfile doesn't support xz in this version of Python tar_cmd = [ 'tar', '-acvf', '/builds/worker/artifacts/profdata.tar.xz', '-C', '/builds/worker/workspace', 'merged.profraw', 'en-US.log', ] subprocess.check_call(tar_cmd) httpd.stop()
try: hostname = URL_TO_FETCH.split('/')[2] except IndexError: hostname = URL_TO_FETCH URL_TO_FETCH = "http://" + hostname print("Run " + str(run) + "/" + str(TIMES) + " - Fetching " + URL_TO_FETCH + " at " + timestamp) time.sleep(1) event = None messages = None try: time.sleep(1) client.navigate(URL_TO_FETCH) except Exception as e: print("Error fetching page " + URL_TO_FETCH + ": " + str(e) + "\n") try: logNavigationTimings(client, URL_TO_FETCH, timestamp, unixtimestamp, LOGDIR + "failed_navtimings.log", scenario=SCENARIO) except Exception as e: print("Could not even log Nav timings for failed page " + str(URL_TO_FETCH) + ": " + str(e)) sys.exit(-1) createDirectory(LOGDIR + "res")
class Firefox(DesktopBrowser): """Firefox""" def __init__(self, path, options, job): DesktopBrowser.__init__(self, path, options, job) self.job = job self.task = None self.options = options self.path = path self.event_name = None self.moz_log = None self.marionette = None def prepare(self, job, task): """Prepare the profile/OS for the browser""" self.moz_log = os.path.join(task['dir'], task['prefix']) + '_moz.log' os.environ["MOZ_LOG_FILE"] = self.moz_log os.environ["MOZ_LOG"] = 'timestamp,sync,nsHttp:5,nsSocketTransport:5,nsStreamPump:5,'\ 'nsHostResolver:5,pipnss:5' DesktopBrowser.prepare(self, job, task) profile_template = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'profiles', 'Firefox') if not task['cached'] and os.path.isdir(profile_template): try: if os.path.isdir(task['profile']): shutil.rmtree(task['profile']) shutil.copytree(profile_template, task['profile']) except Exception: pass def launch(self, _job, task): """Launch the browser""" from marionette_driver.marionette import Marionette args = [ '-profile', '"{0}"'.format(task['profile']), '-no-remote', '-width', str(task['width']), '-height', str(task['height']), '-marionette', 'about:blank' ] if self.path.find(' ') > -1: command_line = '"{0}"'.format(self.path) else: command_line = self.path command_line += ' ' + ' '.join(args) DesktopBrowser.launch_browser(self, command_line) self.marionette = Marionette('localhost', port=2828) self.marionette.start_session(timeout=60) self.marionette.navigate(START_PAGE) def stop(self, job, task): if self.marionette is not None: self.marionette.close() DesktopBrowser.stop(self, job, task) os.environ["MOZ_LOG_FILE"] = '' os.environ["MOZ_LOG"] = '' def run_task(self, task): """Run an individual test""" if self.marionette is not None: self.task = task logging.debug("Running test") end_time = monotonic.monotonic() + task['time_limit'] task['current_step'] = 1 recording = False while len(task['script']) and task['error'] is None and \ monotonic.monotonic() < end_time: self.prepare_task(task) command = task['script'].pop(0) if not recording and command['record']: recording = True self.on_start_recording(task) self.process_command(command) if command['record']: self.wait_for_page_load() if not task['combine_steps'] or not len(task['script']): self.on_stop_recording(task) recording = False self.on_start_processing(task) self.wait_for_processing(task) if task['log_data']: # Move on to the next step task['current_step'] += 1 self.event_name = None # Always navigate to about:blank after finishing in case the tab is # remembered across sessions self.marionette.navigate('about:blank') self.task = None def wait_for_page_load(self): """Wait for the onload event from the extension""" pass def prepare_task(self, task): """Format the file prefixes for multi-step testing""" task['page_data'] = {} if task['current_step'] == 1: task['prefix'] = task['task_prefix'] task['video_subdirectory'] = task['task_video_prefix'] else: task['prefix'] = '{0}_{1:d}'.format(task['task_prefix'], task['current_step']) task['video_subdirectory'] = '{0}_{1:d}'.format( task['task_video_prefix'], task['current_step']) if task['video_subdirectory'] not in task['video_directories']: task['video_directories'].append(task['video_subdirectory']) if self.event_name is not None: task['step_name'] = self.event_name else: task['step_name'] = 'Step_{0:d}'.format(task['current_step']) def on_start_recording(self, task): """Notification that we are about to start an operation that needs to be recorded""" DesktopBrowser.on_start_recording(self, task) def on_stop_recording(self, task): """Notification that we are done with recording""" DesktopBrowser.on_stop_recording(self, task) def on_start_processing(self, task): """Start any processing of the captured data""" DesktopBrowser.on_start_processing(self, task) def wait_for_processing(self, task): """Wait for any background processing threads to finish""" DesktopBrowser.wait_for_processing(self, task) def process_command(self, command): """Process an individual script command""" logging.debug("Processing script command:") logging.debug(command) if command['command'] == 'navigate': self.marionette.navigate(command['target']) elif command['command'] == 'logdata': self.task['combine_steps'] = False if int(re.search(r'\d+', str(command['target'])).group()): logging.debug("Data logging enabled") self.task['log_data'] = True else: logging.debug("Data logging disabled") self.task['log_data'] = False elif command['command'] == 'combinesteps': self.task['log_data'] = True self.task['combine_steps'] = True elif command['command'] == 'seteventname': self.event_name = command['target'] elif command['command'] == 'exec': self.marionette.execute_js_script(command['target']) elif command['command'] == 'sleep': delay = min( 60, max(0, int(re.search(r'\d+', str(command['target'])).group()))) if delay > 0: time.sleep(delay) elif command['command'] == 'setabm': self.task['stop_at_onload'] = \ bool('target' in command and int(re.search(r'\d+', str(command['target'])).group()) == 0) elif command['command'] == 'setactivitytimeout': if 'target' in command: self.task['activity_time'] = \ max(0, min(30, int(re.search(r'\d+', str(command['target'])).group()))) elif command['command'] == 'setuseragent': self.task['user_agent_string'] = command['target'] elif command['command'] == 'setcookie': if 'target' in command and 'value' in command: url = command['target'].strip() cookie = command['value'] pos = cookie.find(';') if pos > 0: cookie = cookie[:pos] pos = cookie.find('=') if pos > 0: name = cookie[:pos].strip() value = cookie[pos + 1:].strip() if len(name) and len(value) and len(url): self.marionette.add_cookie({ 'url': url, 'name': name, 'value': value }) def navigate(self, url): """Navigate to the given URL""" if self.marionette is not None: self.marionette.navigate(url)