def test_to_capabilities(self): opts = Options() assert opts.to_capabilities() == {} profile = FirefoxProfile() opts.profile = profile caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "profile" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["profile"], basestring) assert caps["moz:firefoxOptions"]["profile"] == profile.encoded opts.add_argument("--foo") caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "args" in caps["moz:firefoxOptions"] assert caps["moz:firefoxOptions"]["args"] == ["--foo"] binary = FirefoxBinary() opts.binary = binary caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "binary" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["binary"], basestring) assert caps["moz:firefoxOptions"]["binary"] == binary._start_cmd opts.set_preference("spam", "ham") caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "prefs" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["prefs"], dict) assert caps["moz:firefoxOptions"]["prefs"]["spam"] == "ham"
def test_to_capabilities(self): opts = Options() assert opts.to_capabilities() == {} profile = FirefoxProfile() opts.profile = profile caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "profile" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["profile"], types.StringTypes) assert caps["moz:firefoxOptions"]["profile"] == profile.encoded opts.add_argument("--foo") caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "args" in caps["moz:firefoxOptions"] assert caps["moz:firefoxOptions"]["args"] == ["--foo"] binary = FirefoxBinary() opts.binary = binary caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "binary" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["binary"], types.StringTypes) assert caps["moz:firefoxOptions"]["binary"] == binary._start_cmd
def test_to_capabilities(self): opts = Options() firefox_caps = DesiredCapabilities.FIREFOX.copy() firefox_caps.update({"pageLoadStrategy": "normal"}) assert opts.to_capabilities() == firefox_caps profile = FirefoxProfile() opts.profile = profile caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "profile" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["profile"], str) assert caps["moz:firefoxOptions"]["profile"] == profile.encoded opts.add_argument("--foo") caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "args" in caps["moz:firefoxOptions"] assert caps["moz:firefoxOptions"]["args"] == ["--foo"] binary = FirefoxBinary() opts.binary = binary caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "binary" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["binary"], str) assert caps["moz:firefoxOptions"]["binary"] == binary._start_cmd opts.set_preference("spam", "ham") caps = opts.to_capabilities() assert "moz:firefoxOptions" in caps assert "prefs" in caps["moz:firefoxOptions"] assert isinstance(caps["moz:firefoxOptions"]["prefs"], dict) assert caps["moz:firefoxOptions"]["prefs"]["spam"] == "ham"
def _update_capabilities_with_options(self): new_ff_options = Options() if not self._options: logger.debug( "No options specified, updating capabilities with default firefox settings") else: if "mobileEmulation" in self._options: logger.warning("mobileEmulation is only available for Chrome") if "headless" in self._options and self._options["headless"]: new_ff_options.headless = True if not self._capabilities: logger.debug( "No capabilities specified, creating default firefox capability..") self._capabilities = DesiredCapabilities.FIREFOX.copy() if "marionette" not in self._capabilities: self._capabilities["marionette"] = True new_ff_cap = new_ff_options.to_capabilities() if "moz:firefoxOptions" in self._capabilities and "moz:firefoxOptions" in new_ff_cap: for key, value in new_ff_cap["moz:firefoxOptions"].items(): if not self._capabilities["moz:firefoxOptions"][key] == new_ff_cap["moz:firefoxOptions"][key]: logger.debug( "Updating original capabilities moz:firefoxOptions..") self._capabilities["moz:firefoxOptions"].setdefault( key, []).extend(value) elif "moz:firefoxOptions" not in self._capabilities and "moz:firefoxOptions" in new_ff_cap: logger.debug( "No custom moz:firefoxOptions specified in capabilities, setting default..") self._capabilities["moz:firefoxOptions"] = new_ff_cap["moz:firefoxOptions"]
def test_selenium_grid_headless_firefox(self): ''' selenium ui test with headless mode by grid, and vnc record is disabled (firefox). pre-condition: selenium grid is running. check selenium hub: curl "http://localhost:4444/wd/hub/status" | jq . ''' br_options = FirefoxOptions() br_options.headless = True caps = DesiredCapabilities.FIREFOX caps['platform'] = 'ANY' caps.update(br_options.to_capabilities()) hub_url = 'http://localhost:4444/wd/hub' browser = webdriver.Remote( command_executor=hub_url, desired_capabilities=caps) browser.implicitly_wait(8) b_caps = browser.capabilities print('\nbrowser: %s, version: %s' % (b_caps.get('browserName', 'unknown'), b_caps.get('version', 'unknown'))) try: self.ms_bing_page.open_steps(self, browser) self.utils.save_screenshot(browser, '/tmp/uitest_bing_home_02.png') self.ms_bing_page.search_steps(self, browser) self.utils.save_screenshot( browser, '/tmp/uitest_bing_search_02.png') finally: browser.quit()
def new(cache): global driver options = Options() options.add_argument("--headless") if cache["profile"] != None: options.add_argument(f'user-data-dir={cache["profile"]}') capabilities = options.to_capabilities() driver = webdriver.Firefox(desired_capabilities=capabilities) if cache["profile"] == None: getProfile(cache)
def __init__( self, headless: bool = False, whole_genome_only: bool = True, destination: str = "fastas", ): self.whole_genome_only = whole_genome_only self.destination = destination self.finished = False self.already_downloaded = 0 self.samples_count = None self.new_downloaded = 0 options = Options() options.headless = headless if headless: options.add_argument("--headless") options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') for i in range(30): time.sleep(1) try: self.driver = webdriver.Remote( command_executor="http://selenium:4444/wd/hub", desired_capabilities=options.to_capabilities()) break except MaxRetryError: pass else: self.driver = webdriver.Firefox(options=options) self.driver.implicitly_wait(1000) self.driver.set_window_size(1366, 2000) if not os.path.exists(destination): os.makedirs(destination) self._update_cache() if os.path.isfile(destination + "/metadata.tsv"): self.metadata_handle = open(destination + "/metadata.tsv", "a", encoding='utf-8') else: self.metadata_handle = open(destination + "/metadata.tsv", "w", encoding='utf-8') self.metadata_handle.write("\t".join(METADATA_COLUMNS) + "\n")
def scrape(): problems = [] link = "http://acm-uci.org/Puzzle/" options = Options() options.add_argument("--headless") capabilities = options.to_capabilities() driver = webdriver.Firefox(desired_capabilities=capabilities) driver.get(link) time.sleep(4) buttons = driver.find_elements_by_tag_name('button') for b in buttons: if b.text == "Week 9": b.click() time.sleep(1) links = driver.find_elements_by_tag_name('a') for m in links: l = m.get_attribute("href") if (l and not l.startswith("http://acm-uci.org/") and l != "https://github.com/ACM-UCI/ACM-UCI-Website/issues"): problems.append(l) print("-- Done --") return problems
def get_used_books_from_kleinanzeigen(book): url = "https://www.ebay-kleinanzeigen.de/s-buecher-zeitschriften/c76" options = Options() options.headless = True options.set_capability("javascriptEnabled", True) SELENIUM_HOST = os.getenv("SELENIUM_HOST") SELENIUM_PORT = os.getenv("SELENIUM_PORT") command_executor = f"http://{SELENIUM_HOST}:{SELENIUM_PORT}/wd/hub" driver = webdriver.Remote(command_executor=command_executor, desired_capabilities=options.to_capabilities()) driver.get(url) # books gdpr = WebDriverWait(driver, 10).until( EC.element_to_be_clickable((By.ID, "gdpr-banner-accept"))) gdpr.click() search = WebDriverWait(driver, 10).until( EC.presence_of_element_located((By.ID, "site-search-query"))) try: search.send_keys(book.title) except TypeError as e: raise TypeError(f"TE: {book.title} / {e}") search.send_keys(Keys.RETURN) time.sleep(5) results = driver.find_elements_by_id("srchrslt-adtable") if not results: return [] result = results[0] assert result.tag_name == "ul" return [ single_searchresult_to_bookoffer(item, book.isbn) for item in result.find_elements_by_class_name("lazyload-item") ]
from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.firefox.options import Options options = Options() options.headless = True options.set_capability("javascriptEnabled", True) SELENIUM_HOST = os.getenv("SELENIUM_HOST") SELENIUM_PORT = os.getenv("SELENIUM_PORT") driver = webdriver.Remote( command_executor=f"http://{SELENIUM_HOST}:{SELENIUM_PORT}/wd/hub", desired_capabilities=options.to_capabilities()) wait = WebDriverWait(driver, 10) driver.get("https://www.ebay-kleinanzeigen.de/s-buecher-zeitschriften/c76") # books gdpr = wait.until(EC.element_to_be_clickable((By.ID, "gdpr-banner-accept"))) gdpr.click() search = wait.until(EC.presence_of_element_located((By.ID, "site-search-query"))) search.send_keys("Harry Potter") search.send_keys(Keys.RETURN) time.sleep(2)
def __init__(self, firefox_profile=None, firefox_binary=None, timeout=30, capabilities=None, proxy=None, executable_path="geckodriver", options=None, service_log_path="geckodriver.log", firefox_options=None, service_args=None, desired_capabilities=None, log_path=None, keep_alive=True): """Starts a new local session of Firefox. Based on the combination and specificity of the various keyword arguments, a capabilities dictionary will be constructed that is passed to the remote end. The keyword arguments given to this constructor are helpers to more easily allow Firefox WebDriver sessions to be customised with different options. They are mapped on to a capabilities dictionary that is passed on to the remote end. As some of the options, such as `firefox_profile` and `options.profile` are mutually exclusive, precedence is given from how specific the setting is. `capabilities` is the least specific keyword argument, followed by `options`, followed by `firefox_binary` and `firefox_profile`. In practice this means that if `firefox_profile` and `options.profile` are both set, the selected profile instance will always come from the most specific variable. In this case that would be `firefox_profile`. This will result in `options.profile` to be ignored because it is considered a less specific setting than the top-level `firefox_profile` keyword argument. Similarily, if you had specified a `capabilities["moz:firefoxOptions"]["profile"]` Base64 string, this would rank below `options.profile`. :param firefox_profile: Instance of ``FirefoxProfile`` object or a string. If undefined, a fresh profile will be created in a temporary location on the system. :param firefox_binary: Instance of ``FirefoxBinary`` or full path to the Firefox binary. If undefined, the system default Firefox installation will be used. :param timeout: Time to wait for Firefox to launch when using the extension connection. :param capabilities: Dictionary of desired capabilities. :param proxy: The proxy settings to us when communicating with Firefox via the extension connection. :param executable_path: Full path to override which geckodriver binary to use for Firefox 47.0.1 and greater, which defaults to picking up the binary from the system path. :param options: Instance of ``options.Options``. :param service_log_path: Where to log information from the driver. :param firefox_options: Deprecated argument for options :param service_args: List of args to pass to the driver service :param desired_capabilities: alias of capabilities. In future versions of this library, this will replace 'capabilities'. This will make the signature consistent with RemoteWebDriver. :param log_path: Deprecated argument for service_log_path :param keep_alive: Whether to configure remote_connection.RemoteConnection to use HTTP keep-alive. """ if log_path: warnings.warn('use service_log_path instead of log_path', DeprecationWarning, stacklevel=2) service_log_path = log_path if firefox_options: warnings.warn('use options instead of firefox_options', DeprecationWarning, stacklevel=2) options = firefox_options self.binary = None self.profile = None self.service = None # If desired capabilities is set, alias it to capabilities. # If both are set ignore desired capabilities. if capabilities is None and desired_capabilities: capabilities = desired_capabilities if capabilities is None: capabilities = DesiredCapabilities.FIREFOX.copy() if options is None: options = Options() capabilities = dict(capabilities) if capabilities.get("binary"): self.binary = capabilities["binary"] # options overrides capabilities if options is not None: if options.binary is not None: self.binary = options.binary if options.profile is not None: self.profile = options.profile # firefox_binary and firefox_profile # override options if firefox_binary is not None: if isinstance(firefox_binary, basestring): firefox_binary = FirefoxBinary(firefox_binary) self.binary = firefox_binary options.binary = firefox_binary if firefox_profile is not None: if isinstance(firefox_profile, basestring): firefox_profile = FirefoxProfile(firefox_profile) self.profile = firefox_profile options.profile = firefox_profile # W3C remote # TODO(ato): Perform conformance negotiation if capabilities.get("marionette"): capabilities.pop("marionette") self.service = Service(executable_path, service_args=service_args, log_path=service_log_path) self.service.start() capabilities.update(options.to_capabilities()) executor = FirefoxRemoteConnection( remote_server_addr=self.service.service_url) RemoteWebDriver.__init__(self, command_executor=executor, desired_capabilities=capabilities, keep_alive=True) # Selenium remote else: if self.binary is None: self.binary = FirefoxBinary() if self.profile is None: self.profile = FirefoxProfile() # disable native events if globally disabled self.profile.native_events_enabled = ( self.NATIVE_EVENTS_ALLOWED and self.profile.native_events_enabled) if proxy is not None: proxy.add_to_capabilities(capabilities) executor = ExtensionConnection("127.0.0.1", self.profile, self.binary, timeout) RemoteWebDriver.__init__(self, command_executor=executor, desired_capabilities=capabilities, keep_alive=keep_alive) self._is_remote = False