def generate_firefox_profile( self: "SeleniumTestability", preferences: OptionalDictType = None, accept_untrusted_certs: bool = False, proxy: OptionalStrType = None, ) -> FirefoxProfile: """ Generates a firefox profile that sets up few required preferences for SeleniumTestability to support all necessary features. Parameters: - ``preferences`` - firefox profile preferences in dictionary format. - ``accept_untrusted_certs`` should we accept untrusted/self-signed certificates. - ``proxy`` proxy options Note: If you opt out using this keyword, you are not able to get logs with ``Get Logs`` and Firefox. """ profile = FirefoxProfile() if preferences: for key, value in preferences.items(): # type: ignore profile.set_preference(key, value) profile.set_preference("devtools.console.stdout.content", True) profile.accept_untrusted_certs = accept_untrusted_certs if proxy: profile.set_proxy(proxy) profile.update_preferences() return profile
def create_download_dir_profile_for_firefox(path_to_download, mime_types_file=None, *extensions_files): """ Example use | ${profile} | Create Download Dir Profile For Firefox | Artifacts | Resources/mimeTypes.rdf | Resources/webdriver_element_locator-2.0-fx.xpi | Resources/selenium_ide-2.9.1-fx.xpi | | Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | ff_profile_dir=${profile} | | Click Element | //a[contains(@href,'sample.csv.zip')] | """ path_to_download_check = validate_create_artifacts_dir( path_to_download) fp = FirefoxProfile() fp.set_preference("browser.download.folderList", 2) fp.set_preference("browser.download.manager.showWhenStarting", False) fp.set_preference("browser.download.manager.alertOnEXEOpen", False) fp.set_preference("browser.download.dir", path_to_download_check) fp.set_preference("xpinstall.signatures.required", False) fp.set_preference("browser.helperApps.alwaysAsk.force", False) fp.set_preference( "browser.helperApps.neverAsk.saveToDisk", "application/msword;application/csv;text/csv;image/png;image/jpeg;application/pdf;text/html;text/plain;application/octet-stream" ) fp.set_preference("pdfjs.disabled", True) fp.update_preferences() for single_extension in extensions_files: fp.add_extension(single_extension) if mime_types_file is not None: from shutil import copy2 copy2(os.path.normpath(mime_types_file), fp.profile_dir) logger.info("Firefox Profile Created in dir '" + fp.profile_dir + "'") return fp.profile_dir
def webDriverFirefox(): profile = FirefoxProfile() profile.set_preference('font.default.x-western', 'sans-serif') profile.set_preference('font.name.serif.x-western', 'sans-serif') profile.set_preference('font.size.variable.x-western', 14) profile.set_preference('general.warnOnAboutConfig', False) profile.set_preference('layout.css.devPixelsPerPx', '1.0') profile.set_preference('browser.sessionstore.resume_from_crash', False) profile.set_preference('dom.webnotifications.enabled', False) profile.update_preferences() if system() == "FreeBSD": binary = FirefoxBinary('/usr/local/bin/firefox') elif system() == "Linux": binary = FirefoxBinary('/usr/bin/firefox') elif system() == "Windows": pass caps = DesiredCapabilities.FIREFOX.copy() caps['marionette'] = True caps['screenResolution'] = '2560x1440' # marionette setting is fixed in selenium 3.0 and above by default driver = webdriver.Firefox(firefox_profile=profile, capabilities=caps, firefox_binary=binary) driver.set_window_size(1470, 900) return driver
def test_proxy(proxy): if not proxy: return # r = get('https://www.google.com/', proxy) ip, port, version = proxy.split(',') print(f'Checking proxy {ip}:{port}') options = Options() options.headless = True profile = FirefoxProfile() profile.set_preference('network.proxy.type', 1) profile.set_preference("network.proxy.socks", ip) profile.set_preference("network.proxy.socks_port", int(port)) profile.set_preference("network.proxy.socks_version", int(version)) profile.update_preferences() browser = Firefox(executable_path=r"./geckodriver", firefox_profile=profile, options=options) try: browser.get('https://showip.net/') soup = BeautifulSoup(browser.page_source, 'html.parser') return soup.select_one('#checkip').get('value').strip() == proxy.split( ',')[0].strip() except KeyboardInterrupt: raise KeyboardInterrupt('Abort') except: logging.error(traceback.format_exc()) finally: browser.quit()
def _setup_proxy(self, browser): """Setup proxy for Firefox or Chrome :type browser: str :param browser: Browser name :rtype: selenium.webdriver.FirefoxProfile or selenium.webdriver.ChromeOptions :returns: Proxy settings """ host = "127.0.0.1" port = "9050" if browser == "firefox": firefox_profile = FirefoxProfile() # Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5 firefox_profile.set_preference("network.proxy.type", 1) firefox_profile.set_preference("network.proxy.socks", host) firefox_profile.set_preference("network.proxy.socks_port", int(port)) firefox_profile.update_preferences() return firefox_profile elif browser == "chrome": proxy = f"socks5://{host}:{port}" chrome_options = ChromeOptions() chrome_options.add_argument(f"--proxy-server={proxy}") return chrome_options else: logger.info(f"No proxy setting was done for {browser}")
def get_firefox_driver(): profile = FirefoxProfile() profile.set_preference('browser.cache.disk.enable', False) profile.set_preference('browser.cache.memory.enable', False) profile.set_preference('browser.cache.offline.enable', False) profile.set_preference('network.cookie.cookieBehavior', 2) if os.environ.get("DEV"): profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.socks", "127.0.0.1") profile.set_preference("network.proxy.socks_port", 8050) profile.set_preference("network.proxy.socks_version", 5) profile.update_preferences() caps = DesiredCapabilities().FIREFOX caps["pageLoadStrategy"] = "none" options = firefox_options() options.headless = True options.add_argument('start-maximized') path = os.path.dirname(__file__) if platform.system() == "Linux": path = os.path.join(path, r"driver/geckodriver") else: path = os.path.join(path, r"driver/geckodriver_mac") driver = webdriver.Firefox(executable_path=path, firefox_options=options, desired_capabilities=caps, firefox_profile=profile) driver.set_window_size(1920, 1080) driver.implicitly_wait(1) return driver
def create_download_dir_profile_for_firefox(path_to_download, mime_types_file=None, *extensions_files): """ Example use | ${profile} | Create Download Dir Profile For Firefox | Artifacts | Resources/mimeTypes.rdf | Resources/webdriver_element_locator-2.0-fx.xpi | Resources/selenium_ide-2.9.1-fx.xpi | | Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | ff_profile_dir=${profile} | | Click Element | //a[contains(@href,'sample.csv.zip')] | """ path_to_download_check = validate_create_artifacts_dir(path_to_download) fp = FirefoxProfile() fp.set_preference("browser.download.folderList", 2) fp.set_preference("browser.download.manager.showWhenStarting", False) fp.set_preference("browser.download.manager.alertOnEXEOpen", False) fp.set_preference("browser.download.dir", path_to_download_check) fp.set_preference("xpinstall.signatures.required", False) fp.set_preference("browser.helperApps.alwaysAsk.force", False) fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/msword;application/csv;text/csv;image/png;image/jpeg;application/pdf;text/html;text/plain;application/octet-stream") fp.set_preference("pdfjs.disabled", True) fp.update_preferences() for single_extension in extensions_files: fp.add_extension(single_extension) if mime_types_file is not None: from shutil import copy2 copy2(os.path.normpath(mime_types_file), fp.profile_dir) logger.info("Firefox Profile Created in dir '" + fp.profile_dir + "'") return fp.profile_dir
def _proxy(self): logger.info("proxy: {}".format(self.proxy)) if self.proxy and isinstance(self.proxy, dict) and self.browser == 'chrome': proxy = Proxy() for k, v in self.proxy.items(): proxy.__setattr__(k, v) capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) return capabilities elif self.proxy and isinstance(self.proxy, dict) and self.browser == 'firefox': profile = FirefoxProfile() profile.set_preference('network.proxy.type', 1) for k, v in self.proxy.items(): scheme = k.split("_") host_details = v.split(":") if len(scheme) > 1 and len(host_details) > 1: profile.set_preference("network.proxy.%s" % scheme[0], host_details[0]) profile.set_preference("network.proxy.%s_port" % scheme[0], int(host_details[1])) profile.update_preferences() # profile.accept_untrusted_certs = True return profile
def __init__(self): profile = FirefoxProfile() profile.set_preference("dom.webdriver.enabled", False) profile.set_preference('useAutomationExtension', False) profile.update_preferences() desired = DesiredCapabilities.FIREFOX super().__init__(firefox_profile=profile, desired_capabilities=desired)
def _connection(): o = Options() fp = FirefoxProfile() o.add_argument('-private') fp.set_preference("network.proxy.type", 0) #fp.set_preference("general.useragent.override","iPhone") fp.update_preferences() return Firefox(firefox_profile=fp, firefox_options=o)
def makeProfile(self, config): profile = FirefoxProfile() # Disable Firefox auto update profile.set_preference("app.update.enabled", False) try: if config["fire_bug"]: profile.add_extension(FIREBUG_EXTENSION) domain = "extensions.firebug." # Set default Firebug preferences # Avoid Firebug start page profile.set_preference(domain + "currentVersion", "2.0.6") # Activate everything profile.set_preference(domain + "allPagesActivation", "on") profile.set_preference(domain + "defaultPanelName", "net") # Enable Firebug on all sites profile.set_preference(domain + "net.enableSites", True) self.logger.info("Firebug profile settings enabled") except KeyError: self.logger.warning("Firebug profile settings failed") pass try: if config["net_export"]: profile.add_extension(NETEXPORT_EXTENSION) # Set default NetExport preferences self.har_output = config["net_export_output"] #self.logger.info("Output HAR directory: {0}".format(self.har_output)) profile.set_preference(domain + "netexport.defaultLogDir", self.har_output) profile.set_preference(domain + "netexport.autoExportToFile", True) profile.set_preference(domain + "netexport.alwaysEnableAutoExport", True) # Do not show preview output profile.set_preference(domain + "netexport.showPreview", True) profile.set_preference(domain + "netexport.pageLoadedTimeout", 3000) # Log dir self.logger.info("NetExport profile settings enabled.") # Har ID to check file exists self.har_id = int((datetime.datetime.now()-datetime.datetime(1970,1,1)).total_seconds()*1000000) self.url += "/?{0}".format(self.har_id) #print self.url except KeyError: self.logger.warning("NetExport profile settings failed") pass profile.update_preferences() return profile
def clear_proxy(profile=None): """ :param profile: :return: """ if profile is None: profile = FirefoxProfile() profile.set_preference("network.proxy.type", 0) profile.update_preferences() return profile
def test_that_we_can_accept_a_profile(capabilities, webserver): profile1 = FirefoxProfile() profile1.set_preference("browser.startup.homepage_override.mstone", "") profile1.set_preference("startup.homepage_welcome_url", webserver.where_is('simpleTest.html')) profile1.update_preferences() profile2 = FirefoxProfile(profile1.path) driver = Firefox(capabilities=capabilities, firefox_profile=profile2) title = driver.title driver.quit() assert "Hello WebDriver" == title
def private_connection(PROXY_HOST,PROXY_PORT): fp = FirefoxProfile() o = Options() o.add_argument('-private') fp.set_preference("network.proxy.type", 1) fp.set_preference("network.proxy.http",PROXY_HOST) fp.set_preference("network.proxy.http_port",int(PROXY_PORT)) fp.set_preference("network.proxy.ssl",PROXY_HOST) fp.set_preference("network.proxy.ssl_port",int(PROXY_PORT)) fp.set_preference("general.useragent.override","iPhone") fp.update_preferences() return Firefox(firefox_profile=fp, firefox_options=o)
def create(self, headless=False, devtools=False, incognito=False, proxy_url='', proxy_port=80): """Creates a Firefox browser. :param headless: If the browser should start in headless mode :type headless: bool :param devtools: If the browser should start with opened dev tools :type devtools: bool :param incognito: If the browser should start in incognito mode :type incognito: bool :param proxy_url: The proxy url :type proxy_url: str :param proxy_port: The proxy port :type proxy_port: int :return: A fully prepared Firefox browser :rtype: Firefox """ options = FirefoxOptions() profile = FirefoxProfile() options.profile = profile profile.set_preference("devtools.jsonview.enabled", False) # To parse JSON profile.set_preference("devtools.inspector.show-three-pane-tooltip", False) # Monopolizes focus # profile.set_preference("devtools.inspector.three-pane-first-run", False) # Monopolizes focus if headless: options.headless = True self._log_headless() if devtools: options.add_argument('-devtools') self._log_devtools() if incognito: profile.set_preference("browser.privatebrowsing.autostart", True) self._log_incognito() if not proxy_url == '': profile.set_preference("network.proxy.type", ProxyType.MANUAL['ff_value']) profile.set_preference("network.proxy.http", proxy_url) profile.set_preference("network.proxy.http_port", proxy_port) profile.set_preference("network.proxy.share_proxy_settings", True) proxy_address = "{}:{}".format(proxy_url, proxy_port) self._log_proxy(proxy_address) profile.update_preferences() self.log.debug("Arguments: {}".format(options.arguments)) if platform.system() == 'Windows': return Firefox(options=options, executable_path='geckodriver.exe') else: return Firefox(options=options)
def test_that_we_can_accept_a_profile(capabilities, webserver): profile1 = FirefoxProfile() profile1.set_preference("browser.startup.homepage_override.mstone", "") profile1.set_preference("startup.homepage_welcome_url", webserver.where_is('simpleTest.html')) profile1.update_preferences() profile2 = FirefoxProfile(profile1.path) driver = Firefox( capabilities=capabilities, firefox_profile=profile2) title = driver.title driver.quit() assert "Hello WebDriver" == title
def firefox_profile(request): profile = FirefoxProfile(request.config.getoption('firefox_profile')) for preference in request.config.getoption('firefox_preferences'): name, value = preference if value.isdigit(): # handle integer preferences value = int(value) elif value.lower() in ['true', 'false']: # handle boolean preferences value = value.lower() == 'true' profile.set_preference(name, value) profile.update_preferences() for extension in request.config.getoption('firefox_extensions'): profile.add_extension(extension) return profile
def profile(user_agent: Optional[str] = None, language: str = 'en-us', private: bool = False, disable_images: bool = False, mute_audio: bool = False, proxy: Optional[Proxy] = None, path: Optional[str] = None) -> FirefoxProfile: profile = FirefoxProfile( path if path and os.path.exists(path) else None) if user_agent: profile.set_preference('general.useragent.override', user_agent) if language: profile.set_preference('intl.accept_languages', language) if private: profile.set_preference('browser.privatebrowsing.autostart', True) if disable_images: profile.set_preference('permissions.default.image', 2) profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', False) if mute_audio: profile.set_preference('media.volume_scale', '0.0') if proxy: profile.set_preference('network.proxy.type', 1) profile.set_preference('network.proxy.http', proxy.host) profile.set_preference('network.proxy.http_port', proxy.port) profile.set_preference('network.proxy.ssl', proxy.host) profile.set_preference('network.proxy.ssl_port', proxy.port) profile.set_preference('network.proxy.ftp', proxy.host) profile.set_preference('network.proxy.ftp_port', proxy.port) profile.set_preference('network.proxy.socks', proxy.host) profile.set_preference('network.proxy.socks_port', proxy.port) profile.set_preference('network.proxy.socks_version', 5) profile.set_preference('signon.autologin.proxy', True) profile.set_preference('marionatte', False) profile.set_preference('dom.webdriver.enabled', False) profile.set_preference('media.peerconnection.enabled', False) profile.set_preference('useAutomationExtension', False) profile.set_preference('general.warnOnAboutConfig', False) profile.update_preferences() return profile
def firefox_webdriver_factory( accept_language: str = 'en-us, en') -> FirefoxWebDriver: """ Create a firefox webdriver, in headless mode or not, depending on the setting HEADLESS_TEST_MODE """ options = FirefoxOptions() options.headless = settings.HEADLESS_TEST_MODE profile = FirefoxProfile() profile.set_preference('intl.accept_languages', accept_language) profile.update_preferences() selenium = FirefoxWebDriver(options=options, firefox_profile=profile) selenium.implicitly_wait(15) return selenium
def create_driver(debug: bool) -> WebDriver: firefox_options = Options() if not debug: firefox_options.add_argument("-headless") # disable navigator.webdriver in order to make driver automation # undetectable: https://stackoverflow.com/a/60626696 profile = FirefoxProfile() profile.set_preference("dom.webdriver.enabled", False) profile.set_preference("useAutomationExtension", False) profile.update_preferences() driver = Firefox( executable_path="./selenium-drivers/geckodriver", options=firefox_options, firefox_profile=profile, ) return driver
def firefox_profile(pytestconfig): profile = None if pytestconfig.getoption("firefox_profile"): profile = FirefoxProfile(pytestconfig.getoption("firefox_profile")) warnings.warn( "--firefox-profile has been deprecated and will be removed in " "a future release. Please use the firefox_options fixture to " "set a profile path or FirefoxProfile object using " "firefox_options.profile.", DeprecationWarning, ) if pytestconfig.getoption("firefox_preferences"): profile = profile or FirefoxProfile() warnings.warn( "--firefox-preference has been deprecated and will be removed in " "a future release. Please use the firefox_options fixture to set " "preferences using firefox_options.set_preference. If you are " "using Firefox 47 or earlier then you will need to create a " "FirefoxProfile object with preferences and set this using " "firefox_options.profile.", DeprecationWarning, ) for preference in pytestconfig.getoption("firefox_preferences"): name, value = preference if value.isdigit(): # handle integer preferences value = int(value) elif value.lower() in ["true", "false"]: # handle boolean preferences value = value.lower() == "true" profile.set_preference(name, value) profile.update_preferences() if pytestconfig.getoption("firefox_extensions"): profile = profile or FirefoxProfile() warnings.warn( "--firefox-extensions has been deprecated and will be removed in " "a future release. Please use the firefox_options fixture to " "create a FirefoxProfile object with extensions and set this " "using firefox_options.profile.", DeprecationWarning, ) for extension in pytestconfig.getoption("firefox_extensions"): profile.add_extension(extension) return profile
def test_that_unicode_prefs_are_written_in_the_correct_format(): profile = FirefoxProfile() profile.set_preference('sample.preference.2', unicode('hi there')) profile.update_preferences() assert 'hi there' == profile.default_preferences["sample.preference.2"] encoded = profile.encoded decoded = base64.b64decode(encoded) with BytesIO(decoded) as fp: zip = zipfile.ZipFile(fp, "r") for entry in zip.namelist(): if entry.endswith('user.js'): user_js = zip.read(entry) for line in user_js.splitlines(): if line.startswith(b'user_pref("sample.preference.2",'): assert line.endswith(b'hi there");') # there should be only one user.js break
def test_that_unicode_prefs_are_written_in_the_correct_format(): profile = FirefoxProfile() profile.set_preference('sample.preference.2', unicode('hi there')) profile.update_preferences() assert 'hi there' == profile.default_preferences["sample.preference.2"] encoded = profile.encoded decoded = base64.b64decode(encoded) with BytesIO(decoded) as fp: zip = zipfile.ZipFile(fp, "r") for entry in zip.namelist(): if entry.endswith('user.js'): user_js = zip.read(entry) for line in user_js.splitlines(): if line.startswith(b'user_pref("sample.preference.2",'): assert line.endswith(b'hi there");') # there should be only one user.js break
def generate_firefox_profile( self: "SeleniumTestability", options: OptionalDictType = None, accept_untrusted_certs: bool = False, proxy: OptionalStrType = None, ) -> FirefoxProfile: profile = FirefoxProfile() if options: for key, value in options.items(): # type: ignore profile.set_preference(key, value) profile.set_preference("devtools.console.stdout.content", True) if accept_untrusted_certs: profile.accept_untrusted_certs if proxy: profile.set_proxy(proxy) profile.update_preferences() return profile
def __init__(self,test,browser_name,url,test_data=None): super(FrameworkTests,self).__init__(test) self.test=test self.browser_name=browser_name self.url=url self.driver=None if self.browser_name=='firefox': ffp = FirefoxProfile() ffp.update_preferences() self.driver = Firefox(firefox_profile=ffp) elif self.browser_name=='chrome': chromedriver = load_browser_driver("chromedriver") os.environ["webdriver.chrome.driver"] = chromedriver self.driver=Chrome(chromedriver) elif self.browser_name=='ie': iedriver = load_browser_driver("IEDriverServer") os.environ["webdriver.ie.driver"] = iedriver self.driver=Ie(iedriver) self.verification = [] self.verification.append("Test %s on browser %s" %(self.test,self.browser_name)) self.test_data=test_data self.errors=[]
def firefox_profile(pytestconfig): profile = None if pytestconfig.getoption('firefox_profile'): profile = FirefoxProfile(pytestconfig.getoption('firefox_profile')) warnings.warn( '--firefox-profile has been deprecated and will be removed in ' 'a future release. Please use the firefox_options fixture to ' 'set a profile path or FirefoxProfile object using ' 'firefox_options.profile.', DeprecationWarning) if pytestconfig.getoption('firefox_preferences'): profile = profile or FirefoxProfile() warnings.warn( '--firefox-preference has been deprecated and will be removed in ' 'a future release. Please use the firefox_options fixture to set ' 'preferences using firefox_options.set_preference. If you are ' 'using Firefox 47 or earlier then you will need to create a ' 'FirefoxProfile object with preferences and set this using ' 'firefox_options.profile.', DeprecationWarning) for preference in pytestconfig.getoption('firefox_preferences'): name, value = preference if value.isdigit(): # handle integer preferences value = int(value) elif value.lower() in ['true', 'false']: # handle boolean preferences value = value.lower() == 'true' profile.set_preference(name, value) profile.update_preferences() if pytestconfig.getoption('firefox_extensions'): profile = profile or FirefoxProfile() warnings.warn( '--firefox-extensions has been deprecated and will be removed in ' 'a future release. Please use the firefox_options fixture to ' 'create a FirefoxProfile object with extensions and set this ' 'using firefox_options.profile.', DeprecationWarning) for extension in pytestconfig.getoption('firefox_extensions'): profile.add_extension(extension) return profile
def init_driver_firefox(self, total_width=1600, total_height=900): options = Options() fp = FirefoxProfile() if self.headless: options.add_argument("--headless") if not self.download_images: fp.set_preference('permissions.default.image', 2) fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') if self.proxy_host and self.proxy_port: fp = webdriver.FirefoxProfile() # network.proxy.type: Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5 fp.set_preference("network.proxy.type", 1) fp.set_preference("network.proxy.http", self.proxy_host) fp.set_preference("network.proxy.http_port", int(self.proxy_port)) # WARNING ! proxy_port should be int fp.set_preference("general.useragent.override", self.user_agent) fp.update_preferences() elif self.proxy_port or self.proxy_host: raise(Exception('If you want to use a proxy, please provide proxy_host and proxy_port')) self.driver = Firefox(firefox_options=options, firefox_profile=fp) self.driver.set_window_size(total_width, total_height)
class function_webdriver(): def __init__(self): self.profile = FirefoxProfile() self.profile.set_preference('permissions.default.desktop-notification', 1) if enable_change_proxy: self.profile = self.change_proxy(host, port, self.profile) else: self.profile = self.clear_proxy(self.profile) self.profile.update_preferences() self.driver = webdriver.Firefox(firefox_profile=self.profile) self.driver.maximize_window() def change_proxy(self, host, port, profile: object = None): if profile is None: profile = FirefoxProfile() profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.http", host) profile.set_preference("network.proxy.http_port", int(port)) profile.set_preference("network.proxy.ssl", host) profile.set_preference("network.proxy.ssl_port", int(port)) profile.update_preferences() return profile def clear_proxy(self, profile=None): if profile is None: profile = FirefoxProfile() profile.set_preference("network.proxy.type", 0) profile.update_preferences() return profile def login(self, user_name, password): print("Opened facebook") username_box = self.driver.find_element_by_id('email') username_box.send_keys(user_name) print("Email Id entered") time.sleep(1) password_box = self.driver.find_element_by_id('pass') password_box.send_keys(password) print("Password entered") login_box = self.driver.find_element_by_id('loginbutton') login_box.click() print("Done") return True def logout(self): logout = self.driver.find_element_by_id("userNavigationLabel") logout.click() time.sleep(1) logout2 = self.driver.find_element_by_css_selector( "li._54ni:nth-child(12) > a:nth-child(1) > span:nth-child(1) > span:nth-child(1)" ) logout2.click() return True def get_URL(self, url): return self.driver.get(url) def quit(self): return self.clear_proxy(self.profile) #def wait_for_page_load(self): # try: # element_present = EC.presence_of_element_located((By.ID, "element_id")) # WebDriverWait(self.driver, 10).until(element_present) # except TimeoutException: # print("Timed out waiting for page to load") def scroll_page(self): lenOfPage = self.driver.execute_script( "window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;" ) #self.wait_for_page_load() try: #element_present = EC.presence_of_element_located((By.ID, "element_id")) element_present = EC.presence_of_element_located( (By.TAG_NAME, "html")) WebDriverWait(self.driver, 10).until(element_present) except TimeoutException: print("Timed out waiting for page to load") match = False while (match == False): lastCount = lenOfPage time.sleep(2) lenOfPage = self.driver.execute_script( "window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;" ) if lastCount == lenOfPage: match = True def get_URL_image(self): images = self.driver.find_elements_by_css_selector( "a[rel='theater'][data-render-location='homepage_stream']") #images = self.driver.find_elements_by_css_selector("a[class='_5pcq']") #images = self.driver.find_elements_by_class_name("_5pcq") for image in images: if ("video" in image.get_attribute("href")): continue str = image.get_attribute("href") print(str) self.click_Image(image) try: # element_present = EC.presence_of_element_located((By.ID, "element_id")) element_present = EC.presence_of_element_located( (By.CLASS_NAME, "b=[spotlight]")) WebDriverWait(self.driver, 10).until(element_present) except TimeoutException: print("Timed out waiting for page to load") wait = WebDriverWait(self.driver, 10) get_like_share = wait.until( EC.presence_of_element_located( (By.ID, "fbPhotoSnowliftFeedback"))) # Count the number of likes try: likes = get_like_share.find_element_by_class_name('_4arz') except NoSuchElementException as e: print("The number of likes: 0") pass else: number_of_likes = likes.get_attribute("innerText") print('The number of likes:', number_of_likes) # Count the number of shares try: shares = get_like_share.find_element_by_class_name( 'UFIShareLink') except (NoSuchElementException): print("The number of shares: 0") pass else: number_of_shares = shares.get_attribute("innerText") print('The number of shares:', number_of_shares) return image def click_Image(self, image): self.driver.execute_script("arguments[0].scrollIntoView(true);", image) self.driver.execute_script("arguments[0].click();", image)
def test_that_boolean_prefs_are_written_in_the_correct_format(): profile = FirefoxProfile() profile.set_preference("sample.bool.preference", True) profile.update_preferences() assert profile.default_preferences["sample.bool.preference"] is True
def test_that_integer_prefs_are_written_in_the_correct_format(): profile = FirefoxProfile() profile.set_preference("sample.int.preference", 12345) profile.update_preferences() assert 12345 == profile.default_preferences["sample.int.preference"]
def __init__( self, # webdriver.Firefox parameters start here firefox_profile: webdriver.FirefoxProfile = webdriver. FirefoxProfile(), firefox_binary=None, timeout=30, capabilities=None, proxy=None, executable_path="geckodriver", options=None, service_log_path="geckodriver.log", firefox_options=None, service_args=None, desired_capabilities=None, log_path=None, keep_alive=True, # CamouflageHandler parameters user_agents: list = [], allow_reuse_ip_after: int = 10, time_between_calls: float = 0.25, random_delay: bool = True, # Parameters of this class change_ip_after: int = 42, change_user_agent_after: int = 0, cookie_domain: str = '', persist_cookies_when_close: bool = False, reload_cookies_when_start: bool = False, location_of_cookies: str = 'cookies.pkl'): """Starts a new session of TorFirefoxWebdriver. Keyword arguments: change_ip_after -- Number of calls before changing the IP. (default 42) change_user_agent_after -- Number of calls before changing the user-agent. If the number is zero, the user-agent never will be changed (default 0) cookie_domain -- Domain of the cookie, eg., https://example.com/ (default '') persist_cookies_when_close -- Whether to save cookies when closing the driver (default False) reload_cookies_when_start -- Whether to reload the cookies that were saved when closing the last driver session (default False) location_of_cookies -- Where to save cookies (default 'cookies.pkl') """ CamouflageHandler.__init__(self, user_agents=user_agents, allow_reuse_ip_after=allow_reuse_ip_after, time_between_calls=time_between_calls, random_delay=random_delay) self.number_of_requests_made = 0 self.change_ip_after = change_ip_after self.change_user_agent_after = change_user_agent_after firefox_profile.set_preference("network.proxy.type", 1) firefox_profile.set_preference("network.proxy.socks", '127.0.0.1') firefox_profile.set_preference("network.proxy.socks_port", 9050) firefox_profile.update_preferences() webdriver.Firefox.__init__(self, firefox_profile=firefox_profile, firefox_binary=firefox_binary, timeout=timeout, capabilities=capabilities, proxy=proxy, executable_path=executable_path, options=options, service_log_path=service_log_path, firefox_options=firefox_options, service_args=service_args, desired_capabilities=desired_capabilities, log_path=log_path, keep_alive=keep_alive) # Enables advanced Firefox preferences (to change user-agents) if change_user_agent_after > 0: if len(user_agents) > 0: super().get('about:config') self.find_element_by_id('showWarningNextTime').click() self.find_element_by_id('warningButton').click() else: raise EmptyUserAgentListError( 'You are trying to rotate user-agent with an empty list of user-agents.' ) self.cookie_domain = cookie_domain self.persist_cookies_when_close = persist_cookies_when_close self.location_of_cookies = location_of_cookies if reload_cookies_when_start: if not self.cookie_domain: raise CookieDomainError( 'To reload cookies, you need to specify their domain') self.reload_cookies()
def ClearProxy(self, profile=None): if profile is None: profile = FirefoxProfile() profile.set_preference("network.proxy.type", 0) profile.update_preferences() return profile
def test_that_boolean_prefs_are_written_in_the_correct_format(): profile = FirefoxProfile() profile.set_preference("sample.bool.preference", True) profile.update_preferences() assert profile.default_preferences["sample.bool.preference"] is True
def test_that_integer_prefs_are_written_in_the_correct_format(): profile = FirefoxProfile() profile.set_preference("sample.int.preference", 12345) profile.update_preferences() assert 12345 == profile.default_preferences["sample.int.preference"]
# p.set_preference("browser.download.panel.shown", False) ok p.set_preference("browser.shell.checkDefaultBrowser", False) p.set_preference("browser.download.folderList", 0) p.set_preference("browser.download.manager.showWhenStarting", False) p.set_preference("browser.download.manager.alertOnEXEOpen", False) # Só downloads PDF p.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/pdf") # p.set_preference("browser.download.importedFromSqlite", False) ok # p.set_preference("pdfjs.disabled", True) ok # p.set_preference("pdfjs.showPreviousViewOnLoad", False) ok # p.set_preference("pdfjs.previousHandler.preferredAction", 4) ok # p.set_preference("pdfjs.previousHandler.alwaysAskBeforeHandling", True) p.set_preference("plugin.disable_full_page_plugin_for_types", "application/pdf") p.set_preference("pdfjs.migrationVersion", 2) #Configurações Câmera e audio # p.set_preference("media.webspeech.recognition.enable", True) p.set_preference("camera.control.face_detection.enabled", True) p.set_preference("dom.imagecapture.enabled", True) # Limpa cache e cookies p.set_preference('browser.cache.disk.enable', False) p.set_preference('browser.cache.memory.enable', False) p.set_preference('browser.cache.offline.enable', False) p.set_preference('network.cookie.cookieBehavior', 2) p.update_preferences() #Provavelmente aplica driver = webdriver.Firefox(firefox_binary="caminho/firefox36", firefox_profile=pp)
class Scrapper(): UserAgentList = ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:63.0) Gecko/20100101 Firefox/63.0", "Mozilla/5.0 (X11; Linux x86_64; rv:75.0) Gecko/20100101 Firefox/75.0", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0", "Mozilla/5.0 (Windows NT 10.0; rv:68.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36" ] def __init__(self, name, urls, str_file, test_url = "http://ipv4.plain-text-ip.com/", #test_url = "https://www.york.ac.uk/teaching/cws/wws/webpage1.html", data_folder = "{}/scrapped".format(thispath), proxy = False, headless= False, wait=1, max_nexts=0): logger.info("") logger.info("********************************************************************") logger.info("*** Inicializando el scrapper ***") logger.info("********************************************************************") logger.info("") self.name = name self.urls = urls self.str_file = str_file self.test_url = test_url self.data_folder = "{}/{}".format(data_folder,self.name) self.proxy = proxy if self.proxy: self.actual_proxy = False if isinstance(self.proxy,dict): try: self.proxy_list = proxies(typ=self.proxy["Types"], lim=self.proxy["Limit"], countries_list=self.proxy["Countries"]) except: raise else: self.proxy_list = proxies() else: self.proxy_list = False self.headless = headless self.wait = wait self.max_nexts = max_nexts self.data = [] self.previus_data = [] self.tread = False self.set_dom() self.configure_driver() self.crawl() def set_dom(self): dom_strc = dom(self.str_file) self.main_strc = dom_strc.main_strc self.data_strc = dom_strc.data_strc def configure_driver(self): self.firefox_capabilities = DesiredCapabilities.FIREFOX UserAgent = random.choice(self.UserAgentList) logger.info("User agent: {}".format(UserAgent)) if self.headless: self.options = Options() self.options.add_argument('--headless') self.firefox_profile = FirefoxProfile() self.firefox_profile.set_preference("browser.privatebrowsing.autostart", True) self.firefox_profile.set_preference("general.useragent.override", UserAgent) self.firefox_profile.update_preferences() else: self.options = Options() self.firefox_profile = FirefoxProfile() self.firefox_profile.set_preference("browser.privatebrowsing.autostart", True) # Esta lineas no sirven por ahora, buscar alternativas """self.firefox_profile.set_preference('permissions.default.image',2) self.firefox_profile.set_preference("permissions.default.stylesheet",2)""" self.firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', False) self.firefox_profile.set_preference('media.navigator.video.enabled',False) self.firefox_profile.set_preference('media.encoder.webm.enabled',False) self.firefox_profile.set_preference('media.ffmpeg.enabled',False) self.firefox_profile.set_preference('media.flac.enabled',False) self.firefox_profile.set_preference("general.useragent.override", UserAgent) self.firefox_profile.update_preferences() try: self.driver = Firefox(capabilities=self.firefox_capabilities, options=self.options, firefox_profile=self.firefox_profile, executable_path='geckodriver') except: raise def createDataFile(self): try: if not path.isdir(self.data_folder): makedirs(self.data_folder) if not path.isfile(self.data_file): with open(self.data_file, "w") as datafile: writer = csv.DictWriter(datafile, delimiter="\t", fieldnames=list(self.data_strc.keys())) writer.writeheader() except: raise def crawl(self): logger.info("Starting crawler") try: for enum, url in enumerate(self.urls): logger.info("Crawling {}".format(url["URL"])) filename = url["PARAM"].replace(" ","") self.data_file = "{}/{}.csv".format(self.data_folder,filename) self.createDataFile() self.url = url self.navigate() self.driver.close() logger.info("Finishing crawler") except: raise def navigate(self): try: self.set_proxy() self.driver.get(self.url["URL"]) self.check_containers() self.get_data() self.nexts = 0 self.next() except: self.navigate() def check_containers(self): try: print("---> Check containers") WebDriverWait(self.driver, 30).until(expected.visibility_of_element_located((By.CSS_SELECTOR, '{}'.format(self.main_strc["LiCont"]["Selector"])))) self.main_strc["LiCont"]["Elements"] = self.driver.find_elements_by_css_selector('{}'.format(self.main_strc["LiCont"]["Selector"])) self.main_strc["ElCont"]["Elements"] = [] if self.main_strc["LiCont"]["Elements"] != []: for element in self.main_strc["LiCont"]["Elements"]: self.main_strc["ElCont"]["Elements"] += element.find_elements_by_css_selector('{}'.format(self.main_strc["ElCont"]["Selector"])) if self.actual_proxy: self.proxy_list.proxy_notwork(self.actual_proxy.ip, self.actual_proxy.port) except TimeoutException as e: print("---> Nos cacharon") if self.actual_proxy: self.proxy_list.proxy_notwork(self.actual_proxy.host, self.actual_proxy.port) self.set_proxy() try: self.driver.close() self.navigate(self) except: self.navigate(self) pass except WebDriverException as e: print("---> Ni idea que pasó") if self.actual_proxy: self.proxy_list.proxy_notwork(self.actual_proxy.ip, self.actual_proxy.port) verified_proxy_host = "" self.set_proxy() try: self.driver.close() self.navigate(self) except: self.navigate(self) pass def get_data(self): try: if self.main_strc["ElCont"]["Elements"] != []: for element in self.main_strc["ElCont"]["Elements"]: data_dict = {} for key, selector in self.data_strc.items(): elements = element.find_elements_by_css_selector('{}'.format(selector["Selector"])) data_dict[key] = "" if key == "Link": data_dict[key] += elements[0].get_attribute('href') else: if len(elements) > 1: for e in elements: data_dict[key] += "<e>"+e.text+"</e>" elif len(elements)==1: data_dict[key] = elements[0].text with open(self.data_file, "a") as datafile: if "".join(list(data_dict.values())) != "": writer = csv.DictWriter(datafile, delimiter="\t", fieldnames=list(self.data_strc.keys())) writer.writerow(data_dict) except: raise def next(self): if self.main_strc["Next"]["Selector"]: self.nexts += 1 #print(self.max_nexts, self.nexts, self.max_nexts == 0 or self.nexts < self.max_nexts) if self.max_nexts == 0 or self.nexts < self.max_nexts: try: self.main_strc["Next"]["Elements"] = WebDriverWait(self.driver,2).until(expected.element_to_be_clickable((By.CSS_SELECTOR, '{}'.format(self.main_strc["Next"]["Selector"])))) wait = random.randrange(5,10) sleep(wait) self.main_strc["Next"]["Elements"].click() self.check_containers() self.get_data() self.next() except: raise def set_proxy(self): if self.proxy_list: self.proxy_list.select_proxy() self.actual_proxy = self.proxy_list.selected_proxy logger.info("Setting {}:{} from {} as proxy".format(self.actual_proxy.host, self.actual_proxy.port, self.actual_proxy.geo)) self.driver.execute("SET_CONTEXT", {"context": "chrome"}) try: self.driver.execute_script(""" Services.prefs.setIntPref('network.proxy.type', 1); Services.prefs.setCharPref("network.proxy.http", arguments[0]); Services.prefs.setIntPref("network.proxy.http_port", arguments[1]); Services.prefs.setCharPref("network.proxy.ssl", arguments[0]); Services.prefs.setIntPref("network.proxy.ssl_port", arguments[1]);""", self.actual_proxy.host,self.actual_proxy.port ) finally: self.driver.execute("SET_CONTEXT", {"context": "content"}) self.test_proxy() else: logger.info("------------------------La cagó pedazo e bola!!!!") """ Services.prefs.setIntPref('network.proxy.type', 1); Services.prefs.setCharPref("network.proxy.http", arguments[0]); Services.prefs.setIntPref("network.proxy.http_port", arguments[1]); Services.prefs.setCharPref("network.proxy.ssl", arguments[0]); Services.prefs.setIntPref("network.proxy.ssl_port", arguments[1]); Services.prefs.setCharPref("network.proxy.ftp", arguments[0]); Services.prefs.setIntPref("network.proxy.ftp_port", arguments[1]); """ def test_proxy(self, timeout = 10): if self.actual_proxy: logger.info("Testing {}:{} from {} as proxy".format(self.actual_proxy.host)) try: self.driver.set_page_load_timeout(timeout) self.driver.get(self.test_url) logger.info("{} worked in test page".format(self.actual_proxy.host)) except TimeoutException as e: if timeout == 10: test_proxy(self, timeout = 20) else: logger.info("{} Time out".format(self.actual_proxy.host)) self.proxy_list.proxy_notwork(self.actual_proxy.host self.actual_proxy.port) self.set_proxy() except WebDriverException as e: logger.info("{} Something goes wrong".format(self.actual_proxy.host)) self.proxy_list.proxy_notwork(self.actual_proxy.ip, self.actual_proxy.port) self.set_proxy()
def profile(): profile = FirefoxProfile() profile.set_preference('browser.startup.homepage_override.mstone', '') profile.set_preference('startup.homepage_welcome_url', 'about:') profile.update_preferences() return profile