def get_phantomjs_browser( self, proxy=None, timeout=15, ): """ 创建一个phantomjs浏览器 :param proxy: String "ip:port" :param timeout: Int :return: Phantomjs.Browser 浏览器对象 """ capabilities = webdriver.DesiredCapabilities.PHANTOMJS capabilities['phantomjs.page.settings.userAgent'] = random.choice( USER_AGENTS) capabilities["phantomjs.page.settings.loadImages"] = False if proxy: prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy prox.socks_proxy = proxy prox.ssl_proxy = proxy prox.add_to_capabilities(capabilities) browser = webdriver.PhantomJS(desired_capabilities=capabilities) browser.maximize_window() browser.set_page_load_timeout(timeout) return browser
def get_browser(): # @todo add windows path if platform.system() == "Darwin": os.environ["webdriver.chrome.driver"] = os.path.expanduser( "~" ) + '/Library/Application Support/ZAP/webdriver/macos/64/chromedriver' else: os.environ["webdriver.chrome.driver"] = os.path.expanduser( "~") + '/.ZAP/webdriver/linux/64/chromedriver' proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = "127.0.0.1:8080" proxy.socks_proxy = "127.0.0.1:8080" proxy.ssl_proxy = "127.0.0.1:8080" capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) options = webdriver.ChromeOptions() options.add_argument('--ignore-certificate-errors') options.add_argument("--test-type") return webdriver.Chrome( executable_path=os.environ["webdriver.chrome.driver"], chrome_options=options, desired_capabilities=capabilities)
def proxy_driver(PROXIES, co=co): prox = Proxy() ua = UserAgent() while True: if PROXIES: pxy = PROXIES[-1] break else: print("--- Proxies used up (%s)" % len(PROXIES)) PROXIES = get_proxies() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = pxy #prox.socks_proxy = pxy prox.ssl_proxy = pxy capabilities = dict(DesiredCapabilities.CHROME) capabilities["chrome.page.settings.userAgent"] = (ua.random) prox.add_to_capabilities(capabilities) service_args = ['--ssl-protocol=any', '--ignore-ssl-errors=true'] driver = webdriver.Chrome("chromedriver.exe", options=co, desired_capabilities=capabilities, service_args=service_args) return driver
def __init__(self, songUrl, proxy_url): self.songUrl = songUrl self.id = -1 self.name = '' self.album_id = -1 self.comments_num = -1 self.similar_song_ids = [] self.artists = [] chrome_options = Options() prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy_url prox.ssl_proxy = proxy_url # prox.socks_proxy = proxy_url capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) chrome_options.add_argument('--headless') # chrome_options.add_argument('user-agent={0}'.format(random.choice(uas))) # chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2}) if config_is_ubuntu: chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') # chrome_options.add_argument('--proxy-server=http://111.222.141.127:8118') # chrome_options.add_argument('--proxy-server={}'.format(proxy_url)) # chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # debug_print_thread("we are using proxy sever with url " + proxy_url) # chrome_options.add_argument('--proxy-server=http://114.98.27.147:4216') self.driver = webdriver.Chrome(config_chrome_path, options=chrome_options)
def __init__(self, proxy): """ Initialize the web driver. """ if proxy: PROXY = self.get_proxy() custom_proxy = Proxy() custom_proxy.proxy_type = ProxyType.MANUAL custom_proxy.ssl_proxy = PROXY capabilities = webdriver.DesiredCapabilities.CHROME custom_proxy.add_to_capabilities(capabilities) self.driver = webdriver.Chrome( ChromeDriverManager().install(), desired_capabilities=capabilities ) else: options = Options() options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') self.driver = webdriver.Chrome( ChromeDriverManager().install(), chrome_options=options )
def proxied_driver(addresses, driver_type="chrome", co=None): assert len(addresses) > 0, "At least one proxy address must be provided" prox = Proxy() prox.proxy_type = ProxyType.MANUAL addr = random.choice(addresses) prox.http_proxy = addr prox.ssl_proxy = addr assert driver_type in ["chrome", "firefox"], "proxy_type must be chrome or firefox " if driver_type == "chrome": capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) driver = webdriver.Chrome(chrome_options=co, desired_capabilities=capabilities) # 참고로 이런 .method의 번거로움이 python 단점 중 하나. 간단한 설정이어야 하는게 이상하게 구성하게됨 elif driver_type == "firefox": capabilities = DesiredCapabilities.FIREFOX prox.add_to_capabilities(capabilities) driver = webdriver.Firefox( firefox_options=co, desired_capabilities=capabilities, executable_path="/usr/local/bin/geckodriver") return driver
def load_page(source): prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "127.0.0.1:9090" prox.socks_proxy = "127.0.0.1:9090" prox.ssl_proxy = "127.0.0.1:9090" capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) driver = webdriver.Chrome(desired_capabilities=capabilities) driver.get(source) navigationStart = driver.execute_script( "return window.performance.timing.navigationStart") responseStart = driver.execute_script( "return window.performance.timing.responseStart") domComplete = driver.execute_script( "return window.performance.timing.domComplete") backendPerformance = responseStart - navigationStart frontendPerformance = domComplete - responseStart print "Back End: %s" % backendPerformance print "Front End: %s" % frontendPerformance driver.quit()
def getGoogleChromeDriver(fullproxy): try: proxy = fullproxy.split(' ')[0] conditionalPrint("proxy used : " + proxy) WINDOW_SIZE = "1920,1080" option = webdriver.ChromeOptions() option.add_argument("--incognito") #option.add_argument("--disable-gpu") #option.add_argument("--disable-infobars") #option.add_argument("--disable-notifications") #option.add_argument("--disable-extensions") if hideBrowser == "YES": option.add_argument("--headless") option.add_argument("--window-size=%s" % WINDOW_SIZE) prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy prox.socks_proxy = proxy prox.ssl_proxy = proxy capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) browser = webdriver.Chrome( executable_path="C:\\webdrivers\\chromedriver.exe", chrome_options=option, desired_capabilities=capabilities) #browser.set_window_position(-10000, 0) return browser except Exception: LogError(traceback, "fullproxy = " + fullproxy) return None
def get_capabilities(self): capabilities = DesiredCapabilities.FIREFOX.copy() # capabilities = DesiredCapabilities.CHROME.copy() # print('get_capabilities') proxy = self.profile.proxy if proxy is None or not proxy.active: # use random default proxy proxies = models.Proxy.objects.filter(active=True, default=True).all() if proxies: proxy = random.choice(proxies) else: proxy = None if proxy: # print('setting a proxy') # print(proxy) prox = Proxy() prox.proxy_type = ProxyType.MANUAL if proxy.proxy_type == proxy.HTTP: # print('HTTP proxy') prox.http_proxy = f'{proxy.ip}:{proxy.port}' prox.ssl_proxy = f'{proxy.ip}:{proxy.port}' prox.ftp_proxy = f'{proxy.ip}:{proxy.port}' elif proxy.proxy_type == proxy.SOCKS: # print('Socks proxy') prox.socks_proxy = f'{proxy.ip}:{proxy.port}' prox.socks_username = proxy.username prox.socks_password = proxy.password prox.add_to_capabilities(capabilities) # print(capabilities) return capabilities
def CreateBrowser(proxy, head=False, window=False, ua=False): if proxy: prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.ssl_proxy = proxy capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) # Set random user agent opts = Options() if ua: ua = UserAgent() agent = ua.chrome opts.add_argument("user-agent=" + agent) if not head: opts.add_argument("--headless") if window: opts.add_argument("--window-size=" + window) else: opts.add_argument("--window-size=%s" % "1920,1080") if proxy: _browser = webdriver.Chrome(options=opts, desired_capabilities=capabilities) else: _browser = webdriver.Chrome(options=opts) return _browser
def _start_proxy_(self, ip: str, port: int): prox = Proxy() endpoint = ip + ":" + str(port) prox.proxy_type = ProxyType.MANUAL prox.http_proxy = endpoint prox.socks_proxy = endpoint prox.ssl_proxy = endpoint
def initialize_browser() -> object: chrome_options = Options() chrome_options.add_argument('--window-size=1920x1080') chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument( 'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 ' 'Safari/537.36') proxy_url = "127.0.0.1:24001" proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = proxy_url proxy.ssl_proxy = proxy_url capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) if is_development(): return webdriver.Chrome('./bin/chromedriver.exe', chrome_options=chrome_options, desired_capabilities=capabilities) else: chrome_options.add_argument('--data-path=/tmp/data-path') chrome_options.add_argument('--homedir=/tmp') chrome_options.add_argument('--disk-cache-dir=/tmp/cache-dir') chrome_options.add_argument('--user-data-dir=/tmp/user-data') chrome_options.add_argument('--hide-scrollbars') chrome_options.add_argument('--enable-logging') chrome_options.add_argument('--log-level=0') chrome_options.add_argument('--v=99') chrome_options.add_argument('--single-process') chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--headless') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-gpu') return webdriver.Chrome(chrome_options=chrome_options, desired_capabilities=capabilities)
def get_webdriver(): options = webdriver.ChromeOptions() options.add_argument('window-size=1920x1080') options.add_argument('--headless') options.add_argument('--no-sandbox') # Configure Proxy option proxy = Proxy() if Settings.socks_proxy is None and Settings.https_proxy is None and Settings.https_proxy is None: proxy.proxy_type = ProxyType.DIRECT else: proxy.proxy_type = ProxyType.MANUAL if Settings.socks_proxy is not None: options.add_argument("--proxy-server=" + Settings.get_full_socks_proxy()) else: if Settings.http_proxy is not None: proxy.http_proxy = Settings.http_proxy if Settings.https_proxy is not None: proxy.ssl_proxy = Settings.https_proxy # Configure capabilities capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) return webdriver.Chrome(options=options, desired_capabilities=capabilities)
def create_instances(self): for _proxy in self.proxies: proxy = _proxy.proxy service_args = [] # service_args.append('--proxy={}:{}'.format(proxy.ip, proxy.port)) if proxy.username and proxy.password: service_args.append('--proxy-auth={}:{}'.format( proxy.username, proxy.password)) capabilities = DesiredCapabilities.PHANTOMJS capabilities[ 'phantomjs.page.settings.resourceTimeout'] = self.max_delay_limit * 1000 _proxy_ = Proxy() _proxy_.proxy_type = ProxyType.MANUAL _proxy_.http_proxy = '{}:{}'.format(proxy.ip, proxy.port) _proxy_.socks_proxy = '{}:{}'.format(proxy.ip, proxy.port) _proxy_.ssl_proxy = '{}:{}'.format(proxy.ip, proxy.port) _proxy_.add_to_capabilities(capabilities) driver = webdriver.PhantomJS( service_args=service_args, desired_capabilities=capabilities, service_log_path='/tmp/ghostdriver.log') driver.set_window_size(1120, 1080) driver.set_page_load_timeout(self.max_delay_limit) self.multi_instances.append(driver)
def setup_proxy(proxy): proxy_config = Proxy() proxy_config.proxy_type = ProxyType.MANUAL proxy_config.http_proxy = proxy proxy_config.ssl_proxy = proxy capabilities = webdriver.DesiredCapabilities.CHROME proxy_config.add_to_capabilities(capabilities) return proxy_config
def _add_proxy_support(self, capabilities): if "RX_PROXY" in os.environ: proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = os.environ.get("RX_PROXY") proxy.socks_proxy = os.environ.get("RX_PROXY") proxy.ssl_proxy = os.environ.get("RX_PROXY") proxy.add_to_capabilities(capabilities)
def crawl(minPage,maxPage,outputFormat): # set selenium proxy prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "http://*****:*****@class='agree-button']")) driver.execute_script("arguments[0].click();",cookies_modal) time.sleep(random.randint(5, 10)) #crawl for page in range(minPage,maxPage+1): url = "https://musescore.com/hub/piano?page=" + str(page) print("crawling page: "+ str(page)) #clear if no of pages is too big if page % 5 == 0: subprocess.call("killall -HUP tor", shell = True) time.sleep(1) for index in range(1,21): #select score from grid driver.get(url) # go back to grid of scores score_list = driver.find_elements_by_xpath("//h2[@class='score-info__title']") try: score_list[index].click() #download driver.execute_script(open("musescore-downloader.js").read()) #try twice, if still error move on download_button = WebDriverWait(driver, 10).until( lambda x: x.find_elements_by_xpath("//button[@class='_3L7Ul _3qfU_ _38TLP _3A7i9 _2XPrY _13O-4 _15kzJ']")) #XML if outputFormat == 'XML': download_button[2].click() #MIDI elif outputFormat == 'MIDI': download_button[3].click() else: print("Incorrect usage! please type:" + "musescore-crawler.py -f <format> -s <minPage> -e <maxPage>") sys.exit(2) time.sleep(random.randint(10,30)) time.sleep(5) except: print("Failed opening score, move on to next")
def login(self, headless=True, wait_time=15, proxy=None): """ Logs user in with the provided credentials User session is stored in the 'cred_root' folder and reused so there is no need to login every time. Pinterest sessions lasts for about 15 days Ideally you need to call this method 3-4 times a month at most. :return python dict object describing the pinterest response """ chrome_options = Options() if headless: chrome_options.add_argument("--headless") if proxy is not None: http_proxy = Proxy() http_proxy.proxy_type = ProxyType.MANUAL http_proxy.http_proxy = proxy http_proxy.socks_proxy = proxy http_proxy.ssl_proxy = proxy http_proxy.add_to_capabilities(chrome_options) chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options) driver.get("https://pinterest.com/login") try: WebDriverWait(driver, wait_time).until( EC.element_to_be_clickable((By.ID, 'email'))) driver.find_element_by_id("email").send_keys(self.email) driver.find_element_by_id("password").send_keys(self.password) logins = driver.find_elements_by_xpath( "//*[contains(text(), 'Log in')]") for login in logins: login.click() WebDriverWait(driver, wait_time).until( EC.invisibility_of_element((By.ID, 'email'))) cookies = driver.get_cookies() self.http.cookies.clear() for cookie in cookies: self.http.cookies.set(cookie['name'], cookie['value']) self.registry.update_all(self.http.cookies.get_dict()) except Exception as e: print("Failed to login", e) driver.close()
def get_proxy(self): proxy_spider_dict = ProxySpider().proxy[0] proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = "%(ip)s:%(port)s" % proxy_spider_dict if 'HTTPS' in proxy_spider_dict.get('proxy_type'): proxy.ssl_proxy = "%(ip)s:%(port)s" % proxy_spider_dict print("使用代理", proxy_spider_dict) return proxy
def get_proxy(proxies): print(proxies) prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxies prox.ftp_proxy = proxies prox.ssl_proxy = proxies capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) return capabilities
def test_sets_ssl_proxy(self): self.driver.quit() profile = webdriver.FirefoxProfile() proxy = Proxy() proxy.ssl_proxy = 'https://test.hostname:1234' profile.set_proxy(proxy) assert profile.default_preferences["network.proxy.type"] == str(ProxyType.MANUAL['ff_value']) assert profile.default_preferences["network.proxy.ssl"] == '"test.hostname"' assert profile.default_preferences["network.proxy.ssl_port"] == '1234'
def proxy_driver(PROXIES): prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = PROXIES prox.ssl_proxy = PROXIES capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) driver = webdriver.Chrome(PATH, options=chrome_options, desired_capabilities=capabilities) return driver
def createDriver(self): capabilities = webdriver.DesiredCapabilities.FIREFOX if self.config['settings']['proxy'] and self.config['proxy'] != '' and self.config['proxy'] != ' ': proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = self.config['proxy'] proxy.ssl_proxy = self.config['proxy'] proxy.add_to_capabilities(capabilities) self.driver = webdriver.Firefox(capabilities=capabilities, log_path="modules/geckodriver.log", executable_path='modules/geckodriver')
def installProxy(self, PROXY_HOST, PROXY_PORT): prox = Proxy() prox.proxy_type = ProxyType.MANUAL proxstr = PROXY_HOST + ":" + PROXY_PORT prox.http_proxy = proxstr prox.socks_proxy = proxstr prox.ssl_proxy = proxstr capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) return webdriver.Chrome()
def test_sets_ssl_proxy(self): self.driver.quit() profile = webdriver.FirefoxProfile() proxy = Proxy() proxy.ssl_proxy = 'https://test.hostname:1234' profile.set_proxy(proxy) assert profile.default_preferences["network.proxy.type"] == str( ProxyType.MANUAL['ff_value']) assert profile.default_preferences[ "network.proxy.ssl"] == '"test.hostname"' assert profile.default_preferences["network.proxy.ssl_port"] == '1234'
def set_chrome_proxy(): global DRIVER_CAPABILITIES, PROXY_USED PROXY_USED = repr(random.choice(PROXY_LIST)) proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = PROXY_USED #proxy.socks_proxy = PROXY proxy.ssl_proxy = PROXY_USED DRIVER_CAPABILITIES = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(DRIVER_CAPABILITIES)
def __init__(self, proxyIP='', hideWindow=False, fakeURL='', keyActive='', postion='left', numberThread=1, totalThread=1): option = Options() option.add_argument("--disable-infobars") option.add_argument("start-maximized") option.add_argument("--disable-extensions") option.add_argument("--disable-background-mode") prefs = {"profile.managed_default_content_settings.images": 2} option.add_experimental_option("prefs", prefs) if postion == 'left': option.add_argument("window-size=960,950") option.add_argument("--window-position=0,0") else: # user32 = ctypes.windll.user32 # user32.SetProcessDPIAware() # option.add_argument("window-size=150,300") # maxHeight = round(user32.GetSystemMetrics(1) / 300) # maxWidth = round(user32.GetSystemMetrics(1) / 150) # heightY = int(numberThread) * 300 # widthX = 0 # if maxHeight >= user32.GetSystemMetrics(1): # widthX = int(numberThread) * 150 # option.add_argument("--window-position="+str(int(numberThread) * widthX)+","+str(heightY)+"") # print('goc ne') option.add_argument("window-size=399,677") option.add_argument("--window-position=0,0") # Pass the argument 1 to allow and 2 to block option.add_experimental_option( "prefs", {"profile.default_content_setting_values.notifications": 2}) self.fakeURL = fakeURL self.keyActive = keyActive if hideWindow: option.add_argument("--headless") capabilities = webdriver.DesiredCapabilities.CHROME if proxyIP != '': prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.ssl_proxy = proxyIP prox.add_to_capabilities(capabilities) self.driver = webdriver.Chrome(options=option, executable_path=r".\\chromedriver.exe", desired_capabilities=capabilities)
def vote(): emails = get_emails() for item in emails: print(item) proxies = get_proxies() proxy_index = random.randrange(0, 5) proxy = proxies[proxy_index] print(proxy) full_name = item[0] email = item[1] print(full_name) print(email) proxy_config = Proxy() proxy_config.proxy_type = ProxyType.MANUAL proxy_config.http_proxy = proxy proxy_config.ssl_proxy = proxy capabilities = webdriver.DesiredCapabilities.CHROME proxy_config.add_to_capabilities(capabilities) chrome_options = Options() chrome_options.headless = True driver = webdriver.Chrome("./chromedriver", desired_capabilities=capabilities, options=chrome_options) driver.get("") # FILL IN WITH WEBSITE time.sleep(10) # give page time to load form_full_name = driver.find_element_by_xpath('') #FILL IN WITH X PATH form_full_name.send_keys(full_name) form_radio_button = driver.find_element_by_xpath( '') #FILL IN WITH X PATH form_radio_button.click() form_video_dropdown = driver.find_element_by_xpath( '') #FILL IN WITH X PATH form_video_dropdown.click() form_video_choice = driver.find_element_by_xpath( '') #FILL IN WITH X PATH form_video_choice.click() form_email_address = driver.find_element_by_xpath( '') #FILL IN WITH X PATH form_email_address.send_keys(email) form_submit_button = driver.find_element_by_xpath( '') #FILL IN WITH X PATH form_submit_button.click() time.sleep(10) # give time to submit get_confirmation_text = driver.find_element_by_css_selector( '') #FILL IN WITH X PATH if ((get_confirmation_text.text ) == ""): #FILL IN WITH CONFIRMATION MESSAGE print("Successfully voted!") else: print("Did not vote successfully.") driver.quit() time.sleep(300) # wait 5 mins
def __setup_driver(self): ''' Set up the webdriver with a proxy''' web_opts = Options() web_opts.headless = True web_prox = Proxy() web_prox.proxy_type = ProxyType.MANUAL web_prox.http_proxy = self.useropts['HTTPPROXY'] web_prox.ssl_proxy = self.useropts['HTTPPROXY'] web_capabilities = webdriver.DesiredCapabilities.FIREFOX web_prox.add_to_capabilities(web_capabilities) self.driver = webdriver.Firefox(options=web_opts, desired_capabilities=web_capabilities) self.driver.implicitly_wait(10) # seconds
def _proxy_generator(self): "Randomly generate the deliberate proxy !" if self._browser == 'chrome': _capabilities = DesiredCapabilities.CHROME else: _capabilities = DesiredCapabilities.FIREFOX _proxy = Proxy() _proxy.proxy_type = ProxyType.MANUAL _proxy.http_proxy = next(self._proxy_pool) _proxy.ssl_proxy = next(self._proxy_pool) _proxy.add_to_capabilities(_capabilities) return _capabilities
def setup_proxy(self): PROXY = "159.8.114.34" prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.autodetect = False if self.driver_type == 'firefox': #add more if you add more driver profiles above capabilities = webdriver.DesiredCapabilities.FIREFOX elif self.driver_type == 'chrome': capabilities = webdriver.DesiredCapabilities.CHROME prox.http_proxy = PROXY prox.ssl_proxy = PROXY prox.add_to_capabilities(capabilities) return capabilities
def _setup_driver(self): proxy = Proxy() proxy.proxy_type = ProxyType.DIRECT if 'http' in self.proxy: proxy.http_proxy = self.proxy['http'] if 'https' in self.proxy: proxy.ssl_proxy = self.proxy['https'] capa = self._build_capabilities() proxy.add_to_capabilities(capa) options = self._build_options() # TODO some browsers don't need headless # TODO handle different proxy setting? options.set_headless(self.HEADLESS) if self.DRIVER is webdriver.Firefox: if self.responses_dirname and not os.path.isdir(self.responses_dirname): os.makedirs(self.responses_dirname) options.profile = DirFirefoxProfile(self.responses_dirname) if self.responses_dirname: capa['profile'] = self.responses_dirname self.driver = self.DRIVER(options=options, capabilities=capa) elif self.DRIVER is webdriver.Chrome: self.driver = self.DRIVER(options=options, desired_capabilities=capa) elif self.DRIVER is webdriver.PhantomJS: if self.responses_dirname: if not os.path.isdir(self.responses_dirname): os.makedirs(self.responses_dirname) log_path = os.path.join(self.responses_dirname, 'selenium.log') else: log_path = NamedTemporaryFile(prefix='weboob_selenium_', suffix='.log', delete=False).name self.driver = self.DRIVER(desired_capabilities=capa, service_log_path=log_path) else: raise NotImplementedError() if self.WINDOW_SIZE: self.driver.set_window_size(*self.WINDOW_SIZE)
def start_client(self): capabilities = {} for c in self.capabilities: name, value = c.split(':') # handle integer capabilities if value.isdigit(): value = int(value) # handle boolean capabilities elif value.lower() in ['true', 'false']: value = value.lower() == 'true' capabilities.update({name: value}) if self.proxy_host and self.proxy_port: proxy = Proxy() proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port) proxy.ssl_proxy = proxy.http_proxy proxy.add_to_capabilities(capabilities) profile = None if self.driver.upper() == 'REMOTE': capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper())) if json.loads(self.chrome_options) or self.extension_paths: capabilities = self.create_chrome_options( self.chrome_options, self.extension_paths).to_capabilities() if self.browser_name.upper() == 'FIREFOX': profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) if self.browser_version: capabilities['version'] = self.browser_version capabilities['platform'] = self.platform.upper() executor = 'http://%s:%s/wd/hub' % (self.host, self.port) try: self.selenium = webdriver.Remote(command_executor=executor, desired_capabilities=capabilities or None, browser_profile=profile) except AttributeError: valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')] raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers))) elif self.driver.upper() == 'CHROME': options = None if self.chrome_options or self.extension_paths: options = self.create_chrome_options( self.chrome_options, self.extension_paths) if self.chrome_path: self.selenium = webdriver.Chrome(executable_path=self.chrome_path, chrome_options=options, desired_capabilities=capabilities or None) else: self.selenium = webdriver.Chrome(chrome_options=options, desired_capabilities=capabilities or None) elif self.driver.upper() == 'FIREFOX': binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) self.selenium = webdriver.Firefox( firefox_binary=binary, firefox_profile=profile, capabilities=capabilities or None) elif self.driver.upper() == 'IE': self.selenium = webdriver.Ie() elif self.driver.upper() == 'PHANTOMJS': self.selenium = webdriver.PhantomJS() elif self.driver.upper() == 'OPERA': capabilities.update(webdriver.DesiredCapabilities.OPERA) self.selenium = webdriver.Opera(executable_path=self.opera_path, desired_capabilities=capabilities) elif self.driver.upper() == 'BROWSERSTACK': from cloud import BrowserStack self.cloud = BrowserStack() self.selenium = self.cloud.driver( self.test_id, capabilities, self.options) elif self.driver.upper() == 'SAUCELABS': from cloud import SauceLabs self.cloud = SauceLabs() self.selenium = self.cloud.driver( self.test_id, capabilities, self.options, self.keywords) else: self.selenium = getattr(webdriver, self.driver)() if self.event_listener is not None and not isinstance(self.selenium, EventFiringWebDriver): self.selenium = EventFiringWebDriver(self.selenium, self.event_listener())
def start_webdriver_client(self): capabilities = {} if self.capabilities: capabilities.update(json.loads(self.capabilities)) if self.proxy_host and self.proxy_port: proxy = Proxy() proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port) proxy.ssl_proxy = proxy.http_proxy proxy.add_to_capabilities(capabilities) profile = None if self.driver.upper() == 'REMOTE': capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper())) if json.loads(self.chrome_options) or self.extension_paths: capabilities = self.create_chrome_options( self.chrome_options, self.extension_paths).to_capabilities() if self.browser_name.upper() == 'FIREFOX': profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) if self.browser_version: capabilities['version'] = self.browser_version capabilities['platform'] = self.platform.upper() executor = 'http://%s:%s/wd/hub' % (self.host, self.port) try: self.selenium = webdriver.Remote(command_executor=executor, desired_capabilities=capabilities or None, browser_profile=profile) except AttributeError: valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')] raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers))) elif self.driver.upper() == 'CHROME': options = None if self.chrome_options or self.extension_paths: options = self.create_chrome_options( self.chrome_options, self.extension_paths) if self.chrome_path: self.selenium = webdriver.Chrome(executable_path=self.chrome_path, chrome_options=options, desired_capabilities=capabilities or None) else: self.selenium = webdriver.Chrome(chrome_options=options, desired_capabilities=capabilities or None) elif self.driver.upper() == 'FIREFOX': binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) self.selenium = webdriver.Firefox( firefox_binary=binary, firefox_profile=profile, capabilities=capabilities or None) elif self.driver.upper() == 'IE': self.selenium = webdriver.Ie() elif self.driver.upper() == 'OPERA': capabilities.update(webdriver.DesiredCapabilities.OPERA) self.selenium = webdriver.Opera(executable_path=self.opera_path, desired_capabilities=capabilities) else: self.selenium = getattr(webdriver, self.driver)()