def get_phantomjs_browser( self, proxy=None, timeout=15, ): """ 创建一个phantomjs浏览器 :param proxy: String "ip:port" :param timeout: Int :return: Phantomjs.Browser 浏览器对象 """ capabilities = webdriver.DesiredCapabilities.PHANTOMJS capabilities['phantomjs.page.settings.userAgent'] = random.choice( USER_AGENTS) capabilities["phantomjs.page.settings.loadImages"] = False if proxy: prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy prox.socks_proxy = proxy prox.ssl_proxy = proxy prox.add_to_capabilities(capabilities) browser = webdriver.PhantomJS(desired_capabilities=capabilities) browser.maximize_window() browser.set_page_load_timeout(timeout) return browser
def proxy_driver(PROXIES, co=co): prox = Proxy() ua = UserAgent() while True: if PROXIES: pxy = PROXIES[-1] break else: print("--- Proxies used up (%s)" % len(PROXIES)) PROXIES = get_proxies() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = pxy #prox.socks_proxy = pxy prox.ssl_proxy = pxy capabilities = dict(DesiredCapabilities.CHROME) capabilities["chrome.page.settings.userAgent"] = (ua.random) prox.add_to_capabilities(capabilities) service_args = ['--ssl-protocol=any', '--ignore-ssl-errors=true'] driver = webdriver.Chrome("chromedriver.exe", options=co, desired_capabilities=capabilities, service_args=service_args) return driver
def proxied_driver(addresses, driver_type="chrome", co=None): assert len(addresses) > 0, "At least one proxy address must be provided" prox = Proxy() prox.proxy_type = ProxyType.MANUAL addr = random.choice(addresses) prox.http_proxy = addr prox.ssl_proxy = addr assert driver_type in ["chrome", "firefox"], "proxy_type must be chrome or firefox " if driver_type == "chrome": capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) driver = webdriver.Chrome(chrome_options=co, desired_capabilities=capabilities) # 참고로 이런 .method의 번거로움이 python 단점 중 하나. 간단한 설정이어야 하는게 이상하게 구성하게됨 elif driver_type == "firefox": capabilities = DesiredCapabilities.FIREFOX prox.add_to_capabilities(capabilities) driver = webdriver.Firefox( firefox_options=co, desired_capabilities=capabilities, executable_path="/usr/local/bin/geckodriver") return driver
def test_manual_proxy_is_set_in_profile(self): # The setup gave us a browser but we dont need it self.driver.quit() self.profile = webdriver.FirefoxProfile() proxy = Proxy() proxy.no_proxy = 'localhost, foo.localhost' proxy.http_proxy = 'some.url:1234' proxy.ftp_proxy = None proxy.sslProxy = 'some2.url' self.profile.set_proxy(proxy) assert self.profile.default_preferences[ "network.proxy.type"] == ProxyType.MANUAL['ff_value'] assert self.profile.default_preferences[ "network.proxy.no_proxies_on"] == 'localhost, foo.localhost' assert self.profile.default_preferences[ "network.proxy.http"] == 'some.url' assert self.profile.default_preferences[ "network.proxy.http_port"] == 1234 assert self.profile.default_preferences[ "network.proxy.ssl"] == 'some2.url' assert "network.proxy.ssl_port" not in self.profile.default_preferences assert "network.proxy.ftp" not in self.profile.default_preferences
def __get_driver_desired_capabilities(self): prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "103.109.58.245:8080" capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) return capabilities
def __init__(self, email, password, category): super().__init__() self.email = email self.password = password self.category = category self.count = 1 logging.basicConfig( handlers=[logging.FileHandler('./Logs/scraper.log', 'w', 'utf-8')], format=': %(asctime)s : %(levelname)s : %(message)s : ', ) self.logger = logging.getLogger() self.logger.setLevel(logging.INFO) self.links = link_generator.generate(self.category) self.proxy = proxy_generator.get_proxy() self.scrapedData = [] try: prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = self.proxy capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) self.driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_settings, desired_capabilities=capabilities) except Exception as e: self.logger.critical("Driver Error: " + str(e))
def create_instances(self): for _proxy in self.proxies: proxy = _proxy.proxy service_args = [] # service_args.append('--proxy={}:{}'.format(proxy.ip, proxy.port)) if proxy.username and proxy.password: service_args.append('--proxy-auth={}:{}'.format( proxy.username, proxy.password)) capabilities = DesiredCapabilities.PHANTOMJS capabilities[ 'phantomjs.page.settings.resourceTimeout'] = self.max_delay_limit * 1000 _proxy_ = Proxy() _proxy_.proxy_type = ProxyType.MANUAL _proxy_.http_proxy = '{}:{}'.format(proxy.ip, proxy.port) _proxy_.socks_proxy = '{}:{}'.format(proxy.ip, proxy.port) _proxy_.ssl_proxy = '{}:{}'.format(proxy.ip, proxy.port) _proxy_.add_to_capabilities(capabilities) driver = webdriver.PhantomJS( service_args=service_args, desired_capabilities=capabilities, service_log_path='/tmp/ghostdriver.log') driver.set_window_size(1120, 1080) driver.set_page_load_timeout(self.max_delay_limit) self.multi_instances.append(driver)
def load_page(source): prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "127.0.0.1:9090" prox.socks_proxy = "127.0.0.1:9090" prox.ssl_proxy = "127.0.0.1:9090" capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) driver = webdriver.Chrome(desired_capabilities=capabilities) driver.get(source) navigationStart = driver.execute_script( "return window.performance.timing.navigationStart") responseStart = driver.execute_script( "return window.performance.timing.responseStart") domComplete = driver.execute_script( "return window.performance.timing.domComplete") backendPerformance = responseStart - navigationStart frontendPerformance = domComplete - responseStart print "Back End: %s" % backendPerformance print "Front End: %s" % frontendPerformance driver.quit()
def __init__(self, songUrl, proxy_url): self.songUrl = songUrl self.id = -1 self.name = '' self.album_id = -1 self.comments_num = -1 self.similar_song_ids = [] self.artists = [] chrome_options = Options() prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy_url prox.ssl_proxy = proxy_url # prox.socks_proxy = proxy_url capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) chrome_options.add_argument('--headless') # chrome_options.add_argument('user-agent={0}'.format(random.choice(uas))) # chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2}) if config_is_ubuntu: chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') # chrome_options.add_argument('--proxy-server=http://111.222.141.127:8118') # chrome_options.add_argument('--proxy-server={}'.format(proxy_url)) # chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # debug_print_thread("we are using proxy sever with url " + proxy_url) # chrome_options.add_argument('--proxy-server=http://114.98.27.147:4216') self.driver = webdriver.Chrome(config_chrome_path, options=chrome_options)
def get_browser(): # @todo add windows path if platform.system() == "Darwin": os.environ["webdriver.chrome.driver"] = os.path.expanduser( "~" ) + '/Library/Application Support/ZAP/webdriver/macos/64/chromedriver' else: os.environ["webdriver.chrome.driver"] = os.path.expanduser( "~") + '/.ZAP/webdriver/linux/64/chromedriver' proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = "127.0.0.1:8080" proxy.socks_proxy = "127.0.0.1:8080" proxy.ssl_proxy = "127.0.0.1:8080" capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) options = webdriver.ChromeOptions() options.add_argument('--ignore-certificate-errors') options.add_argument("--test-type") return webdriver.Chrome( executable_path=os.environ["webdriver.chrome.driver"], chrome_options=options, desired_capabilities=capabilities)
def get_capabilities(self): capabilities = DesiredCapabilities.FIREFOX.copy() # capabilities = DesiredCapabilities.CHROME.copy() # print('get_capabilities') proxy = self.profile.proxy if proxy is None or not proxy.active: # use random default proxy proxies = models.Proxy.objects.filter(active=True, default=True).all() if proxies: proxy = random.choice(proxies) else: proxy = None if proxy: # print('setting a proxy') # print(proxy) prox = Proxy() prox.proxy_type = ProxyType.MANUAL if proxy.proxy_type == proxy.HTTP: # print('HTTP proxy') prox.http_proxy = f'{proxy.ip}:{proxy.port}' prox.ssl_proxy = f'{proxy.ip}:{proxy.port}' prox.ftp_proxy = f'{proxy.ip}:{proxy.port}' elif proxy.proxy_type == proxy.SOCKS: # print('Socks proxy') prox.socks_proxy = f'{proxy.ip}:{proxy.port}' prox.socks_username = proxy.username prox.socks_password = proxy.password prox.add_to_capabilities(capabilities) # print(capabilities) return capabilities
def _start_proxy_(self, ip: str, port: int): prox = Proxy() endpoint = ip + ":" + str(port) prox.proxy_type = ProxyType.MANUAL prox.http_proxy = endpoint prox.socks_proxy = endpoint prox.ssl_proxy = endpoint
def getGoogleChromeDriver(fullproxy): try: proxy = fullproxy.split(' ')[0] conditionalPrint("proxy used : " + proxy) WINDOW_SIZE = "1920,1080" option = webdriver.ChromeOptions() option.add_argument("--incognito") #option.add_argument("--disable-gpu") #option.add_argument("--disable-infobars") #option.add_argument("--disable-notifications") #option.add_argument("--disable-extensions") if hideBrowser == "YES": option.add_argument("--headless") option.add_argument("--window-size=%s" % WINDOW_SIZE) prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy prox.socks_proxy = proxy prox.ssl_proxy = proxy capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) browser = webdriver.Chrome( executable_path="C:\\webdrivers\\chromedriver.exe", chrome_options=option, desired_capabilities=capabilities) #browser.set_window_position(-10000, 0) return browser except Exception: LogError(traceback, "fullproxy = " + fullproxy) return None
def initialize_browser() -> object: chrome_options = Options() chrome_options.add_argument('--window-size=1920x1080') chrome_options.add_argument('--ignore-certificate-errors') chrome_options.add_argument( 'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 ' 'Safari/537.36') proxy_url = "127.0.0.1:24001" proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = proxy_url proxy.ssl_proxy = proxy_url capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) if is_development(): return webdriver.Chrome('./bin/chromedriver.exe', chrome_options=chrome_options, desired_capabilities=capabilities) else: chrome_options.add_argument('--data-path=/tmp/data-path') chrome_options.add_argument('--homedir=/tmp') chrome_options.add_argument('--disk-cache-dir=/tmp/cache-dir') chrome_options.add_argument('--user-data-dir=/tmp/user-data') chrome_options.add_argument('--hide-scrollbars') chrome_options.add_argument('--enable-logging') chrome_options.add_argument('--log-level=0') chrome_options.add_argument('--v=99') chrome_options.add_argument('--single-process') chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--headless') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-gpu') return webdriver.Chrome(chrome_options=chrome_options, desired_capabilities=capabilities)
def __open_browser(context): chrm = context.config.userdata['chromedriver_path'] try: # if there is a proxy, we'll use it. Otherwise, we won't. requests.get("http://localhost:8888", timeout=0.01) # if there was no exception, we continue here. PROXY = "localhost:8888" proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = PROXY capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) if (chrm): context.driver = webdriver.Chrome( desired_capabilities=capabilities, executable_path=chrm) else: context.driver = webdriver.Chrome( desired_capabilities=capabilities) return context.driver except: if (chrm): context.driver = webdriver.Chrome(executable_path=chrm) else: context.driver = webdriver.Chrome() return context.driver
def get_webdriver(): options = webdriver.ChromeOptions() options.add_argument('window-size=1920x1080') options.add_argument('--headless') options.add_argument('--no-sandbox') # Configure Proxy option proxy = Proxy() if Settings.socks_proxy is None and Settings.https_proxy is None and Settings.https_proxy is None: proxy.proxy_type = ProxyType.DIRECT else: proxy.proxy_type = ProxyType.MANUAL if Settings.socks_proxy is not None: options.add_argument("--proxy-server=" + Settings.get_full_socks_proxy()) else: if Settings.http_proxy is not None: proxy.http_proxy = Settings.http_proxy if Settings.https_proxy is not None: proxy.ssl_proxy = Settings.https_proxy # Configure capabilities capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) return webdriver.Chrome(options=options, desired_capabilities=capabilities)
def get_driver(proxy=False, login=False): if proxy: #set up proxy prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "10.211.55.4:808" capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) else: capabilities = webdriver.DesiredCapabilities.CHROME chrome_options = webdriver.ChromeOptions() #chrome_options.add_argument('--headless') #chrome_options.add_argument('--disable-gpu') chrome_options.add_argument( '--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36' ) chrome_options.add_argument('--window-size=1024,768') driver = webdriver.Chrome('./chromedriver', chrome_options=chrome_options, desired_capabilities=capabilities) if login: driver.get(login_url) name = driver.find_element_by_xpath(xpaths.login_username) name.send_keys(username) pw = driver.find_element_by_xpath(xpaths.login_password) pw.send_keys(password) login = driver.find_element_by_xpath(xpaths.login_btn) login.click() return driver
def get_browser(PROXY): import os luminati_host = os.environ.get('LUMINATI_HOST') luminati_port = os.environ.get('LUMINATI_PORT') PROXY = 'http://' + luminati_host + ':' + luminati_port print(PROXY) proxy = Proxy() proxy.http_proxy = PROXY proxy.ftp_proxy = PROXY proxy.sslProxy = PROXY proxy.no_proxy = "localhost" #etc... ;) proxy.proxy_type = ProxyType.MANUAL capabilities = webdriver.DesiredCapabilities.CHROME proxy.add_to_capabilities(capabilities) # path = '/home/balu/balu/work/Courses/luminati+selinium/testapp/app/helper_files/chromedriver' # driver = webdriver.Chrome(executable_path = path, desired_capabilities=capabilities) driver = webdriver.Remote("http://172.20.128.1:4444/wd/hub", desired_capabilities=capabilities) url = 'https://lumtest.com/myip.json' driver.get(url) print(driver.page_source) return driver
def main(): """ This is the entry for the command which makes it convenient to install the proxy certificate """ commandArgs = sys.argv[1:] proxyPort = findFreePort() proxyThread = threading.Thread(target=runProxy, args=[proxyPort], daemon=True) proxyThread.start() capabilities = webdriver.DesiredCapabilities.CHROME capabilities['loggingPrefs'] = {'browser': 'ALL'} proxyConfig = Proxy() proxyConfig.proxy_type = ProxyType.MANUAL proxyConfig.http_proxy = f"localhost:{proxyPort}" proxyConfig.add_to_capabilities(capabilities) driver = webdriver.Chrome(desired_capabilities=capabilities) driver.get("http://mitm.it/") print( "Please kill the command with Ctrl-C or (Cmd-C on macOS) when you are finished installing the certificates. Timeout in 600 seconds..." ) timeout = 600 if len(commandArgs) > 0: timeout = int(str(commandArgs[0])) time.sleep(timeout)
def _add_proxy_support(self, capabilities): if "RX_PROXY" in os.environ: proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = os.environ.get("RX_PROXY") proxy.socks_proxy = os.environ.get("RX_PROXY") proxy.ssl_proxy = os.environ.get("RX_PROXY") proxy.add_to_capabilities(capabilities)
def setup_proxy(proxy): proxy_config = Proxy() proxy_config.proxy_type = ProxyType.MANUAL proxy_config.http_proxy = proxy proxy_config.ssl_proxy = proxy capabilities = webdriver.DesiredCapabilities.CHROME proxy_config.add_to_capabilities(capabilities) return proxy_config
def set_selenium_proxy(self, selenium_proxy): proxy = Proxy() proxy.http_proxy = selenium_proxy proxy.ftp_proxy = selenium_proxy proxy.sslProxy = selenium_proxy proxy.no_proxy = None proxy.proxy_type = ProxyType.MANUAL proxy.add_to_capabilities(self.capabilities) self.capabilities["acceptSslCerts"] = True
def crawl(minPage,maxPage,outputFormat): # set selenium proxy prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = "http://*****:*****@class='agree-button']")) driver.execute_script("arguments[0].click();",cookies_modal) time.sleep(random.randint(5, 10)) #crawl for page in range(minPage,maxPage+1): url = "https://musescore.com/hub/piano?page=" + str(page) print("crawling page: "+ str(page)) #clear if no of pages is too big if page % 5 == 0: subprocess.call("killall -HUP tor", shell = True) time.sleep(1) for index in range(1,21): #select score from grid driver.get(url) # go back to grid of scores score_list = driver.find_elements_by_xpath("//h2[@class='score-info__title']") try: score_list[index].click() #download driver.execute_script(open("musescore-downloader.js").read()) #try twice, if still error move on download_button = WebDriverWait(driver, 10).until( lambda x: x.find_elements_by_xpath("//button[@class='_3L7Ul _3qfU_ _38TLP _3A7i9 _2XPrY _13O-4 _15kzJ']")) #XML if outputFormat == 'XML': download_button[2].click() #MIDI elif outputFormat == 'MIDI': download_button[3].click() else: print("Incorrect usage! please type:" + "musescore-crawler.py -f <format> -s <minPage> -e <maxPage>") sys.exit(2) time.sleep(random.randint(10,30)) time.sleep(5) except: print("Failed opening score, move on to next")
def login(self, headless=True, wait_time=15, proxy=None): """ Logs user in with the provided credentials User session is stored in the 'cred_root' folder and reused so there is no need to login every time. Pinterest sessions lasts for about 15 days Ideally you need to call this method 3-4 times a month at most. :return python dict object describing the pinterest response """ chrome_options = Options() if headless: chrome_options.add_argument("--headless") if proxy is not None: http_proxy = Proxy() http_proxy.proxy_type = ProxyType.MANUAL http_proxy.http_proxy = proxy http_proxy.socks_proxy = proxy http_proxy.ssl_proxy = proxy http_proxy.add_to_capabilities(chrome_options) chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options) driver.get("https://pinterest.com/login") try: WebDriverWait(driver, wait_time).until( EC.element_to_be_clickable((By.ID, 'email'))) driver.find_element_by_id("email").send_keys(self.email) driver.find_element_by_id("password").send_keys(self.password) logins = driver.find_elements_by_xpath( "//*[contains(text(), 'Log in')]") for login in logins: login.click() WebDriverWait(driver, wait_time).until( EC.invisibility_of_element((By.ID, 'email'))) cookies = driver.get_cookies() self.http.cookies.clear() for cookie in cookies: self.http.cookies.set(cookie['name'], cookie['value']) self.registry.update_all(self.http.cookies.get_dict()) except Exception as e: print("Failed to login", e) driver.close()
def test_sets_http_proxy(self): self.driver.quit() profile = webdriver.FirefoxProfile() proxy = Proxy() proxy.http_proxy = 'http://test.hostname:1234' profile.set_proxy(proxy) assert profile.default_preferences["network.proxy.type"] == str(ProxyType.MANUAL['ff_value']) assert profile.default_preferences["network.proxy.http"] == '"test.hostname"' assert profile.default_preferences["network.proxy.http_port"] == '1234'
def get_proxy(self): proxy_spider_dict = ProxySpider().proxy[0] proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = "%(ip)s:%(port)s" % proxy_spider_dict if 'HTTPS' in proxy_spider_dict.get('proxy_type'): proxy.ssl_proxy = "%(ip)s:%(port)s" % proxy_spider_dict print("使用代理", proxy_spider_dict) return proxy
def get_browser(): """start and quit the browser with proxy""" prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxy.PROXIES capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) browser = webdriver.Firefox(desired_capabilities=capabilities, executable_path=path) wait = WebDriverWait(browser, 30) return browser, wait
def get_proxy(proxies): print(proxies) prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = proxies prox.ftp_proxy = proxies prox.ssl_proxy = proxies capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) return capabilities
def setup_capabilities(): capabilities = DesiredCapabilities.PHANTOMJS.copy() proxy = Proxy() proxy.proxy_type = ProxyType.MANUAL proxy.http_proxy = '1.0.0.25:80' # prox.socks_proxy = "ip_addr:port" # prox.ssl_proxy = "ip_addr:port" proxy.add_to_capabilities(capabilities) return capabilities
def proxy_driver(PROXIES): prox = Proxy() prox.proxy_type = ProxyType.MANUAL prox.http_proxy = PROXIES prox.ssl_proxy = PROXIES capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) driver = webdriver.Chrome(PATH, options=chrome_options, desired_capabilities=capabilities) return driver
def installProxy(self, PROXY_HOST, PROXY_PORT): prox = Proxy() prox.proxy_type = ProxyType.MANUAL proxstr = PROXY_HOST + ":" + PROXY_PORT prox.http_proxy = proxstr prox.socks_proxy = proxstr prox.ssl_proxy = proxstr capabilities = webdriver.DesiredCapabilities.CHROME prox.add_to_capabilities(capabilities) return webdriver.Chrome()
def testCanAddToDesiredCapabilities(self): desired_capabilities = {} proxy = Proxy() proxy.http_proxy = 'some.url:1234' proxy.add_to_capabilities(desired_capabilities) expected_capabilities = { 'proxy': { 'proxyType': 'manual', 'httpProxy': 'some.url:1234' } } self.assertEqual(expected_capabilities, desired_capabilities)
def test_manual_proxy_is_set_in_profile(): profile = FirefoxProfile() proxy = Proxy() proxy.no_proxy = 'localhost, foo.localhost' proxy.http_proxy = 'some.url:1234' proxy.ftp_proxy = None proxy.sslProxy = 'some2.url' profile.set_proxy(proxy) assert profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value'] assert profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost' assert profile.default_preferences["network.proxy.http"] == 'some.url' assert profile.default_preferences["network.proxy.http_port"] == 1234 assert profile.default_preferences["network.proxy.ssl"] == 'some2.url' assert "network.proxy.ssl_port" not in profile.default_preferences assert "network.proxy.ftp" not in profile.default_preferences
def testCanAddManualProxyToDesiredCapabilities(self): proxy = Proxy() proxy.http_proxy = self.MANUAL_PROXY["httpProxy"] proxy.ftp_proxy = self.MANUAL_PROXY["ftpProxy"] proxy.no_proxy = self.MANUAL_PROXY["noProxy"] proxy.sslProxy = self.MANUAL_PROXY["sslProxy"] proxy.socksProxy = self.MANUAL_PROXY["socksProxy"] proxy.socksUsername = self.MANUAL_PROXY["socksUsername"] proxy.socksPassword = self.MANUAL_PROXY["socksPassword"] desired_capabilities = {} proxy.add_to_capabilities(desired_capabilities) proxy_capabilities = self.MANUAL_PROXY.copy() proxy_capabilities["proxyType"] = "MANUAL" expected_capabilities = {"proxy": proxy_capabilities} self.assertEqual(expected_capabilities, desired_capabilities)
def testCanAddManualProxyToDesiredCapabilities(self): proxy = Proxy() proxy.http_proxy = self.MANUAL_PROXY['httpProxy'] proxy.ftp_proxy = self.MANUAL_PROXY['ftpProxy'] proxy.no_proxy = self.MANUAL_PROXY['noProxy'] proxy.sslProxy = self.MANUAL_PROXY['sslProxy'] proxy.socksProxy = self.MANUAL_PROXY['socksProxy'] proxy.socksUsername = self.MANUAL_PROXY['socksUsername'] proxy.socksPassword = self.MANUAL_PROXY['socksPassword'] desired_capabilities = {} proxy.add_to_capabilities(desired_capabilities) proxy_capabilities = self.MANUAL_PROXY.copy() proxy_capabilities['proxyType'] = 'MANUAL' expected_capabilities = {'proxy': proxy_capabilities} self.assertEqual(expected_capabilities, desired_capabilities)
def test_manual_proxy_is_set_in_profile(self): # The setup gave us a browser but we dont need it self.driver.quit() self.profile = webdriver.FirefoxProfile() proxy = Proxy() proxy.no_proxy = 'localhost, foo.localhost' proxy.http_proxy = 'some.url:1234' proxy.ftp_proxy = None proxy.sslProxy = 'some2.url' self.profile.set_proxy(proxy) assert self.profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value'] assert self.profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost' assert self.profile.default_preferences["network.proxy.http"] == 'some.url' assert self.profile.default_preferences["network.proxy.http_port"] == 1234 assert self.profile.default_preferences["network.proxy.ssl"] == 'some2.url' assert "network.proxy.ssl_port" not in self.profile.default_preferences assert "network.proxy.ftp" not in self.profile.default_preferences
def _setup_driver(self): proxy = Proxy() proxy.proxy_type = ProxyType.DIRECT if 'http' in self.proxy: proxy.http_proxy = self.proxy['http'] if 'https' in self.proxy: proxy.ssl_proxy = self.proxy['https'] capa = self._build_capabilities() proxy.add_to_capabilities(capa) options = self._build_options() # TODO some browsers don't need headless # TODO handle different proxy setting? options.set_headless(self.HEADLESS) if self.DRIVER is webdriver.Firefox: if self.responses_dirname and not os.path.isdir(self.responses_dirname): os.makedirs(self.responses_dirname) options.profile = DirFirefoxProfile(self.responses_dirname) if self.responses_dirname: capa['profile'] = self.responses_dirname self.driver = self.DRIVER(options=options, capabilities=capa) elif self.DRIVER is webdriver.Chrome: self.driver = self.DRIVER(options=options, desired_capabilities=capa) elif self.DRIVER is webdriver.PhantomJS: if self.responses_dirname: if not os.path.isdir(self.responses_dirname): os.makedirs(self.responses_dirname) log_path = os.path.join(self.responses_dirname, 'selenium.log') else: log_path = NamedTemporaryFile(prefix='weboob_selenium_', suffix='.log', delete=False).name self.driver = self.DRIVER(desired_capabilities=capa, service_log_path=log_path) else: raise NotImplementedError() if self.WINDOW_SIZE: self.driver.set_window_size(*self.WINDOW_SIZE)
def testCanAddToDesiredCapabilities(self): desired_capabilities = {} proxy = Proxy() proxy.http_proxy = 'some.url:1234' proxy.ftp_proxy = 'ftp.proxy:1234' proxy.no_proxy = 'localhost, foo.localhost' proxy.sslProxy = 'ssl.proxy:1234' proxy.autodetect = 'True' proxy.add_to_capabilities(desired_capabilities) expected_capabilities = { 'proxy': { 'proxyType': 'MANUAL', 'httpProxy': 'some.url:1234', 'ftpProxy': 'ftp.proxy:1234', 'noProxy': 'localhost, foo.localhost', 'sslProxy': 'ssl.proxy:1234', 'autodetect': 'True' } } print 'descap', desired_capabilities self.assertEqual(expected_capabilities, desired_capabilities)
def start_client(self): capabilities = {} for c in self.capabilities: name, value = c.split(':') # handle integer capabilities if value.isdigit(): value = int(value) # handle boolean capabilities elif value.lower() in ['true', 'false']: value = value.lower() == 'true' capabilities.update({name: value}) if self.proxy_host and self.proxy_port: proxy = Proxy() proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port) proxy.ssl_proxy = proxy.http_proxy proxy.add_to_capabilities(capabilities) profile = None if self.driver.upper() == 'REMOTE': capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper())) if json.loads(self.chrome_options) or self.extension_paths: capabilities = self.create_chrome_options( self.chrome_options, self.extension_paths).to_capabilities() if self.browser_name.upper() == 'FIREFOX': profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) if self.browser_version: capabilities['version'] = self.browser_version capabilities['platform'] = self.platform.upper() executor = 'http://%s:%s/wd/hub' % (self.host, self.port) try: self.selenium = webdriver.Remote(command_executor=executor, desired_capabilities=capabilities or None, browser_profile=profile) except AttributeError: valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')] raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers))) elif self.driver.upper() == 'CHROME': options = None if self.chrome_options or self.extension_paths: options = self.create_chrome_options( self.chrome_options, self.extension_paths) if self.chrome_path: self.selenium = webdriver.Chrome(executable_path=self.chrome_path, chrome_options=options, desired_capabilities=capabilities or None) else: self.selenium = webdriver.Chrome(chrome_options=options, desired_capabilities=capabilities or None) elif self.driver.upper() == 'FIREFOX': binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) self.selenium = webdriver.Firefox( firefox_binary=binary, firefox_profile=profile, capabilities=capabilities or None) elif self.driver.upper() == 'IE': self.selenium = webdriver.Ie() elif self.driver.upper() == 'PHANTOMJS': self.selenium = webdriver.PhantomJS() elif self.driver.upper() == 'OPERA': capabilities.update(webdriver.DesiredCapabilities.OPERA) self.selenium = webdriver.Opera(executable_path=self.opera_path, desired_capabilities=capabilities) elif self.driver.upper() == 'BROWSERSTACK': from cloud import BrowserStack self.cloud = BrowserStack() self.selenium = self.cloud.driver( self.test_id, capabilities, self.options) elif self.driver.upper() == 'SAUCELABS': from cloud import SauceLabs self.cloud = SauceLabs() self.selenium = self.cloud.driver( self.test_id, capabilities, self.options, self.keywords) else: self.selenium = getattr(webdriver, self.driver)() if self.event_listener is not None and not isinstance(self.selenium, EventFiringWebDriver): self.selenium = EventFiringWebDriver(self.selenium, self.event_listener())
def start_webdriver_client(self): capabilities = {} if self.capabilities: capabilities.update(json.loads(self.capabilities)) if self.proxy_host and self.proxy_port: proxy = Proxy() proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port) proxy.ssl_proxy = proxy.http_proxy proxy.add_to_capabilities(capabilities) profile = None if self.driver.upper() == 'REMOTE': capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper())) if json.loads(self.chrome_options) or self.extension_paths: capabilities = self.create_chrome_options( self.chrome_options, self.extension_paths).to_capabilities() if self.browser_name.upper() == 'FIREFOX': profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) if self.browser_version: capabilities['version'] = self.browser_version capabilities['platform'] = self.platform.upper() executor = 'http://%s:%s/wd/hub' % (self.host, self.port) try: self.selenium = webdriver.Remote(command_executor=executor, desired_capabilities=capabilities or None, browser_profile=profile) except AttributeError: valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')] raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers))) elif self.driver.upper() == 'CHROME': options = None if self.chrome_options or self.extension_paths: options = self.create_chrome_options( self.chrome_options, self.extension_paths) if self.chrome_path: self.selenium = webdriver.Chrome(executable_path=self.chrome_path, chrome_options=options, desired_capabilities=capabilities or None) else: self.selenium = webdriver.Chrome(chrome_options=options, desired_capabilities=capabilities or None) elif self.driver.upper() == 'FIREFOX': binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None profile = self.create_firefox_profile( self.firefox_preferences, self.profile_path, self.extension_paths) self.selenium = webdriver.Firefox( firefox_binary=binary, firefox_profile=profile, capabilities=capabilities or None) elif self.driver.upper() == 'IE': self.selenium = webdriver.Ie() elif self.driver.upper() == 'OPERA': capabilities.update(webdriver.DesiredCapabilities.OPERA) self.selenium = webdriver.Opera(executable_path=self.opera_path, desired_capabilities=capabilities) else: self.selenium = getattr(webdriver, self.driver)()