def getDriver(self, driverType='firefox'):
try:
if driverType == 'phantomjs':
#capabilities = webdriver.DesiredCapabilities.PHANTOMJS.copy()
#capabilities['platform'] = "LINUX"
driver = webdriver.PhantomJS('/usr/local/share/phantomjs/bin/phantomjs')
else:
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
self.dis = Display(visible=0,size=(800,600))
self.dis.start()
prx = Proxy()
prx.httpProxy= '199.200.120.36:8089' #'112.196.3.254:9064' india
prx.proxyType = ProxyType.MANUAL
prof = FirefoxProfile('./ffprofile')
driver = webdriver.Firefox(proxy=prx, firefox_profile=prof)
driver.implicitly_wait(30)
driver.set_page_load_timeout(30)
print 'started the driver successfully...'
return driver
except Exception as e:
print 'error has occured while starting the driver....'
print type(e)
print e
self.quit()
#sys.exit(0)
return None
def initialize(driver_url):
if len(Config.DESIRED_CAPABILITIES) == 0:
options = webdriver.ChromeOptions()
options.add_experimental_option('prefs',
{'intl.accept_languages': 'ja_JP'})
cap = options.to_capabilities()
else:
cap = {}
for k, v in [cap.split("=") for cap in Config.DESIRED_CAPABILITIES]:
k = k.strip("\"'")
v = maybe_bool(v.strip("\"'"))
merge(cap, construct_dict(k, v))
if Config.HTTP_PROXY or Config.HTTPS_PROXY or Config.NO_PROXY:
proxy = Proxy()
proxy.sslProxy = Config.HTTPS_PROXY
proxy.httpProxy = Config.HTTP_PROXY
proxy.noProxy = Config.NO_PROXY
proxy.proxyType = ProxyType.MANUAL
proxy.add_to_capabilities(cap)
driver = webdriver.Remote(command_executor=driver_url,
desired_capabilities=cap)
return driver
def init_env():
codes = ['aftes110:', 'aftes200:', 'aftes300:', 'aftes500:', 'aftes750C:']
code = codes[4]
print('Hotels.com Generator')
PROXY = '3.14.8.17:3128'
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = PROXY
chrome_options.Proxy = proxy
chrome_options.add_argument("ignore-certificate-errors")
driver = get_driver()
return driver, code
def configure_driver(self, ip_address):
options = webdriver.ChromeOptions(
) # add options to chrome such as hide mode, change user settings, etc.
options.add_argument('headless') # hide chrome mode while scraping
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = ip_address
options.Proxy = proxy
options.add_argument("ignore-certificate-errors")
driver = webdriver.Chrome(self.__filepath_chrome,
chrome_options=options)
return (driver)
def setupSelenium(self):
"""Set up connection with the (remote) selenium server"""
self.debug('Setting up selenium')
desired_capabilities = DesiredCapabilities.FIREFOX.copy()
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.httpProxy = "localhost:8085"
proxy.sslProxy = "localhost:8085"
proxy.add_to_capabilities(desired_capabilities)
driver = webdriver.Remote(
command_executor='http://127.0.0.1:4444/wd/hub',
desired_capabilities=desired_capabilities)
return driver
def __activate_proxy(self, use_proxy):
if use_proxy:
# Initialize the proxy
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
# Load a random proxy
self.proxy = choice(PROXIES)
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = self.proxy
# Apply it to self.options
self.options.Proxy = proxy
self.options.add_argument("ignore-certificate-errors")
def initialize(driver_url):
options = webdriver.ChromeOptions()
options.add_experimental_option('prefs', {'intl.accept_languages': 'ja_JP'})
cap = options.to_capabilities()
if Config.HTTP_PROXY or Config.HTTPS_PROXY or Config.NO_PROXY:
proxy = Proxy()
proxy.sslProxy = Config.HTTPS_PROXY
proxy.httpProxy = Config.HTTP_PROXY
proxy.noProxy = Config.NO_PROXY
proxy.proxyType = ProxyType.MANUAL
proxy.add_to_capabilities(cap)
driver = webdriver.Remote(
command_executor=driver_url,
desired_capabilities=cap)
return driver
def proxy_driver(PROXIES, co=co):
prox = Proxy()
if PROXIES:
pxy = PROXIES[-1]
else:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = pxy
co.Proxy = proxy
co.add_argument("ignore-certificate-errors")
driver = webdriver.Chrome(executable_path="D:/chromedriver.exe",
chrome_options=co)
return driver
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
options = webdriver.ChromeOptions()
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
from random_proxies import random_proxy
PROXY = '66.97.120.123:3128'
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = PROXY
options.Proxy = proxy
options.add_argument("ignore-certificate-errors")
driver = webdriver.Chrome('../../data/chromedriver.exe', chrome_options=options)
driver.get('https://lumtest.com/myip.json')
def set_proxy(self):
proxy = Proxy()
proxy.proxyType=ProxyType.MANUAL
proxy.http_proxy = "%s:%s" % (self.HTTP_PROXY, self.HTTP_PROXY_PORT)
proxy.no_proxy = self.NOPROXY
proxy.add_to_capabilities(self.capabilities)
