def get_phantomjs_browser(
self,
proxy=None,
timeout=15,
):
"""
创建一个phantomjs浏览器
:param proxy: String "ip:port"
:param timeout: Int
:return: Phantomjs.Browser 浏览器对象
"""
capabilities = webdriver.DesiredCapabilities.PHANTOMJS
capabilities['phantomjs.page.settings.userAgent'] = random.choice(
USER_AGENTS)
capabilities["phantomjs.page.settings.loadImages"] = False
if proxy:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
prox.add_to_capabilities(capabilities)
browser = webdriver.PhantomJS(desired_capabilities=capabilities)
browser.maximize_window()
browser.set_page_load_timeout(timeout)
return browser
def get_browser():
# @todo add windows path
if platform.system() == "Darwin":
os.environ["webdriver.chrome.driver"] = os.path.expanduser(
"~"
) + '/Library/Application Support/ZAP/webdriver/macos/64/chromedriver'
else:
os.environ["webdriver.chrome.driver"] = os.path.expanduser(
"~") + '/.ZAP/webdriver/linux/64/chromedriver'
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = "127.0.0.1:8080"
proxy.socks_proxy = "127.0.0.1:8080"
proxy.ssl_proxy = "127.0.0.1:8080"
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument("--test-type")
return webdriver.Chrome(
executable_path=os.environ["webdriver.chrome.driver"],
chrome_options=options,
desired_capabilities=capabilities)
def proxy_driver(PROXIES, co=co):
prox = Proxy()
ua = UserAgent()
while True:
if PROXIES:
pxy = PROXIES[-1]
break
else:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
#prox.socks_proxy = pxy
prox.ssl_proxy = pxy
capabilities = dict(DesiredCapabilities.CHROME)
capabilities["chrome.page.settings.userAgent"] = (ua.random)
prox.add_to_capabilities(capabilities)
service_args = ['--ssl-protocol=any', '--ignore-ssl-errors=true']
driver = webdriver.Chrome("chromedriver.exe",
options=co,
desired_capabilities=capabilities,
service_args=service_args)
return driver
def __init__(self, songUrl, proxy_url):
self.songUrl = songUrl
self.id = -1
self.name = ''
self.album_id = -1
self.comments_num = -1
self.similar_song_ids = []
self.artists = []
chrome_options = Options()
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy_url
prox.ssl_proxy = proxy_url
# prox.socks_proxy = proxy_url
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
chrome_options.add_argument('--headless')
# chrome_options.add_argument('user-agent={0}'.format(random.choice(uas)))
# chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
if config_is_ubuntu:
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
# chrome_options.add_argument('--proxy-server=http://111.222.141.127:8118')
# chrome_options.add_argument('--proxy-server={}'.format(proxy_url))
# chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
# debug_print_thread("we are using proxy sever with url " + proxy_url)
# chrome_options.add_argument('--proxy-server=http://114.98.27.147:4216')
self.driver = webdriver.Chrome(config_chrome_path,
options=chrome_options)
def __init__(self, proxy):
"""
Initialize the web driver.
"""
if proxy:
PROXY = self.get_proxy()
custom_proxy = Proxy()
custom_proxy.proxy_type = ProxyType.MANUAL
custom_proxy.ssl_proxy = PROXY
capabilities = webdriver.DesiredCapabilities.CHROME
custom_proxy.add_to_capabilities(capabilities)
self.driver = webdriver.Chrome(
ChromeDriverManager().install(),
desired_capabilities=capabilities
)
else:
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
self.driver = webdriver.Chrome(
ChromeDriverManager().install(),
chrome_options=options
)
def proxied_driver(addresses, driver_type="chrome", co=None):
assert len(addresses) > 0, "At least one proxy address must be provided"
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
addr = random.choice(addresses)
prox.http_proxy = addr
prox.ssl_proxy = addr
assert driver_type in ["chrome",
"firefox"], "proxy_type must be chrome or firefox "
if driver_type == "chrome":
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(chrome_options=co,
desired_capabilities=capabilities)
# 참고로 이런 .method의 번거로움이 python 단점 중 하나. 간단한 설정이어야 하는게 이상하게 구성하게됨
elif driver_type == "firefox":
capabilities = DesiredCapabilities.FIREFOX
prox.add_to_capabilities(capabilities)
driver = webdriver.Firefox(
firefox_options=co,
desired_capabilities=capabilities,
executable_path="/usr/local/bin/geckodriver")
return driver
def load_page(source):
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "127.0.0.1:9090"
prox.socks_proxy = "127.0.0.1:9090"
prox.ssl_proxy = "127.0.0.1:9090"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
driver.get(source)
navigationStart = driver.execute_script(
"return window.performance.timing.navigationStart")
responseStart = driver.execute_script(
"return window.performance.timing.responseStart")
domComplete = driver.execute_script(
"return window.performance.timing.domComplete")
backendPerformance = responseStart - navigationStart
frontendPerformance = domComplete - responseStart
print "Back End: %s" % backendPerformance
print "Front End: %s" % frontendPerformance
driver.quit()
def getGoogleChromeDriver(fullproxy):
try:
proxy = fullproxy.split(' ')[0]
conditionalPrint("proxy used : " + proxy)
WINDOW_SIZE = "1920,1080"
option = webdriver.ChromeOptions()
option.add_argument("--incognito")
#option.add_argument("--disable-gpu")
#option.add_argument("--disable-infobars")
#option.add_argument("--disable-notifications")
#option.add_argument("--disable-extensions")
if hideBrowser == "YES":
option.add_argument("--headless")
option.add_argument("--window-size=%s" % WINDOW_SIZE)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
browser = webdriver.Chrome(
executable_path="C:\\webdrivers\\chromedriver.exe",
chrome_options=option,
desired_capabilities=capabilities)
#browser.set_window_position(-10000, 0)
return browser
except Exception:
LogError(traceback, "fullproxy = " + fullproxy)
return None
def get_capabilities(self):
capabilities = DesiredCapabilities.FIREFOX.copy()
# capabilities = DesiredCapabilities.CHROME.copy()
# print('get_capabilities')
proxy = self.profile.proxy
if proxy is None or not proxy.active:
# use random default proxy
proxies = models.Proxy.objects.filter(active=True,
default=True).all()
if proxies:
proxy = random.choice(proxies)
else:
proxy = None
if proxy:
# print('setting a proxy')
# print(proxy)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
if proxy.proxy_type == proxy.HTTP:
# print('HTTP proxy')
prox.http_proxy = f'{proxy.ip}:{proxy.port}'
prox.ssl_proxy = f'{proxy.ip}:{proxy.port}'
prox.ftp_proxy = f'{proxy.ip}:{proxy.port}'
elif proxy.proxy_type == proxy.SOCKS:
# print('Socks proxy')
prox.socks_proxy = f'{proxy.ip}:{proxy.port}'
prox.socks_username = proxy.username
prox.socks_password = proxy.password
prox.add_to_capabilities(capabilities)
# print(capabilities)
return capabilities
def CreateBrowser(proxy, head=False, window=False, ua=False):
if proxy:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.ssl_proxy = proxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
# Set random user agent
opts = Options()
if ua:
ua = UserAgent()
agent = ua.chrome
opts.add_argument("user-agent=" + agent)
if not head:
opts.add_argument("--headless")
if window:
opts.add_argument("--window-size=" + window)
else:
opts.add_argument("--window-size=%s" % "1920,1080")
if proxy:
_browser = webdriver.Chrome(options=opts,
desired_capabilities=capabilities)
else:
_browser = webdriver.Chrome(options=opts)
return _browser
def _start_proxy_(self, ip: str, port: int):
prox = Proxy()
endpoint = ip + ":" + str(port)
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = endpoint
prox.socks_proxy = endpoint
prox.ssl_proxy = endpoint
def initialize_browser() -> object:
chrome_options = Options()
chrome_options.add_argument('--window-size=1920x1080')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument(
'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 '
'Safari/537.36')
proxy_url = "127.0.0.1:24001"
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = proxy_url
proxy.ssl_proxy = proxy_url
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
if is_development():
return webdriver.Chrome('./bin/chromedriver.exe',
chrome_options=chrome_options,
desired_capabilities=capabilities)
else:
chrome_options.add_argument('--data-path=/tmp/data-path')
chrome_options.add_argument('--homedir=/tmp')
chrome_options.add_argument('--disk-cache-dir=/tmp/cache-dir')
chrome_options.add_argument('--user-data-dir=/tmp/user-data')
chrome_options.add_argument('--hide-scrollbars')
chrome_options.add_argument('--enable-logging')
chrome_options.add_argument('--log-level=0')
chrome_options.add_argument('--v=99')
chrome_options.add_argument('--single-process')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
return webdriver.Chrome(chrome_options=chrome_options,
desired_capabilities=capabilities)
def get_webdriver():
options = webdriver.ChromeOptions()
options.add_argument('window-size=1920x1080')
options.add_argument('--headless')
options.add_argument('--no-sandbox')
# Configure Proxy option
proxy = Proxy()
if Settings.socks_proxy is None and Settings.https_proxy is None and Settings.https_proxy is None:
proxy.proxy_type = ProxyType.DIRECT
else:
proxy.proxy_type = ProxyType.MANUAL
if Settings.socks_proxy is not None:
options.add_argument("--proxy-server=" +
Settings.get_full_socks_proxy())
else:
if Settings.http_proxy is not None:
proxy.http_proxy = Settings.http_proxy
if Settings.https_proxy is not None:
proxy.ssl_proxy = Settings.https_proxy
# Configure capabilities
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
return webdriver.Chrome(options=options, desired_capabilities=capabilities)
def create_instances(self):
for _proxy in self.proxies:
proxy = _proxy.proxy
service_args = []
# service_args.append('--proxy={}:{}'.format(proxy.ip, proxy.port))
if proxy.username and proxy.password:
service_args.append('--proxy-auth={}:{}'.format(
proxy.username, proxy.password))
capabilities = DesiredCapabilities.PHANTOMJS
capabilities[
'phantomjs.page.settings.resourceTimeout'] = self.max_delay_limit * 1000
_proxy_ = Proxy()
_proxy_.proxy_type = ProxyType.MANUAL
_proxy_.http_proxy = '{}:{}'.format(proxy.ip, proxy.port)
_proxy_.socks_proxy = '{}:{}'.format(proxy.ip, proxy.port)
_proxy_.ssl_proxy = '{}:{}'.format(proxy.ip, proxy.port)
_proxy_.add_to_capabilities(capabilities)
driver = webdriver.PhantomJS(
service_args=service_args,
desired_capabilities=capabilities,
service_log_path='/tmp/ghostdriver.log')
driver.set_window_size(1120, 1080)
driver.set_page_load_timeout(self.max_delay_limit)
self.multi_instances.append(driver)
def setup_proxy(proxy):
proxy_config = Proxy()
proxy_config.proxy_type = ProxyType.MANUAL
proxy_config.http_proxy = proxy
proxy_config.ssl_proxy = proxy
capabilities = webdriver.DesiredCapabilities.CHROME
proxy_config.add_to_capabilities(capabilities)
return proxy_config
def _add_proxy_support(self, capabilities):
if "RX_PROXY" in os.environ:
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = os.environ.get("RX_PROXY")
proxy.socks_proxy = os.environ.get("RX_PROXY")
proxy.ssl_proxy = os.environ.get("RX_PROXY")
proxy.add_to_capabilities(capabilities)
def crawl(minPage,maxPage,outputFormat):
# set selenium proxy
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "http://*****:*****@class='agree-button']"))
driver.execute_script("arguments[0].click();",cookies_modal)
time.sleep(random.randint(5, 10))
#crawl
for page in range(minPage,maxPage+1):
url = "https://musescore.com/hub/piano?page=" + str(page)
print("crawling page: "+ str(page))
#clear if no of pages is too big
if page % 5 == 0:
subprocess.call("killall -HUP tor", shell = True)
time.sleep(1)
for index in range(1,21):
#select score from grid
driver.get(url) # go back to grid of scores
score_list = driver.find_elements_by_xpath("//h2[@class='score-info__title']")
try:
score_list[index].click()
#download
driver.execute_script(open("musescore-downloader.js").read())
#try twice, if still error move on
download_button = WebDriverWait(driver, 10).until(
lambda x: x.find_elements_by_xpath("//button[@class='_3L7Ul _3qfU_ _38TLP _3A7i9 _2XPrY _13O-4 _15kzJ']"))
#XML
if outputFormat == 'XML':
download_button[2].click()
#MIDI
elif outputFormat == 'MIDI':
download_button[3].click()
else:
print("Incorrect usage! please type:"
+ "musescore-crawler.py -f <format> -s <minPage> -e <maxPage>")
sys.exit(2)
time.sleep(random.randint(10,30))
time.sleep(5)
except:
print("Failed opening score, move on to next")
def login(self, headless=True, wait_time=15, proxy=None):
"""
Logs user in with the provided credentials
User session is stored in the 'cred_root' folder
and reused so there is no need to login every time.
Pinterest sessions lasts for about 15 days
Ideally you need to call this method 3-4 times a month at most.
:return python dict object describing the pinterest response
"""
chrome_options = Options()
if headless:
chrome_options.add_argument("--headless")
if proxy is not None:
http_proxy = Proxy()
http_proxy.proxy_type = ProxyType.MANUAL
http_proxy.http_proxy = proxy
http_proxy.socks_proxy = proxy
http_proxy.ssl_proxy = proxy
http_proxy.add_to_capabilities(chrome_options)
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('chromedriver',
chrome_options=chrome_options)
driver.get("https://pinterest.com/login")
try:
WebDriverWait(driver, wait_time).until(
EC.element_to_be_clickable((By.ID, 'email')))
driver.find_element_by_id("email").send_keys(self.email)
driver.find_element_by_id("password").send_keys(self.password)
logins = driver.find_elements_by_xpath(
"//*[contains(text(), 'Log in')]")
for login in logins:
login.click()
WebDriverWait(driver, wait_time).until(
EC.invisibility_of_element((By.ID, 'email')))
cookies = driver.get_cookies()
self.http.cookies.clear()
for cookie in cookies:
self.http.cookies.set(cookie['name'], cookie['value'])
self.registry.update_all(self.http.cookies.get_dict())
except Exception as e:
print("Failed to login", e)
driver.close()
def get_proxy(self):
proxy_spider_dict = ProxySpider().proxy[0]
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = "%(ip)s:%(port)s" % proxy_spider_dict
if 'HTTPS' in proxy_spider_dict.get('proxy_type'):
proxy.ssl_proxy = "%(ip)s:%(port)s" % proxy_spider_dict
print("使用代理", proxy_spider_dict)
return proxy
def get_proxy(proxies):
print(proxies)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxies
prox.ftp_proxy = proxies
prox.ssl_proxy = proxies
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
return capabilities
def test_sets_ssl_proxy(self):
self.driver.quit()
profile = webdriver.FirefoxProfile()
proxy = Proxy()
proxy.ssl_proxy = 'https://test.hostname:1234'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == str(ProxyType.MANUAL['ff_value'])
assert profile.default_preferences["network.proxy.ssl"] == '"test.hostname"'
assert profile.default_preferences["network.proxy.ssl_port"] == '1234'
def proxy_driver(PROXIES):
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = PROXIES
prox.ssl_proxy = PROXIES
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(PATH, options=chrome_options, desired_capabilities=capabilities)
return driver
def createDriver(self):
capabilities = webdriver.DesiredCapabilities.FIREFOX
if self.config['settings']['proxy'] and self.config['proxy'] != '' and self.config['proxy'] != ' ':
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = self.config['proxy']
proxy.ssl_proxy = self.config['proxy']
proxy.add_to_capabilities(capabilities)
self.driver = webdriver.Firefox(capabilities=capabilities, log_path="modules/geckodriver.log",
executable_path='modules/geckodriver')
def installProxy(self, PROXY_HOST, PROXY_PORT):
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
proxstr = PROXY_HOST + ":" + PROXY_PORT
prox.http_proxy = proxstr
prox.socks_proxy = proxstr
prox.ssl_proxy = proxstr
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
return webdriver.Chrome()
def test_sets_ssl_proxy(self):
self.driver.quit()
profile = webdriver.FirefoxProfile()
proxy = Proxy()
proxy.ssl_proxy = 'https://test.hostname:1234'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == str(
ProxyType.MANUAL['ff_value'])
assert profile.default_preferences[
"network.proxy.ssl"] == '"test.hostname"'
assert profile.default_preferences["network.proxy.ssl_port"] == '1234'
def set_chrome_proxy():
global DRIVER_CAPABILITIES, PROXY_USED
PROXY_USED = repr(random.choice(PROXY_LIST))
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY_USED
#proxy.socks_proxy = PROXY
proxy.ssl_proxy = PROXY_USED
DRIVER_CAPABILITIES = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(DRIVER_CAPABILITIES)
def __init__(self,
proxyIP='',
hideWindow=False,
fakeURL='',
keyActive='',
postion='left',
numberThread=1,
totalThread=1):
option = Options()
option.add_argument("--disable-infobars")
option.add_argument("start-maximized")
option.add_argument("--disable-extensions")
option.add_argument("--disable-background-mode")
prefs = {"profile.managed_default_content_settings.images": 2}
option.add_experimental_option("prefs", prefs)
if postion == 'left':
option.add_argument("window-size=960,950")
option.add_argument("--window-position=0,0")
else:
# user32 = ctypes.windll.user32
# user32.SetProcessDPIAware()
# option.add_argument("window-size=150,300")
# maxHeight = round(user32.GetSystemMetrics(1) / 300)
# maxWidth = round(user32.GetSystemMetrics(1) / 150)
# heightY = int(numberThread) * 300
# widthX = 0
# if maxHeight >= user32.GetSystemMetrics(1):
# widthX = int(numberThread) * 150
# option.add_argument("--window-position="+str(int(numberThread) * widthX)+","+str(heightY)+"")
# print('goc ne')
option.add_argument("window-size=399,677")
option.add_argument("--window-position=0,0")
# Pass the argument 1 to allow and 2 to block
option.add_experimental_option(
"prefs",
{"profile.default_content_setting_values.notifications": 2})
self.fakeURL = fakeURL
self.keyActive = keyActive
if hideWindow:
option.add_argument("--headless")
capabilities = webdriver.DesiredCapabilities.CHROME
if proxyIP != '':
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.ssl_proxy = proxyIP
prox.add_to_capabilities(capabilities)
self.driver = webdriver.Chrome(options=option,
executable_path=r".\\chromedriver.exe",
desired_capabilities=capabilities)
def vote():
emails = get_emails()
for item in emails:
print(item)
proxies = get_proxies()
proxy_index = random.randrange(0, 5)
proxy = proxies[proxy_index]
print(proxy)
full_name = item[0]
email = item[1]
print(full_name)
print(email)
proxy_config = Proxy()
proxy_config.proxy_type = ProxyType.MANUAL
proxy_config.http_proxy = proxy
proxy_config.ssl_proxy = proxy
capabilities = webdriver.DesiredCapabilities.CHROME
proxy_config.add_to_capabilities(capabilities)
chrome_options = Options()
chrome_options.headless = True
driver = webdriver.Chrome("./chromedriver",
desired_capabilities=capabilities,
options=chrome_options)
driver.get("") # FILL IN WITH WEBSITE
time.sleep(10) # give page time to load
form_full_name = driver.find_element_by_xpath('') #FILL IN WITH X PATH
form_full_name.send_keys(full_name)
form_radio_button = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_radio_button.click()
form_video_dropdown = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_video_dropdown.click()
form_video_choice = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_video_choice.click()
form_email_address = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_email_address.send_keys(email)
form_submit_button = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_submit_button.click()
time.sleep(10) # give time to submit
get_confirmation_text = driver.find_element_by_css_selector(
'') #FILL IN WITH X PATH
if ((get_confirmation_text.text
) == ""): #FILL IN WITH CONFIRMATION MESSAGE
print("Successfully voted!")
else:
print("Did not vote successfully.")
driver.quit()
time.sleep(300) # wait 5 mins
def __setup_driver(self):
''' Set up the webdriver with a proxy'''
web_opts = Options()
web_opts.headless = True
web_prox = Proxy()
web_prox.proxy_type = ProxyType.MANUAL
web_prox.http_proxy = self.useropts['HTTPPROXY']
web_prox.ssl_proxy = self.useropts['HTTPPROXY']
web_capabilities = webdriver.DesiredCapabilities.FIREFOX
web_prox.add_to_capabilities(web_capabilities)
self.driver = webdriver.Firefox(options=web_opts,
desired_capabilities=web_capabilities)
self.driver.implicitly_wait(10) # seconds
def _proxy_generator(self):
"Randomly generate the deliberate proxy !"
if self._browser == 'chrome':
_capabilities = DesiredCapabilities.CHROME
else:
_capabilities = DesiredCapabilities.FIREFOX
_proxy = Proxy()
_proxy.proxy_type = ProxyType.MANUAL
_proxy.http_proxy = next(self._proxy_pool)
_proxy.ssl_proxy = next(self._proxy_pool)
_proxy.add_to_capabilities(_capabilities)
return _capabilities
def setup_proxy(self):
PROXY = "159.8.114.34"
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.autodetect = False
if self.driver_type == 'firefox': #add more if you add more driver profiles above
capabilities = webdriver.DesiredCapabilities.FIREFOX
elif self.driver_type == 'chrome':
capabilities = webdriver.DesiredCapabilities.CHROME
prox.http_proxy = PROXY
prox.ssl_proxy = PROXY
prox.add_to_capabilities(capabilities)
return capabilities
def _setup_driver(self):
proxy = Proxy()
proxy.proxy_type = ProxyType.DIRECT
if 'http' in self.proxy:
proxy.http_proxy = self.proxy['http']
if 'https' in self.proxy:
proxy.ssl_proxy = self.proxy['https']
capa = self._build_capabilities()
proxy.add_to_capabilities(capa)
options = self._build_options()
# TODO some browsers don't need headless
# TODO handle different proxy setting?
options.set_headless(self.HEADLESS)
if self.DRIVER is webdriver.Firefox:
if self.responses_dirname and not os.path.isdir(self.responses_dirname):
os.makedirs(self.responses_dirname)
options.profile = DirFirefoxProfile(self.responses_dirname)
if self.responses_dirname:
capa['profile'] = self.responses_dirname
self.driver = self.DRIVER(options=options, capabilities=capa)
elif self.DRIVER is webdriver.Chrome:
self.driver = self.DRIVER(options=options, desired_capabilities=capa)
elif self.DRIVER is webdriver.PhantomJS:
if self.responses_dirname:
if not os.path.isdir(self.responses_dirname):
os.makedirs(self.responses_dirname)
log_path = os.path.join(self.responses_dirname, 'selenium.log')
else:
log_path = NamedTemporaryFile(prefix='weboob_selenium_', suffix='.log', delete=False).name
self.driver = self.DRIVER(desired_capabilities=capa, service_log_path=log_path)
else:
raise NotImplementedError()
if self.WINDOW_SIZE:
self.driver.set_window_size(*self.WINDOW_SIZE)
def start_client(self):
capabilities = {}
for c in self.capabilities:
name, value = c.split(':')
# handle integer capabilities
if value.isdigit():
value = int(value)
# handle boolean capabilities
elif value.lower() in ['true', 'false']:
value = value.lower() == 'true'
capabilities.update({name: value})
if self.proxy_host and self.proxy_port:
proxy = Proxy()
proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port)
proxy.ssl_proxy = proxy.http_proxy
proxy.add_to_capabilities(capabilities)
profile = None
if self.driver.upper() == 'REMOTE':
capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper()))
if json.loads(self.chrome_options) or self.extension_paths:
capabilities = self.create_chrome_options(
self.chrome_options,
self.extension_paths).to_capabilities()
if self.browser_name.upper() == 'FIREFOX':
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
if self.browser_version:
capabilities['version'] = self.browser_version
capabilities['platform'] = self.platform.upper()
executor = 'http://%s:%s/wd/hub' % (self.host, self.port)
try:
self.selenium = webdriver.Remote(command_executor=executor,
desired_capabilities=capabilities or None,
browser_profile=profile)
except AttributeError:
valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')]
raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers)))
elif self.driver.upper() == 'CHROME':
options = None
if self.chrome_options or self.extension_paths:
options = self.create_chrome_options(
self.chrome_options,
self.extension_paths)
if self.chrome_path:
self.selenium = webdriver.Chrome(executable_path=self.chrome_path,
chrome_options=options,
desired_capabilities=capabilities or None)
else:
self.selenium = webdriver.Chrome(chrome_options=options,
desired_capabilities=capabilities or None)
elif self.driver.upper() == 'FIREFOX':
binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
self.selenium = webdriver.Firefox(
firefox_binary=binary,
firefox_profile=profile,
capabilities=capabilities or None)
elif self.driver.upper() == 'IE':
self.selenium = webdriver.Ie()
elif self.driver.upper() == 'PHANTOMJS':
self.selenium = webdriver.PhantomJS()
elif self.driver.upper() == 'OPERA':
capabilities.update(webdriver.DesiredCapabilities.OPERA)
self.selenium = webdriver.Opera(executable_path=self.opera_path,
desired_capabilities=capabilities)
elif self.driver.upper() == 'BROWSERSTACK':
from cloud import BrowserStack
self.cloud = BrowserStack()
self.selenium = self.cloud.driver(
self.test_id, capabilities, self.options)
elif self.driver.upper() == 'SAUCELABS':
from cloud import SauceLabs
self.cloud = SauceLabs()
self.selenium = self.cloud.driver(
self.test_id, capabilities, self.options, self.keywords)
else:
self.selenium = getattr(webdriver, self.driver)()
if self.event_listener is not None and not isinstance(self.selenium, EventFiringWebDriver):
self.selenium = EventFiringWebDriver(self.selenium, self.event_listener())
def start_webdriver_client(self):
capabilities = {}
if self.capabilities:
capabilities.update(json.loads(self.capabilities))
if self.proxy_host and self.proxy_port:
proxy = Proxy()
proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port)
proxy.ssl_proxy = proxy.http_proxy
proxy.add_to_capabilities(capabilities)
profile = None
if self.driver.upper() == 'REMOTE':
capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper()))
if json.loads(self.chrome_options) or self.extension_paths:
capabilities = self.create_chrome_options(
self.chrome_options,
self.extension_paths).to_capabilities()
if self.browser_name.upper() == 'FIREFOX':
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
if self.browser_version:
capabilities['version'] = self.browser_version
capabilities['platform'] = self.platform.upper()
executor = 'http://%s:%s/wd/hub' % (self.host, self.port)
try:
self.selenium = webdriver.Remote(command_executor=executor,
desired_capabilities=capabilities or None,
browser_profile=profile)
except AttributeError:
valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')]
raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers)))
elif self.driver.upper() == 'CHROME':
options = None
if self.chrome_options or self.extension_paths:
options = self.create_chrome_options(
self.chrome_options,
self.extension_paths)
if self.chrome_path:
self.selenium = webdriver.Chrome(executable_path=self.chrome_path,
chrome_options=options,
desired_capabilities=capabilities or None)
else:
self.selenium = webdriver.Chrome(chrome_options=options,
desired_capabilities=capabilities or None)
elif self.driver.upper() == 'FIREFOX':
binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
self.selenium = webdriver.Firefox(
firefox_binary=binary,
firefox_profile=profile,
capabilities=capabilities or None)
elif self.driver.upper() == 'IE':
self.selenium = webdriver.Ie()
elif self.driver.upper() == 'OPERA':
capabilities.update(webdriver.DesiredCapabilities.OPERA)
self.selenium = webdriver.Opera(executable_path=self.opera_path,
desired_capabilities=capabilities)
else:
self.selenium = getattr(webdriver, self.driver)()
