def _start_proxy_(self, ip: str, port: int):
prox = Proxy()
endpoint = ip + ":" + str(port)
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = endpoint
prox.socks_proxy = endpoint
prox.ssl_proxy = endpoint
def create_instances(self):
for _proxy in self.proxies:
proxy = _proxy.proxy
service_args = []
# service_args.append('--proxy={}:{}'.format(proxy.ip, proxy.port))
if proxy.username and proxy.password:
service_args.append('--proxy-auth={}:{}'.format(
proxy.username, proxy.password))
capabilities = DesiredCapabilities.PHANTOMJS
capabilities[
'phantomjs.page.settings.resourceTimeout'] = self.max_delay_limit * 1000
_proxy_ = Proxy()
_proxy_.proxy_type = ProxyType.MANUAL
_proxy_.http_proxy = '{}:{}'.format(proxy.ip, proxy.port)
_proxy_.socks_proxy = '{}:{}'.format(proxy.ip, proxy.port)
_proxy_.ssl_proxy = '{}:{}'.format(proxy.ip, proxy.port)
_proxy_.add_to_capabilities(capabilities)
driver = webdriver.PhantomJS(
service_args=service_args,
desired_capabilities=capabilities,
service_log_path='/tmp/ghostdriver.log')
driver.set_window_size(1120, 1080)
driver.set_page_load_timeout(self.max_delay_limit)
self.multi_instances.append(driver)
def getGoogleChromeDriver(fullproxy):
try:
proxy = fullproxy.split(' ')[0]
conditionalPrint("proxy used : " + proxy)
WINDOW_SIZE = "1920,1080"
option = webdriver.ChromeOptions()
option.add_argument("--incognito")
#option.add_argument("--disable-gpu")
#option.add_argument("--disable-infobars")
#option.add_argument("--disable-notifications")
#option.add_argument("--disable-extensions")
if hideBrowser == "YES":
option.add_argument("--headless")
option.add_argument("--window-size=%s" % WINDOW_SIZE)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
browser = webdriver.Chrome(
executable_path="C:\\webdrivers\\chromedriver.exe",
chrome_options=option,
desired_capabilities=capabilities)
#browser.set_window_position(-10000, 0)
return browser
except Exception:
LogError(traceback, "fullproxy = " + fullproxy)
return None
def get_phantomjs_browser(
self,
proxy=None,
timeout=15,
):
"""
创建一个phantomjs浏览器
:param proxy: String "ip:port"
:param timeout: Int
:return: Phantomjs.Browser 浏览器对象
"""
capabilities = webdriver.DesiredCapabilities.PHANTOMJS
capabilities['phantomjs.page.settings.userAgent'] = random.choice(
USER_AGENTS)
capabilities["phantomjs.page.settings.loadImages"] = False
if proxy:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
prox.add_to_capabilities(capabilities)
browser = webdriver.PhantomJS(desired_capabilities=capabilities)
browser.maximize_window()
browser.set_page_load_timeout(timeout)
return browser
def get_browser():
# @todo add windows path
if platform.system() == "Darwin":
os.environ["webdriver.chrome.driver"] = os.path.expanduser(
"~"
) + '/Library/Application Support/ZAP/webdriver/macos/64/chromedriver'
else:
os.environ["webdriver.chrome.driver"] = os.path.expanduser(
"~") + '/.ZAP/webdriver/linux/64/chromedriver'
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = "127.0.0.1:8080"
proxy.socks_proxy = "127.0.0.1:8080"
proxy.ssl_proxy = "127.0.0.1:8080"
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument("--test-type")
return webdriver.Chrome(
executable_path=os.environ["webdriver.chrome.driver"],
chrome_options=options,
desired_capabilities=capabilities)
def load_page(source):
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "127.0.0.1:9090"
prox.socks_proxy = "127.0.0.1:9090"
prox.ssl_proxy = "127.0.0.1:9090"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
driver.get(source)
navigationStart = driver.execute_script(
"return window.performance.timing.navigationStart")
responseStart = driver.execute_script(
"return window.performance.timing.responseStart")
domComplete = driver.execute_script(
"return window.performance.timing.domComplete")
backendPerformance = responseStart - navigationStart
frontendPerformance = domComplete - responseStart
print "Back End: %s" % backendPerformance
print "Front End: %s" % frontendPerformance
driver.quit()
def get_capabilities(self):
capabilities = DesiredCapabilities.FIREFOX.copy()
# capabilities = DesiredCapabilities.CHROME.copy()
# print('get_capabilities')
proxy = self.profile.proxy
if proxy is None or not proxy.active:
# use random default proxy
proxies = models.Proxy.objects.filter(active=True,
default=True).all()
if proxies:
proxy = random.choice(proxies)
else:
proxy = None
if proxy:
# print('setting a proxy')
# print(proxy)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
if proxy.proxy_type == proxy.HTTP:
# print('HTTP proxy')
prox.http_proxy = f'{proxy.ip}:{proxy.port}'
prox.ssl_proxy = f'{proxy.ip}:{proxy.port}'
prox.ftp_proxy = f'{proxy.ip}:{proxy.port}'
elif proxy.proxy_type == proxy.SOCKS:
# print('Socks proxy')
prox.socks_proxy = f'{proxy.ip}:{proxy.port}'
prox.socks_username = proxy.username
prox.socks_password = proxy.password
prox.add_to_capabilities(capabilities)
# print(capabilities)
return capabilities
def _add_proxy_support(self, capabilities):
if "RX_PROXY" in os.environ:
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = os.environ.get("RX_PROXY")
proxy.socks_proxy = os.environ.get("RX_PROXY")
proxy.ssl_proxy = os.environ.get("RX_PROXY")
proxy.add_to_capabilities(capabilities)
def login(self, headless=True, wait_time=15, proxy=None):
"""
Logs user in with the provided credentials
User session is stored in the 'cred_root' folder
and reused so there is no need to login every time.
Pinterest sessions lasts for about 15 days
Ideally you need to call this method 3-4 times a month at most.
:return python dict object describing the pinterest response
"""
chrome_options = Options()
if headless:
chrome_options.add_argument("--headless")
if proxy is not None:
http_proxy = Proxy()
http_proxy.proxy_type = ProxyType.MANUAL
http_proxy.http_proxy = proxy
http_proxy.socks_proxy = proxy
http_proxy.ssl_proxy = proxy
http_proxy.add_to_capabilities(chrome_options)
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('chromedriver',
chrome_options=chrome_options)
driver.get("https://pinterest.com/login")
try:
WebDriverWait(driver, wait_time).until(
EC.element_to_be_clickable((By.ID, 'email')))
driver.find_element_by_id("email").send_keys(self.email)
driver.find_element_by_id("password").send_keys(self.password)
logins = driver.find_elements_by_xpath(
"//*[contains(text(), 'Log in')]")
for login in logins:
login.click()
WebDriverWait(driver, wait_time).until(
EC.invisibility_of_element((By.ID, 'email')))
cookies = driver.get_cookies()
self.http.cookies.clear()
for cookie in cookies:
self.http.cookies.set(cookie['name'], cookie['value'])
self.registry.update_all(self.http.cookies.get_dict())
except Exception as e:
print("Failed to login", e)
driver.close()
def installProxy(self, PROXY_HOST, PROXY_PORT):
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
proxstr = PROXY_HOST + ":" + PROXY_PORT
prox.http_proxy = proxstr
prox.socks_proxy = proxstr
prox.ssl_proxy = proxstr
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
return webdriver.Chrome()
def setup_driver(proxy=False, detached=False):
try:
chrome_options = Options()
#chrome_options.add_argument('--headless')
chrome_options.add_argument('--mute-audio')
prefs = {'profile.managed_default_content_settings.images': 2}
chrome_options.add_experimental_option('prefs', prefs)
chrome_options.add_experimental_option('detach', False)
dc = DesiredCapabilities.CHROME
dc['loggingPrefs'] = {'browser': 'ALL'}
dc['unexpectedAlertBehaviour'] = 'accept'
if proxy:
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.socks_proxy = config.PROXY
proxy.http_proxy = config.PROXY
proxy.ssl_proxy = config.PROXY
proxy.add_to_capabilities(dc)
driver = webdriver.Chrome(chrome_options=chrome_options,
desired_capabilities=dc)
max_wait = config.LOAD_TIMEOUT
driver.set_window_size(500, 500)
driver.set_page_load_timeout(max_wait)
driver.set_script_timeout(max_wait)
except Exception as e:
config.DEBUG and ipdb.set_trace()
if not detached:
def handler(*args):
if config.DEBUG:
ipdb.set_trace()
else:
try:
if driver:
driver.stop_client()
driver.close()
driver.quit()
except:
log('quitting')
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGINT, handler)
return driver
def _init_browser(self):
chrome_options = Options()
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("--mute-audio")
chrome_options.add_argument('--dns-prefetch-disable')
chrome_options.add_argument('--disable-setuid-sandbox')
if self.headless_browser:
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
if self.disable_image_load:
chrome_options.add_argument(
'--blink-settings=imagesEnabled=false')
# Replaces browser User Agent from "HeadlessChrome".
user_agent = "Chrome"
chrome_options.add_argument(
'user-agent={user_agent}'.format(user_agent=user_agent))
capabilities = DesiredCapabilities.CHROME
if self.proxy_address and self.proxy_port:
prox = Proxy()
proxy = ":".join([self.proxy_address, str(self.proxy_port)])
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
prox.add_to_capabilities(capabilities)
if self.browser_profile_path is not None:
chrome_options.add_argument('user-data-dir={}'.format(
self.browser_profile_path))
chrome_prefs = {
'intl.accept_languages': Settings.browser_language,
}
if self.disable_image_load:
chrome_prefs['profile.managed_default_content_settings.images'] = 2
chrome_options.add_experimental_option('prefs', chrome_prefs)
try:
self.browser = webdriver.Chrome(desired_capabilities=capabilities,
chrome_options=chrome_options)
except WebDriverException as exc:
self.logger.exception(exc)
raise UtomarketError('初始化chrome失败')
self.browser.implicitly_wait(self.page_delay)
def setup_driver(proxy=False, detached=False, driver=None):
try:
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--mute-audio')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-logging')
prefs = {'profile.managed_default_content_settings.images': 2}
chrome_options.add_experimental_option('prefs', prefs)
chrome_options.add_experimental_option('detach', False)
dc = DesiredCapabilities.CHROME
dc['loggingPrefs'] = {'browser': 'ALL'}
if proxy:
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.socks_proxy = config.PROXY
proxy.http_proxy = config.PROXY
proxy.ssl_proxy = config.PROXY
proxy.add_to_capabilities(dc)
driver = webdriver.Chrome(chrome_options=chrome_options,
desired_capabilities=dc)
max_wait = config.PROXY_LOAD_TIMEOUT if proxy else config.LOAD_TIMEOUT
driver.set_window_size(1000, 1000)
driver.set_page_load_timeout(max_wait)
driver.set_script_timeout(max_wait)
setattr(driver, 'proxy', proxy)
except Exception as e:
log(e, type='error')
if not detached:
def handler(*args, **kwargs):
try:
if driver:
driver.stop_client()
driver.close()
driver.quit()
except:
log('quitting')
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGINT, handler)
return driver
def ChromeProxy(self):
proxy=Proxy()
proxy.proxy_type=ProxyType.MANUAL
proxy.http_proxy="ipAddress:port"
proxy.socks_proxy="ipAddress:port"
proxy.ssl_proxy="ipAddress:port"
cap=webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(cap)
driver=webdriver.Chrome(desired_capabilities=cap)
def __init__(self, logger=None, proxy=False, headless=False):
self.logger = logger
self.setup_log()
geckdriver_path = r'/home/ghost/Drivers/geckodriver'
# Initialize class variables
self.fb_url = 'https://mbasic.facebook.com/'
# Bot directive variables
self.likes_monthly = 0
self.tags_list = None
self.groups_list = None
self.like_vids_monthly = 0
self.accept_all_requests = False
self.send_requests = True
self.post_imgs = 0
browser_profile = webdriver.FirefoxProfile()
browser_profile.set_preference('dom.webnotifications.enabled', False)
options = Options()
options.headless = headless
# Prob create selenium instance here
if proxy_conf.LIST is None:
self.driver = webdriver.Firefox(executable_path=geckdriver_path,
firefox_profile=browser_profile,
firefox_options=options)
self.bot_logger.info('Browser driver has been initialized')
else:
# Set up proxy if 'proxy' variables' value is True
proxy_server = Proxy()
proxy_server.proxy_type = ProxyType.MANUAL
proxy_server.http_proxy = proxy_conf.LIST['http']
proxy_server.socks_proxy = proxy_conf.LIST['sock']
proxy_server.ssl_proxy = proxy_conf.LIST['ssl']
capabilities = webdriver.DesiredCapabilities.FIREFOX
proxy_server.add_to_capabilities(capabilities)
self.driver = webdriver.Firefox(executable_path=geckdriver_path,
firefox_profile=browser_profile,
desired_capabilities=capabilities)
bot_logger.info('Browser driver has been initialized')
self.bot_utils = BotUtils(self.driver, self.fb_url)
def driver(request):
#wd = webdriver.Ie()
#wd = webdriver.Firefox(firefox_binary="c:\\Program Files (x86)\\Firefox Developer Edition\\firefox.exe", desired_capabilities={"proxy": {"proxyType": "MANUAL", "httpProxy": "localhost:8888"}})
#wd = webdriver.Chrome()
#wd = webdriver.Chrome(desired_capabilities={"proxy": {"proxyType": "MANUAL", "httpsProxy": "127.0.0.1:8888"}})
#wd = webdriver.Firefox()
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "127.0.0.1:8888"
prox.socks_proxy = "127.0.0.1:8888"
prox.ssl_proxy = "127.0.0.1:8888"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
wd = webdriver.Chrome(desired_capabilities=capabilities)
request.addfinalizer(wd.quit)
return wd
def openChrome(paydetailsO, itemdetsO, timeO, strictO, skipO, nextO, cdloc,
capabilities, useProxy, PROXY):
global driver, strict, password, reg, items, droptime, pDescr, paydetails, category, skipS, nextS
chrome_options = webdriver.ChromeOptions()
if useProxy:
prx = Proxy()
prx.proxy_type = ProxyType.MANUAL
prx.http_proxy = PROXY
prx.socks_proxy = PROXY
prx.ssl_proxy = PROXY
prx.add_to_capabilities(capabilities)
else:
prx = Proxy()
prx.proxy_type = ProxyType.SYSTEM
prx.add_to_capabilities(capabilities)
chrome_options.binary_location = capabilities['chrome.binary']
driver = webdriver.Chrome(cdloc, desired_capabilities=capabilities)
openTab('https://www.google.com', driver)
paydetails = paydetailsO
reg = paydetailsO['Region']
strict = strictO
skipS = skipO
nextS = nextO
droptime = timeO
items = []
for x in itemdetsO:
print(x[0], x[1], x[2], x[3])
items.append({
'selectedCategory': x[0],
'keywords': x[1].split(','),
'selectedColour': x[2],
'selectedSize': x[3]
})
returnTime()
try:
for it in items:
searchItem(it)
cart()
except NoSuchWindowException:
print('[!] Chrome window closed. Click GO! to re-start')
return None
def get_proxy_capabilities():
"Function to add proxy to Browser Capabilities"
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
try:
proxy_url = get_anonymous_ip()
print("proxy_url : ",proxy_url)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy_url
prox.socks_proxy = proxy_url
prox.ssl_proxy = proxy_url
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
return capabilities
except Exception as e:
print("Exception in setting proxy capabilities : ",e)
pass
def getIEDriver(fullproxy):
try:
ExecutablePath = 'C:\\webdrivers\\IEDriverServer.exe'
BinaryPath = 'C:\\webdrivers\\IEDriverServer.exe'
#create capabilities
capabilities = DesiredCapabilities.INTERNETEXPLORER
#capabilities.pop("ignoreZoomSetting", True);
#delete platform and version keys
capabilities.pop("platform", "WIN8")
capabilities.pop("version", "11")
capabilities.pop("browserName", "internet explorer")
capabilities.pop("ie.usePerProcessProxy", True)
capabilities.pop("usePerProcessProxy", True)
capabilities.pop("INTRODUCE_FLAKINESS_BY_IGNORING_SECURITY_DOMAINS",
True)
capabilities.pop(
"InternetExplorerDriver.INTRODUCE_FLAKINESS_BY_IGNORING_SECURITY_DOMAINS",
True)
#capabilities.pop("nativeEvents", False)
#capabilities.pop("ignoreProtectedModeSettings", True)
capabilities.pop("ie.setProxyByServer", True)
capabilities.pop("setProxyByServer", True)
#capabilities.setCapability(InternetExplorerDriver.IE_USE_PRE_PROCESS_PROXY,true);
if fullproxy != "":
proxy = fullproxy.split(' ')[0]
conditionalPrint("proxy used : " + proxy)
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
prox.noProxy = None
prox.autodetect = None
#prox["class"] ="org.openqa.selenium.Proxy"
prox.add_to_capabilities(capabilities)
driver = webdriver.Ie(executable_path=ExecutablePath,
capabilities=capabilities)
return driver
except Exception:
LogError(traceback, "fullproxy = " + fullproxy)
return None
def proxy_driver(PROXIES, co=co):
prox = Proxy()
if PROXIES:
pxy = PROXIES[-1]
else:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(chrome_options=co, desired_capabilities=capabilities, executable_path=pathWebdriver)
return driver
def getFirefoxDriver(host, port):
try:
path = 'C:\\webdrivers\\geckodriver.exe'
fullproxy = host + ":" + port
#https://stackoverflow.com/questions/17082425/running-selenium-webdriver-with-a-proxy-in-python
firefox_capabilities = DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
"httpProxy": fullproxy,
"ftpProxy": fullproxy,
"sslProxy": fullproxy,
"noProxy": None,
"proxyType": "MANUAL",
"autodetect": False
}
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = fullproxy
prox.socks_proxy = fullproxy
prox.ssl_proxy = fullproxy
#prox.add_to_capabilities(firefox_capabilities)
fp = webdriver.FirefoxProfile()
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http", host)
fp.set_preference("network.proxy.http_port", int(port))
fp.set_preference("network.proxy.https", host)
fp.set_preference("network.proxy.https_port", int(port))
fp.set_preference("network.proxy.ssl", host)
fp.set_preference("network.proxy.ssl_port", int(port))
fp.set_preference("network.proxy.ftp", host)
fp.set_preference("network.proxy.ftp_port", int(port))
fp.set_preference("network.proxy.socks", host)
fp.set_preference("network.proxy.socks_port", int(port))
#fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")
fp.update_preferences()
driver = webdriver.Firefox(executable_path=path)
return driver
except Exception:
LogError(traceback, "host = " + host + " and port = " + port)
return getGoogleChromeDriver(host + ":" + port)
def scrap_linkedin(request):
"""
test: http://localhost:8000/api/scrap_linkedin
:return:
"""
proxy=request.GET.get("proxy","")
capabilities = DesiredCapabilities.CHROME
if len(proxy) > 0:
p = Proxy()
p.proxy_type = ProxyType.MANUAL
p.http_proxy = proxy
p.socks_proxy = proxy
p.ssl_proxy = proxy
p.add_to_capabilities(capabilities)
url="https://www.linkedin.com/in/hhoareau/"
driver:WebDriver = WebDriver(command_executor="http://127.0.0.1:9515", desired_capabilities=capabilities)
driver.implicitly_wait(1)
driver.get(url)
sections=driver.find_elements_by_tag_name("section")
return Response("scrapped",200)
def proxy_driver(self, p, ua):
co = webdriver.ChromeOptions()
co.add_argument('--no-sandbox')
co.add_argument('--disable-dev-shm-usage')
co.add_argument(f"user-agent={ua}")
co.add_argument("--headless")
co.add_argument(f"--window-size={choice(('1024,768', '1280,960', '1280,1024', '1600,1200', '1920,1080'))}")
pxy = p
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
capabilities = webdriver.DesiredCapabilities.CHROME
# capabilities['resolution'] = '1920x1080'
# prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome('/home/jurgeon/projects/jcb/materials/jcb/chromedriver', options = co, desired_capabilities = capabilities)
return driver
def __init__(self, login, password, debug=False, indocker=False):
self.closed = False
if not indocker:
options = webdriver.ChromeOptions()
if not debug:
options.add_argument("--headless")
options.add_argument("--disable-gpu")
self.driver = webdriver.Chrome(chrome_options=options)
else:
started = False
while not started:
try:
capabilities = dict(webdriver.DesiredCapabilities.CHROME.copy())
proxy = Proxy()
proxy.socks_proxy = "socks5://tor:9055"
self.driver = webdriver.Remote("http://chromebrowser:4444/wd/hub", desired_capabilities=capabilities)
started = True
except Exception as e:
logging.exception(e)
time.sleep(10)
self.driver.get('http://forum.watch.ru')
def get_ie_driver():
"""
https://docs.seleniumhq.org/docs/04_webdriver_advanced.jsp#using-a-proxy
https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities#proxy-json-object
使用方式：先用正常页面不带代理的，启动多个窗口登录。登录成功后，再切换回代理。
TODO: 需要设置刷新的时候也走代理，本地模式。 服务模式不需要。
:return:
"""
proxy = get_proxy()
driver = None
if proxy:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
prox.ftp_proxy = proxy
LOGGER.info("try to use proxy: %s", proxy)
caps = webdriver.DesiredCapabilities.INTERNETEXPLORER.copy()
# caps['proxy'] = {
# "httpProxy": proxy,
# "ftpProxy": proxy,
# "sslProxy": proxy,
# "socks_proxy": proxy,
# "noProxy": None,
# "proxyType": "MANUAL",
# "autodetect": False
# }
prox.add_to_capabilities(caps)
# caps["proxy"] = {"proxyType": "manual", "httpProxy": proxy}
opt = options.Options()
opt.use_per_process_proxy = True
driver = webdriver.Ie(capabilities=caps, options=opt)
else:
driver = webdriver.Ie()
return driver
def proxy_driver(PROXIES, index, co=co):
prox = Proxy()
if PROXIES:
pxy = PROXIES[index]
print("pxy:" + str(pxy))
else:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(chrome_options=co2,
desired_capabilities=capabilities)
return driver
def capabilities(ip=None, port='3128', ext=False):
capabs = {}
if ip:
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = ip + ':' + port
proxy.socks_proxy = ip + ':' + port
proxy.ssl_proxy = ip + ':' + port
capabs = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabs)
if ext: # FIXME
capabs = {
**capabs,
**{
'chromeOptions': {
'useAutomationExtension': False,
'forceDevToolsScreenshot': True,
'args': ['--start-maximized', '--disable-infobars']
}
}
}
return capabs or None
def proxy_driver(PROXIES, ua):
print('\n----CONNECTING TO PROXY----')
options = webdriver.ChromeOptions()
prox = Proxy()
if len(PROXIES) < 1:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
pxy = PROXIES.pop()
#pxy = '80.25.87.49:32850'
print(f'Current Proxy ({pxy})')
prox.proxy_type = ProxyType.MANUAL
prox.autodetect = False
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
options.Proxy = prox
print('\n----CHANGING USER AGENT----')
userAgent = ua.random
print(f'Current UserAgent\n{userAgent}')
options.add_argument("--start-maximized")
options.add_argument("--proxy-server=%s" % pxy)
options.add_argument("user-agent={userAgent}")
options.add_argument("ignore-certificate-errors")
options.add_argument(
"--disable-bundled-ppapi-flash") # Disable internal Flash player
options.add_argument(
"--disable-plugins-discovery"
) # Disable external Flash player (by not allowing it to load)
options.add_extension('mpbjkejclgfgadiemmefgebjfooflfhl.crx')
driver = webdriver.Chrome('chromedriver.exe', options=options)
return driver, PROXIES
def proxy_driver(PROXIES, co=co):
prox = Proxy()
if PROXIES:
pxy = PROXIES[-1]
else:
print("--- Proxies used up. Downloading New List (%s)" % len(PROXIES))
PROXIES = get_proxies()
pxy = PROXIES[-1]
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
prox.no_proxy = None
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(options=co,
desired_capabilities=capabilities,
executable_path='chromedriver_mc21')
return driver
def proxy_driver(PROXIES):
co = webdriver.ChromeOptions()
co.add_argument("log-level=3")
#co.add_argument("--headless")
prox = Proxy()
if PROXIES:
pxy = PROXIES[-1]
else:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(chrome_options=co,
desired_capabilities=capabilities)
return driver
