def testCanNotChangeInitializedProxyType(self):
proxy = Proxy(raw={'proxyType': 'direct'})
try:
proxy.proxy_type = ProxyType.SYSTEM
raise Exception("Change of already initialized proxy type should raise exception")
except Exception as e:
pass
def getDriver(self, driverType='firefox'):
try:
if driverType == 'phantomjs':
#capabilities = webdriver.DesiredCapabilities.PHANTOMJS.copy()
#capabilities['platform'] = "LINUX"
driver = webdriver.PhantomJS('/usr/local/share/phantomjs/bin/phantomjs')
else:
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
self.dis = Display(visible=0,size=(800,600))
self.dis.start()
prx = Proxy()
prx.httpProxy= '199.200.120.36:8089' #'112.196.3.254:9064' india
prx.proxyType = ProxyType.MANUAL
prof = FirefoxProfile('./ffprofile')
driver = webdriver.Firefox(proxy=prx, firefox_profile=prof)
driver.implicitly_wait(30)
driver.set_page_load_timeout(30)
print 'started the driver successfully...'
return driver
except Exception as e:
print 'error has occured while starting the driver....'
print type(e)
print e
self.quit()
#sys.exit(0)
return None
def tt1():
desired_capabilities = DesiredCapabilities.PHANTOMJS.copy()
headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Host': 'www.dianping.com',
}
for key, value in headers.iteritems():
desired_capabilities['phantomjs.page.customHeaders.{}'.format(key)] = value
desired_capabilities[
'phantomjs.page.customHeaders.User-Agent'] = \
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) ' \
'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 ' \
'Safari/604.1.38'
ip_port = random.choice(redis_conn1())
print ip_port
proxy = Proxy(
{
'proxyType': ProxyType.MANUAL,
'httpProxy': '%s' % ip_port # 代理ip和端口
}
)
proxy.add_to_capabilities(desired_capabilities)
print desired_capabilities
driver = webdriver.PhantomJS(desired_capabilities=desired_capabilities)
driver.set_page_load_timeout(10)
driver.get("http://www.dianping.com/shop/%s" % ['76964345', '15855144', ])
list1 = driver.find_elements_by_xpath('//div[@class="comment-condition J-comment-condition Fix"]/div/span/a')
for l in list1:
print l.text
if '403 Forbidden' in driver.page_source:
print driver.page_source
driver.close()
def test_autodetect_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.auto_detect = True
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']
def test_pac_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.proxy_autoconfig_url = 'http://some.url:12345/path'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
assert profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'
def testCanNotChangeInitializedProxyType():
proxy = Proxy(raw={'proxyType': 'direct'})
with pytest.raises(Exception):
proxy.proxy_type = ProxyType.SYSTEM
proxy = Proxy(raw={'proxyType': ProxyType.DIRECT})
with pytest.raises(Exception):
proxy.proxy_type = ProxyType.SYSTEM
def test_what_things_look_like(self):
bmp_capabilities = copy.deepcopy(selenium.webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX)
self.client.add_to_capabilities(bmp_capabilities)
proxy_capabilities = copy.deepcopy(selenium.webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX)
proxy = Proxy({'httpProxy': 'localhost:%d' % self.client.port})
proxy.add_to_capabilities(proxy_capabilities)
assert bmp_capabilities == proxy_capabilities
def test_sets_ssl_proxy(self):
self.driver.quit()
profile = webdriver.FirefoxProfile()
proxy = Proxy()
proxy.ssl_proxy = 'https://test.hostname:1234'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == str(ProxyType.MANUAL['ff_value'])
assert profile.default_preferences["network.proxy.ssl"] == '"test.hostname"'
assert profile.default_preferences["network.proxy.ssl_port"] == '1234'
def testCanAddAutodetectProxyToDesiredCapabilities():
proxy = Proxy()
proxy.auto_detect = AUTODETECT_PROXY['autodetect']
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = AUTODETECT_PROXY.copy()
proxy_capabilities['proxyType'] = 'AUTODETECT'
expected_capabilities = {'proxy': proxy_capabilities}
assert expected_capabilities == desired_capabilities
def testCanAddPACProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.proxy_autoconfig_url = self.PAC_PROXY['proxyAutoconfigUrl']
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.PAC_PROXY.copy()
proxy_capabilities['proxyType'] = 'PAC'
expected_capabilities = {'proxy': proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def test_autodetect_proxy_is_set_in_profile(self):
# The setup gave us a browser but we dont need it
self.driver.quit()
self.profile = webdriver.FirefoxProfile()
proxy = Proxy()
proxy.auto_detect = True
self.profile.set_proxy(proxy)
assert self.profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']
def testCanAddPACProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.proxy_autoconfig_url = self.PAC_PROXY["proxyAutoconfigUrl"]
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.PAC_PROXY.copy()
proxy_capabilities["proxyType"] = "PAC"
expected_capabilities = {"proxy": proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def testCanAddAutodetectProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.auto_detect = self.AUTODETECT_PROXY['autodetect']
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.AUTODETECT_PROXY.copy()
proxy_capabilities['proxyType'] = 'AUTODETECT'
expected_capabilities = {'proxy': proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def testCanAddPACProxyToDesiredCapabilities():
proxy = Proxy()
proxy.proxy_autoconfig_url = PAC_PROXY['proxyAutoconfigUrl']
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = PAC_PROXY.copy()
proxy_capabilities['proxyType'] = 'PAC'
expected_capabilities = {'proxy': proxy_capabilities}
assert expected_capabilities == desired_capabilities
def testCanAddAutodetectProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.auto_detect = self.AUTODETECT_PROXY["autodetect"]
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.AUTODETECT_PROXY.copy()
proxy_capabilities["proxyType"] = "AUTODETECT"
expected_capabilities = {"proxy": proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def test_pac_proxy_is_set_in_profile(self):
# The setup gave us a browser but we dont need it
self.driver.quit()
self.profile = webdriver.FirefoxProfile()
proxy = Proxy()
proxy.proxy_autoconfig_url = 'http://some.url:12345/path'
self.profile.set_proxy(proxy)
assert self.profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
assert self.profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'
def testCanNotChangeInitializedProxyType(self):
proxy = Proxy(raw={"proxyType": "direct"})
try:
proxy.proxy_type = ProxyType.SYSTEM
raise Exception("Change of already initialized proxy type should raise exception")
except Exception:
pass
proxy = Proxy(raw={"proxyType": ProxyType.DIRECT})
try:
proxy.proxy_type = ProxyType.SYSTEM
raise Exception("Change of already initialized proxy type should raise exception")
except Exception:
pass
def testCanAddToDesiredCapabilities(self):
desired_capabilities = {}
proxy = Proxy()
proxy.http_proxy = 'some.url:1234'
proxy.add_to_capabilities(desired_capabilities)
expected_capabilities = {
'proxy': {
'proxyType': 'manual',
'httpProxy': 'some.url:1234'
}
}
self.assertEqual(expected_capabilities, desired_capabilities)
def test_manual_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.no_proxy = 'localhost, foo.localhost'
proxy.http_proxy = 'some.url:1234'
proxy.ftp_proxy = None
proxy.sslProxy = 'some2.url'
profile.set_proxy(proxy)
assert profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
assert profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
assert profile.default_preferences["network.proxy.http"] == 'some.url'
assert profile.default_preferences["network.proxy.http_port"] == 1234
assert profile.default_preferences["network.proxy.ssl"] == 'some2.url'
assert "network.proxy.ssl_port" not in profile.default_preferences
assert "network.proxy.ftp" not in profile.default_preferences
def testCanAddManualProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.http_proxy = self.MANUAL_PROXY['httpProxy']
proxy.ftp_proxy = self.MANUAL_PROXY['ftpProxy']
proxy.no_proxy = self.MANUAL_PROXY['noProxy']
proxy.sslProxy = self.MANUAL_PROXY['sslProxy']
proxy.socksProxy = self.MANUAL_PROXY['socksProxy']
proxy.socksUsername = self.MANUAL_PROXY['socksUsername']
proxy.socksPassword = self.MANUAL_PROXY['socksPassword']
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.MANUAL_PROXY.copy()
proxy_capabilities['proxyType'] = 'MANUAL'
expected_capabilities = {'proxy': proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def __init__(self, host="127.0.0.1", server="./selenium-server.jar"):
if server and path.isfile(server) and not _server_started():
self.selenium = _Selenium(server)
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': host,
'ftpProxy': host,
'sslProxy': host,
'noProxy': host
})
caps = webdriver.DesiredCapabilities.FIREFOX
proxy.add_to_capabilities(caps)
try:
self.driver = webdriver.Remote(desired_capabilities=caps)
except URLError:
raise SeleniumServerError
def testCanAddManualProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.http_proxy = self.MANUAL_PROXY["httpProxy"]
proxy.ftp_proxy = self.MANUAL_PROXY["ftpProxy"]
proxy.no_proxy = self.MANUAL_PROXY["noProxy"]
proxy.sslProxy = self.MANUAL_PROXY["sslProxy"]
proxy.socksProxy = self.MANUAL_PROXY["socksProxy"]
proxy.socksUsername = self.MANUAL_PROXY["socksUsername"]
proxy.socksPassword = self.MANUAL_PROXY["socksPassword"]
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.MANUAL_PROXY.copy()
proxy_capabilities["proxyType"] = "MANUAL"
expected_capabilities = {"proxy": proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def __init__(self, host="127.0.0.1", server="./selenium-server.jar"):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = sock.connect_ex(("127.0.0.1", 4444)) == 0
if server and path.isfile(server) and not port:
self.selenium = _Selenium(server)
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': host,
'ftpProxy': host,
'sslProxy': host,
'noProxy': host
})
caps = webdriver.DesiredCapabilities.FIREFOX
proxy.add_to_capabilities(caps)
try:
self.driver = webdriver.Remote(desired_capabilities=caps)
except URLError:
raise SeleniumServerError
def testCanInitEmptyProxy():
proxy = Proxy()
assert ProxyType.UNSPECIFIED == proxy.proxy_type
assert '' == proxy.http_proxy
assert '' == proxy.ftp_proxy
assert '' == proxy.no_proxy
assert '' == proxy.sslProxy
assert '' == proxy.socksProxy
assert '' == proxy.socksUsername
assert '' == proxy.socksPassword
assert proxy.auto_detect is False
assert '' == proxy.proxy_autoconfig_url
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = {}
proxy_capabilities['proxyType'] = 'UNSPECIFIED'
expected_capabilities = {'proxy': proxy_capabilities}
assert expected_capabilities == desired_capabilities
def testCanInitEmptyProxy(self):
proxy = Proxy()
self.assertEqual(ProxyType.UNSPECIFIED, proxy.proxy_type)
self.assertEqual('', proxy.http_proxy)
self.assertEqual('', proxy.ftp_proxy)
self.assertEqual('', proxy.no_proxy)
self.assertEqual('', proxy.sslProxy)
self.assertEqual('', proxy.socksProxy)
self.assertEqual('', proxy.socksUsername)
self.assertEqual('', proxy.socksPassword)
self.assertEqual(False, proxy.auto_detect)
self.assertEqual('', proxy.proxy_autoconfig_url)
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = {}
proxy_capabilities['proxyType'] = 'UNSPECIFIED'
expected_capabilities = {'proxy': proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
def testCanAddToDesiredCapabilities(self):
desired_capabilities = {}
proxy = Proxy()
proxy.http_proxy = 'some.url:1234'
proxy.ftp_proxy = 'ftp.proxy:1234'
proxy.no_proxy = 'localhost, foo.localhost'
proxy.sslProxy = 'ssl.proxy:1234'
proxy.autodetect = 'True'
proxy.add_to_capabilities(desired_capabilities)
expected_capabilities = {
'proxy': {
'proxyType': 'MANUAL',
'httpProxy': 'some.url:1234',
'ftpProxy': 'ftp.proxy:1234',
'noProxy': 'localhost, foo.localhost',
'sslProxy': 'ssl.proxy:1234',
'autodetect': 'True'
}
}
print 'descap', desired_capabilities
self.assertEqual(expected_capabilities, desired_capabilities)
def test_manual_proxy_is_set_in_profile():
profile = FirefoxProfile()
proxy = Proxy()
proxy.no_proxy = 'localhost, foo.localhost'
proxy.http_proxy = 'some.url:1234'
proxy.ftp_proxy = None
proxy.sslProxy = 'some2.url'
profile.set_proxy(proxy)
assert profile.default_preferences[
"network.proxy.type"] == ProxyType.MANUAL['ff_value']
assert profile.default_preferences[
"network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
assert profile.default_preferences["network.proxy.http"] == 'some.url'
assert profile.default_preferences["network.proxy.http_port"] == 1234
assert profile.default_preferences["network.proxy.ssl"] == 'some2.url'
assert "network.proxy.ssl_port" not in profile.default_preferences
assert "network.proxy.ftp" not in profile.default_preferences
def testCanNotChangeInitializedProxyType(self):
proxy = Proxy(raw={'proxyType': 'direct'})
try:
proxy.proxy_type = ProxyType.SYSTEM
raise Exception(
"Change of already initialized proxy type should raise exception"
)
except Exception as e:
pass
proxy = Proxy(raw={'proxyType': ProxyType.DIRECT})
try:
proxy.proxy_type = ProxyType.SYSTEM
raise Exception(
"Change of already initialized proxy type should raise exception"
)
except Exception as e:
pass
def __init__(self, Url, proxy_url):
self.songUrl = Url
self.id = int(self.songUrl.split('=')[1])
self.lv = -1
self.gender = -1
self.homeUrl = 'https://music.163.com/#/user/home?id=' + str(self.id)
self.followersUrl = 'https://music.163.com/#/user/follows?id=' + str(
self.id)
self.songRankUrl = 'https://music.163.com/#/user/songs/rank?id=' + str(
self.id)
self.recent_song_list = []
self.follows_id_list = []
self.playlists = []
chrome_options = Options()
chrome_options.add_argument('--headless')
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy_url
prox.ssl_proxy = proxy_url
# prox.socks_proxy = proxy_url
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
if (config_is_ubuntu):
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
# chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
# chrome_options.add_argument('user-agent={0}'.format(random.choice(uas)))
# debug_print_thread("we are using proxy sever with url " + proxy_url)
self.driver_home = webdriver.Chrome(config_chrome_path,
options=chrome_options)
self.driver_recent_songs = webdriver.Chrome(config_chrome_path,
options=chrome_options)
#self.driver_recent_songs.set_page_load_timeout(10)
self.driver_follows = webdriver.Chrome(config_chrome_path,
options=chrome_options)
def get_browser(self, proxy):
# 启动浏览器
# 禁止加载image
firefox_profile = webdriver.FirefoxProfile()
#firefox_profile.set_preference('permissions.default.stylesheet', 2)
#firefox_profile.set_preference('permissions.default.image', 2)
#firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
# 代理
if proxy and proxy.is_valid():
myProxy = '%s:%s' % (proxy.host, proxy.port)
ff_proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': ''
})
browser = webdriver.Firefox(firefox_profile=firefox_profile,
proxy=ff_proxy)
else:
browser = webdriver.Firefox(firefox_profile=firefox_profile)
return browser
def save_screenshot(self, main_url, screenshot_path):
try:
from pyvirtualdisplay import Display
display = Display(visible=0, size=(1024, 768))
display.start()
from selenium import webdriver
try:
if '127.0.0.1' in main_url or 'localhost' in main_url:
br = webdriver.Firefox()
else:
from selenium.webdriver.common.proxy import Proxy, ProxyType
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': HTTP_PROXY
})
br = webdriver.Firefox(proxy=proxy)
except Exception, e:
LOG.exception(e)
br = webdriver.Firefox()
br.get(main_url)
br.save_screenshot(screenshot_path)
br.quit()
display.stop()
return screenshot_path
def ready(proxy_ip_port=None):
'''
:param proxy_ip_port:
:return:
'''
if system == 'windows':
executable_path = 'D:/geckodriver/geckodriver.exe'
elif system == 'linux':
executable_path = '../temp_file/geckodriver'
else:
logger.warning('没有这个系统：%s,暂时默认为windows' % system)
executable_path = 'D:/geckodriver/geckodriver.exe'
fireFoxOptions = webdriver.FirefoxOptions()
fireFoxOptions.set_headless()
# fireFoxOptions.add_argument("--proxy-server=http://{}".format(random.choice(ip_list)))
# driver = webdriver.Firefox( executable_path='temp_file/geckodriver',options=fireFoxOptions)
if proxy_ip_port:
proxy = Proxy({'httpProxy': proxy_ip_port})
else:
proxy = None
if mode == 'product':
driver = webdriver.Firefox(executable_path=executable_path, options=fireFoxOptions,
proxy=proxy)
elif mode == 'debug':
if system == 'windows':
binary_path = 'C:/Program Files (x86)/Mozilla Firefox/firefox.exe'
else:
binary_path = '/usr/bin/firefox'
binary = FirefoxBinary(binary_path)
driver = webdriver.Firefox(firefox_binary=binary, executable_path=executable_path, proxy=proxy)
else:
raise Exception('ready方法的参数里面没有这种mode：%s' % mode)
driver.implicitly_wait(10) # 设置寻找（包括异步加载的）element所等待的时间（如果不设置，则异步加载的element有可能会找不到）
return driver
def __init__(self, *args, **kwargs):
"""
Initializes spider. Starts xsession in virtual display and
webdriver instance in this xsession. If PROXY_PARAMS defined in
settings.py it'll specify proxy for webdriver instance.
"""
super(IrrSpider, self).__init__(*args, **kwargs)
settings = get_project_settings()
self.xsession = Display(
visible=settings.get('XSESSION_VISIBLE', False),
size=settings.get('XSESSION_DISPLAY_RESOLUTION', (800, 600)),
)
self.xsession.start()
proxy = settings.get('PROXY_PARAMS')
if proxy:
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy,
'ftpProxy': proxy,
'sslProxy': proxy,
'noProxy': proxy,
})
self.browser = webdriver.Firefox(proxy=proxy)
self.browser.maximize_window()
def find_proxy(self, ip):
random.shuffle(self.ip_list)
proxy_ip = self.ip_list[0]
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_ip
})
opt = webdriver.ChromeOptions()
opt.add_argument('--proxy-server=http://'+proxy_ip)
self.browser = webdriver.Chrome(self.chrome_driver, chrome_options=opt)
flag = True
locator = (By.LINK_TEXT, '丁香园')
try:
self.browser.get(ip)
self.browser.find_element_by_id('head')
WebDriverWait(self.browser, 10, 0.5).until(EC.presence_of_element_located(locator))
except:
flag = False
self.browser.quit()
finally:
ip = proxy_ip
return flag
'''
def make_options(self, headless=True, proxing=False):
'''Конфигурирование браузера'''
dict_options = {}
options = Options()
if headless:
options.add_argument("--headless")
if self.binary_location:
options.binary_location = (self.binary_location)
options.page_load_strategy = 'eager'
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('user-agent={}'.format(load_ua()))
dict_options['options'] = options
if proxing:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = 'socks5://127.0.0.1:9050'
prox.ssl_proxy = 'socks5://127.0.0.1:9050'
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
dict_options['desired_capabilities'] = capabilities
return dict_options
def proxy_driver(PROXIES, ua):
print('\n----CONNECTING TO PROXY----')
options = webdriver.ChromeOptions()
prox = Proxy()
if len(PROXIES) < 1:
print("--- Proxies used up (%s)" % len(PROXIES))
PROXIES = get_proxies()
pxy = PROXIES.pop()
#pxy = '80.25.87.49:32850'
print(f'Current Proxy ({pxy})')
prox.proxy_type = ProxyType.MANUAL
prox.autodetect = False
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
options.Proxy = prox
print('\n----CHANGING USER AGENT----')
userAgent = ua.random
print(f'Current UserAgent\n{userAgent}')
options.add_argument("--start-maximized")
options.add_argument("--proxy-server=%s" % pxy)
options.add_argument("user-agent={userAgent}")
options.add_argument("ignore-certificate-errors")
options.add_argument(
"--disable-bundled-ppapi-flash") # Disable internal Flash player
options.add_argument(
"--disable-plugins-discovery"
) # Disable external Flash player (by not allowing it to load)
options.add_extension('mpbjkejclgfgadiemmefgebjfooflfhl.crx')
driver = webdriver.Chrome('chromedriver.exe', options=options)
return driver, PROXIES
def __open_browser(context):
chrm = context.config.userdata['chromedriver_path']
try:
# if there is a proxy, we'll use it. Otherwise, we won't.
requests.get("http://localhost:8888", timeout=0.01)
# if there was no exception, we continue here.
PROXY = "localhost:8888"
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
if (chrm):
context.driver = webdriver.Chrome(
desired_capabilities=capabilities, executable_path=chrm)
else:
context.driver = webdriver.Chrome(
desired_capabilities=capabilities)
return context.driver
except:
if (chrm):
context.driver = webdriver.Chrome(executable_path=chrm)
else:
# adding the service args as described below will cause Chromedriver
# to create a log of the communication between it and the Chrome
# browser. It's eye-opening.
#
# for instance:
# [1568045962.076][INFO]: [e18882b1f2abbda89f232f777f98f686] COMMAND TypeElement {
# "id": "0.47079920350295135-1",
# "sessionId": "e18882b1f2abbda89f232f777f98f686",
# "text": "Byron",
# "value": [ "B", "y", "r", "o", "n" ]
# }
#context.driver = webdriver.Chrome(service_args=["--verbose","--logepath=C:\\temp\\qc1.log"])
context.driver = webdriver.Chrome()
return context.driver
def test_manual_proxy_is_set_in_profile(self):
# The setup gave us a browser but we dont need it
self.driver.quit()
self.profile = webdriver.FirefoxProfile()
proxy = Proxy()
proxy.no_proxy = 'localhost, foo.localhost'
proxy.http_proxy = 'some.url:1234'
proxy.ftp_proxy = None
proxy.sslProxy = 'some2.url'
self.profile.set_proxy(proxy)
assert self.profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
assert self.profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
assert self.profile.default_preferences["network.proxy.http"] == 'some.url'
assert self.profile.default_preferences["network.proxy.http_port"] == 1234
assert self.profile.default_preferences["network.proxy.ssl"] == 'some2.url'
assert "network.proxy.ssl_port" not in self.profile.default_preferences
assert "network.proxy.ftp" not in self.profile.default_preferences
def open_browser():
try:
# if there is a proxy, we'll use it. Otherwise, we won't.
requests.get("http://localhost:8888", timeout=0.01)
# if there was no exception, we continue here.
PROXY = "localhost:8888"
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
return driver
except:
driver = webdriver.Chrome()
return driver
def getFirefoxDriver(host, port):
try:
path = 'C:\\webdrivers\\geckodriver.exe'
fullproxy = host + ":" + port
#https://stackoverflow.com/questions/17082425/running-selenium-webdriver-with-a-proxy-in-python
firefox_capabilities = DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
"httpProxy": fullproxy,
"ftpProxy": fullproxy,
"sslProxy": fullproxy,
"noProxy": None,
"proxyType": "MANUAL",
"autodetect": False
}
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = fullproxy
prox.socks_proxy = fullproxy
prox.ssl_proxy = fullproxy
#prox.add_to_capabilities(firefox_capabilities)
fp = webdriver.FirefoxProfile()
fp.set_preference("network.proxy.type", 1)
fp.set_preference("network.proxy.http", host)
fp.set_preference("network.proxy.http_port", int(port))
fp.set_preference("network.proxy.https", host)
fp.set_preference("network.proxy.https_port", int(port))
fp.set_preference("network.proxy.ssl", host)
fp.set_preference("network.proxy.ssl_port", int(port))
fp.set_preference("network.proxy.ftp", host)
fp.set_preference("network.proxy.ftp_port", int(port))
fp.set_preference("network.proxy.socks", host)
fp.set_preference("network.proxy.socks_port", int(port))
#fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")
fp.update_preferences()
driver = webdriver.Firefox(executable_path=path)
return driver
except Exception:
LogError(traceback, "host = " + host + " and port = " + port)
return getGoogleChromeDriver(host + ":" + port)
def driver(request):
# wd = webdriver.Chrome()
# wd = webdriver.Chrome(desired_capabilities={"chromeOptions": {"args": ["--start-fullscreen"]}})
# wd = webdriver.Chrome(desired_capabilities={"proxy": {"proxyType": "MANUAL", "httpProxy": "localhost:8888"}})
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "127.0.0.1:8888"
# prox.socks_proxy = "127.0.0.1:8888" # Does not work with this
prox.ssl_proxy = "127.0.0.1:8888"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
wd = webdriver.Chrome(desired_capabilities=capabilities)
# wd = webdriver.Firefox()
# options = webdriver.FirefoxOptions()
# options.binary_location = "C:\\Program Files\\Firefox Nightly\\firefox.exe"
# options.add_argument("start-maximized")
# wd = webdriver.Firefox(firefox_options=options)
# new method
# wd = webdriver.Firefox()
# new method more obviously
# wd = webdriver.Firefox(capabilities={"marionette": True})
# old method
# wd = webdriver.Firefox(capabilities={"marionette": False})
# wd = webdriver.Edge(executable_path="C:\Webdrivers\msedgedriver")
# wd = webdriver.Ie()
# wd = webdriver.Ie(capabilities={"unexpectedAlertBehaviour": "dismiss"})
# wd = webdriver.Ie(capabilities={"requireWindowFocus": True})
# wd = webdriver.Ie(capabilities={"IntroduceInstabilityByIgnoringProtectedModeSettings": True, "requireWindowFocus": True, "unexpectedAlertBehaviour": "dismiss", "ignoreZoomSetting": True})
# print(f'\nCAPABILITIES: {wd.capabilities}\nEND CAPABILITIES')
print(f'\nWD capabilities: {wd.capabilities}\n')
request.addfinalizer(wd.quit)
return wd
def proxy_driver(self, p, ua):
co = webdriver.ChromeOptions()
co.add_argument('--no-sandbox')
co.add_argument('--disable-dev-shm-usage')
co.add_argument(f"user-agent={ua}")
co.add_argument("--headless")
co.add_argument(f"--window-size={choice(('1024,768', '1280,960', '1280,1024', '1600,1200', '1920,1080'))}")
pxy = p
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = pxy
prox.socks_proxy = pxy
prox.ssl_proxy = pxy
capabilities = webdriver.DesiredCapabilities.CHROME
# capabilities['resolution'] = '1920x1080'
# prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome('/home/jurgeon/projects/jcb/materials/jcb/chromedriver', options = co, desired_capabilities = capabilities)
return driver
def _setup_driver(self):
proxy = Proxy()
proxy.proxy_type = ProxyType.DIRECT
if 'http' in self.proxy:
proxy.http_proxy = self.proxy['http']
if 'https' in self.proxy:
proxy.ssl_proxy = self.proxy['https']
capa = self._build_capabilities()
proxy.add_to_capabilities(capa)
options = self._build_options()
# TODO some browsers don't need headless
# TODO handle different proxy setting?
options.set_headless(self.HEADLESS)
if self.DRIVER is webdriver.Firefox:
if self.responses_dirname and not os.path.isdir(self.responses_dirname):
os.makedirs(self.responses_dirname)
options.profile = DirFirefoxProfile(self.responses_dirname)
if self.responses_dirname:
capa['profile'] = self.responses_dirname
self.driver = self.DRIVER(options=options, capabilities=capa)
elif self.DRIVER is webdriver.Chrome:
self.driver = self.DRIVER(options=options, desired_capabilities=capa)
elif self.DRIVER is webdriver.PhantomJS:
if self.responses_dirname:
if not os.path.isdir(self.responses_dirname):
os.makedirs(self.responses_dirname)
log_path = os.path.join(self.responses_dirname, 'selenium.log')
else:
log_path = NamedTemporaryFile(prefix='weboob_selenium_', suffix='.log', delete=False).name
self.driver = self.DRIVER(desired_capabilities=capa, service_log_path=log_path)
else:
raise NotImplementedError()
if self.WINDOW_SIZE:
self.driver.set_window_size(*self.WINDOW_SIZE)
def main():
"""
This is the entry for the command which makes it convenient to install the proxy certificate
"""
setupLocalLogging()
commandArgs = sys.argv[1:]
proxyPort = findFreePort()
proxyThread = threading.Thread(target=runProxy,
args=[proxyPort],
daemon=True)
proxyThread.start()
capabilities = webdriver.DesiredCapabilities.CHROME
capabilities['loggingPrefs'] = {'browser': 'ALL'}
proxyConfig = Proxy()
proxyConfig.proxy_type = ProxyType.MANUAL
proxyConfig.http_proxy = f"localhost:{proxyPort}"
proxyConfig.add_to_capabilities(capabilities)
chrome_options = webdriver.chrome.options.Options()
if len(commandArgs) > 0:
chrome_options.headless = True
chrome_options.add_argument(f"--no-sandbox")
driver = webdriver.Chrome(desired_capabilities=capabilities,
chrome_options=chrome_options)
driver.get("http://mitm.it/")
print(
"Please kill the command with Ctrl-C or (Cmd-C on macOS) when you are finished installing the certificates. Timeout in 600 seconds..."
)
timeout = 600
if len(commandArgs) > 0:
timeout = int(str(commandArgs[0]))
time.sleep(timeout)
def open_browser():
try:
# if there is a proxy, we'll use it. Otherwise, we won't.
requests.get("http://localhost:8888", timeout=0.01)
# if there was no exception, we continue here.
PROXY = "localhost:8888"
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
return driver
except:
# if we got an exception while trying to hit a proxy URL,
# it probably means the proxy isn't available, so we run
# without a proxy, as follows.
driver = webdriver.Chrome()
return driver
def test2(self):
driverLocation = '/Users/jamesbaker/Desktop/footlocker_notes/chromedriver'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "34.93.147.196:80"
# prox.socks_proxy = "ip_addr:port"
# prox.ssl_proxy = "ip_addr:port"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(driverLocation,
chrome_options=chrome_options,
desired_capabilities=capabilities)
# driver.get("https://www.footlocker.com/product/nike-air-force-1-07-low-womens/O2132602.html")
driver.get("https://www.google.com")
print(driver.page_source)
# elem = driver.find_element_by_xpath("//button[contains('Add To Cart')]")
# elem = driver.find_element_by_css_selector(".Button.ProductDetails-form__action")
# elem.click()
# def test(self):
# driverLocation = '/Users/jamesbaker/Desktop/footlocker_notes/chromedriver'
# chrome_options = webdriver.ChromeOptions()
PROXY = "smart-us.jeru035.com:23774:jerursvpking8588:rsvpking8588"
PROXY = "jerursvpking8588:[email protected]:23774"
PROXY = "174.127.155.118:32505"
# # chrome_options.add_argument('--headless')
# chrome_options.add_argument('window-size=1200x600') # optional
#
# chrome_options.add_argument('--incognito')
# chrome_options.add_argument('--no-sandbox')
# chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--proxy-server=%s' % PROXY)
from selenium.webdriver.support.ui import WebDriverWait
proxy = "12.345.678.910:8080"
options = WebDriverWait.ChromeOptions()
options.add_argument('--proxy-server=%s' % proxy)
chrome = webdriver.Chrome(chrome_options=options)
chrome.get("https://www ... ")
# or
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "ip_addr:port"
prox.socks_proxy = "ip_addr:port"
prox.ssl_proxy = "ip_addr:port"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
# or Proxy for FireFox
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "proxy.server.address")
def get_browser(user_agent, proxy_address, cert_path):
""" Set up a Selenium browser with given user agent, proxy and SSL cert. """
# PhantomJS
if settings.CAPTURE_BROWSER == 'PhantomJS':
desired_capabilities = dict(DesiredCapabilities.PHANTOMJS)
desired_capabilities["phantomjs.page.settings.userAgent"] = user_agent
browser = webdriver.PhantomJS(
executable_path=getattr(settings, 'PHANTOMJS_BINARY', 'phantomjs'),
desired_capabilities=desired_capabilities,
service_args=[
"--proxy=%s" % proxy_address,
"--ssl-certificates-path=%s" % cert_path,
"--ignore-ssl-errors=true",
"--local-url-access=false",
"--local-storage-path=.",
],
service_log_path=settings.PHANTOMJS_LOG)
# Firefox
elif settings.CAPTURE_BROWSER == 'Firefox':
desired_capabilities = dict(DesiredCapabilities.FIREFOX)
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_address,
'ftpProxy': proxy_address,
'sslProxy': proxy_address,
})
proxy.add_to_capabilities(desired_capabilities)
profile = webdriver.FirefoxProfile()
profile.accept_untrusted_certs = True
profile.assume_untrusted_cert_issuer = True
browser = webdriver.Firefox(
capabilities=desired_capabilities,
firefox_profile=profile)
# Chrome
elif settings.CAPTURE_BROWSER == 'Chrome':
# http://blog.likewise.org/2015/01/setting-up-chromedriver-and-the-selenium-webdriver-python-bindings-on-ubuntu-14-dot-04/
download_dir = os.path.abspath('./downloads')
os.mkdir(download_dir)
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % proxy_address)
chrome_options.add_argument('--test-type')
chrome_options.add_experimental_option("prefs", {"profile.default_content_settings.popups": "0",
"download.default_directory": download_dir,
"download.prompt_for_download": "false"})
desired_capabilities = chrome_options.to_capabilities()
desired_capabilities["acceptSslCerts"] = True
# for more detailed progress updates
# desired_capabilities["loggingPrefs"] = {'performance': 'INFO'}
# then:
# performance_log = browser.get_log('performance')
browser = webdriver.Chrome(desired_capabilities=desired_capabilities)
else:
assert False, "Invalid value for CAPTURE_BROWSER."
browser.implicitly_wait(ELEMENT_DISCOVERY_TIMEOUT)
browser.set_page_load_timeout(ROBOTS_TXT_TIMEOUT)
return browser
def testCanAddAutodetectProxyToDesiredCapabilities(self):
proxy = Proxy(raw=self.AUTODETECT_PROXY)
self.assertEqual(ProxyType.AUTODETECT, proxy.proxy_type)
self.assertEqual(self.AUTODETECT_PROXY['autodetect'],
proxy.auto_detect)
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.common.proxy import ProxyType
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': '114.235.82.16:8118' # 代理ip和端口
})
# 新建一个代理IP对象
desired_capabilities = DesiredCapabilities.PHANTOMJS.copy()
# 加入代理IP
proxy.add_to_capabilities(desired_capabilities)
driver = webdriver.PhantomJS(executable_path='E:/phantomjs/bin/phantomjs.exe',
desired_capabilities=desired_capabilities)
# 测试一下，打开使用的代理IP地址信息
driver.get('http://www.whatismyip.com.tw/')
print(driver.page_source)
requesting the driver to refresh the page after 1-minute sleep...')
time.sleep(60)
canvas = _wait_for_page_to_present(driver)
continue
if __name__ == '__main__':
logging_.initialize(module_name='game_abstract_crawler',
service_name='crawler')
options = Options()
options.headless = True
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--window-size=800,600')
proxy = Proxy()
proxy.http_proxy = 'localhost:8080'
proxy.https_proxy = 'localhost:8080'
capabilities = DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
with Chrome(options=options,
desired_capabilities=capabilities) as driver:
try:
main(driver)
except Exception as e:
_get_screenshot(driver, '99-エラー.png')
logging.exception('Abort with an unhandled exception.')
raise
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium import webdriver
from scroll import scroll
browser = None
proxy_address = "127.0.0.1:8118"
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': proxy_address,
})
gecko = "./geckodriver"
firefox = "/usr/bin/firefox"
firefox_binary = FirefoxBinary(firefox)
def callback(driver):
driver.quit()
def driver(binary=None, proxy=None):
return webdriver.Firefox(executable_path=gecko,
firefox_binary=binary,
proxy=proxy)
if __name__ == "__main__":
def _search(self, start, visible=0):
payload = {}
try:
payload['q'] = self.query.encode('utf8') # query to lookup
except:
payload['q'] = self.query # query to lookup
payload['start'] = start # start point
payload['gl'] = self.country # query from country
payload['hl'] = self.language # user query language
payload['lr'] = 'lang_%s' % self.language # restrict language pages
payload['num'] = GoogleSeleniumPlus.PAGE_LIMIT
payload['safe'] = 'off'
params = urllib.urlencode(payload)
display = Display(visible=visible, size=(800, 600))
display.start()
try:
proxyInfo = ProxyManager.getNextProxy()
myProxy = '%s:%s' % (proxyInfo.host, proxyInfo.port)
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': '' # set this value as desired
})
browser = webdriver.Firefox(proxy=proxy)
browser.set_page_load_timeout(30)
try:
params = urllib.urlencode(payload)
browser.implicitly_wait(10)
browser.get('%s#%s' % (self.googleHost, params))
app_logger.info(u"%s" % browser.current_url)
h3List = browser.find_elements_by_xpath("//h3[@class='r']")
results = []
for h3 in h3List:
link = h3.find_element_by_tag_name('a')
results.append(link.get_attribute("href"))
box = browser.find_element_by_id('lst-ib')
partialQuery = ' '.join(self.query.split()[1:])
for _letter in partialQuery:
box.send_keys(Keys.BACKSPACE)
randomSleep(0.03, 0.05)
typeQuery(box, partialQuery)
randomSleep(0.05, 0.25)
print('-' * 80)
finally:
browser.close()
except Exception as ex:
raise ex
finally:
display.stop()
if not results:
ProxyManager.invalidateProxy()
return results
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.proxy import Proxy, ProxyType
input_email = '*****@*****.**'
input_password = '******'
# Not working right now
myProxy = ''
proxyF = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
'ftpProxy': myProxy,
'sslProxy': myProxy,
'noProxy': '' # set this value as desired
})
def await_element(driver, selector):
try:
element_present = EC.presence_of_element_located(
(By.CSS_SELECTOR, selector))
WebDriverWait(driver, 180).until(element_present)
except TimeoutException:
raise
# Overwrite Selenium headers
HTML_INJECTION_STORED_BLOG = '/htmli_stored.php'
IFRAME_INJECTION = '/iframei.php'
LDAP_INJECTION = ''
MAIL_HEADER_INJECTION = ''
OS_COMMAND_INJECTION = ''
OS_COMMAND_INJECTION_BLIND = ''
PHP_CODE_INJECTION = ''
# set zed attack proxy
zap = ZAPv2(apikey=APIKEY)
# Use the line below if ZAP is not listening on port 8080, for example, if listening on port 8090
# zap = ZAPv2(apikey=apikey, proxies={'http': 'http://127.0.0.1:8090', 'https': 'http://127.0.0.1:8090'})
# Set zap proxy with defined docker ip address
PROXY_ADDRESS = "172.17.0.6:8080"
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY_ADDRESS
proxy.ssl_proxy = PROXY_ADDRESS
# set desired capabilities for firefox client
capabilities = webdriver.DesiredCapabilities.FIREFOX
proxy.add_to_capabilities(capabilities)
driver = webdriver.Remote(command_executor=SELENIUM_HUB_ADDRESS,
desired_capabilities=capabilities)
def login_bwapp(target):
driver.get(target + '/login.php')
driver.find_element_by_id("login").send_keys("bee")
def start_client(self):
capabilities = {}
for c in self.capabilities:
name, value = c.split(':')
# handle integer capabilities
if value.isdigit():
value = int(value)
# handle boolean capabilities
elif value.lower() in ['true', 'false']:
value = value.lower() == 'true'
capabilities.update({name: value})
if self.proxy_host and self.proxy_port:
proxy = Proxy()
proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port)
proxy.ssl_proxy = proxy.http_proxy
proxy.add_to_capabilities(capabilities)
profile = None
if self.driver.upper() == 'REMOTE':
capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper()))
if json.loads(self.chrome_options) or self.extension_paths:
capabilities = self.create_chrome_options(
self.chrome_options,
self.extension_paths).to_capabilities()
if self.browser_name.upper() == 'FIREFOX':
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
if self.browser_version:
capabilities['version'] = self.browser_version
capabilities['platform'] = self.platform.upper()
executor = 'http://%s:%s/wd/hub' % (self.host, self.port)
try:
self.selenium = webdriver.Remote(command_executor=executor,
desired_capabilities=capabilities or None,
browser_profile=profile)
except AttributeError:
valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')]
raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers)))
elif self.driver.upper() == 'CHROME':
options = None
if self.chrome_options or self.extension_paths:
options = self.create_chrome_options(
self.chrome_options,
self.extension_paths)
if self.chrome_path:
self.selenium = webdriver.Chrome(executable_path=self.chrome_path,
chrome_options=options,
desired_capabilities=capabilities or None)
else:
self.selenium = webdriver.Chrome(chrome_options=options,
desired_capabilities=capabilities or None)
elif self.driver.upper() == 'FIREFOX':
binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None
profile = self.create_firefox_profile(
self.firefox_preferences,
self.profile_path,
self.extension_paths)
self.selenium = webdriver.Firefox(
firefox_binary=binary,
firefox_profile=profile,
capabilities=capabilities or None)
elif self.driver.upper() == 'IE':
self.selenium = webdriver.Ie()
elif self.driver.upper() == 'PHANTOMJS':
self.selenium = webdriver.PhantomJS()
elif self.driver.upper() == 'OPERA':
capabilities.update(webdriver.DesiredCapabilities.OPERA)
self.selenium = webdriver.Opera(executable_path=self.opera_path,
desired_capabilities=capabilities)
elif self.driver.upper() == 'BROWSERSTACK':
from cloud import BrowserStack
self.cloud = BrowserStack()
self.selenium = self.cloud.driver(
self.test_id, capabilities, self.options)
elif self.driver.upper() == 'SAUCELABS':
from cloud import SauceLabs
self.cloud = SauceLabs()
self.selenium = self.cloud.driver(
self.test_id, capabilities, self.options, self.keywords)
else:
self.selenium = getattr(webdriver, self.driver)()
if self.event_listener is not None and not isinstance(self.selenium, EventFiringWebDriver):
self.selenium = EventFiringWebDriver(self.selenium, self.event_listener())
def testCanAddPACProxyToDesiredCapabilities(self):
proxy = Proxy(raw=self.PAC_PROXY)
self.assertEqual(ProxyType.PAC, proxy.proxy_type)
self.assertEqual(self.PAC_PROXY['proxyAutoconfigUrl'],
proxy.proxy_autoconfig_url)
def testCanAddManualProxyToDesiredCapabilities(self):
proxy = Proxy()
proxy.http_proxy = self.MANUAL_PROXY['httpProxy']
proxy.ftp_proxy = self.MANUAL_PROXY['ftpProxy']
proxy.no_proxy = self.MANUAL_PROXY['noProxy']
proxy.sslProxy = self.MANUAL_PROXY['sslProxy']
proxy.socksProxy = self.MANUAL_PROXY['socksProxy']
proxy.socksUsername = self.MANUAL_PROXY['socksUsername']
proxy.socksPassword = self.MANUAL_PROXY['socksPassword']
desired_capabilities = {}
proxy.add_to_capabilities(desired_capabilities)
proxy_capabilities = self.MANUAL_PROXY.copy()
proxy_capabilities['proxyType'] = 'MANUAL'
expected_capabilities = {'proxy': proxy_capabilities}
self.assertEqual(expected_capabilities, desired_capabilities)
#! usr/bin/env/python3
# coding:utf-8
# @Time: 2019-04-03 15:55
# Author: turpure
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
# prox.http_proxy = "127.0.0.1:1080"
# prox.socks_proxy = "127.0.0.1:1080"
prox.ssl_proxy = "175.155.249.57:8888"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
def work():
base_url = 'https://www.trackingmore.com/sprintpack-tracking/cn.html?number=0B048028400019917237E'
options = webdriver.ChromeOptions()
prefs = {