def get_web_driver(url, has_proxy=True):
"""
Selenium 使用代理请求
:param url:
:return driver:
"""
if has_proxy:
proxies = get_proxies()
if proxies.has_key('http'):
myProxy = proxies['http']
elif proxies.has_key('https'):
myProxy = proxies['https']
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
# 'ftpProxy': myProxy,
# 'sslProxy': myProxy,
# 'noProxy':d ''
})
print encode_wrap("使用代理:"), myProxy
driver = webdriver.Firefox(proxy=proxy)
else:
driver = webdriver.Firefox()
#driver.set_page_load_timeout(30)
driver.get(url)
return driver
def get_session(url, has_proxy=True, cookie=None):
"""
requests Session使用代理请求
:param url:
:param has_proxy:
:param cookie:
:return requests:
"""
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36'}
proxy = None
if has_proxy:
proxy = get_proxies()
cf = ConfigParser.ConfigParser()
cf.read('../config.ini')
timeout = int(cf.get('web', 'timeout'))
s = requests.Session()
r = s.get(url, proxies=proxy, headers=headers, cookies=cookie,timeout=timeout)
output = 'Code:{0} Proxy:{1}'.format(r.status_code, proxy)
print encode_wrap(output)
if int(r.status_code) != 200:
raise Exception('request fail')
return r, s