def get_session(url, has_proxy=True, cookie=None):
"""
requests Session使用代理请求
:param url:
:param has_proxy:
:param cookie:
:return requests:
"""
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36'
}
proxy = None
if has_proxy:
proxy = get_proxies()
cf = ConfigParser.ConfigParser()
cf.read('../config.ini')
timeout = int(cf.get('web', 'timeout'))
s = requests.Session()
r = s.get(url,
proxies=proxy,
headers=headers,
cookies=cookie,
timeout=timeout)
output = 'Code:{0} Proxy:{1}'.format(r.status_code, proxy)
print encode_wrap(output)
if int(r.status_code) != 200:
raise Exception('request fail')
return r, s
def get_web_driver_phantomjs(url, has_proxy=True):
"""
Selenium 使用代理请求
:param url:
:return driver:
"""
if has_proxy:
proxies = get_proxies()
if proxies.has_key('http'):
myProxy = proxies['http']
elif proxies.has_key('https'):
myProxy = proxies['https']
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
# 'ftpProxy': myProxy,
# 'sslProxy': myProxy,
# 'noProxy':d ''
})
print encode_wrap("使用代理:"), myProxy
driver = webdriver.Chrome()(proxy=proxy)
else:
driver = webdriver.Chrome()()
#driver.set_page_load_timeout(30)
driver.get(url)
return driver
def _get_driver():
if has_proxy:
proxies = get_proxies()
if proxies.has_key('http'):
myProxy = proxies['http']
elif proxies.has_key('https'):
myProxy = proxies['https']
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': myProxy,
# 'ftpProxy': myProxy,
# 'sslProxy': myProxy,
# 'noProxy':d ''
})
print encode_wrap("使用代理:"), myProxy
if simulator == 'Firefox':
driver = webdriver.Firefox(proxy=proxy)
elif simulator == 'Chrome':
driver = webdriver.Chrome()(proxy=proxy)
else:
driver = webdriver.PhantomJS(proxy=proxy)
else:
if simulator == 'Firefox':
driver = webdriver.Firefox()
elif simulator == 'Chrome':
driver = webdriver.Chrome()()
else:
driver = webdriver.PhantomJS()
return driver
def get_session(url, has_proxy=True, cookie=None):
"""
requests Session使用代理请求
:param url:
:param has_proxy:
:param cookie:
:return requests:
"""
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36"
}
proxy = None
if has_proxy:
proxy = get_proxies()
cf = ConfigParser.ConfigParser()
cf.read("../config.ini")
timeout = int(cf.get("web", "timeout"))
s = requests.Session()
r = s.get(url, proxies=proxy, headers=headers, cookies=cookie, timeout=timeout)
output = "Code:{0} Proxy:{1}".format(r.status_code, proxy)
print encode_wrap(output)
if int(r.status_code) != 200:
raise Exception("request fail")
return r, s
def get_web_driver_phantomjs(url, has_proxy=True):
"""
Selenium 使用代理请求
:param url:
:return driver:
"""
if has_proxy:
proxies = get_proxies()
if proxies.has_key("http"):
myProxy = proxies["http"]
elif proxies.has_key("https"):
myProxy = proxies["https"]
proxy = Proxy(
{
"proxyType": ProxyType.MANUAL,
"httpProxy": myProxy,
# 'ftpProxy': myProxy,
# 'sslProxy': myProxy,
# 'noProxy':d ''
}
)
print encode_wrap("使用代理:"), myProxy
driver = webdriver.Chrome()(proxy=proxy)
else:
driver = webdriver.Chrome()()
# driver.set_page_load_timeout(30)
driver.get(url)
return driver
def _get_driver():
if has_proxy:
proxies = get_proxies()
if proxies.has_key("http"):
myProxy = proxies["http"]
elif proxies.has_key("https"):
myProxy = proxies["https"]
proxy = Proxy(
{
"proxyType": ProxyType.MANUAL,
"httpProxy": myProxy,
# 'ftpProxy': myProxy,
# 'sslProxy': myProxy,
# 'noProxy':d ''
}
)
print encode_wrap("使用代理:"), myProxy
if simulator == "Firefox":
driver = webdriver.Firefox(proxy=proxy)
elif simulator == "Chrome":
driver = webdriver.Chrome()(proxy=proxy)
else:
driver = webdriver.PhantomJS(proxy=proxy)
else:
if simulator == "Firefox":
driver = webdriver.Firefox()
elif simulator == "Chrome":
driver = webdriver.Chrome()()
else:
driver = webdriver.PhantomJS()
return driver
