def connect(url, proxy):
try:
start_point = time.time()
response = requests.get(
url,
proxies={
'http': 'http://%s:%s' % (proxy['ip'], proxy['port']),
'https': 'http://%s:%s' % (proxy['ip'], proxy['port'])
},
**{
'timeout': 10,
'headers': config.get_http_header()
})
if response.ok:
interval = round(time.time() - start_point, 2)
res_json = json.loads(response.text)
ip = res_json['origin']
if ',' in ip:
anonymity = 'transparent'
else:
anonymity = 'anonymous'
return True, anonymity, interval
else:
return False, False, False
except (Exception, ) as e:
return False, False, False
def __request(self, proxy, url):
"""
使用代理请求 url
:param proxy:
:param url:
:return:
"""
# 请求设置
kwargs = {'timeout': 10, 'headers': config.get_http_header()}
# 有代理使用
if isinstance(proxy, dict) and proxy:
kwargs['proxies'] = {
'http': 'http://%s:%s' % (proxy['ip'], proxy['port']),
'https': 'http://%s:%s' % (proxy['ip'], proxy['port'])
}
response = requests.get(url, **kwargs)
if response.ok and response.status_code == 200:
return response.text
else:
config.console_log(
'请求返回的状态码: %s URL: %s 内容: %s' %
(url, str(response.status_code), response.text), 'red')
return None
def _text(self, url):
response = requests.get(
url, **{
'timeout': 5,
'headers': config.get_http_header()
})
if response.ok:
return response.text
else:
# TODO 使用代理重新尝试下载
return None
def _text(self, url):
try:
response = requests.get(url, **{'timeout': 10, 'headers': config.get_http_header()})
if response.ok:
return response.text
else:
# todo 使用代理重新尝试下载
config.console_log('请求返回的状态码: ' + str(response.status_code), 'red')
return None
except Timeout as e:
config.console_log('请求超时: ' + str(e), 'red')
return None