def download(url):
try:
r = requests.get(url=url, headers=config.get_header(), timeout=config.TIMEOUT)
r.encoding = chardet.detect(r.content)['encoding']
if (not r.ok) or len(r.content) < 500:
raise ConnectionError
else:
return r.text
except Exception:
count = 0 # 重试次数
proxylist = sqlhelper.select(10)
if not proxylist:
return None
while count < config.RETRY_TIME:
try:
proxy = random.choice(proxylist)
ip = proxy[0]
port = proxy[1]
proxies = {
"http": "http://%s:%s" %(ip, port),
"https": "https://%s:%s" %(ip, port)}
r = requests.get(url=url, headers=config.get_header(), timeout=config.TIMEOUT, proxies=proxies)
r.encoding = chardet.detect(r.content)['encoding']
if (not r.ok) or len(r.content) < 500:
raise ConnectionError
else:
return r.text
except Exception:
count += 1
return None
def _checkHttpProxy(selfip, proxies, isHttp=True):
types = -1
speed = -1
if isHttp:
test_url = config.TEST_HTTP_HEADER
else:
test_url = config.TEST_HTTPS_HEADER
try:
start = time.time()
r = requests.get(url=test_url,
headers=config.get_header(),
timeout=config.TIMEOUT,
proxies=proxies)
if r.ok:
speed = round(time.time() - start, 2)
content = json.loads(r.text)
headers = content['headers']
ip = content['origin']
proxy_connection = headers.get('Proxy-Connection', None)
if ',' in ip:
types = 2
elif proxy_connection:
types = 1
else:
types = 0
return True, types, speed
else:
return False, types, speed
except Exception as e:
return False, types, speed
def getMyIP():
try:
r = requests.get(url=config.TEST_IP,
headers=config.get_header(),
timeout=config.TIMEOUT)
#r = requests.get(url=config.TEST_IP, timeout=config.TIMEOUT)
ip = json.loads(r.text)
return ip['origin']
except Exception as e:
raise TestUrlFail
def baidu_check(selfip, proxies):
'''
用来检测代理的类型，突然发现，免费网站写的信息不靠谱，还是要自己检测代理的类型
:param
:return:
'''
protocol = -1
types = -1
speed = -1
# try:
# #http://ip.chinaz.com/getip.aspx挺稳定，可以用来检测ip
# r = requests.get(url=config.TEST_URL, headers=config.get_header(), timeout=config.TIMEOUT,
# proxies=proxies)
# r.encoding = chardet.detect(r.content)['encoding']
# if r.ok:
# if r.text.find(selfip)>0:
# return protocol, types, speed
# else:
# return protocol,types,speed
#
#
# except Exception as e:
# return protocol, types, speed
try:
start = time.time()
r = requests.get(url='https://www.baidu.com',
headers=config.get_header(),
timeout=config.TIMEOUT,
proxies=proxies)
r.encoding = chardet.detect(r.content)['encoding']
if r.ok:
speed = round(time.time() - start, 2)
protocol = 0
types = 0
else:
speed = -1
protocol = -1
types = -1
except Exception as e:
speed = -1
protocol = -1
types = -1
return protocol, types, speed
return protocol, types, speed
def getMyIP():
try:
r = requests.get(url=config.TEST_IP,
headers=config.get_header(),
timeout=config.TIMEOUT)
#r = requests.get(url=config.TEST_IP, timeout=config.TIMEOUT)
ip = json.loads(r.text)
return ip['origin']
except Exception as e:
raise TestUrlFail
if __name__ == '__main__':
'''
ip = '222.186.161.132'
port = 3128
proxies = {"http": "http://%s:%s" % (ip, port), "https": "http://%s:%s" % (ip, port)}
_checkHttpProxy(None,proxies)
# getMyIP()
# str="{ip:'61.150.43.121',address:'陕西省西安市 西安电子科技大学'}"
# j = json.dumps(str)
# str = j['ip']
# print str
'''
print(config.TEST_IP)
print(config.get_header())
print('timeout: %s' % config.TIMEOUT)
print(getMyIP())