def getMyIP():
try:
r = requests.get(url=config.TEST_IP, headers=config.get_header(), timeout=config.TIMEOUT)
ip = json.loads(r.text)
return ip['origin']
except Exception as e:
raise Test_URL_Fail
def _checkHttpProxy(selfip, proxies, isHttp=True):
types = -1
speed = -1
if isHttp:
test_url = config.TEST_HTTP_HEADER
else:
test_url = config.TEST_HTTPS_HEADER
try:
start = time.time()
r = requests.get(url=test_url, headers=config.get_header(), timeout=config.TIMEOUT, proxies=proxies)
if r.ok:
speed = round(time.time() - start, 2)
content = json.loads(r.text)
headers = content['headers']
ip = content['origin']
proxy_connection = headers.get('Proxy-Connection', None)
if ',' in ip:
types = 2
elif proxy_connection:
types = 1
else:
types = 0
return True, types, speed
else:
return False, types, speed
except Exception as e:
return False, types, speed
def download(url):
try:
r = requests.get(url=url,
headers=config.get_header(),
timeout=config.TIMEOUT)
r.encoding = chardet.detect(r.content)['encoding']
if (not r.ok) or len(r.content) < 500:
raise ConnectionError
else:
return r.text
except Exception:
count = 0 # 重试次数
proxylist = sqlhelper.select(10)
if not proxylist:
return None
while count < config.RETRY_TIME:
try:
proxy = random.choice(proxylist)
ip = proxy[0]
port = proxy[1]
proxies = {
"http": "http://%s:%s" % (ip, port),
"https": "http://%s:%s" % (ip, port)
}
r = requests.get(url=url,
headers=config.get_header(),
timeout=config.TIMEOUT,
proxies=proxies)
r.encoding = chardet.detect(r.content)['encoding']
if (not r.ok) or len(r.content) < 500:
raise ConnectionError
else:
return r.text
except Exception:
count += 1
return None
def baidu_check(selfip, proxies):
'''
用来检测代理的类型，突然发现，免费网站写的信息不靠谱，还是要自己检测代理的类型
:param
:return:
'''
protocol = -1
types = -1
speed = -1
# try:
# #http://ip.chinaz.com/getip.aspx挺稳定，可以用来检测ip
# r = requests.get(url=config.TEST_URL, headers=config.get_header(), timeout=config.TIMEOUT,
# proxies=proxies)
# r.encoding = chardet.detect(r.content)['encoding']
# if r.ok:
# if r.text.find(selfip)>0:
# return protocol, types, speed
# else:
# return protocol,types,speed
#
#
# except Exception as e:
# return protocol, types, speed
try:
start = time.time()
r = requests.get(url='https://www.baidu.com', headers=config.get_header(), timeout=config.TIMEOUT, proxies=proxies)
r.encoding = chardet.detect(r.content)['encoding']
if r.ok:
speed = round(time.time() - start, 2)
protocol= 0
types=0
else:
speed = -1
protocol= -1
types=-1
except Exception as e:
speed = -1
protocol = -1
types = -1
return protocol, types, speed