def getMyIP(): try: r = requests.get(url=config.TEST_IP, headers=config.get_header(), timeout=config.TIMEOUT) ip = json.loads(r.text) return ip['origin'] except Exception as e: raise Test_URL_Fail
def _checkHttpProxy(selfip, proxies, isHttp=True): types = -1 speed = -1 if isHttp: test_url = config.TEST_HTTP_HEADER else: test_url = config.TEST_HTTPS_HEADER try: start = time.time() r = requests.get(url=test_url, headers=config.get_header(), timeout=config.TIMEOUT, proxies=proxies) if r.ok: speed = round(time.time() - start, 2) content = json.loads(r.text) headers = content['headers'] ip = content['origin'] proxy_connection = headers.get('Proxy-Connection', None) if ',' in ip: types = 2 elif proxy_connection: types = 1 else: types = 0 return True, types, speed else: return False, types, speed except Exception as e: return False, types, speed
def download(url): try: r = requests.get(url=url, headers=config.get_header(), timeout=config.TIMEOUT) r.encoding = chardet.detect(r.content)['encoding'] if (not r.ok) or len(r.content) < 500: raise ConnectionError else: return r.text except Exception: count = 0 # 重试次数 proxylist = sqlhelper.select(10) if not proxylist: return None while count < config.RETRY_TIME: try: proxy = random.choice(proxylist) ip = proxy[0] port = proxy[1] proxies = { "http": "http://%s:%s" % (ip, port), "https": "http://%s:%s" % (ip, port) } r = requests.get(url=url, headers=config.get_header(), timeout=config.TIMEOUT, proxies=proxies) r.encoding = chardet.detect(r.content)['encoding'] if (not r.ok) or len(r.content) < 500: raise ConnectionError else: return r.text except Exception: count += 1 return None
def baidu_check(selfip, proxies): ''' 用来检测代理的类型,突然发现,免费网站写的信息不靠谱,还是要自己检测代理的类型 :param :return: ''' protocol = -1 types = -1 speed = -1 # try: # #http://ip.chinaz.com/getip.aspx挺稳定,可以用来检测ip # r = requests.get(url=config.TEST_URL, headers=config.get_header(), timeout=config.TIMEOUT, # proxies=proxies) # r.encoding = chardet.detect(r.content)['encoding'] # if r.ok: # if r.text.find(selfip)>0: # return protocol, types, speed # else: # return protocol,types,speed # # # except Exception as e: # return protocol, types, speed try: start = time.time() r = requests.get(url='https://www.baidu.com', headers=config.get_header(), timeout=config.TIMEOUT, proxies=proxies) r.encoding = chardet.detect(r.content)['encoding'] if r.ok: speed = round(time.time() - start, 2) protocol= 0 types=0 else: speed = -1 protocol= -1 types=-1 except Exception as e: speed = -1 protocol = -1 types = -1 return protocol, types, speed