def get_page(url): headers = {'User-Agent': myAgents.get_agent()} try: req = requests.get(url, headers=headers, timeout=6) time.sleep(random.randint(2, 4)) if req.status_code == 200: return req.text except Exception as e: print('ydl 连接失败', e)
def with_handler(url): headers = {'User-Agent': myAgents.get_agent()} proxy = {'http': ''} cookiejar = http.cookiejar.CookieJar() proxy_handler = urllib.request.ProxyHandler(proxies=proxy) cookies_handler = urllib.request.HTTPCookieProcessor(cookiejar) opener = urllib.request.build_opener(cookies_handler, proxy_handler) req = urllib.request.Request(url, headers=headers) res = opener.open(req, timeout=6.66) return res
def get_page(url): headers = {'User-Agent': myAgents.get_agent(), 'Referer': 'http://www.yun-daili.com/free.asp?stype=1&page=1'} try: req = requests.get(url, headers=headers, timeout=6) time.sleep(random.randint(2, 4)) if req.status_code == 200: return req.text except Exception as e: print('xiaoma 连接失败', e)
def get_page(url): headers = {'User-Agent': myAgents.get_agent()} req = urllib.request.Request(url, headers=headers) time.sleep(random.randint(2, 4)) try: res = urllib.request.urlopen(req, timeout=6) if res.status == 200: return res.read().decode() except Exception as e: print('xcdl 连接失败', e)
def request_baidu(ip_tup): url = 'https://www.baidu.com/' proxies = { 'https': 'https://' + ip_tup[0] + ':' + ip_tup[1], 'http': 'http://' + ip_tup[0] + ':' + ip_tup[1] } headers = {'User-Agent': get_agent()} try: time.sleep(random.randint(2, 3)) with requests.get(url, headers=headers, timeout=6.66, proxies=proxies) as request: return request.text except Exception as e: print('*****', e)
def get_page(url): headers = { 'User-Agent': myAgents.get_agent(), 'Host': 'www.66ip.cn', 'Referer': 'http://www.66ip.cn/4.html' } req = urllib.request.Request(url, headers=headers) time.sleep(random.randint(1, 3)) try: res = urllib.request.urlopen(req, timeout=5) if res.status == 200: return res.read().decode('gbk') except Exception as e: print('ip66 连接失败', e)
def get_page(url): proxies_list = [ '182.253.71.227:80', '122.72.18.34:80', '177.136.252.7:3128', '182.41.3.223:8118', '182.42.244.205:808' ] headers = {'User-Agent': myAgents.get_agent()} proxy = {'http': random.choice(proxies_list)} proxies_handler = urllib.request.ProxyHandler(proxy) opener = urllib.request.build_opener(proxies_handler) req = urllib.request.Request(url, headers=headers) time.sleep(random.randint(2, 4)) try: res = opener.open(req, timeout=6) if res.status == 200: return res.read().decode() except Exception as e: print('xcdl 连接失败', e)