def fetch_xici(self, num): """抓取http://www.xicidaili.com/,质量10%""" page = 1 proxyes = [] while len(proxyes) <= num and page <= 2: url = "http://www.xicidaili.com/nn/%s" % page req = requests.get(url, headers=self.headers) html = req.text selector = etree.HTML(html) tbody = selector.xpath('//tr[@class]') for line in tbody: tds = line.xpath('td/text()') ip = tds[0] port = tds[1] speed = line.xpath('td[7]/div/@title')[0][:-1] latency = line.xpath('td[8]/div/@title')[0][:-1] # print('%s,%s,%s,%s'%(ip, port, speed, latency)) if float(speed) < 3 and float(latency) < 1: proxy = "%s:%s" % (ip, port) proxy_dict = {'http': proxy, 'https': proxy} valid_res = self.proxy_vaild(proxy_dict) if valid_res[0]: proxyes.append(valid_res[1]) logger.info('抓取 xicidaili 第 %d 页,有效代理 %d 个' % (page, len(proxyes))) page += 1 return proxyes
def scheduler(self): """根据状态码实现调度""" deal_code = ['10000', '10001', '10003', '10004', '10016', '10020','10021','10022', '10023'] pass_code = ['20800', '20801', '20802', '20803', '20003'] self.status_dict = { "10000": self.status_ok, "10001": self.status_change_key, "10003": self.status_change_key, "10004": self.status_change_user_agent, "10010": self.status_change_proxy, "10016": self.status_change_user_agent, "10020": self.status_change_key, "10021": self.status_change_proxy, "10022": self.status_change_proxy, "10023": self.status_change_key, "20003": self.status_pass } status = self.respond['status'] infocode = self.respond['infocode'] print(infocode) if infocode in deal_code: return self.status_dict[infocode]() elif infocode in pass_code: self.status_pass() else: print(infocode) logger.info(infocode) self.status_invalid_request()
def get_duration(self): respond = self.process() if respond == None: logger.info(u'响应为None的链接是%s'%(self.req_url)) elif respond['duration'] == None: logger.info(u'duration为None的链接是%s'%(self.req_url)) else: duration = respond['duration'] return int(duration)
def process(self): if self.change_num < len(self.key_list): self.key_dict['key'] = self.key_list[self.change_num] self.change_num += 1 logger.info("========已更换Key=========") print("========已更换Key=========") return self.key_dict else: logger.info("========Key已用完,随机选取Key=========") self.key_dict['key'] = random.choice(self.key_list) return self.key_dict
def proxy_vaild(self, proxy_dict): url = "http://ip.chinaz.com/getip.aspx" #用来测试IP是否可用的url try: r = requests.get(url, proxies=proxy_dict, headers=self.headers, timeout=3, allow_redirects=False) if r.status_code == 200: print(r.text) return (True, proxy_dict) else: logger.info('_______%s 无效代理________' % r.status_code) return (False, ) except (req_e.ReadTimeout, req_e.ConnectTimeout, req_e.ProxyError, req_e.ConnectionError, req_e.ChunkedEncodingError): logger.info('_______连接超时 无效代理________') return (False, )
def fetch_66ip(self, num): """抓取http://www.66ip.cn/,质量25%""" proxyes = [] url = "http://www.66ip.cn/nmtq.php?getnum=100&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=0&api=66ip" req = requests.get(url, headers=self.headers) html = req.text urls = html.split("</script>")[1].split("<br />") for u in urls[:-1]: if u.strip(): proxy = u.strip() proxy_dict = {'http': proxy, 'https': proxy} valid_res = self.proxy_vaild(proxy_dict) if valid_res[0]: proxyes.append(valid_res[1]) if len(proxyes) >= num: break else: continue logger.info('抓取 66ip,有效代理 %d 个' % (len(proxyes))) return proxyes
def fetch_new_proxyes(self, num): crawls = [ self.fetch_ip181, self.fetch_66ip, self.fetch_xici, self.fetch_kxdaili ] valid_proxyes = [] if os.path.exists('proxy.csv'): local_proxyes = self.read_proxy_file('proxy.csv') for proxy in local_proxyes: valid_res = self.proxy_vaild(proxy) if valid_res[0]: valid_proxyes.append(valid_res[1]) else: pass if len(valid_proxyes) < num: demand_num = num - len(valid_proxyes) logger.info('_______有效代理%s,需要抓取%s________' % (len(valid_proxyes), demand_num)) for crawl in crawls: new_proxyes = crawl(demand_num) logger.info('_______抓取新代理%s________' % len(new_proxyes)) valid_proxyes += new_proxyes demand_num = demand_num - len(new_proxyes) if demand_num <= 0: logger.info('_______代理抓取完毕,共%s________' % len(valid_proxyes)) self.save_proxy(valid_proxyes) break else: continue return valid_proxyes
def fetch_ip181(self, num): """抓取http://www.ip181.com/,10分钟更新100个,质量55%""" proxyes = [] url = 'http://www.ip181.com/' req = requests.get(url, headers=self.headers) html = req.text selector = etree.HTML(html) tbody = selector.xpath('//tr') for line in tbody[1:]: tds = line.xpath('td/text()') ip = tds[0] port = tds[1] latency = tds[4].split(' ')[0] if float(latency) < 0.5: proxy = "%s:%s" % (ip, port) proxy_dict = {'http': proxy, 'https': proxy} valid_res = self.proxy_vaild(proxy_dict) if valid_res[0]: proxyes.append(valid_res[1]) if len(proxyes) >= num: break logger.info('抓取 ip181,有效代理 %d 个' % (len(proxyes))) return proxyes
def fetch_kxdaili(self, num): """抓取http://www.kxdaili.com/,质量 5%""" page = 1 proxyes = [] while len(proxyes) <= num and page <= 10: url = "http://www.kxdaili.com/dailiip/1/%d.html" % page req = requests.get(url, headers=self.headers) html = req.text selector = etree.HTML(html) tbody = selector.xpath('//tr') for line in tbody: tds = line.xpath('td/text()') ip = tds[0] port = tds[1] latency = tds[4].split(' ')[0] if float(latency) < 0.5: proxy = "%s:%s" % (ip, port) proxy_dict = {'http': proxy, 'https': proxy} valid_res = self.proxy_vaild(proxy_dict) if valid_res[0]: proxyes.append(valid_res[1]) logger.info('抓取 kxdaili 第 %d 页,有效代理 %d 个' % (page, len(proxyes))) page += 1 return proxyes
def save_proxy(self, res_list): df = pd.DataFrame(res_list) df.to_csv('proxy.csv') logger.info('_______代理已储存________')