def crawl(): urls = [ 'http://www.xicidaili.com/nn/', 'http://www.xicidaili.com/nn/2', 'http://www.xicidaili.com/wn/' ] result = [] for url in urls: try: req = logic_common.build_request(url) table = BeautifulSoup(req.text, 'lxml').find('table', id='ip_list').find_all('tr') except Exception as e: error_log.error('Spider xicidaili error.[msg]={}'.format(e)) continue for tr in table[1:]: try: tds = tr.find_all('td') ip = tds[1].get_text() + ':' + tds[2].get_text() result.append(ip) except: pass info_log.info('Spider xicidaili success.Crawled IP Count:{}'.format( len(result))) return result
def crawl(): urls = [ 'http://www.data5u.com/free/gngn/index.shtml', 'http://www.data5u.com/free/gwgn/index.shtml', 'http://www.data5u.com/free/gnpt/index.shtml', 'http://www.data5u.com/free/gwpt/index.shtml' ] result = [] for url in urls: try: req = logic_common.build_request(url) table = BeautifulSoup(req.text, 'lxml').find_all('ul', {"class": 'l2'}) except Exception as e: error_log.error('Spider data5u error.[msg]={}'.format(e)) continue for item in table[1:]: try: spans = item.find_all('span') ip = spans[0].get_text() port = spans[1].get_text() except: continue line = ip + ':' + port result.append( line.replace('\r', '').replace('\n', '').replace('\t', '').replace(' ', '')) info_log.info('Spider data5u success.Crawled IP Count:{}'.format( len(result))) return result
def verify_ip(ip_item): """ :param ip_item: :return: """ status = is_available('%s:%s' % (ip_item['ip'], ip_item['ip_port'])) if status: ip_item['utime'] = int(time.time()) try: proxypool.update_ip(ip_item) except Exception as e: error_log.error('update_ip fail.[item]={} [msg]={}'.format( json.dumps(ip_item), e)) return info_log.info('update_ip success.[ip_item]={}'.format( json.dumps(ip_item))) else: try: proxypool.delete_ip(ip_item) except Exception as e: error_log.error('delete_ip fail.[item]={} [msg]={}'.format( json.dumps(ip_item), e)) return info_log.info('delete_ip success.[ip_item]={}'.format( json.dumps(ip_item)))
def crawl(): result = [] for page in range(5): url = 'https://proxy.coderbusy.com/classical/anonymous-type/highanonymous.aspx?page=%s' % ( page + 1) try: req = logic_common.build_request(url) table = BeautifulSoup(req.text, 'lxml').find('div', { 'class': 'table-responsive' }).find_all('tr') except Exception as e: error_log.error('Spider CoderBusy error.[msg]={}'.format(e)) continue for item in table[1:]: try: tds = item.find_all('td') ip = tds[0].get_text() port = tds[2].get_text() except: continue line = ip + ':' + port result.append( line.replace('\r', '').replace('\n', '').replace('\t', '').replace(' ', '')) info_log.info('Spider CoderBusy success.Crawled IP Count:{}'.format( len(result))) return result
def crawl(): urls = ['http://www.89ip.cn/tiqv.php?sxb=&tqsl=300&ports=&ktip=&xl=on&submit=%CC%E1++%C8%A1'] result = [] for pageurl in urls: try: req = logic_common.build_request(pageurl) html = req.text except Exception as e: error_log.error('Spider 89ip error.[msg]={}'.format(e)) continue ips = re.findall('\d+\.\d+\.\d+\.\d+:\d+', html) result += ips time.sleep(2) info_log.info('Spider 89ip success.Crawled IP Count:{}'.format(len(result))) return result
def crawl(): urls = [ 'http://www.66ip.cn/nmtq.php?getnum=600&isp=0&anonymoustype=0&start=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip' ] result = [] for pageurl in urls: try: req = logic_common.build_request(pageurl) html = req.text except Exception as e: error_log.error('Spider 66ip error.[msg]={}'.format(e)) continue ips = re.findall('\d+\.\d+\.\d+\.\d+:\d+', html) result += ips time.sleep(2) info_log.info('Spider 66ip success.Crawled IP Count:{}'.format( len(result))) return result
def insert_into_proxypool(ip): """代理池中插入ip""" status = is_available(ip) if not status: return ip_item = { 'ip': ip.split(':')[0], 'ip_port': ip.split(':')[-1], 'ctime': int(time.time()), 'utime': int(time.time()) } try: proxypool.insert_into_proxypool(ip_item) except Exception as e: error_log.error( 'insert_into_proxypool fail.[ip_item]={} [msg]={}'.format( json.dumps(ip_item), e)) return info_log.info('insert_into_proxypool success.[ip_item]={}'.format( json.dumps(ip_item)))
def crawl(): result = [] for page in range(1, 10): url = 'https://www.kuaidaili.com/ops/proxylist/{}/'.format(page) try: req = logic_common.build_request(url) table = BeautifulSoup(req.text, 'lxml').find( 'div', {'id': 'freelist'}).find('table').find_all('tr') except Exception as e: error_log.error('Spider kuaidaili error.[msg]={}'.format(e)) continue for tr in table[1:]: try: ip = tr.find('td', {'data-title': 'IP'}).get_text() port = tr.find('td', {'data-title': 'PORT'}).get_text() ip = ip + ':' + port result.append(ip) except: pass info_log.info('Spider kuaidaili success.Crawled IP Count:{}'.format(len(result))) return result