def work(self, task_queue, result_queue, url_queue): downloader = HtmlDownloader() parser = HtmlParser() while True: while not task_queue.empty(): new_url = task_queue.get() print('获得新任务: %s' % new_url) response = downloader.download(new_url) items = parser.parser(response) if len(items) > 1: for i in range(0, 60): product_rate_url = items[i].get('product_rate_url') print('获得链接:%s' % product_rate_url) other_store_url = items[i].get('other_store_url') print('获得链接:%s' % other_store_url) url_queue.put(product_rate_url) url_queue.put(other_store_url) print('获取结果:%s' % str(items[i])) result_queue.put(items[i]) next_page_url = items[-1] if next_page_url == 'No next page': print('已经爬取到最后一页,工作节点准备结束') result_queue.put('end') return url_queue.put(next_page_url) print('获得链接:%s' % next_page_url) else: print('获取结果:%s' % str(items[0])) result_queue.put(items[0])
class SpiderMan(object): def __init__(self): self.manager = UrlManager() self.downloader = HtmlDownloader() self.parser = HtmlParser() self.output = DataOutput() # 实例化时连接到数据库 self.s = Settings().setting def crawl(self): self.output.create_table() # 创建表 total_page = self.s["Index"][1] - self.s["Index"][0] total_data = total_page * self.s["Page"] total_errors = 0 total_duplicates = 0 old_total = self.output.get_total() for Index in range(self.s["Index"][0], self.s["Index"][1]): duplicates = self.manager.add_urls(Index, self.output) urls = self.manager.get_urls() bar = pyprind.ProgBar(self.s["Page"] - duplicates, title="Crawling " + "Page " + str(Index) + " ......") # 进度条 for url in urls: try: bar.update() html = self.downloader.download(url) data = self.parser.parse(html) self.output.insert_into_db(data) # 插入数据库 except Exception: continue new_total = self.output.get_total() self.output.close_cursor() # 关闭数据库连接 print("本次爬取", new_total - old_total, "条")
class SpiderMan(object): def __init__(self): self.manager = UrlManager() self.downloader = HtmlDownloader() self.parser = HtmlParser() self.output = DataOutput() # 实例化时连接到数据库 def crawl(self): self.output.create_table() # 创建表 self.manager.add_new_urls() # 创建url total = self.manager.new_urls_size() bar = pyprind.ProgBar(30, title="Crawling......") # 进度条 while (self.manager.new_urls_size()): url = self.manager.get_new_url() html = self.downloader.download(url) data = self.parser.parse(html) errors, errors_messages = self.output.insert_into_db( data) # 插入数据库 bar.update() ''' sys.stdout.write( str(self.manager.old_urls_size() / total * 100) + "%") sys.stdout.flush() # print('爬取', self.manager.old_urls_size(), '条。') ''' self.output.close_cursor() # 关闭数据库连接 print("本次共爬取", total, "条") if errors: print("其中", errors, "条数据出错") print("错误:" + str(errors_messages))
def work(self, task_queue, result_queue, url_queue): downloader = HtmlDownloader() parser = HtmlParser() while True: while not task_queue.empty(): new_url = task_queue.get() if new_url == 'end': print('爬虫爬取完成') return print('获得新任务: %s' % new_url) response = downloader.download(new_url) items, next_page = parser.parser(response) url_queue.put(next_page) for item in items: print('任务完成: %s' % item) result_queue.put(item)
class SpiderMan(object): '''爬虫调度器 Attributes: manager: URL管理器 downloader: HTML下载器 parser: HTML解析器 output: 数据存储器 ''' def __init__(self): self.manager = UrlManager() self.downloader = HtmlDownloader() self.parser = HtmlParser() self.output = DataOutput() def crawl(self, root_url): '''爬虫调度函数 Args: root_url: 爬虫入口URL Raises: Expection: 'NoneType' object has no attribute ''' self.manager.add_new_url(root_url) while (self.manager.has_new_url() and self.manager.old_url_size() < 100): try: new_url = self.manager.get_new_url() html = self.downloader.download(new_url) new_urls, data = self.parser.parser(new_url, html) self.manager.add_new_urls(new_urls) self.output.store_data(data) print('已经抓取了%s个链接' % self.manager.old_url_size()) except Exception as e: print('Crawl failed: %s' % e) self.output.output_html()
def __init__(self): self.manager = UrlManager() self.downloader = HtmlDownloader() self.parser = HtmlParser() self.output = DataOutput() # 实例化时连接到数据库 self.s = Settings().setting
爬虫调度器(入口) ''' from URLManager import UrlManager from HTMLDownloader import HtmlDownloader from HTMLParser import HtmlParser from DataOutput import DataOutput from settings import Settings from random import random import time import datetime import threading import multiprocessing manager = UrlManager() downloader = HtmlDownloader() parser = HtmlParser() s = Settings().setting max_threads = 3 base_url = "http://wenshu.court.gov.cn/CreateContentJS/CreateContentJS.aspx?DocID=" # 逐条爬取 def crawl(): db = DataOutput() # 连接到数据库 old_total = db.get_total() while db.has_unvisited(): docid = manager.get_one_docid(db) url = base_url + docid for _ in range(3): try: html = downloader.download(url)
def __init__(self): self.manager = UrlManager() self.downloader = HtmlDownloader() self.parser = HtmlParser() self.output = DataOutput()