def spider_opened(self, spider): self.wx_num, self.req_data_dict, self.req_data_list = TidyReqData.get_gzh_req_data( ) if self.wx_num == 0: self.wx_num = 1 self.pre_crawl_time = time.time()
def spider_opened(self, spider): self.wx_num, self.req_data_dict, self.req_data_list = TidyReqData.get_gzh_req_data( ) if self.wx_num == 0: self.wx_num = 1
class ArticleListSpider(scrapy.Spider): name = 'article_list' allowed_domains = ['mp.weixin.qq.com'] start_url = [] custom_settings = get_global_settings() wx_num, _, _ = TidyReqData.get_gzh_req_data() if wx_num == 0: wx_num = 1 custom_settings['DOWNLOAD_DELAY'] = round(2.0 / wx_num, 2) custom_settings['ITEM_PIPELINES'] = { 'crawler.crawler.pipelines.load_more.ResponseArticleListPipeline': 300, } custom_settings['DOWNLOADER_MIDDLEWARES'] = { 'crawler.crawler.middlewares.load_more.LoadMoreMiddleware': 543, } counter = 0 list_offset = 0 def __init__(self, *args, **kwargs): """ :param args: :param kwargs: 实例化爬虫需要调用的函数 """ self.current_nickname = '' def start_requests(self): """ :return:重新爬虫的入口函数, 否者直接请求start_urls中的各个url 重写之后手动调用Request并指定回调函数例如self.parse """ yield Request(url='http://www.aii.com', meta={"list_offset": self.list_offset}, callback=self.parse, dont_filter=True) def parse(self, response): """ :param response: :return:请求完成之后的回调函数 """ self.counter += 1 cmc = response.get_ext_data['can_msg_continue'] next_offset = response.get_ext_data['next_offset'] item = LoadMoreItem() item['article_list'] = response.get_ext_data['data'] item['nickname'] = response.get_ext_data['nickname'] self.current_nickname = response.get_ext_data['nickname'] gc.report_crawling({ 'nickname': item['nickname'], 'percent': 'UNK', 'more': cmc, 'title': len(item['article_list']) }) yield item if cmc == 1: yield Request(url='http://www.aii.com', meta={"list_offset": next_offset}, callback=self.parse, dont_filter=True) def close(self, reason): """ :param reason: :return:所有url请求完毕之后关闭爬虫的回调函数 """ # 删除被删除的公众号 被删除的公众号content_url为空 from db import delete delete(self.current_nickname, content_url="") print(self.name, "爬虫关闭")
class ArticleSpider(scrapy.Spider): """ 公众号文章内容爬虫 """ name = 'article' allowed_domains = ['mp.weixin.qq.com'] start_url = [] custom_settings = get_global_settings() wx_num, _, _ = TidyReqData.get_gzh_req_data() # 担心ip被封 设置请求间隔 # custom_settings['DOWNLOAD_DELAY'] = 0.5 custom_settings['DOWNLOADER_MIDDLEWARES'] = { 'crawler.crawler.middlewares.crawl_article.CrawlArticleMiddleware': 543, } custom_settings['ITEM_PIPELINES'] = { 'crawler.crawler.pipelines.crawl_article.ResponseArticlePipeline': 300, } custom_settings['DOWNLOAD_TIMEOUT'] = 10 custom_settings['CONCURRENT_REQUESTS'] = 16 def __init__(self, *args, **kwargs): """ :param args: :param kwargs: 实例化爬虫需要调用的函数 """ # 包含当前公众号所有不存在文本内容数据的生成器 self.current_nickname = TidyReqData.get_nickname() self.articles_list = get_collection_article(self.current_nickname, article={"$exists": False}, title={"$exists": True}) self.crawler_begin_time = time() self.crawler_parse_counter = 0 def start_requests(self): """ :return:重新爬虫的入口函数, 否者直接请求start_urls中的各个url 重写之后手动调用Request并指定回调函数例如self.parse """ for article in self.articles_list: if "weixin" in article['content_url']: yield Request(url=article['content_url'], callback=self.parse) def parse(self, response): """ :param response: :return:请求完成之后的回调函数 """ item = CrawlArticleItem() item['article_data'] = response.get_ext_data['article_data'] item['nickname'] = response.get_ext_data['nickname'] item['raw_url'] = response.get_ext_data['raw_url'] self.crawler_parse_counter += 1 time_gap = time() - self.crawler_begin_time print(round(time_gap / self.crawler_parse_counter, 3), item['article_data']['article'].replace('\n', '')) # 发送状态给前端 crawling_item = {} crawling_item['nickname'] = item['nickname'] crawling_item['percent'] = self.crawler_parse_counter crawling_item['more'] = round(time_gap / self.crawler_parse_counter, 3) crawling_item['title'] = find_one(item['nickname'], item['raw_url'])['title'][:10] gc.report_crawling(crawling_item) yield item def close(self, reason): """ :param reason: :return:所有url请求完毕之后关闭爬虫的回调函数 """ time_gap = time() - self.crawler_begin_time if self.crawler_parse_counter != 0: print("%s爬虫关闭 用时%d 共计爬取%d 平均%f" % (self.name, time_gap, self.crawler_parse_counter, time_gap / self.crawler_parse_counter)) from instance.global_instance import gs print("正在为 %s 创建索引..." % (self.current_nickname)) index_result = gs.index_db_docs(self.current_nickname) print("索引完成", index_result) from db.meta_data import insert_article_metadata insert_article_metadata( self.current_nickname, { 'date': datetime.datetime.now(), 'articles_num': self.crawler_parse_counter })
class ArticleReadDataSpider(scrapy.Spider): """ 公众号文章阅读数据爬虫 """ name = 'read_data' allowed_domains = ['mp.weixin.qq.com'] start_url = [] custom_settings = get_global_settings() wx_num, _, _ = TidyReqData.get_gzh_req_data() if wx_num == 0: wx_num = 1 custom_settings['DOWNLOAD_DELAY'] = round(2.5 / wx_num, 2) custom_settings['DOWNLOADER_MIDDLEWARES'] = { 'crawler.crawler.middlewares.crawl_article.ArticleReadDataMiddleware': 543, } custom_settings['ITEM_PIPELINES'] = { 'crawler.crawler.pipelines.crawl_article.ResponseArticleReadDataPipeline': 300, } custom_settings['CONCURRENT_REQUESTS'] = 1 def __init__(self, *args, **kwargs): """ :param args: :param kwargs: 实例化爬虫需要调用的函数 """ # 包含当前公众号所有不存在文本内容数据的生成器 self.current_nickname = TidyReqData.get_nickname() print(self.current_nickname) articles_list = get_collection_article(self.current_nickname, read_num={"$exists": False}, comment_id={"$exists": True}) self.articles_list = [] for article in articles_list: self.articles_list.append(article) self.task_num = len(self.articles_list) self.task_counter = 0 self.begin_time = time() self.pre_time = time() def start_requests(self): """ :return:重新爬虫的入口函数, 否者直接请求start_urls中的各个url 重写之后手动调用Request并指定回调函数例如self.parse """ for article in self.articles_list: if ':' in article['content_url']: request = Request(url=article['content_url'], callback=self.parse, dont_filter=False) request.set_ext_data({ 'content_url': article['content_url'], 'comment_id': article['comment_id'] }) yield request def parse(self, response): """ :param response: :return:请求完成之后的回调函数 """ item = CrawlArticleReadDataItem() item['read_data'] = response.get_ext_data['read_data'] item['nickname'] = response.get_ext_data['nickname'] item['content_url'] = response.get_ext_data['content_url'] # 打印状爬虫状态信息 self.task_counter += 1 pre_time_gap = time() - self.pre_time total_time_gap = time() - self.begin_time time_need = (self.task_num - self.task_counter) * (total_time_gap / self.task_counter) print(round(pre_time_gap, 2), round(total_time_gap / self.task_counter, 2), "%d/%d" % (self.task_counter, self.task_num), response.get_ext_data['read_data']['read_num'], response.get_ext_data['read_data']['like_num'], response.get_ext_data['read_data']['nick_name'], str(datetime.timedelta(seconds=time_need)).split('.')[0]) self.pre_time = time() crawling_item = {} crawling_item['nickname'] = item['nickname'] crawling_item['percent'] = '%d/%d' % (self.task_counter, self.task_num) crawling_item['more'] = response.get_ext_data['read_data']['read_num'] crawling_item['title'] = find_one(item['nickname'], item['content_url'])['title'][:10] gc.report_crawling(crawling_item) yield item def close(self, reason): """ :param reason: :return:所有url请求完毕之后关闭爬虫的回调函数 """ print(self.name, "爬虫关闭")
def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) self.wx_num, self.req_data_dict, self.req_data_list = TidyReqData.get_gzh_req_data( ) if self.wx_num == 0: self.wx_num = 1