def parse(self, res): items_data = RankingItem.get_items(html=res.html) result = [] res_dic = {} for item in items_data: each_book_list = [] # 只取排名前十的书籍数据 for index, value in enumerate(item.book_list[:10]): item_data = NameItem.get_item(html_etree=value) name = item_data.get('top_name') or item_data.get('other_name') each_book_list.append({ 'num': index + 1, 'name': name }) data = { 'title': item.ranking_title, 'more': item.more, 'book_list': each_book_list, 'updated_at': time.strftime("%Y-%m-%d %X", time.localtime()), } result.append(data) res_dic['data'] = result res_dic['target_url'] = res.url res_dic['type'] = self.qidian_type.get(res.url.split('=')[-1]) res_dic['spider'] = "qidian" async_callback(self.save, res_dic=res_dic)
def parse(self, res): item_data = HYNovelInfoItem.get_item(html=res.html) item_data['target_url'] = res.url item_data['spider'] = 'heiyan' item_data['updated_at'] = time.strftime("%Y-%m-%d %X", time.localtime()) print('获取 {} 小说信息成功'.format(item_data['novel_name'])) print(item_data) self.all_novels_info_col.update({'novel_name': item_data['novel_name'], 'spider': 'heiyan'}, item_data, upsert=True) async_callback(self.save, res_dic=item_data)
def parse(self, res): data = res.html result = [] res_dic = {} if data: for each_data in data: data = { 'name': each_data.get('bookName', ''), 'type': each_data.get('bookShortCateName', ''), 'num': each_data.get('orderNo', ''), 'updated_at': time.strftime("%Y-%m-%d %X", time.localtime()), } result.append(data) res_dic['data'] = result res_dic['target_url'] = res.url res_dic['type'] = "全部类别" res_dic['spider'] = "zh_bd_novels" async_callback(self.save, res_dic=res_dic)
def parse_item(self, res): items_data = QidianNovelsItem.get_items(html=res.html) for item in items_data: res_dic = { 'novel_url': item.novel_url, 'novel_name': item.novel_name, 'novel_author': item.novel_author, 'novel_author_home_url': item.novel_author_home_url, 'spider': 'qidian', 'updated_at': time.strftime("%Y-%m-%d %X", time.localtime()), } if self.all_novels_col.find_one({ "novel_name": item.novel_name, 'novel_author': item.novel_author }) is None: self.all_novels_col.insert_one(res_dic) async_callback(self.save, res_dic=res_dic) print(item.novel_name + ' - 抓取成功')