def parseDocument(self, doc): doc = pq(doc); wrapparent = self.articleRule.wrapparent pageparent = self.articleRule.pageparent content_re = ""; #子页面url urls = [] #文本数据内容 content = "" article = doc.find(wrapparent); #pages if pageparent: urls = self.parsePage(article, pageparent) #need title, tags extrarules = self.articleRule.extrarules #只有文章是有content if len(extrarules): for key, rule, fetch_all in extrarules: field = Field(field_id=key, rule=rule); value = getElementData(doc, rule, self.data["images"], fetch_all) self.data[field.get('name')] = field if field.is_article_content(): content_re = field.get("rule") content = value elif field.is_gallery_content(): content_re = field.get("rule") content = [] if (isinstance(value, list)): content += value else: field.value = value #采集分页内容 if len(urls) > 0 and content_re: for next_url in urls: next_page = Fetch(next_url, charset = self.seed["charset"], timeout = self.seed["timeout"]).read() if next_page is not None: next_page = self._getContent(next_page, wrapparent, content_re); if next_page: if isinstance(content, list): content.append(next_page) else: content += next_page if content and content_re: if isinstance(content, list): self.data['content'].value = content self.data['images'] += content else: content = Readability(content, self.url, self.articleRule.filters) images = content.getImages(); self.data['content'].value = content.getContent(); self.data['images'] += images