def parseDocument(self, doc):
doc = pq(doc);
wrapparent = self.articleRule.wrapparent
pageparent = self.articleRule.pageparent
content_re = "";
#子页面url
urls = []
#文本数据内容
content = ""
article = doc.find(wrapparent);
#pages
if pageparent:
urls = self.parsePage(article, pageparent)
#need title, tags
extrarules = self.articleRule.extrarules
#只有文章是有content
if len(extrarules):
for key, rule, fetch_all in extrarules:
field = Field(field_id=key, rule=rule);
value = getElementData(doc, rule, self.data["images"], fetch_all)
self.data[field.get('name')] = field
if field.is_article_content():
content_re = field.get("rule")
content = value
elif field.is_gallery_content():
content_re = field.get("rule")
content = []
if (isinstance(value, list)):
content += value
else:
field.value = value
#采集分页内容
if len(urls) > 0 and content_re:
for next_url in urls:
next_page = Fetch(next_url, charset = self.seed["charset"], timeout = self.seed["timeout"]).read()
if next_page is not None:
next_page = self._getContent(next_page, wrapparent, content_re);
if next_page:
if isinstance(content, list):
content.append(next_page)
else:
content += next_page
if content and content_re:
if isinstance(content, list):
self.data['content'].value = content
self.data['images'] += content
else:
content = Readability(content, self.url, self.articleRule.filters)
images = content.getImages();
self.data['content'].value = content.getContent();
self.data['images'] += images
