def parse_title(text, proxy=None):
encoding = tools.get_encoding(r_meta, text)
r = r_subject.search(text)
if r:
subject = r.group(1)
else:
subject = 'unknow'
return tools.to_utf8(subject, encoding)
def parse_title(text, proxy=None):
encoding = tools.get_encoding(r_meta, text)
r = r_subject.search(text)
if r:
subject = r.group(1)
else:
subject = 'unknow'
return tools.to_utf8(subject, encoding)
def parse_index(text, proxy=None):
encoding = tools.get_encoding(r_meta, text)
b = r_bookid.findall(text)
bookid = b[0]
s = []
for (url, title) in r_index.findall(text):
title = title.replace(' ', ' ').strip()
# url = bookid + url
yield url, tools.to_utf8(title, encoding)
def parse_index(text, proxy=None):
encoding = tools.get_encoding(r_meta, text)
b = r_bookid.findall(text)
bookid = b[0]
s = []
for (url, title) in r_index.findall(text):
title = title.replace(' ', ' ').strip()
# url = bookid + url
yield url, tools.to_utf8(title, encoding)
def parse_page(title, text, proxy=None):
encoding = tools.get_encoding(r_meta, text)
r = r_title.search(text)
if r:
title = r.group(1).strip()
title = tools.to_utf8(title, encoding)
r = r_content.search(text)
if r:
text = tools.format_html_text(r.group(1), encoding)
else:
text = ''
return title + '\r\n' * 2 + text
def parse_page(title, text, proxy=None):
encoding = tools.get_encoding(r_meta, text)
r = r_title.search(text)
if r:
title = r.group(1).strip()
title = tools.to_utf8(title, encoding)
r = r_content.search(text)
if r:
text = tools.format_html_text(r.group(1), encoding)
else:
text = ''
return title + '\r\n'*2 + text
