def init_database(self):
try:
self.mydb = MysqlManager(**self.conf.database)
logger.info("init_database() success.")
except Exception as e:
logger.error("init database failed, error:[{0}]".format(str(e)))
sys.exit(-1)
def default_parse(self, site):
url = site.get("url")
chapters = OrderedDict()
try:
html = self.get_html_text(url, encode="gbk")
soup = BeautifulSoup(html, "html.parser")
subNode = soup.body.dl
for child in subNode.children:
try:
pos = child.a["href"].rfind(r"/")
chapterUrl = "{0}{1}".format(url,
child.a["href"][pos + 1:])
chapterName = child.a.string
#去掉重复的章节
if chapterUrl in chapters:
chapters.move_to_end(chapterUrl)
else:
chapters[chapterUrl] = chapterName
except:
pass
except Exception as e:
logger.error("parse ({0} failed, error:[{1}])".format(url, str(e)))
chapters = None
return chapters
def parse_novel_thread(self, novel):
novelName = novel.get("name")
table = novel.get("table")
siteList = novel.get("site")
receiver = novel.get("receiver")
sendContent = novel.get("send_content")
for site in siteList:
try:
if not site.get("status"): continue
siteName = site.get("name")
parseFunc = self.parseMethod.get(siteName, self.default_parse)
timeStart = time.time()
chapters = parseFunc(site)
lastChapterUrl, lastChapterName = chapters.popitem()
if not self.chapter_is_exist(
(lastChapterUrl, lastChapterName), table):
chapters[lastChapterUrl] = lastChapterName
if sendContent:
self.send_chapter_has_content(table, chapters,
novelName, siteName,
receiver)
else:
self.send_chapter(table,
(lastChapterUrl, lastChapterName),
novelName, siteName, receiver)
timeEnd = time.time()
logger.info("{0}({1})最新章节: {2}, 耗时:{3}".format(
novelName, siteName, lastChapterName, timeEnd - timeStart))
except Exception as e:
logger.error(
"parse_novel_thread({0}) failed, error:[{1}]".format(
site.get("url"), str(e)))
def save_chapter(self, siteName, chapter, table):
try:
sql = "insert into {0}(site, site_name, chapter_name, send_time) values('{1}', '{2}', '{3}', now())".format(
table, chapter[0], siteName, self.get_chapter_name(chapter[1]))
self.mydb.execute(sql)
return True
except Exception as e:
logger.error("save chapter failed, error:[{0}]".format(str(e)))
return False
def init_novel(self):
self.novelList = []
try:
for novel in self.conf.novel:
if not novel.get("status"): continue
table = novel.get("table")
if self.make_table(table):
self.novelList.append(novel)
else:
logger.error(
"init novel failed, error:[创建表{}失败!]".format(table))
sys.exit(-1)
logger.info("init_novel() success.")
except Exception as e:
logger.error("init novel failed, error:[{0}]".format(str(e)))
sys.exit(-1)
def send(self, receivers, subject, content, mimeType="plain"):
receiverList = self.list_split(receivers, nums=30) # 一封邮件最多有三十个收件人
for receiver in receiverList:
# 对于发送失败的邮件, 换个发送人再次发送
for myemail in self.conf.emails:
myemail["receiver"] = receiver
myemail["subject"] = subject
myemail["content"] = content
myemail["mimeType"] = mimeType
stderr = self.send_email(**myemail)
if not stderr:
logger.info("({0})[{1}] send email to {2} success.".format(
subject, myemail.get("sender"), str(receiver)))
break
else:
logger.error(
"({0})[{1}] send email to {2} failed!, error:{3}".
format(subject, myemail.get("sender"), str(receiver),
stderr))