def init_database(self): try: self.mydb = MysqlManager(**self.conf.database) logger.info("init_database() success.") except Exception as e: logger.error("init database failed, error:[{0}]".format(str(e))) sys.exit(-1)
def default_parse(self, site): url = site.get("url") chapters = OrderedDict() try: html = self.get_html_text(url, encode="gbk") soup = BeautifulSoup(html, "html.parser") subNode = soup.body.dl for child in subNode.children: try: pos = child.a["href"].rfind(r"/") chapterUrl = "{0}{1}".format(url, child.a["href"][pos + 1:]) chapterName = child.a.string #去掉重复的章节 if chapterUrl in chapters: chapters.move_to_end(chapterUrl) else: chapters[chapterUrl] = chapterName except: pass except Exception as e: logger.error("parse ({0} failed, error:[{1}])".format(url, str(e))) chapters = None return chapters
def parse_novel_thread(self, novel): novelName = novel.get("name") table = novel.get("table") siteList = novel.get("site") receiver = novel.get("receiver") sendContent = novel.get("send_content") for site in siteList: try: if not site.get("status"): continue siteName = site.get("name") parseFunc = self.parseMethod.get(siteName, self.default_parse) timeStart = time.time() chapters = parseFunc(site) lastChapterUrl, lastChapterName = chapters.popitem() if not self.chapter_is_exist( (lastChapterUrl, lastChapterName), table): chapters[lastChapterUrl] = lastChapterName if sendContent: self.send_chapter_has_content(table, chapters, novelName, siteName, receiver) else: self.send_chapter(table, (lastChapterUrl, lastChapterName), novelName, siteName, receiver) timeEnd = time.time() logger.info("{0}({1})最新章节: {2}, 耗时:{3}".format( novelName, siteName, lastChapterName, timeEnd - timeStart)) except Exception as e: logger.error( "parse_novel_thread({0}) failed, error:[{1}]".format( site.get("url"), str(e)))
def save_chapter(self, siteName, chapter, table): try: sql = "insert into {0}(site, site_name, chapter_name, send_time) values('{1}', '{2}', '{3}', now())".format( table, chapter[0], siteName, self.get_chapter_name(chapter[1])) self.mydb.execute(sql) return True except Exception as e: logger.error("save chapter failed, error:[{0}]".format(str(e))) return False
def init_novel(self): self.novelList = [] try: for novel in self.conf.novel: if not novel.get("status"): continue table = novel.get("table") if self.make_table(table): self.novelList.append(novel) else: logger.error( "init novel failed, error:[创建表{}失败!]".format(table)) sys.exit(-1) logger.info("init_novel() success.") except Exception as e: logger.error("init novel failed, error:[{0}]".format(str(e))) sys.exit(-1)
def send(self, receivers, subject, content, mimeType="plain"): receiverList = self.list_split(receivers, nums=30) # 一封邮件最多有三十个收件人 for receiver in receiverList: # 对于发送失败的邮件, 换个发送人再次发送 for myemail in self.conf.emails: myemail["receiver"] = receiver myemail["subject"] = subject myemail["content"] = content myemail["mimeType"] = mimeType stderr = self.send_email(**myemail) if not stderr: logger.info("({0})[{1}] send email to {2} success.".format( subject, myemail.get("sender"), str(receiver))) break else: logger.error( "({0})[{1}] send email to {2} failed!, error:{3}". format(subject, myemail.get("sender"), str(receiver), stderr))