def blogJobDay(self):
logger.info('blogJob-startTime:%s' %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
messages = Engine().getYesterdayUrls()
MailUtil().senHtml(messages)
logger.info('blogJob-endTime:%s' %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
def deduplication(self):
if len(self.urls) == 0:
logger.info('urls is null')
else:
# python 字典遍历时不能做删除，改为 list
for url in list(self.urls):
if 'xz' in url:
src = 'xz'
elif 'anquanke' in url:
src = 'anquanke'
md5hash = hashlib.md5(url.encode('utf-8')).hexdigest()
title = self.urls[url]
logger.info('url info : {src} {url} {md5hash} {title}'.format(src=src, url=url, md5hash=md5hash, title=title))
result = MySqlHandle().selectDB('select * from blogurl where md5hash = "' + md5hash + '"')
logger.info('selct result : ' + str( result))
if len(result) > 0:
if result[0][5] == '1':
self.urls.pop(url)
logger.info('url: {url} already exists and downloaded'.format(url=url))
logger.info('url: {url} already exists but not downloaded'.format(url=url))
else:
status = 0
insertResult = MySqlHandle().insertDB('insert into blogurl(src, url, title, md5hash, status, inserttime) values '
'("' + src + '","' + url + '","' + title + '","' + md5hash + '","' + str(status) +'","' + time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) + '")')
logger.info('insertResult : ' + str(insertResult))
if insertResult == 0:
MailUtil().sendError('插入数据库异常 : ' + src + ' ' + title + ' ' + url)
return self.urls
def updateDB(self, sql):
logger.info("sql : {sql}".format(sql=sql))
'''更新数据库操作'''
self.cursor = self.conn.cursor()
try:
result = self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
# 回滚
self.conn.rollback()
logger.error("update data error : {e}".format(e=e))
MailUtil().sendError('更新数据库异常 : ' + e)
finally:
self.cursor.close()
self.conn.close()
def senMail(self):
for url in self.urls:
if 'xz' in url:
src = 'xz'
elif 'anquanke' in url:
src = 'anquanke'
md5hash = hashlib.md5(url.encode('utf-8')).hexdigest()
result = MySqlHandle().selectDB('select * from blogurl where md5hash = "' + md5hash + '"')
if len(result) > 0:
if result[0][5] == '0':
if MailUtil().sendPdf(src, self.urls[url], url, (project_path + '/pdf/' + src + '/' + self.urls[url].replace('\u200b', '').strip() + '.pdf')) == True:
MySqlHandle().updateDB('update blogurl set status = "1" where md5hash = "' + md5hash + '"')
logger.info('Send pdf email successfully : ' + src + ' ' + self.urls[url] + '.pdf')
else:
logger.info('Send pdf email failed : ' + src + ' ' + self.urls[url] + '.pdf')
def selectDB(self, sql):
''' 数据库查询 '''
logger.info("sql : {sql}".format(sql=sql))
self.cursor = self.conn.cursor()
result = 0
try:
self.cursor.execute(sql)
result = self.cursor.fetchall()
except Exception as e:
logger.error("select data error : {e}".format(e=e))
MailUtil().sendError('查询数据库异常 : ' + e)
finally:
self.cursor.close()
self.conn.close()
return result
def download_file(self):
for url in list(self.urls):
if 'xz' in url:
src = 'xz'
elif 'anquanke' in url:
src = 'anquanke'
filepath = project_path + '/pdf/' + src + '/' + self.urls[url].replace('\u200b', '').strip() + '.pdf'
if not os.path.exists(filepath):
if (make_pdf_from_url(url, src, self.urls[url]) == False):
# html 转换 pdf 异常就不发送邮件了，从 url 中删除
self.urls.pop(url)
MailUtil().sendError(src + ' 文章转换 pdf 异常: ' + url)
else:
logger.info('file already exists : ' + url)
return True
def deleteDB(self, sql):
''' 操作数据库数据删除 '''
logger.info("sql : {sql}".format(sql=sql))
self.cursor = self.conn.cursor()
try:
# 执行sql
result = self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
# 发生错误时回滚
self.conn.rollback()
logger.error("insert data error : {e}".format(e=e))
MailUtil().sendError('删除数据库异常 : ' + e)
finally:
self.cursor.close()
def insertDB(self, sql):
'''插入数据库操作'''
logger.info("sql : {sql}".format(sql=sql))
self.cursor = self.conn.cursor()
result = 0
try:
result = self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
# 回滚
self.conn.rollback()
result = 0
logger.error("insert data error : {e}".format(e=e))
MailUtil().sendError('插入数据库异常 : ' + e)
finally:
self.cursor.close()
self.conn.close()
return result