def GET(self):
user = self.getcurrentuser()
#为了简单起见，就不用其他库生成xml，而直接使用字符串格式化生成
opmlTpl = u"""<?xml version="1.0" encoding="utf-8" ?>
<opml version="2.0">
<head>
<title>KindleEar.opml</title>
<dateCreated>%s</dateCreated>
<dateModified>%s</dateModified>
<ownerName>KindleEar</ownerName>
</head>
<body>
%s
</body>
</opml>"""
date = local_time('%a, %d %b %Y %H:%M:%S GMT', user.timezone)
#添加时区信息
if user.timezone != 0:
date += '+%02d00' % user.timezone if user.timezone > 0 else '-%02d00' % abs(user.timezone)
outlines = []
for feed in Feed.all().filter('book = ', user.ownfeeds):
outlines.append(' <outline type="rss" text="%s" xmlUrl="%s" isFulltext="%d" />' %
(feed.title, urllib.quote_plus(feed.url), feed.isfulltext))
outlines = '\n'.join(outlines)
opmlfile = opmlTpl % (date, date, outlines)
web.header("Content-Type","text/xml;charset=utf-8")
web.header("Content-Disposition","attachment;filename=KindleEar_subscription.xml")
return opmlfile.encode('utf-8')
def GET(self):
user = self.getcurrentuser()
#为了简单起见，就不用其他库生成xml，而直接使用字符串格式化生成
opmlTpl = u"""<?xml version="1.0" encoding="utf-8" ?>
<opml version="2.0">
<head>
<title>KindleEar.opml</title>
<dateCreated>%s</dateCreated>
<dateModified>%s</dateModified>
<ownerName>KindleEar</ownerName>
</head>
<body>
%s
</body>
</opml>"""
date = local_time('%a, %d %b %Y %H:%M:%S GMT', user.timezone)
#添加时区信息
if user.timezone != 0:
date += '+%02d00' % user.timezone if user.timezone > 0 else '-%02d00' % abs(
user.timezone)
outlines = []
for feed in Feed.all().filter('book = ', user.ownfeeds):
outlines.append(
' <outline type="rss" text="%s" xmlUrl="%s" isFulltext="%d" />'
% (feed.title, urllib.quote_plus(
feed.url.encode('utf8')), feed.isfulltext))
outlines = '\n'.join(outlines)
opmlfile = opmlTpl % (date, date, outlines)
web.header("Content-Type", "text/xml;charset=utf-8")
web.header("Content-Disposition",
"attachment;filename=KindleEar_subscription.xml")
return opmlfile.encode('utf-8')
def setMetaData(
oeb,
title='Feeds',
lang='zh-cn',
date=None,
creator=local_time("%d-%b-%y"),
pubtype='periodical:magazine:KindleEar'
): #pubtype='periodical:magazine:KindleEar' | 'book:book:KindleEar'
oeb.metadata.add('language', lang if lang else 'zh-cn')
oeb.metadata.add('creator', creator)
oeb.metadata.add('title', title)
oeb.metadata.add('identifier',
str(uuid.uuid4()),
id='uuid_id',
scheme='uuid')
oeb.uid = oeb.metadata.identifier[0]
oeb.metadata.add("publication_type", pubtype)
if not date:
import datetime
date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
oeb.metadata.add("date", date)
def GET(self):
username = web.input().get('u')
id_ = web.input().get('id')
feedsId = web.input().get('feedsId')
if id_:
id_ = [int(item) for item in id_.split('|') if item.isdigit()]
self.queue2push = defaultdict(list)
books = Book.all()
if username: #现在投递【测试使用】，不需要判断时间和星期
user = KeUser.all().filter("name = ", username).get()
if not user or not user.kindle_email:
return self.render('autoback.html', "Delivering", tips=_('The username not exist or the email of kindle is empty.'))
sent = []
if id_: #推送特定账号指定的书籍，这里不判断特定账号是否已经订阅了指定的书籍，只要提供就推送
books2push = [Book.get_by_id(item) for item in id_ if Book.get_by_id(item)]
else: #推送特定账号所有的书籍
books2push = [item for item in books if username in item.users]
for book in books2push:
self.queueit(user, book.key().id(), book.separate, feedsId)
sent.append(book.title)
self.flushqueue()
if len(sent):
tips = _("Book(s) (%s) put to queue!") % u', '.join(sent)
else:
tips = _("No book to deliver!")
return self.render('autoback.html', "Delivering", tips=tips)
#定时cron调用
sentcnt = 0
for book in books:
if not book.users: #没有用户订阅此书
continue
bkcls = None
if book.builtin:
bkcls = BookClass(book.title)
if not bkcls:
continue
#确定此书是否需要下载
for u in book.users:
user = KeUser.all().filter("enable_send = ",True).filter("name = ", u).get()
if not user or not user.kindle_email:
continue
#先判断当天是否需要推送
day = local_time('%A', user.timezone)
usrdays = user.send_days
if bkcls and bkcls.deliver_days: #按星期推送
days = bkcls.deliver_days
if not isinstance(days, list):
days = [days]
if day not in days:
continue
elif usrdays and day not in usrdays: #为空也表示每日推送
continue
#时间判断
h = int(local_time("%H", user.timezone)) + 1
if h >= 24:
h -= 24
if bkcls and bkcls.deliver_times:
times = bkcls.deliver_times
if not isinstance(times, list):
times = [times]
if h not in times:
continue
elif user.send_time != h:
continue
#到了这里才是需要推送的
self.queueit(user, book.key().id(), book.separate)
sentcnt += 1
self.flushqueue()
return "Put <strong>%d</strong> books to queue!" % sentcnt
def receive(self, message):
#如果有多个收件人的话，只解释第一个收件人
to = parseaddr(message.to)[1]
to = to.split('@')[0] if to and '@' in to else 'xxx'
if '__' in to:
listto = to.split('__')
username = listto[0] if listto[0] else 'admin'
to = listto[1]
else:
username = '******'
user = KeUser.all().filter('name = ', username).get()
if not user:
username = '******'
user = KeUser.all().filter('name = ', username).get()
if not user or not user.kindle_email:
self.response.out.write('No account or no email configured!')
return
sender = parseaddr(message.sender)[1]
mailhost = sender.split('@')[1] if sender and '@' in sender else None
if (not sender or not mailhost) or \
(not user.whitelist.filter('mail = ', '*').get()
and not user.whitelist.filter('mail = ', sender.lower()).get()
and not user.whitelist.filter('mail = ', '@' + mailhost.lower()).get()):
self.response.out.write("Spam mail!")
log.warn('Spam mail from : %s' % sender)
return
if hasattr(message, 'subject'):
subject = decode_subject(message.subject).strip()
else:
subject = u"NoSubject"
#邮件主题中如果在最后添加一个 !links，则强制提取邮件中的链接然后生成电子书
forceToLinks = False
forceToArticle = False
if subject.endswith('!links'):
subject = subject.replace('!links', '').rstrip()
forceToLinks = True
elif subject.find(' !links ') >= 0:
subject = subject.replace(' !links ', '')
forceToLinks = True
#如果邮件主题在最后添加一个 !article，则强制转换邮件内容为电子书，忽略其中的链接
if not forceToLinks:
if subject.endswith('!article'):
subject = subject.replace('!article', '').rstrip()
forceToArticle = True
elif subject.find(' !article ') >= 0:
subject = subject.replace(' !article ', '')
forceToArticle = True
#通过邮件触发一次“现在投递”
if to.lower() == 'trigger':
return self.TrigDeliver(subject, username)
#获取和解码邮件内容
txt_bodies = message.bodies('text/plain')
html_bodies = message.bodies('text/html')
try:
allBodies = [body.decode() for ctype, body in html_bodies]
except:
log.warn('Decode html bodies of mail failed.')
allBodies = []
#此邮件为纯文本邮件
if len(allBodies) == 0:
log.info('no html body, use text body.')
try:
allBodies = [body.decode() for ctype, body in txt_bodies]
except:
log.warn('Decode text bodies of mail failed.')
allBodies = []
bodies = u''.join(allBodies)
if not bodies:
return
bodyurls = []
for l in bodies.split('\n'):
l = l.strip()
if not l:
continue
link = IsHyperLink(l)
if link:
bodyurls.append('<a href="%s">%s</a><br />' % (link, link))
else:
break
bodies = u"""<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>%s</title></head><body>%s</body></html>""" % (
subject, ''.join(bodyurls) if bodyurls else bodies)
allBodies = [bodies.encode('utf-8')]
#开始处理邮件内容
soup = BeautifulSoup(allBodies[0], 'lxml')
#合并多个邮件文本段
if len(allBodies) > 1:
for o in allBodies[1:]:
so = BeautifulSoup(o, 'lxml')
b = so.find('body')
if not b:
continue
for c in b.contents:
soup.body.append(c)
#判断邮件内容是文本还是链接（包括多个链接的情况）
links = []
body = soup.body if soup.find('body') else soup
if not forceToArticle: #如果强制转正文就不分析链接了，否则先分析和提取链接
for s in body.stripped_strings:
link = IsHyperLink(s)
if link:
if link not in links:
links.append(link)
#如果是多个链接，则必须一行一个，不能留空，除非强制提取链接
#这个处理是为了去除部分邮件客户端在邮件末尾添加的一个广告链接
elif not forceToLinks:
break
if not links and not forceToArticle: #如果通过正常字符（显示出来的）判断没有链接，则看html的a标签
links = [
link['href']
for link in soup.find_all('a', attrs={'href': True})
]
text = ' '.join([s for s in body.stripped_strings])
#如果有相对路径，则在里面找一个绝对路径，然后转换其他
hasRelativePath = False
fullPath = ''
for link in links:
text = text.replace(link, '')
if not link.startswith('http'):
hasRelativePath = True
if not fullPath and link.startswith('http'):
fullPath = link
if hasRelativePath and fullPath:
for idx, link in enumerate(links):
if not link.startswith('http'):
links[idx] = urllib.urljoin(fullPath, link)
#如果字数太多，则认为直接推送正文内容
if not forceToLinks and (len(links) != 1 or
len(text) > WORDCNT_THRESHOLD_FOR_APMAIL):
links = []
if links:
#判断是下载文件还是转发内容
isBook = bool(to.lower() in ('book', 'file', 'download'))
if not isBook:
isBook = bool(link[-5:].lower() in ('.mobi', '.epub', '.docx'))
if not isBook:
isBook = bool(link[-4:].lower() in ('.pdf', '.txt', '.doc',
'.rtf'))
isDebug = bool(to.lower() == 'debug')
if isDebug:
bookType = 'Debug'
elif isBook:
bookType = 'Download'
else:
bookType = user.book_type
param = {
'u': username,
'urls':
base64.urlsafe_b64encode(zlib.compress('|'.join(links), 9)),
'type': bookType,
'to': user.kindle_email,
'tz': user.timezone,
'subject': subject[:SUBJECT_WORDCNT_FOR_APMAIL],
'lng': user.ownfeeds.language,
'keepimage': '1' if user.ownfeeds.keep_image else '0'
}
taskqueue.add(url='/url2book',
queue_name="deliverqueue1",
method='GET',
params=param,
target='worker')
else: #直接转发邮件正文
#先判断是否有图片
from lib.makeoeb import MimeFromFilename
hasimage = False
if hasattr(message, 'attachments'):
for f, c in message.attachments:
if MimeFromFilename(f):
hasimage = True
break
#先修正不规范的HTML邮件
h = soup.find('head')
if not h:
h = soup.new_tag('head')
soup.html.insert(0, h)
t = soup.head.find('title')
if not t:
t = soup.new_tag('title')
t.string = subject
soup.head.insert(0, t)
#有图片的话，要生成MOBI或EPUB才行
#而且多看邮箱不支持html推送，也先转换epub再推送
if hasimage or (user.book_type == "epub"):
from main import local_time
from lib.makeoeb import (getOpts, CreateOeb, setMetaData,
ServerContainer, byteStringIO,
EPUBOutput, MOBIOutput)
#仿照Amazon的转换服务器的处理，去掉CSS
if DELETE_CSS_FOR_APPSPOTMAIL:
tag = soup.find('style', attrs={'type': 'text/css'})
if tag:
tag.extract()
for tag in soup.find_all(attrs={'style': True}):
del tag['style']
#将图片的src的文件名调整好
for img in soup.find_all('img', attrs={'src': True}):
if img['src'].lower().startswith('cid:'):
img['src'] = img['src'][4:]
opts = getOpts()
oeb = CreateOeb(log, None, opts)
setMetaData(oeb,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
user.ownfeeds.language,
local_time(tz=user.timezone),
pubtype='book:book:KindleEar')
oeb.container = ServerContainer(log)
id_, href = oeb.manifest.generate(id='page', href='page.html')
item = oeb.manifest.add(id_,
href,
'application/xhtml+xml',
data=unicode(soup))
oeb.spine.add(item, False)
oeb.toc.add(subject, href)
if hasattr(message, 'attachments'):
for filename, content in message.attachments:
mimetype = MimeFromFilename(filename)
if mimetype:
try:
content = content.decode()
except:
pass
else:
id_, href = oeb.manifest.generate(
id='img', href=filename)
item = oeb.manifest.add(id_,
href,
mimetype,
data=content)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, log)
BaseHandler.SendToKindle(username, user.kindle_email,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
user.book_type, str(oIO.getvalue()),
user.timezone)
else: #没有图片则直接推送HTML文件，阅读体验更佳
m = soup.find('meta', attrs={"http-equiv": "Content-Type"})
if not m:
m = soup.new_tag('meta',
content="text/html; charset=utf-8")
m["http-equiv"] = "Content-Type"
soup.html.head.insert(0, m)
else:
m['content'] = "text/html; charset=utf-8"
html = unicode(soup).encode('utf-8')
BaseHandler.SendToKindle(username, user.kindle_email,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
'html', html, user.timezone, False)
self.response.out.write('Done')
def ProcessComicRSS(self, username, user, feed):
opts = None
oeb = None
# 创建 OEB
#global log
opts = getOpts(user.device, 'comic')
oeb = CreateOeb(main.log, None, opts)
pubtype = 'book:book:KindleEar'
language = 'zh-cn'
setMetaData(oeb,
feed.title,
language,
local_time("%Y-%m-%d", user.timezone),
pubtype=pubtype)
oeb.container = ServerContainer(main.log)
#guide
id_, href = oeb.manifest.generate('masthead',
DEFAULT_MASTHEAD) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
id_, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
if feed.url.startswith(("http://ac.qq.com", "http://m.ac.qq.com")):
book = TencentBaseBook(imgindex=imgindex, opts=opts, user=user)
elif feed.url.startswith("http://www.cartoonmad.com"):
book = CartoonMadBaseBook(imgindex=imgindex, opts=opts, user=user)
else:
return "Failed to push book <%s>!" % title
book.title = feed.title
book.description = feed.title
book.language = language
book.keep_image = True
book.oldest_article = 7
book.fulltext_by_readability = True
book.feeds = [(feed.title, feed.url)]
book.url_filters = [flt.url for flt in user.urlfilter]
try: #书的质量可能不一，一本书的异常不能影响其他书籍的推送
for sec_or_media, url, title, content, brief, thumbnail in book.Items(
):
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_,
href,
sec_or_media,
data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append(
(title, brief, thumbnail, content))
itemcnt += 1
except Exception as e:
excFileName, excFuncName, excLineNo = get_exc_location()
main.log.warn(
"Failed to push <%s> : %s, in file '%s', %s (line %d)" %
(book.title, str(e), excFileName, excFuncName, excLineNo))
return "Failed to push book <%s>!" % title
volumeTitle = ''
if itemcnt > 0:
insertHtmlToc = False
insertThumbnail = False
volumeTitle = book.LastDeliveredVolume()
oeb.metadata.clear('title')
oeb.metadata.add('title', feed.title + volumeTitle)
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc,
insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order('-time').get()
except:
ultima_log = sorted(DeliverLog.all(),
key=attrgetter('time'),
reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(username, user.kindle_email,
feed.title + volumeTitle, user.book_type,
str(oIO.getvalue()), user.timezone)
rs = "%s(%s).%s Sent!" % (feed.title, local_time(tz=user.timezone),
user.book_type)
main.log.info(rs)
return rs
else:
self.deliverlog(username,
str(user.kindle_email),
feed.title + volumeTitle,
0,
status='nonews',
tz=user.timezone)
rs = "No new feeds."
main.log.info(rs)
return rs
def GET(self):
username = web.input().get('u')
id_ = web.input().get('id') #for debug
self.queue2push = defaultdict(list)
books = Book.all()
if username: #现在投递【测试使用】，不需要判断时间和星期
sent = []
books2push = Book.get_by_id(int(id_)) if id_ and id_.isdigit() else None
books2push = [books2push] if books2push else books
for book in books2push:
if not id_ and username not in book.users:
continue
user = KeUser.all().filter("name = ", username).get()
if user and user.kindle_email:
self.queueit(user, book.key().id(), book.separate)
sent.append(book.title)
self.flushqueue()
if len(sent):
tips = _("Book(s) (%s) put to queue!") % u', '.join(sent)
else:
tips = _("No book to deliver!")
return self.render('autoback.html', "Delivering",tips=tips)
#定时cron调用
sentcnt = 0
for book in books:
if not book.users: #没有用户订阅此书
continue
bkcls = None
if book.builtin:
bkcls = BookClass(book.title)
if not bkcls:
continue
#确定此书是否需要下载
for u in book.users:
user = KeUser.all().filter("enable_send = ",True).filter("name = ", u).get()
if not user or not user.kindle_email:
continue
#先判断当天是否需要推送
day = local_time('%A', user.timezone)
usrdays = user.send_days
if bkcls and bkcls.deliver_days: #按星期推送
days = bkcls.deliver_days
if not isinstance(days, list):
days = [days]
if day not in days:
continue
elif usrdays and day not in usrdays: #为空也表示每日推送
continue
#时间判断
h = int(local_time("%H", user.timezone)) + 1
if h >= 24:
h -= 24
if bkcls and bkcls.deliver_times:
times = bkcls.deliver_times
if not isinstance(times, list):
times = [times]
if h not in times:
continue
elif user.send_time != h:
continue
#到了这里才是需要推送的
self.queueit(user, book.key().id(), book.separate)
sentcnt += 1
self.flushqueue()
return "Put <strong>%d</strong> books to queue!" % sentcnt
def GET(self):
username = web.input().get("u")
bookid = web.input().get("id")
user = KeUser.all().filter("name = ", username).get()
if not user:
return "User not exist!<br />"
to = user.kindle_email
booktype = user.book_type
titlefmt = user.titlefmt
tz = user.timezone
bookid = bookid.split(',') if ',' in bookid else [bookid]
bks = []
for id_ in bookid:
try:
bks.append(Book.get_by_id(int(id_)))
except:
continue
#return "id of book is invalid or book not exist!<br />"
if len(bks) == 0:
return "No have book to push!"
elif len(bks) == 1:
book4meta = BookClass(bks[0].title) if bks[0].builtin else bks[0]
else: #多本书合并推送时使用“自定义RSS”的元属性
book4meta = user.ownfeeds
if not book4meta:
return "No have book to push.<br />"
opts = oeb = None
# 创建 OEB
#global log
opts = getOpts(user.device)
oeb = CreateOeb(main.log, None, opts)
title = "%s %s" % (book4meta.title, local_time(titlefmt, tz)) if titlefmt else book4meta.title
setMetaData(oeb, title, book4meta.language, local_time("%Y-%m-%d",tz), 'KindleEar')
oeb.container = ServerContainer(main.log)
#guide
if len(bks) == 1:
if bks[0].builtin:
mhfile = book4meta.mastheadfile
coverfile = book4meta.coverfile
else: #单独的推送自定义RSS
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER
else:
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER_BV if user.merge_books else DEFAULT_COVER
if mhfile:
id_, href = oeb.manifest.generate('masthead', mhfile) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(mhfile))
oeb.guide.add('masthead', 'Masthead Image', href)
if coverfile:
id_, href = oeb.manifest.generate('cover', coverfile)
item = oeb.manifest.add(id_, href, MimeFromFilename(coverfile))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
elif len(bks) > 1 and DEFAULT_COVER:
#将所有书籍的封面拼贴成一个
#如果DEFAULT_COVER=None说明用户不需要封面
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_, href, 'image/jpeg', data=self.MergeCovers(bks, opts))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt,imgindex = 0,0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
for bk in bks:
if bk.builtin:
book = BookClass(bk.title)
if not book:
main.log.warn('not exist book <%s>' % bk.title)
continue
book = book(imgindex=imgindex)
book.url_filters = [flt.url for flt in user.urlfilter]
if bk.needs_subscription: #需要登录
subs_info = user.subscription_info(bk.title)
if subs_info:
book.account = subs_info.account
book.password = subs_info.password
else: # 自定义RSS
if bk.feedscount == 0:
continue #return "the book has no feed!<br />"
book = BaseFeedBook(imgindex=imgindex)
book.title = bk.title
book.description = bk.description
book.language = bk.language
book.keep_image = bk.keep_image
book.oldest_article = bk.oldest_article
book.fulltext_by_readability = True
feeds = bk.feeds
book.feeds = [(feed.title, feed.url, feed.isfulltext) for feed in feeds]
book.url_filters = [flt.url for flt in user.urlfilter]
# 对于html文件，变量名字自文档,thumbnail为文章第一个img的url
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容,
# img的thumbail仅当其为article的第一个img为True
try: #书的质量可能不一，一本书的异常不能影响推送
for sec_or_media, url, title, content, brief, thumbnail in book.Items(opts,user):
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append((title, brief, thumbnail, content))
itemcnt += 1
except Exception as e:
main.log.warn("Failure in pushing book '%s' : %s" % (book.title, str(e)))
continue
if itemcnt > 0:
InsertToc(oeb, sections, toc_thumbnails)
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
ultima_log = DeliverLog.all().order('-time').get()
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 5:
time.sleep(8)
self.SendToKindle(username, to, book4meta.title, booktype, str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!"%(book4meta.title, local_time(tz=tz), booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username, to, book4meta.title, 0, status='nonews',tz=tz)
rs = "No new feeds."
main.log.info(rs)
return rs
def GET(self):
username = web.input().get("u")
bookid = web.input().get("id")
user = KeUser.all().filter("name = ", username).get()
if not user:
return "User not exist!<br />"
to = user.kindle_email
if (';' in to) or (',' in to):
to = to.replace(',', ';').replace(' ', '').split(';')
booktype = user.book_type #mobi,epub
bookmode = user.book_mode or 'periodical' #periodical,comic
titlefmt = user.titlefmt
tz = user.timezone
bookid = bookid.split(',') if ',' in bookid else [bookid]
bks = []
for id_ in bookid:
try:
bks.append(Book.get_by_id(int(id_)))
except:
continue
#return "id of book is invalid or book not exist!<br />"
book4meta = None
if len(bks) == 0:
return "No have book to push!"
elif len(bks) == 1:
if bks[0].builtin:
book4meta = BookClass(bks[0].title)
mhfile = book4meta.mastheadfile
coverfile = book4meta.coverfile
if issubclass(book4meta, BaseComicBook
): #如果单独推送一个继承自BaseComicBook的书籍，则自动设置为漫画模式
bookmode = 'comic'
else: #单独的推送自定义RSS
book4meta = bks[0]
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER
else: #多本书合并推送时使用“自定义RSS”的元属性
book4meta = user.ownfeeds
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER_BV if user.merge_books else DEFAULT_COVER
if not book4meta:
return "No have book to push.<br />"
opts = None
oeb = None
# 创建 OEB
#global log
opts = getOpts(user.device, bookmode)
oeb = CreateOeb(main.log, None, opts)
bookTitle = "%s %s" % (book4meta.title, local_time(
titlefmt, tz)) if titlefmt else book4meta.title
if bookmode == 'comic':
pubtype = 'book:book:KindleEar'
else:
pubtype = 'periodical:magazine:KindleEar'
setMetaData(oeb,
bookTitle,
book4meta.language,
local_time("%Y-%m-%d", tz),
pubtype=pubtype)
oeb.container = ServerContainer(main.log)
#guide
if mhfile:
id_, href = oeb.manifest.generate('masthead',
mhfile) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(mhfile))
oeb.guide.add('masthead', 'Masthead Image', href)
if coverfile:
imgData = None
imgMime = ''
#使用保存在数据库的用户上传的封面
if coverfile == DEFAULT_COVER and user.cover:
imgData = user.cover
imgMime = 'image/jpeg' #保存在数据库中的只可能是jpeg格式
elif callable(coverfile): #如果封面需要回调的话
try:
imgData = book4meta().coverfile()
if imgData:
imgType = imghdr.what(None, imgData)
if imgType: #如果是合法图片
imgMime = r"image/" + imgType
else:
main.log.warn(
'content of cover is invalid : [%s].' %
bookTitle)
imgData = None
except Exception as e:
main.log.warn(
'Failed to fetch cover for book [%s]. [Error: %s]' %
(bookTitle, str(e)))
coverfile = DEFAULT_COVER
imgData = None
imgMime = ''
if imgData and imgMime:
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_, href, imgMime, data=imgData)
else:
id_, href = oeb.manifest.generate('cover', coverfile)
item = oeb.manifest.add(id_, href, MimeFromFilename(coverfile))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
elif len(bks) > 1 and DEFAULT_COVER:
#将所有书籍的封面拼贴成一个
#如果DEFAULT_COVER=None说明用户不需要封面
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_,
href,
'image/jpeg',
data=self.MergeCovers(bks, opts, user))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
for bk in bks:
if bk.builtin:
cbook = BookClass(bk.title)
if not cbook:
main.log.warn('not exist book <%s>' % bk.title)
continue
book = cbook(imgindex=imgindex, opts=opts, user=user)
book.url_filters = [flt.url for flt in user.urlfilter]
if bk.needs_subscription: #需要登录
subs_info = user.subscription_info(bk.title)
if subs_info:
book.account = subs_info.account
book.password = subs_info.password
else: # 自定义RSS
if bk.feedscount == 0:
continue #return "the book has no feed!<br />"
book = BaseFeedBook(imgindex=imgindex, opts=opts, user=user)
book.title = bk.title
book.description = bk.description
book.language = bk.language
book.keep_image = bk.keep_image
book.oldest_article = bk.oldest_article
book.fulltext_by_readability = True
feeds = bk.feeds
book.feeds = []
for feed in feeds:
if feed.url.startswith(
("http://www.cartoonmad.com", "http://ac.qq.com",
"http://m.ac.qq.com")):
self.ProcessComicRSS(username, user, feed)
else:
book.feeds.append(
(feed.title, feed.url, feed.isfulltext))
book.url_filters = [flt.url for flt in user.urlfilter]
# 对于html文件，变量名字自文档,thumbnail为文章第一个img的url
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容,
# img的thumbail仅当其为article的第一个img为True
try: #书的质量可能不一，一本书的异常不能影响其他书籍的推送
for sec_or_media, url, title, content, brief, thumbnail in book.Items(
):
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_,
href,
sec_or_media,
data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append(
(title, brief, thumbnail, content))
itemcnt += 1
except Exception as e:
excFileName, excFuncName, excLineNo = get_exc_location()
main.log.warn(
"Failed to push <%s> : %s, in file '%s', %s (line %d)" %
(book.title, str(e), excFileName, excFuncName, excLineNo))
continue
volumeTitle = ''
if itemcnt > 0:
#漫画模式不需要TOC和缩略图
if bookmode == 'comic':
insertHtmlToc = False
insertThumbnail = False
if len(bks) == 1 and book: #因为漫画模式没有目录，所以在标题中添加卷号
volumeTitle = book.LastDeliveredVolume()
oeb.metadata.clear('title')
oeb.metadata.add('title', bookTitle + volumeTitle)
else:
insertHtmlToc = GENERATE_HTML_TOC
insertThumbnail = GENERATE_TOC_THUMBNAIL
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc,
insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order('-time').get()
except:
ultima_log = sorted(DeliverLog.all(),
key=attrgetter('time'),
reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(username, to, book4meta.title + volumeTitle,
booktype, str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!" % (book4meta.title, local_time(tz=tz),
booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username,
str(to),
book4meta.title + volumeTitle,
0,
status='nonews',
tz=tz)
rs = "No new feeds."
main.log.info(rs)
return rs
def GET(self):
username = web.input().get("u")
urls = web.input().get("urls")
subject = web.input().get("subject")
to = web.input().get("to")
language = web.input().get("lng")
keepimage = bool(web.input().get("keepimage") == '1')
booktype = web.input().get("type", "mobi")
tz = int(web.input().get("tz", TIMEZONE))
if not all((username, urls, subject, to, language, booktype, tz)):
return "Some parameter missing!<br />"
if (';' in to) or (',' in to):
to = to.replace(',', ';').replace(' ', '').split(';')
to = list(filter(lambda x: x.find('@', 1,
len(x) - 1) > 0,
to)) #最简单的判断是否是EMAIL
if type(urls) is unicode:
urls = urls.encode('utf-8')
urls = zlib.decompress(base64.urlsafe_b64decode(urls))
if booktype == 'Download': #直接下载电子书并推送
from lib.filedownload import Download
for url in urls.split('|'):
dlinfo, filename, content = Download(url)
#如果标题已经给定了文件名，则使用标题文件名
if '.' in subject and (1 < len(subject.split('.')[-1]) < 5):
filename = subject
if content:
self.SendToKindle(username, to, filename, '', content, tz)
else:
if not dlinfo:
dlinfo = 'download failed'
self.deliverlog(username,
str(to),
filename,
0,
status=dlinfo,
tz=tz)
main.log.info("%s Sent!" % filename)
return "%s Sent!" % filename
elif booktype == 'Debug': #调试目的，将链接直接下载，发送到管理员邮箱
from books.base import debug_fetch
#如果标题已经给定了文件名，则使用标题文件名，否则为默认文件名(page.html)
filename = None
if '.' in subject and (1 < len(subject.split('.')[-1]) < 5):
filename = subject
for url in urls.split('|'):
debug_fetch(url, filename)
main.log.info('[DEBUG] debug file sent!')
return 'Debug file sent!'
user = KeUser.all().filter("name = ", username).get()
if not user or not user.kindle_email:
return "User not exist!<br />"
opts = getOpts(user.device)
book = BaseUrlBook(opts=opts, user=user)
book.title = book.description = subject
book.language = language
book.keep_image = keepimage
book.network_timeout = 60
book.feeds = [(subject, url) for url in urls.split('|')]
book.url_filters = [flt.url for flt in user.urlfilter]
# 创建 OEB
oeb = CreateOeb(main.log, None, opts)
oeb.container = ServerContainer(main.log)
if len(book.feeds) > 1:
setMetaData(oeb, subject, language, local_time(tz=tz))
id_, href = oeb.manifest.generate('masthead', DEFAULT_MASTHEAD)
oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
else:
setMetaData(oeb,
subject,
language,
local_time(tz=tz),
pubtype='book:book:KindleEar')
# id, href = oeb.manifest.generate('cover', DEFAULT_COVER)
# item = oeb.manifest.add(id, href, MimeFromFilename(DEFAULT_COVER))
# oeb.guide.add('cover', 'Cover', href)
# oeb.metadata.add('cover', id)
# 对于html文件，变量名字自文档
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容
itemcnt, hasimage = 0, False
sections = {subject: []}
toc_thumbnails = {} #map img-url -> manifest-href
for sec_or_media, url, title, content, brief, thumbnail in book.Items(
):
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
itemcnt += 1
hasimage = True
else:
if len(book.feeds) > 1:
sections[subject].append(
(title, brief, thumbnail, content))
else:
id_, href = oeb.manifest.generate(id='page',
href='page.html')
item = oeb.manifest.add(id_,
href,
'application/xhtml+xml',
data=content)
oeb.spine.add(item, False)
oeb.toc.add(title, href)
itemcnt += 1
if itemcnt > 0:
if len(book.feeds) > 1:
InsertToc(oeb, sections, toc_thumbnails, GENERATE_HTML_TOC,
GENERATE_TOC_THUMBNAIL)
# elif not hasimage: #单文章没有图片则去掉封面
# href = oeb.guide['cover'].href
# oeb.guide.remove('cover')
# item = oeb.manifest.hrefs[href]
# oeb.manifest.remove(item)
# oeb.metadata.clear('cover')
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
self.SendToKindle(username, to, book.title, booktype,
str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!" % (book.title, local_time(tz=tz), booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username,
str(to),
book.title,
0,
status='fetch failed',
tz=tz)
rs = "[Url2Book]Fetch url failed."
main.log.info(rs)
return rs
def GET(self):
username = web.input().get("u")
bookid = web.input().get("id")
user = KeUser.all().filter("name = ", username).get()
if not user:
return "User not exist!<br />"
to = user.kindle_email
booktype = user.book_type
titlefmt = user.titlefmt
tz = user.timezone
bookid = bookid.split(',') if ',' in bookid else [bookid]
bks = []
for id in bookid:
try:
bks.append(Book.get_by_id(int(id)))
except:
continue
#return "id of book is invalid or book not exist!<br />"
if len(bks) == 0:
return "No have book to push!"
elif len(bks) == 1:
book4meta = BookClass(bks[0].title) if bks[0].builtin else bks[0]
else: #多本书合并推送时使用“自定义RSS”的元属性
book4meta = user.ownfeeds
if not book4meta:
return "No have book to push.<br />"
opts = oeb = None
# 创建 OEB
#global log
opts = getOpts(user.device)
oeb = CreateOeb(main.log, None, opts)
title = "%s %s" % (book4meta.title, local_time(titlefmt, tz)) if titlefmt else book4meta.title
setMetaData(oeb, title, book4meta.language, local_time("%Y-%m-%d",tz), 'KindleEar')
oeb.container = ServerContainer(main.log)
#guide
if len(bks)==1 and bks[0].builtin:
mhfile = book4meta.mastheadfile
coverfile = book4meta.coverfile
else:
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER_BV if user.merge_books else DEFAULT_COVER
if mhfile:
id_, href = oeb.manifest.generate('masthead', mhfile) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(mhfile))
oeb.guide.add('masthead', 'Masthead Image', href)
if coverfile:
id_, href = oeb.manifest.generate('cover', coverfile)
item = oeb.manifest.add(id_, href, MimeFromFilename(coverfile))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
elif len(bks)>1 and DEFAULT_COVER:
#将所有书籍的封面拼贴成一个
#如果DEFAULT_COVER=None说明用户不需要封面
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_, href, 'image/jpeg', data=self.MergeCovers(bks,opts))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt,imgindex = 0,0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
for bk in bks:
if bk.builtin:
book = BookClass(bk.title)
if not book:
main.log.warn('not exist book <%s>' % bk.title)
continue
book = book(imgindex=imgindex)
book.url_filters = [flt.url for flt in user.urlfilter]
if bk.needs_subscription: #需要登录
subs_info = user.subscription_info(bk.title)
if subs_info:
book.account = subs_info.account
book.password = subs_info.password
else: # 自定义RSS
if bk.feedscount == 0:
continue #return "the book has no feed!<br />"
book = BaseFeedBook(imgindex=imgindex)
book.title = bk.title
book.description = bk.description
book.language = bk.language
book.keep_image = bk.keep_image
book.oldest_article = bk.oldest_article
book.fulltext_by_readability = True
feeds = bk.feeds
book.feeds = [(feed.title, feed.url, feed.isfulltext) for feed in feeds]
book.url_filters = [flt.url for flt in user.urlfilter]
# 对于html文件，变量名字自文档,thumbnail为文章第一个img的url
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容,
# img的thumbail仅当其为article的第一个img为True
for sec_or_media, url, title, content, brief, thumbnail in book.Items(opts,user):
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append((title, brief, thumbnail, content))
itemcnt += 1
if itemcnt > 0:
InsertToc(oeb, sections, toc_thumbnails)
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
ultima_log = DeliverLog.all().order('-time').get()
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 5:
time.sleep(8)
self.SendToKindle(username, to, book4meta.title, booktype, str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!"%(book4meta.title, local_time(tz=tz), booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username, to, book4meta.title, 0, status='nonews',tz=tz)
rs = "No new feeds."
main.log.info(rs)
return rs
def receive(self, message):
# 如果有多个收件人的话，只解释第一个收件人
to = parseaddr(message.to)[1]
to = to.split("@")[0] if to and "@" in to else "xxx"
if "__" in to:
listto = to.split("__")
username = listto[0] if listto[0] else "admin"
to = listto[1]
else:
username = "******"
user = KeUser.all().filter("name = ", username).get()
if not user:
username = "******"
user = KeUser.all().filter("name = ", username).get()
if not user or not user.kindle_email:
self.response.out.write("No account or no email configured!")
return
sender = parseaddr(message.sender)[1]
mailhost = sender.split("@")[1] if sender and "@" in sender else None
if (not sender or not mailhost) or (
not user.whitelist.filter("mail = ", "*").get()
and not user.whitelist.filter("mail = ", sender.lower()).get()
and not user.whitelist.filter("mail = ", "@" + mailhost.lower()).get()
):
self.response.out.write("Spam mail!")
log.warn("Spam mail from : %s" % sender)
return
if hasattr(message, "subject"):
subject = decode_subject(message.subject).strip()
else:
subject = u"NoSubject"
# 邮件主题中如果在最后添加一个 !links，则强制提取邮件中的链接然后生成电子书
forceToLinks = False
forceToArticle = False
if subject.endswith("!links"):
subject = subject.replace("!links", "").rstrip()
forceToLinks = True
elif subject.find(" !links ") >= 0:
subject = subject.replace(" !links ", "")
forceToLinks = True
if subject.endswith("!article"):
subject = subject.replace("!article", "").rstrip()
forceToArticle = True
elif subject.find(" !article ") >= 0:
subject = subject.replace(" !article ", "")
forceToArticle = True
# 通过邮件触发一次“现在投递”
if to.lower() == "trigger":
return self.TrigDeliver(subject, username)
# 获取和解码邮件内容
txt_bodies = message.bodies("text/plain")
html_bodies = message.bodies("text/html")
try:
allBodies = [body.decode() for ctype, body in html_bodies]
except:
log.warn("Decode html bodies of mail failed.")
allBodies = []
# 此邮件为纯文本邮件
if len(allBodies) == 0:
log.info("no html body, use text body.")
try:
allBodies = [body.decode() for ctype, body in txt_bodies]
except:
log.warn("Decode text bodies of mail failed.")
allBodies = []
bodies = u"".join(allBodies)
if not bodies:
return
bodyurls = []
for l in bodies.split("\n"):
l = l.strip()
if not l:
continue
link = IsHyperLink(l)
if link:
bodyurls.append('<a href="%s">%s</a><br />' % (link, link))
else:
break
bodies = u"""<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>%s</title></head><body>%s</body></html>""" % (
subject,
"".join(bodyurls) if bodyurls else bodies,
)
allBodies = [bodies.encode("utf-8")]
# 开始处理邮件内容
soup = BeautifulSoup(allBodies[0], "lxml")
# 合并多个邮件文本段
if len(allBodies) > 1:
for o in allBodies[1:]:
so = BeautifulSoup(o, "lxml")
b = so.find("body")
if not b:
continue
for c in b.contents:
soup.body.append(c)
# 判断邮件内容是文本还是链接（包括多个链接的情况）
links = []
body = soup.body if soup.find("body") else soup
if not forceToArticle:
for s in body.stripped_strings:
link = IsHyperLink(s)
if link:
if link not in links:
links.append(link)
elif not forceToLinks: # 如果是多个链接，则必须一行一个，除非强制提取链接
break
if not links and not forceToArticle: # 正常字符判断没有链接，看html的a标签
links = [link["href"] for link in soup.find_all("a", attrs={"href": True})]
text = " ".join([s for s in body.stripped_strings])
# 如果有相对路径，则在里面找一个绝对路径，然后转换其他
hasRelativePath = False
fullPath = ""
for link in links:
text = text.replace(link, "")
if not link.startswith("http"):
hasRelativePath = True
if not fullPath and link.startswith("http"):
fullPath = link
if hasRelativePath and fullPath:
for idx, link in enumerate(links):
if not link.startswith("http"):
links[idx] = urllib.urljoin(fullPath, link)
# 如果字数太多，则认为直接推送正文内容
if not forceToLinks and (len(links) != 1 or len(text) > WORDCNT_THRESHOLD_FOR_APMAIL):
links = []
if links:
# 判断是下载文件还是转发内容
isbook = bool(to.lower() in ("book", "file", "download"))
isbook = link[-5:].lower() in (".mobi", ".epub", ".docx") if not isbook else isbook
isbook = link[-4:].lower() in (".pdf", ".txt", ".doc", ".rtf") if not isbook else isbook
param = {
"u": username,
"urls": base64.urlsafe_b64encode(zlib.compress("|".join(links), 9)),
"type": "Download" if isbook else user.book_type,
"to": user.kindle_email,
"tz": user.timezone,
"subject": subject[:SUBJECT_WORDCNT_FOR_APMAIL],
"lng": user.ownfeeds.language,
"keepimage": "1" if user.ownfeeds.keep_image else "0",
}
taskqueue.add(url="/url2book", queue_name="deliverqueue1", method="GET", params=param, target="worker")
else: # 直接转发邮件正文
# 先判断是否有图片
from lib.makeoeb import MimeFromFilename
hasimage = False
if hasattr(message, "attachments"):
for f, c in message.attachments:
if MimeFromFilename(f):
hasimage = True
break
# 先修正不规范的HTML邮件
h = soup.find("head")
if not h:
h = soup.new_tag("head")
soup.html.insert(0, h)
t = soup.head.find("title")
if not t:
t = soup.new_tag("title")
t.string = subject
soup.head.insert(0, t)
# 有图片的话，要生成MOBI或EPUB才行
# 而且多看邮箱不支持html推送，也先转换epub再推送
if hasimage or (user.book_type == "epub"):
from main import local_time
from lib.makeoeb import (
getOpts,
CreateOeb,
setMetaData,
ServerContainer,
byteStringIO,
EPUBOutput,
MOBIOutput,
)
# 仿照Amazon的转换服务器的处理，去掉CSS
if DELETE_CSS_FOR_APPSPOTMAIL:
tag = soup.find("style", attrs={"type": "text/css"})
if tag:
tag.extract()
for tag in soup.find_all(attrs={"style": True}):
del tag["style"]
# 将图片的src的文件名调整好
for img in soup.find_all("img", attrs={"src": True}):
if img["src"].lower().startswith("cid:"):
img["src"] = img["src"][4:]
opts = getOpts()
oeb = CreateOeb(log, None, opts)
setMetaData(
oeb,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
user.ownfeeds.language,
local_time(tz=user.timezone),
pubtype="book:book:KindleEar",
)
oeb.container = ServerContainer(log)
id, href = oeb.manifest.generate(id="page", href="page.html")
item = oeb.manifest.add(id, href, "application/xhtml+xml", data=unicode(soup))
oeb.spine.add(item, False)
oeb.toc.add(subject, href)
if hasattr(message, "attachments"):
for filename, content in message.attachments:
mimetype = MimeFromFilename(filename)
if mimetype:
try:
content = content.decode()
except:
pass
else:
id, href = oeb.manifest.generate(id="img", href=filename)
item = oeb.manifest.add(id, href, mimetype, data=content)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, log)
BaseHandler.SendToKindle(
username,
user.kindle_email,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
user.book_type,
str(oIO.getvalue()),
user.timezone,
)
else: # 没有图片则直接推送HTML文件，阅读体验更佳
m = soup.find("meta", attrs={"http-equiv": "Content-Type"})
if not m:
m = soup.new_tag("meta", content="text/html; charset=utf-8")
m["http-equiv"] = "Content-Type"
soup.html.head.insert(0, m)
else:
m["content"] = "text/html; charset=utf-8"
html = unicode(soup).encode("utf-8")
BaseHandler.SendToKindle(
username,
user.kindle_email,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
"html",
html,
user.timezone,
False,
)
self.response.out.write("Done")
def GET(self):
username = web.input().get("u")
urls = web.input().get("urls")
subject = web.input().get("subject")
to = web.input().get("to")
language = web.input().get("lng")
keepimage = bool(web.input().get("keepimage") == '1')
booktype = web.input().get("type", "mobi")
tz = int(web.input().get("tz", TIMEZONE))
if not all((username, urls, subject, to, language, booktype, tz)):
return "Some parameter missing!<br />"
#global log
if booktype == 'Download': #直接下载电子书并推送
from lib.filedownload import Download
for url in urls.split('|'):
dlinfo, filename, content = Download(url)
#如果标题已经给定了文件名，则使用标题文件名
if '.' in subject and (1 < len(subject.split('.')[-1]) < 5):
filename = subject
if content:
self.SendToKindle(username, to, filename, '', content, tz)
else:
if not dlinfo:
dlinfo = 'download failed'
self.deliverlog(username,
to,
filename,
0,
status=dlinfo,
tz=tz)
main.log.info("%s Sent!" % filename)
return "%s Sent!" % filename
user = KeUser.all().filter("name = ", username).get()
if not user or not user.kindle_email:
return "User not exist!<br />"
book = BaseUrlBook()
book.title = book.description = subject
book.language = language
book.keep_image = keepimage
book.network_timeout = 60
book.feeds = [(subject, url) for url in urls.split('|')]
book.url_filters = [flt.url for flt in user.urlfilter]
opts = oeb = None
# 创建 OEB
opts = getOpts(user.device)
oeb = CreateOeb(main.log, None, opts)
oeb.container = ServerContainer(main.log)
if len(book.feeds) > 1:
setMetaData(oeb, subject, language, local_time(tz=tz))
id, href = oeb.manifest.generate('masthead', DEFAULT_MASTHEAD)
oeb.manifest.add(id, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
else:
setMetaData(oeb,
subject,
language,
local_time(tz=tz),
pubtype='book:book:KindleEar')
id, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id)
# 对于html文件，变量名字自文档
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容
itemcnt, hasimage = 0, False
sections = {subject: []}
toc_thumbnails = {} #map img-url -> manifest-href
for sec_or_media, url, title, content, brief, thumbnail in book.Items(
opts, user):
if sec_or_media.startswith(r'image/'):
id, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
itemcnt += 1
hasimage = True
else:
if len(book.feeds) > 1:
sections[subject].append(
(title, brief, thumbnail, content))
else:
id, href = oeb.manifest.generate(id='page',
href='page.html')
item = oeb.manifest.add(id,
href,
'application/xhtml+xml',
data=content)
oeb.spine.add(item, False)
oeb.toc.add(title, href)
itemcnt += 1
if itemcnt > 0:
if len(book.feeds) > 1:
InsertToc(oeb, sections, toc_thumbnails)
elif not hasimage: #单文章没有图片则去掉封面
href = oeb.guide['cover'].href
oeb.guide.remove('cover')
item = oeb.manifest.hrefs[href]
oeb.manifest.remove(item)
oeb.metadata.clear('cover')
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
self.SendToKindle(username, to, book.title, booktype,
str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!" % (book.title, local_time(tz=tz), booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username,
to,
book.title,
0,
status='fetch failed',
tz=tz)
rs = "[Url2Book]Fetch url failed."
main.log.info(rs)
return rs
def GET(self):
username = web.input().get("u")
urls = web.input().get("urls")
subject = web.input().get("subject")
to = web.input().get("to")
language = web.input().get("lng")
keepimage = bool(web.input().get("keepimage") == '1')
booktype = web.input().get("type", "mobi")
tz = int(web.input().get("tz", TIMEZONE))
if not all((username,urls,subject,to,language,booktype,tz)):
return "Some parameter missing!<br />"
#global log
if booktype == 'Download': #直接下载电子书并推送
from lib.filedownload import Download
for url in urls.split('|'):
dlinfo, filename, content = Download(url)
#如果标题已经给定了文件名，则使用标题文件名
if '.' in subject and (1 < len(subject.split('.')[-1]) < 5):
filename = subject
if content:
self.SendToKindle(username, to, filename, '', content, tz)
else:
if not dlinfo:
dlinfo = 'download failed'
self.deliverlog(username, to, filename, 0, status=dlinfo,tz=tz)
main.log.info("%s Sent!" % filename)
return "%s Sent!" % filename
user = KeUser.all().filter("name = ", username).get()
if not user or not user.kindle_email:
return "User not exist!<br />"
book = BaseUrlBook()
book.title = book.description = subject
book.language = language
book.keep_image = keepimage
book.network_timeout = 60
book.feeds = [(subject,url) for url in urls.split('|')]
book.url_filters = [flt.url for flt in user.urlfilter]
opts = oeb = None
# 创建 OEB
opts = getOpts(user.device)
oeb = CreateOeb(main.log, None, opts)
oeb.container = ServerContainer(main.log)
if len(book.feeds) > 1:
setMetaData(oeb, subject, language, local_time(tz=tz))
id, href = oeb.manifest.generate('masthead', DEFAULT_MASTHEAD)
oeb.manifest.add(id, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
else:
setMetaData(oeb, subject, language, local_time(tz=tz), pubtype='book:book:KindleEar')
id, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id)
# 对于html文件，变量名字自文档
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容
itemcnt,hasimage = 0,False
sections = {subject:[]}
toc_thumbnails = {} #map img-url -> manifest-href
for sec_or_media, url, title, content, brief, thumbnail in book.Items(opts,user):
if sec_or_media.startswith(r'image/'):
id, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
itemcnt += 1
hasimage = True
else:
if len(book.feeds) > 1:
sections[subject].append((title,brief,thumbnail,content))
else:
id, href = oeb.manifest.generate(id='page', href='page.html')
item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
oeb.spine.add(item, False)
oeb.toc.add(title, href)
itemcnt += 1
if itemcnt > 0:
if len(book.feeds) > 1:
InsertToc(oeb, sections, toc_thumbnails)
elif not hasimage: #单文章没有图片则去掉封面
href = oeb.guide['cover'].href
oeb.guide.remove('cover')
item = oeb.manifest.hrefs[href]
oeb.manifest.remove(item)
oeb.metadata.clear('cover')
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
self.SendToKindle(username, to, book.title, booktype, str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!"%(book.title, local_time(tz=tz), booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username, to, book.title, 0, status='fetch failed',tz=tz)
rs = "[Url2Book]Fetch url failed."
main.log.info(rs)
return rs
def receive(self, message):
#如果有多个收件人的话，只解释第一个收件人
to = parseaddr(message.to)[1]
to = to.split('@')[0] if to and '@' in to else 'xxx'
if '__' in to:
listto = to.split('__')
username = listto[0] if listto[0] else 'admin'
to = listto[1]
else:
username = '******'
user = KeUser.all().filter('name = ', username).get()
if not user:
username = '******'
user = KeUser.all().filter('name = ', username).get()
if not user or not user.kindle_email:
self.response.out.write('No account or no email configured!')
return
sender = parseaddr(message.sender)[1]
mailhost = sender.split('@')[1] if sender and '@' in sender else None
if (not sender or not mailhost) or \
(not user.whitelist.filter('mail = ', '*').get()
and not user.whitelist.filter('mail = ', sender.lower()).get()
and not user.whitelist.filter('mail = ', '@' + mailhost.lower()).get()):
self.response.out.write("Spam mail!")
log.warn('Spam mail from : %s' % sender)
return
if hasattr(message, 'subject'):
subject = decode_subject(message.subject)
else:
subject = u"NoSubject"
#通过邮件触发一次“现在投递”
if to.lower() == 'trigger':
return self.TrigDeliver(subject, username)
#获取和解码邮件内容
txt_bodies = message.bodies('text/plain')
html_bodies = message.bodies('text/html')
try:
allBodies = [body.decode() for ctype, body in html_bodies]
except:
log.warn('Decode html bodies of mail failed.')
allBodies = []
#此邮件为纯文本邮件
if len(allBodies) == 0:
log.info('no html body, use text body.')
try:
allBodies = [body.decode() for ctype, body in txt_bodies]
except:
log.warn('Decode text bodies of mail failed.')
allBodies = []
bodies = u''.join(allBodies)
if not bodies:
return
bodyurls = []
for l in bodies.split('\n'):
l = l.strip()
if not l:
continue
link = IsHyperLink(l)
if link:
bodyurls.append('<a href="%s">%s</a><br />' % (link,link))
else:
break
bodies = u"""<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>%s</title></head><body>%s</body></html>""" %(subject,
''.join(bodyurls) if bodyurls else bodies)
allBodies = [bodies.encode('utf-8')]
#开始处理邮件内容
soup = BeautifulSoup(allBodies[0], 'lxml')
#合并多个邮件文本段
if len(allBodies) > 1:
for o in allBodies[1:]:
so = BeautifulSoup(o, 'lxml')
b = so.find('body')
if not b:
continue
for c in b.contents:
soup.body.append(c)
#判断邮件内容是文本还是链接（包括多个链接的情况）
links = []
body = soup.body if soup.find('body') else soup
for s in body.stripped_strings:
link = IsHyperLink(s)
if link:
links.append(link)
else: #如果是多个链接，则必须一行一个
break
if not links: #正常字符判断没有链接，看html的a标签
links = list(soup.find_all('a',attrs={'href':True}))
link = links[0]['href'] if links else ''
text = ' '.join([s for s in body.stripped_strings])
text = text.replace(link, '')
#如果字数太多，则认为直接推送正文内容
if len(links) != 1 or len(text) > WORDCNT_THRESHOLD_FOR_APMAIL:
links = []
if links:
#判断是下载文件还是转发内容
isbook = bool(to.lower() in ('book', 'file', 'download'))
isbook = link[-5:].lower() in ('.mobi','.epub','.docx') if not isbook else isbook
isbook = link[-4:].lower() in ('.pdf','.txt','.doc','.rtf') if not isbook else isbook
param = {'u':username,
'urls':base64.urlsafe_b64encode(zlib.compress('|'.join(links), 9)),
'type':'Download' if isbook else user.book_type,
'to':user.kindle_email,
'tz':user.timezone,
'subject':subject[:SUBJECT_WORDCNT_FOR_APMAIL],
'lng':user.ownfeeds.language,
'keepimage':'1' if user.ownfeeds.keep_image else '0'
}
taskqueue.add(url='/url2book',queue_name="deliverqueue1",method='GET',
params=param,target='worker')
else: #直接转发邮件正文
#先判断是否有图片
from lib.makeoeb import MimeFromFilename
hasimage = False
if hasattr(message, 'attachments'):
for f,c in message.attachments:
if MimeFromFilename(f):
hasimage = True
break
#先修正不规范的HTML邮件
h = soup.find('head')
if not h:
h = soup.new_tag('head')
soup.html.insert(0, h)
t = soup.head.find('title')
if not t:
t = soup.new_tag('title')
t.string = subject
soup.head.insert(0, t)
#有图片的话，要生成MOBI或EPUB才行
#而且多看邮箱不支持html推送，也先转换epub再推送
if hasimage or (user.book_type == "epub"):
from main import local_time
from lib.makeoeb import (getOpts, CreateOeb, setMetaData,
ServerContainer, byteStringIO,
EPUBOutput, MOBIOutput)
#仿照Amazon的转换服务器的处理，去掉CSS
if DELETE_CSS_FOR_APPSPOTMAIL:
tag = soup.find('style', attrs={'type':'text/css'})
if tag:
tag.extract()
for tag in soup.find_all(attrs={'style':True}):
del tag['style']
#将图片的src的文件名调整好
for img in soup.find_all('img',attrs={'src':True}):
if img['src'].lower().startswith('cid:'):
img['src'] = img['src'][4:]
opts = getOpts()
oeb = CreateOeb(log, None, opts)
setMetaData(oeb, subject[:SUBJECT_WORDCNT_FOR_APMAIL],
user.ownfeeds.language, local_time(tz=user.timezone),
pubtype='book:book:KindleEar')
oeb.container = ServerContainer(log)
id, href = oeb.manifest.generate(id='page', href='page.html')
item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=unicode(soup))
oeb.spine.add(item, False)
oeb.toc.add(subject, href)
if hasattr(message, 'attachments'):
for filename,content in message.attachments:
mimetype = MimeFromFilename(filename)
if mimetype:
try:
content = content.decode()
except:
pass
else:
id, href = oeb.manifest.generate(id='img', href=filename)
item = oeb.manifest.add(id, href, mimetype, data=content)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, log)
BaseHandler.SendToKindle(username, user.kindle_email,
subject[:SUBJECT_WORDCNT_FOR_APMAIL],
user.book_type, str(oIO.getvalue()), user.timezone)
else: #没有图片则直接推送HTML文件，阅读体验更佳
m = soup.find('meta', attrs={"http-equiv":"Content-Type"})
if not m:
m = soup.new_tag('meta', content="text/html; charset=utf-8")
m["http-equiv"] = "Content-Type"
soup.html.head.insert(0,m)
else:
m['content'] = "text/html; charset=utf-8"
html = unicode(soup).encode('utf-8')
BaseHandler.SendToKindle(username, user.kindle_email,
subject[:SUBJECT_WORDCNT_FOR_APMAIL], 'html', html, user.timezone, False)
self.response.out.write('Done')
def ProcessComicRSS(self, username, user, feed):
opts = None
oeb = None
# 创建 OEB
#global log
opts = getOpts(user.device, 'comic')
oeb = CreateOeb(main.log, None, opts)
pubtype = 'book:book:KindleEar'
language = 'zh-cn'
setMetaData(oeb, feed.title, language, local_time("%Y-%m-%d", user.timezone), pubtype=pubtype)
oeb.container = ServerContainer(main.log)
#guide
id_, href = oeb.manifest.generate('masthead', DEFAULT_MASTHEAD) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
id_, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
book = CartoonMadBaseBook(imgindex=imgindex, opts=opts, user=user)
book.title = feed.title
book.description = feed.title
book.language = language
book.keep_image = True
book.oldest_article = 7
book.fulltext_by_readability = True
book.feeds = [(feed.title, feed.url)]
book.url_filters = [flt.url for flt in user.urlfilter]
try: #书的质量可能不一，一本书的异常不能影响其他书籍的推送
for sec_or_media, url, title, content, brief, thumbnail in book.Items():
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append((title, brief, thumbnail, content))
itemcnt += 1
except Exception as e:
excFileName, excFuncName, excLineNo = get_exc_location()
main.log.warn("Failed to push <%s> : %s, in file '%s', %s (line %d)" % (
book.title, str(e), excFileName, excFuncName, excLineNo))
return "Failed to push book <%s>!"%title
volumeTitle = ''
if itemcnt > 0:
insertHtmlToc = False
insertThumbnail = False
volumeTitle = book.LastDeliveredVolume()
oeb.metadata.clear('title')
oeb.metadata.add('title', feed.title + volumeTitle)
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc, insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order('-time').get()
except:
ultima_log = sorted(DeliverLog.all(), key=attrgetter('time'), reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(username, user.kindle_email, feed.title + volumeTitle, user.book_type, str(oIO.getvalue()), user.timezone)
rs = "%s(%s).%s Sent!"%(feed.title, local_time(tz=user.timezone), user.book_type)
main.log.info(rs)
return rs
else:
self.deliverlog(username, str(user.kindle_email), feed.title + volumeTitle, 0, status='nonews', tz=user.timezone)
rs = "No new feeds."
main.log.info(rs)
return rs
def GET(self):
username = web.input().get("u")
bookid = web.input().get("id")
user = KeUser.all().filter("name = ", username).get()
if not user:
return "User not exist!<br />"
to = user.kindle_email
if (';' in to) or (',' in to):
to = to.replace(',', ';').replace(' ', '').split(';')
booktype = user.book_type #mobi,epub
bookmode = user.book_mode or 'periodical' #periodical,comic
titlefmt = user.titlefmt
tz = user.timezone
bookid = bookid.split(',') if ',' in bookid else [bookid]
bks = []
for id_ in bookid:
try:
bks.append(Book.get_by_id(int(id_)))
except:
continue
#return "id of book is invalid or book not exist!<br />"
book4meta = None
if len(bks) == 0:
return "No have book to push!"
elif len(bks) == 1:
if bks[0].builtin:
book4meta = BookClass(bks[0].title)
mhfile = book4meta.mastheadfile
coverfile = book4meta.coverfile
if issubclass(book4meta, BaseComicBook): #如果单独推送一个继承自BaseComicBook的书籍，则自动设置为漫画模式
bookmode = 'comic'
else: #单独的推送自定义RSS
book4meta = bks[0]
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER
else: #多本书合并推送时使用“自定义RSS”的元属性
book4meta = user.ownfeeds
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER_BV if user.merge_books else DEFAULT_COVER
if not book4meta:
return "No have book to push.<br />"
opts = None
oeb = None
# 创建 OEB
#global log
opts = getOpts(user.device, bookmode)
oeb = CreateOeb(main.log, None, opts)
bookTitle = "%s %s" % (book4meta.title, local_time(titlefmt, tz)) if titlefmt else book4meta.title
if bookmode == 'comic':
pubtype = 'book:book:KindleEar'
else:
pubtype = 'periodical:magazine:KindleEar'
setMetaData(oeb, bookTitle, book4meta.language, local_time("%Y-%m-%d",tz), pubtype=pubtype)
oeb.container = ServerContainer(main.log)
#guide
if mhfile:
id_, href = oeb.manifest.generate('masthead', mhfile) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(mhfile))
oeb.guide.add('masthead', 'Masthead Image', href)
if coverfile:
imgData = None
imgMime = ''
#使用保存在数据库的用户上传的封面
if coverfile == DEFAULT_COVER and user.cover:
imgData = user.cover
imgMime = 'image/jpeg' #保存在数据库中的只可能是jpeg格式
elif callable(coverfile): #如果封面需要回调的话
try:
imgData = book4meta().coverfile()
if imgData:
imgType = imghdr.what(None, imgData)
if imgType: #如果是合法图片
imgMime = r"image/" + imgType
else:
main.log.warn('content of cover is invalid : [%s].' % bookTitle)
imgData = None
except Exception as e:
main.log.warn('Failed to fetch cover for book [%s]. [Error: %s]' % (bookTitle, str(e)))
coverfile = DEFAULT_COVER
imgData = None
imgMime = ''
if imgData and imgMime:
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_, href, imgMime, data=imgData)
else:
id_, href = oeb.manifest.generate('cover', coverfile)
item = oeb.manifest.add(id_, href, MimeFromFilename(coverfile))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
elif len(bks) > 1 and DEFAULT_COVER:
#将所有书籍的封面拼贴成一个
#如果DEFAULT_COVER=None说明用户不需要封面
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_, href, 'image/jpeg', data=self.MergeCovers(bks, opts, user))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
for bk in bks:
if bk.builtin:
cbook = BookClass(bk.title)
if not cbook:
main.log.warn('not exist book <%s>' % bk.title)
continue
book = cbook(imgindex=imgindex, opts=opts, user=user)
book.url_filters = [flt.url for flt in user.urlfilter]
if bk.needs_subscription: #需要登录
subs_info = user.subscription_info(bk.title)
if subs_info:
book.account = subs_info.account
book.password = subs_info.password
else: # 自定义RSS
if bk.feedscount == 0:
continue #return "the book has no feed!<br />"
book = BaseFeedBook(imgindex=imgindex, opts=opts, user=user)
book.title = bk.title
book.description = bk.description
book.language = bk.language
book.keep_image = bk.keep_image
book.oldest_article = bk.oldest_article
book.fulltext_by_readability = True
feeds = bk.feeds
book.feeds = []
for feed in feeds:
if feed.url.startswith("http://www.cartoonmad.com"):
self.ProcessComicRSS(username, user, feed)
else:
book.feeds.append((feed.title, feed.url, feed.isfulltext))
book.url_filters = [flt.url for flt in user.urlfilter]
# 对于html文件，变量名字自文档,thumbnail为文章第一个img的url
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容,
# img的thumbail仅当其为article的第一个img为True
try: #书的质量可能不一，一本书的异常不能影响其他书籍的推送
for sec_or_media, url, title, content, brief, thumbnail in book.Items():
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_, href, sec_or_media, data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append((title, brief, thumbnail, content))
itemcnt += 1
except Exception as e:
excFileName, excFuncName, excLineNo = get_exc_location()
main.log.warn("Failed to push <%s> : %s, in file '%s', %s (line %d)" % (
book.title, str(e), excFileName, excFuncName, excLineNo))
continue
volumeTitle = ''
if itemcnt > 0:
#漫画模式不需要TOC和缩略图
if bookmode == 'comic':
insertHtmlToc = False
insertThumbnail = False
if len(bks) == 1 and book: #因为漫画模式没有目录，所以在标题中添加卷号
volumeTitle = book.LastDeliveredVolume()
oeb.metadata.clear('title')
oeb.metadata.add('title', bookTitle + volumeTitle)
else:
insertHtmlToc = GENERATE_HTML_TOC
insertThumbnail = GENERATE_TOC_THUMBNAIL
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc, insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order('-time').get()
except:
ultima_log = sorted(DeliverLog.all(), key=attrgetter('time'), reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(username, to, book4meta.title + volumeTitle, booktype, str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!"%(book4meta.title, local_time(tz=tz), booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username, str(to), book4meta.title + volumeTitle, 0, status='nonews', tz=tz)
rs = "No new feeds."
main.log.info(rs)
return rs
def push_comic_book(self, username, user, book, opts=None):
if not opts:
opts = getOpts(user.device, "comic")
oeb = CreateOeb(main.log, None, opts)
pubtype = 'book:book:KindleEar'
language = 'zh-cn'
setMetaData(
oeb,
book.title,
language,
local_time("%Y-%m-%d", user.timezone),
pubtype=pubtype,
)
oeb.container = ServerContainer(main.log)
#guide
id_, href = oeb.manifest.generate('masthead', DEFAULT_MASTHEAD) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
id_, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
chapters = book.ParseFeedUrls()
if not chapters:
self.deliverlog(
username,
str(user.kindle_email),
book.title,
0,
status="nonews",
tz=user.timezone,
)
return
for (
bookname,
chapter_title,
img_list,
chapter_url,
next_chapter_index,
) in chapters:
try:
image_count = 0
for (
mime_or_section,
url,
filename,
content,
brief,
thumbnail,
) in book.gen_image_items(img_list, chapter_url):
if not mime_or_section or not filename or not content:
continue
if mime_or_section.startswith(r"image/"):
id_, href = oeb.manifest.generate(id="img", href=filename)
item = oeb.manifest.add(
id_, href, mime_or_section, data=content
)
if thumbnail:
toc_thumbnails[url] = href
else:
sections.setdefault(mime_or_section, [])
sections[mime_or_section].append(
(filename, brief, thumbnail, content)
)
image_count += 1
title = book.title + " " + chapter_title
if not image_count:
self.deliverlog(
username,
str(user.kindle_email),
title,
0,
status="can't download image",
tz=user.timezone,
)
rs = "No new feeds."
main.log.info(rs)
continue
insertHtmlToc = False
insertThumbnail = False
oeb.metadata.clear("title")
oeb.metadata.add("title", title)
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc, insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order("-time").get()
except:
ultima_log = sorted(
DeliverLog.all(), key=attrgetter("time"), reverse=True
)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(
username,
user.kindle_email,
title,
user.book_type,
str(oIO.getvalue()),
user.timezone,
)
book.UpdateLastDelivered(bookname, chapter_title, next_chapter_index)
rs = "%s(%s).%s Sent!" % (
title,
local_time(tz=user.timezone),
user.book_type,
)
main.log.info(rs)
except:
main.log.exception(
u"Failed to push {} {}".format(bookname, chapter_title)
)
def GET(self):
username = web.input().get('u')
id_ = web.input().get('id') #for debug
self.queue2push = defaultdict(list)
books = Book.all()
if username: #现在投递【测试使用】，不需要判断时间和星期
sent = []
books2push = Book.get_by_id(
int(id_)) if id_ and id_.isdigit() else None
books2push = [books2push] if books2push else books
for book in books2push:
if not id_ and username not in book.users:
continue
user = KeUser.all().filter("name = ", username).get()
if user and user.kindle_email:
self.queueit(user, book.key().id(), book.separate)
sent.append(book.title)
self.flushqueue()
if len(sent):
tips = _("Book(s) (%s) put to queue!") % u', '.join(sent)
else:
tips = _("No book to deliver!")
return self.render('autoback.html', "Delivering", tips=tips)
#定时cron调用
sentcnt = 0
for book in books:
if not book.users: #没有用户订阅此书
continue
bkcls = None
if book.builtin:
bkcls = BookClass(book.title)
if not bkcls:
continue
#确定此书是否需要下载
for u in book.users:
user = KeUser.all().filter("enable_send = ",
True).filter("name = ", u).get()
if not user or not user.kindle_email:
continue
#先判断当天是否需要推送
day = local_time('%A', user.timezone)
usrdays = user.send_days
if bkcls and bkcls.deliver_days: #按星期推送
days = bkcls.deliver_days
if not isinstance(days, list):
days = [days]
if day not in days:
continue
elif usrdays and day not in usrdays: #为空也表示每日推送
continue
#时间判断
h = int(local_time("%H", user.timezone)) + 1
if h >= 24:
h -= 24
if bkcls and bkcls.deliver_times:
times = bkcls.deliver_times
if not isinstance(times, list):
times = [times]
if h not in times:
continue
elif user.send_time != h:
continue
#到了这里才是需要推送的
self.queueit(user, book.key().id(), book.separate)
sentcnt += 1
self.flushqueue()
return "Put <strong>%d</strong> books to queue!" % sentcnt
def push_comic_book(self, username, user, book, opts=None):
if not opts:
opts = getOpts(user.device, "comic")
oeb = CreateOeb(main.log, None, opts)
pubtype = 'book:book:KindleEar'
language = 'zh-cn'
setMetaData(
oeb,
book.title,
language,
local_time("%Y-%m-%d", user.timezone),
pubtype=pubtype,
)
oeb.container = ServerContainer(main.log)
#guide
id_, href = oeb.manifest.generate('masthead',
DEFAULT_MASTHEAD) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
id_, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
chapters = book.ParseFeedUrls()
if not chapters:
self.deliverlog(
username,
str(user.kindle_email),
book.title,
0,
status="nonews",
tz=user.timezone,
)
return
for (
bookname,
chapter_title,
img_list,
chapter_url,
next_chapter_index,
) in chapters:
try:
image_count = 0
for (
mime_or_section,
url,
filename,
content,
brief,
thumbnail,
) in book.gen_image_items(img_list, chapter_url):
if not mime_or_section or not filename or not content:
continue
if mime_or_section.startswith(r"image/"):
id_, href = oeb.manifest.generate(id="img",
href=filename)
item = oeb.manifest.add(id_,
href,
mime_or_section,
data=content)
if thumbnail:
toc_thumbnails[url] = href
else:
sections.setdefault(mime_or_section, [])
sections[mime_or_section].append(
(filename, brief, thumbnail, content))
image_count += 1
title = book.title + " " + chapter_title
if not image_count:
self.deliverlog(
username,
str(user.kindle_email),
title,
0,
status="can't download image",
tz=user.timezone,
)
rs = "No new feeds."
main.log.info(rs)
continue
insertHtmlToc = False
insertThumbnail = False
oeb.metadata.clear("title")
oeb.metadata.add("title", title)
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc,
insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order("-time").get()
except:
ultima_log = sorted(DeliverLog.all(),
key=attrgetter("time"),
reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(
username,
user.kindle_email,
title,
user.book_type,
str(oIO.getvalue()),
user.timezone,
)
book.UpdateLastDelivered(bookname, chapter_title,
next_chapter_index)
rs = "%s(%s).%s Sent!" % (
title,
local_time(tz=user.timezone),
user.book_type,
)
main.log.info(rs)
except:
main.log.exception(u"Failed to push {} {}".format(
bookname, chapter_title))