def post(self):
name = self.get_argument('name', '')
password = self.get_argument('password', '')
email = self.get_argument('email', '')
domain = self.get_argument('domain', '')
password = utils.make_password(password)
if domain == '':
domain = name
if User.by_email(email):
self.redirect('/login') # self.write('用户已经存在，请直接登录')
else:
res = User
res['id'] = res.find().count() + 1
res['user_name'] = name
res['user_pass'] = password
res['user_email'] = email
res['user_domain'] = domain
res.new(res)
send_joinus_email(email, name)
user_info = {
'user_id': res.id,
'user_name': res.user_name,
'user_email': res.user_email,
'user_domain': res.user_domain
}
self.set_secure_cookie('user',
tornado.escape.json_encode(user_info))
self.redirect(self.get_argument('next', '/'))
def post(self):
name = self.get_argument("name", "")
password = self.get_argument("password", "")
email = self.get_argument("email", "")
domain = self.get_argument("domain", "")
password = utils.make_password(password)
if domain == "":
domain = name
if User.by_email(email):
self.redirect("/login") # self.write('用户已经存在，请直接登录')
else:
res = User
res["id"] = res.find().count() + 1
res["user_name"] = name
res["user_pass"] = password
res["user_email"] = email
res["user_domain"] = domain
res.new(res)
send_joinus_email(email, name)
user_info = {
"user_id": res.id,
"user_name": res.user_name,
"user_email": res.user_email,
"user_domain": res.user_domain,
}
self.set_secure_cookie("user", tornado.escape.json_encode(user_info))
self.redirect(self.get_argument("next", "/"))
def post(self):
name = self.get_argument('name', '')
password = self.get_argument('password', '')
email = self.get_argument('email', '')
domain = self.get_argument('domain', '')
password = utils.make_password(password)
if domain == '':
domain = name
if User.by_email(email):
self.redirect('/login') # self.write('用户已经存在，请直接登录')
else:
res = User
res['id'] = res.find().count() + 1
print(name, type)
res['user_name'] = name
res['user_pass'] = password
res['user_email'] = email
res['user_domain'] = domain
res.new(res)
send_joinus_email(email, name)
user_info = {
'user_id': res.id,
'user_name': res.user_name,
'user_email': res.user_email,
'user_domain': res.user_domain}
self.set_secure_cookie(
'user', tornado.escape.json_encode(user_info))
self.redirect(self.get_argument('next', '/'))
def add_from_file(rss_url, rss_hostname, rss_name):
doc = adb.User_Col.find_one({'user_domain': rss_hostname})
if doc:
user_name = doc['user_name']
assert user_name == rss_name
user_url = doc['user_url']
assert user_url != rss_url
return
# rss_file = 'content/gen/qdaily_2019-04-20 15:07:12.xml'
feeds = feedparser.parse(rss_url)
print(feeds.keys())
if hasattr(feeds.feed, 'description'):
if feeds.feed.description != feeds.feed.subtitle:
print('!' * 88)
print('description != subtitle')
print(feeds.feed.description)
print(feeds.feed.subtitle)
# assert feeds.feed.description == feeds.feed.subtitle
# print(feeds.description)
email = '{}@anwensf.com'.format(rss_hostname)
if hasattr(feeds.feed, 'title'):
print(feeds.feed.title, rss_name)
# assert feeds.feed.title == rss_name
if rss_name in '创业邦 国家地理 Flipboard-Books'.split():
pass
else:
assert rss_name.replace('-',
'') in feeds.feed.title.replace(' ', '')
else:
print('no title', rss_name)
if User.by_email(email):
print(email)
return
else:
res = User
res['id'] = res.find().count() + 1
res['user_name'] = rss_name
# podcast
# https://anyway.fm/rss.xml
# print(dir(feeds.feed))
# print(feeds.feed.keys())
# print(dir(feeds))
# print(feeds.keys())
# print(len(feeds.entries))
res['user_url'] = feeds.feed.link
if hasattr(feeds.feed, 'description'):
res['user_say'] = feeds.feed.subtitle # feeds.feed.description
if hasattr(feeds.feed, 'language'):
res['user_lang'] = feeds.feed.language.lower() # zh-cn
res['user_pass'] = random_string()
res['user_email'] = email
res['user_rss'] = rss_url
res['user_domain'] = rss_hostname
res.new(res)
def get(self):
if self.get_argument("openid.mode", None):
user = yield self.get_authenticated_user()
doc = User.by_email(user["email"])
if not doc:
doc = User
doc["id"] = doc.find().count() + 1
doc["user_name"] = user["name"]
doc["user_pass"] = utils.make_emailverify()
doc["user_email"] = user["email"]
doc["user_domain"] = user["name"]
doc.new(doc)
send_joinus_email(user["email"], user["name"])
user_info = {
"user_id": doc.id,
"user_name": doc.user_name,
"user_email": doc.user_email,
"user_domain": doc.user_domain,
}
self.set_secure_cookie("user", tornado.escape.json_encode(user_info))
self.redirect("/")
return
self.authenticate_redirect()
def get(self):
if self.get_argument("openid.mode", None):
user = yield self.get_authenticated_user()
doc = User.by_email(user['email'])
if not doc:
doc = User
doc['id'] = doc.find().count() + 1
doc['user_name'] = user['name']
doc['user_pass'] = utils.make_emailverify()
doc['user_email'] = user['email']
doc['user_domain'] = user['name']
doc.new(doc)
send_joinus_email(user['email'], user['name'])
user_info = {
'user_id': doc.id,
'user_name': doc.user_name,
'user_email': doc.user_email,
'user_domain': doc.user_domain}
self.set_secure_cookie("user",
tornado.escape.json_encode(user_info))
self.redirect("/")
return
self.authenticate_redirect()
def get(self):
if self.get_argument("openid.mode", None):
user = yield self.get_authenticated_user()
doc = User.by_email(user['email'])
if not doc:
doc = User
doc['id'] = doc.find().count() + 1
doc['user_name'] = user['name']
doc['user_pass'] = utils.make_emailverify()
doc['user_email'] = user['email']
doc['user_domain'] = user['name']
doc.new(doc)
send_joinus_email(user['email'], user['name'])
user_info = {
'user_id': doc.id,
'user_name': doc.user_name,
'user_email': doc.user_email,
'user_domain': doc.user_domain
}
self.set_secure_cookie("user",
tornado.escape.json_encode(user_info))
self.redirect("/")
return
self.authenticate_redirect()
def get(self):
wx_code = self.get_argument("code", '')
appname = self.get_argument("appname", '')
if not wx_code or not appname:
return self.write_error(401)
wx_api = 'https://api.weixin.qq.com/sns/jscode2session'
params = {}
params['appid'] = appinfo[appname]['WX_APPID']
params['secret'] = appinfo[appname]['WX_SECRET']
params['js_code'] = wx_code
params['grant_type'] = 'authorization_code'
r = requests.get(wx_api, params=params, timeout=5)
r = r.json()
session_key = r.get('session_key')
openid = r.get('openid')
if not session_key or not openid:
print(r)
# return self.write_json()
return self.write_error(402)
# 复用email password
doc = User.by_email(openid + '@wechat')
if doc:
doc.update(
{
'user_pass': session_key,
}
)
doc.save()
user_info = {
'user_id': doc.id,
'user_email': doc.user_email,
'user_name': doc.user_name,
'user_domain': doc.user_domain,
}
token = self.create_signed_value(
'user', tornado.escape.json_encode(user_info))
self.res['token'] = token.decode('u8')
self.res['is_admin'] = admin.is_admin(doc['id'])
if doc['id'] in wx_admin_ids:
self.res['is_admin'] = True
return self.write_json()
else:
# res['id'] = User.find().count() + 1
res = {}
res['user_email'] = openid + '@wechat'
res['user_pass'] = session_key
res['user_name'] = 'null'
user = User.new(res)
user_info = {
'user_id': user.id,
'user_email': user.user_email,
'user_name': user.user_name,
'user_domain': user.user_domain,
}
token = self.create_signed_value(
'user', tornado.escape.json_encode(user_info))
self.res['token'] = token.decode('u8')
return self.write_json()
def add_from_file(rss_url, rss_hostname, rss_name):
# rss_file = 'content/gen/qdaily_2019-04-20 15:07:12.xml'
n = Share.find().count()
print(n)
print(rss_name)
feeds = feedparser.parse(rss_url)
for post in feeds.entries[::-1]:
# authors
# itunes_episodetype full
# itunes_episode
# itunes_explicit
# itunes_title
# itunes_duration
# published link subtitle id image title tags
# links title_detail author_detail summary_detail guidislink published_parsed summary content author
# subtitle_detail
# title title_detail
# published published_parsed
# summary summary_detail
# author
# link links guidislink
# authors
# 'itunes_title', 'itunes_episode'
# 'author_detail', 'id', 'itunes_duration'
# <itunes:duration>6957</itunes:duration>
# TODO
# 修正内容 目前暂时不支持
# <enclosure type="audio/mpeg" url="https://kernelpanic.fm/55/audio.mp3"/>
# <media:content url="https://cdn.flipboard.com/telegraph.co.uk/1356d637c7438f6fcffda0d5de177b6058904de6/original.jpg" medium="image" type="image/jpeg" width="480" height="300" />
# media_content
# print(post.keys())
if hasattr(post, 'summary'):
summary = post.summary
assert post.summary == post.description
else:
summary = ''
# 部分rss没有content
if hasattr(post, 'content'):
content = post.content[0]['value']
else:
if hasattr(post, 'summary'):
content = post.summary
else:
print('no content', rss_url, rss_hostname, rss_name)
continue
if content.startswith('<![CDATA[') and content.endswith(']]>'):
# m = rgx.search(content)
# content = m.group(1)
content = content[9:-3]
if summary.startswith('<![CDATA[') and summary.endswith(']]>'):
summary = summary[9:-3]
if hasattr(post, 'published'):
if 'GMT' == post.published[-3:]:
published = datetime.strptime(post.published,
"%a, %d %b %Y %H:%M:%S GMT")
elif ',' in post.published:
if post.published.endswith('2019'):
pass
# May 19, 2019
published = datetime.strptime(post.published, "%b %d, %Y")
else:
published = datetime.strptime(post.published,
"%a, %d %b %Y %H:%M:%S %z")
# Thu, 18 Apr 2019 19:32:58 +0800
elif '/' in post.published:
published = datetime.strptime(post.published,
"%Y/%m/%d %H:%M:%S %z")
elif 'Z' == post.published[-1]:
post.published = post.published.replace('.000Z', 'Z')
published = datetime.strptime(post.published,
"%Y-%m-%dT%H:%M:%SZ")
# <pubDate>15 Jun 2019 06:30:00 EST</pubDate>
elif 'EST' in post.published:
post.published = post.published[:-4]
published = datetime.strptime(post.published,
"%d %b %Y %H:%M:%S")
elif 'T' in post.published:
# 2019-05-24T15:05:50-04:00
post.published = post.published[:-6]
# tz = post.published[-6:].replace(':', '')
published = datetime.strptime(post.published,
"%Y-%m-%dT%H:%M:%S")
# published = published.replace(tzinfo=FixedOffset(tz))
elif post.published.count(' ') == 1:
published = datetime.strptime(post.published,
"%Y-%m-%d %H:%M:%S")
else:
published = datetime.strptime(post.published,
"%Y-%m-%d %H:%M:%S %z")
published = published.timestamp()
else:
if random.random() > 0.9:
print('no published time')
published = time.time()
title = post.title
link = post.link
author = ''
if hasattr(post, 'source'):
source_title = post.source.title
# print(source_title)
print(rss_name, source_title)
if rss_name == '虎嗅':
pass
author = source_title
else:
assert rss_name in source_title
# assert rss_name == source_title
source = rss_name
if hasattr(post, 'category_title'):
category = post.category_title
assert ' ' not in category
assert ',' not in category
tags = [category]
elif hasattr(post, 'tags'):
tags = post.tags
# print(tags)
# assert len(tags) == 1
# tags = tags[0]['term']
tags = ','.join([t['term'] for t in tags])
category = ''
if '-' in tags:
print(tags)
tags = tags.replace(' ', '-')
tags = tags.split(',')
for tag in tags:
if ' ' in tag:
print(tag)
else:
# print('no category')
category = ''
tags = []
sharetype = 'rss'
try:
markdown = html2text.html2text(content)
except Exception as e:
print('error in html-to-markdown: {}'.format(e))
continue
assert link
res = {
'title': title,
'link': link,
'source': source,
'category': category,
'content': content,
'summary': summary,
'sharetype': sharetype,
'tags': tags,
'markdown': markdown,
'published': published,
'updated': time.time(),
}
# print(post.keys())
if hasattr(post, 'author'):
# TODO
print('author: ', post.author)
res['author'] = post.author
else:
res['author'] = author
# 去重方案
# - 标题重复
found = Share.find({'title': title})
if found.count():
if found.count() > 1:
print('!! repeated article title: {}'.format(title))
elif found.count() == 1:
# continue
share = Share.by_sid(found[0].id)
if share and summary and not share.link and link:
print(res['link'])
print('title {} updated'.format(title))
share.update(res)
share.save()
else:
print('title {} adding'.format(title))
email = '{}@anwensf.com'.format(rss_hostname)
auser = User.by_email(email)
assert auser
share = Share
user_id = auser.id
res['user_id'] = user_id # just use 1 as default
# continue
assert res['link']
share = share.new(res)
user = User.by_sid(user_id)
user.user_leaf += 10
user.save()
for i in tags:
doc = {'name': i, 'share_ids': share.id}
Tag.new(doc)
def add_from_file(rss_url, rss_hostname, rss_name):
doc = adb.User_Col.find_one({'user_domain': rss_hostname})
if doc:
user_name = doc['user_name']
assert user_name == rss_name
user_url = doc['user_url']
assert user_url != rss_url
# 通常情况下，已经收录的不再解析，除非需要更新字段
user_id = doc['id']
avatar_dir = 'static/avatar'
size = 'raw'
avatar_path = '%s/feed_%s_%s.jpg' % (avatar_dir, str(user_id), size)
if os.path.isfile(avatar_path):
print('feed img got', avatar_path)
return
# return
# print(user_id)
# return
# rss_file = 'content/gen/qdaily_2019-04-20 15:07:12.xml'
feeds = feedparser.parse(rss_url)
# print(feeds.keys())
# 更新图片
if hasattr(feeds.feed, 'image'):
# print(feeds.feed.image)
# print(type(feeds.feed.image))
# print(feeds.feed.image['href'])
# print(11)
if feeds.feed.image.get('href'):
href = feeds.feed.image['href']
ext = href.split('.')[-1]
img_name = '{}/feed_{}.{}'.format(avatar_dir, user_id, ext)
r = requests.get(href)
href = href.replace('https', 'http')
if r.status_code == 200:
print(img_name, 'status_code 200')
with open(img_name, 'wb') as f:
for chunk in r.iter_content():
f.write(chunk)
make_post_thumb(img_name, sizes=[[132, 132]])
else:
print(r.status_code)
print('wget {} -O {}'.format(href, img_name))
# <image>
# <url>https://www.huxiu.com/static_2015/img/logo.png</url>
# <title>虎嗅网</title>
# <link>http://www.huxiu.com</link>
# </image>
return
if hasattr(feeds.feed, 'description'):
if feeds.feed.description != feeds.feed.subtitle:
print('!'*88)
print('description != subtitle')
print(feeds.feed.description)
print(feeds.feed.subtitle)
# assert feeds.feed.description == feeds.feed.subtitle
# print(feeds.description)
email = '{}@anwensf.com'.format(rss_hostname)
if hasattr(feeds.feed, 'title'):
print(feeds.feed.title, rss_name)
# assert feeds.feed.title == rss_name
if rss_name in '创业邦 国家地理 Flipboard-Books'.split():
pass
else:
assert rss_name.replace('-', '') in feeds.feed.title.replace(' ', '')
else:
print('no title', rss_name)
if User.by_email(email):
print(email)
return
else:
res = User
res['id'] = res.find().count() + 1
res['user_name'] = rss_name
# podcast
# https://anyway.fm/rss.xml
# print(dir(feeds.feed))
# print(feeds.feed.keys())
# print(dir(feeds))
# print(feeds.keys())
# print(len(feeds.entries))
res['user_url'] = feeds.feed.link
if hasattr(feeds.feed, 'description'):
res['user_say'] = feeds.feed.subtitle # feeds.feed.description
if hasattr(feeds.feed, 'language'):
res['user_lang'] = feeds.feed.language.lower() # zh-cn
res['user_pass'] = random_string()
res['user_email'] = email
res['user_rss'] = rss_url
res['user_domain'] = rss_hostname
res.new(res)