def POST(self):
web.header('Content-Type', 'application/json')
data = json.loads(web.data())
if 'tag_id' not in data:
return json.dumps({
'response': "404, but not really",
'message': "missing tag_id"
})
password_secret = "moja_mami"
password = data['tag_id'] + password_secret
password = password.encode()
salt = os.urandom(16)
kdf = PBKDF2HMAC(algorithm=hashes.SHA256(),
length=32,
salt=salt,
iterations=100000,
backend=default_backend())
key = base64.urlsafe_b64encode(kdf.derive(password))
string_key = key.decode("utf-8")
if 'location' in data:
new_tag = Tag(tag_id=data['tag_id'],
location=data['location'],
tag_key=string_key)
else:
new_tag = Tag(tag_id=data['tag_id'], tag_key=string_key)
return json.dumps({
'response': "SAVE SUCCESS.",
'new_tag': str(new_tag)
})
def post(self):
share_id = self.get_argument("share_id", None)
tags = self.get_argument("tags", '')
# user_id = self.current_user["user_id"]
tags = tags.strip()
if share_id:
share = Share.by_sid(share_id)
if share and tags not in share.tags:
tags = share.tags + ' ' + tags
res = {
'tags': tags,
'updated': time.time(),
}
share.update(res)
share.save()
tags = tags.split(' ')
tags = list(set(tags))
for i in tags:
doc = {
'name': i,
'share_ids': share.id
}
Tag.new(doc)
def get(self, name=None):
if not name:
tags = Tag.find()
self.render("tag.html", tags=tags)
else:
tag = Tag.find_one({'name': name})
shares = []
share_ids = tag.share_ids.split(' ')
share_ids = list(set(share_ids))
cond = {}
cond['_id'] = 0
cond['id'] = 1
cond['user_id'] = 1
cond['published'] = 1
cond['sharetype'] = 1
cond['title'] = 1
cond['commentnum'] = 1
for share_id in share_ids:
# share = Share.by_sid(share_id)
share = Share.find_one({'id': int(share_id)}, cond)
print(share)
# <!-- <p class="info">{{ escape(share.markdown) }} ...</p> -->
user = User.by_sid(share.user_id)
share.user_name = user.user_name
share.user_domain = user.user_domain
share.published = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(share.published))
# share.markdown = filter_tags(
# markdown2.markdown(share.markdown))[:100]
shares.append(share)
self.render("tage.html", tag=tag, name=name, shares=shares)
def post(self):
id = self.get_argument("id", None)
tags = self.get_argument("tags", '')
user_id = self.current_user["user_id"]
res = {
'title': self.get_argument("title"),
'markdown': self.get_argument("markdown"),
'sharetype': self.get_argument("type"),
'slug': self.get_argument("slug", ''),
'tags': tags,
'updated': time.time(),
}
if id:
share = Share.by_sid(id)
if not share:
self.redirect("/404")
share.update(res)
share.save()
else:
share = Share
res['user_id'] = user_id
share = share.new(res)
user = User.by_sid(user_id)
user.user_leaf += 10
user.save()
for i in tags.split(' '):
Tag.new(i, share.id)
self.redirect("/share/" + str(share.id))
def create_tag(label):
new_tag = Tag(
label=label
)
db.session.add(new_tag)
db.session.commit()
return new_tag.serialize()
def create_tag():
body = json.loads(request.data)
title = body.get("title")
if title is None:
return failure_response("Invalid field!")
new_tag = Tag(title=title)
db.session.add(new_tag)
db.session.commit()
return success_response(new_tag.serialize(), 201)
def post(self):
# print self.request.arguments
share_id = self.get_argument("id", None)
title = self.get_argument("title", '')
markdown = self.get_argument("markdown", '')
content = self.get_argument("content", '')
sharetype = self.get_argument("type", '')
slug = self.get_argument("slug", '')
status = 1 if self.get_argument("dosubmit", None) == u'保存草稿' else 0
tags = self.get_argument("tags", '')
upload_img = self.get_argument("uploadImg", '')
post_img = self.get_argument("post_Img", '')
post_img = '' if post_img == 'None' else post_img
user_id = self.current_user["user_id"]
res = {
'title': title,
'markdown': markdown,
'content': content,
'sharetype': sharetype,
'slug': slug,
'tags': tags,
'status': status,
'upload_img': upload_img,
'post_img': post_img,
'updated': time.time(),
}
if share_id:
share = Share.by_sid(share_id)
if not share:
self.redirect("/404")
share.update(res)
share.save()
else:
share = Share
res['user_id'] = user_id
share = share.new(res)
user = User.by_sid(user_id)
user.user_leaf += 10
user.save()
for i in tags.split(' '):
doc = {
'name': i,
'share_ids': share.id
}
Tag.new(doc)
if status == 1:
self.redirect("/share/?id=" + str(share.id))
self.redirect("/share/" + str(share.id))
def save_photos(photos):
c = 0
for media in photos:
if media.created_time > settings.period['start'] and media.location.point != None:
try:
photo = Photo.create(
insta_id = media.id,
thumb = media.get_thumbnail_url(),
url = media.link,
username = media.user.username,
insta_filter = media.filter,
date = media.created_time,
longitude = media.location.point.longitude,
latitude = media.location.point.latitude,
message = media.caption,
like_count = media.like_count,
user_in_photo_count = len(media.users_in_photo)
)
except peewee.IntegrityError:
pass
else:
if hasattr(media, 'tags'):
for t in media.tags:
tag, tag_created = Tag.get_or_create(name = pattern.sub('', t.name))
PhotoTag.get_or_create(tag=tag, photo=photo)
c += 1
print 'New media: %d / %d' % (c, len(photos))
def make_tag_if_not_exists_and_return_tag_object(self, tag_name):
res = self.get_tag(tag_name)
if not res:
res = Tag(seq('tags'), tag_name)
self.session.add(res)
return res
def create_tag(tag: RequestTagObject) -> ResultWithData[str]:
tag_id = short_unique_id()
tag_created = Tag(tag_id=tag_id,
name=tag.name,
pretty_name_sv=tag.pretty_name_sv,
pretty_name_en=tag.pretty_name_en)
return get_result_with_data(tag_created.tag_id)
def remove_tag(tag_id: str) -> ResultWithData:
tag = Tag.get(tag_id=tag_id)
if tag is None:
return get_result_with_error(TAG_ID_NOT_EXIST)
else:
tag.delete()
return get_result_with_data("The tag was successfully deleted")
def post(self):
# print self.request.arguments
share_id = self.get_argument("id", None)
title = self.get_argument("title", '')
markdown = self.get_argument("markdown", '')
content = self.get_argument("content", '')
sharetype = self.get_argument("sharetype", '')
slug = self.get_argument("slug", '')
tags = self.get_argument("tags", '')
upload_img = self.get_argument("uploadImg", '')
post_img = self.get_argument("post_Img", '')
link = self.get_argument("link", '')
user_id = self.current_user["user_id"]
res = {
'title': title,
'markdown': markdown,
'content': content,
'sharetype': sharetype,
'slug': slug,
'tags': tags,
'upload_img': upload_img,
'post_img': post_img,
'link': link,
'updated': time.time(),
}
if share_id:
share = Share.by_sid(share_id)
if not share:
self.redirect("/404")
share.update(res)
share.save()
else:
share = Share
res['user_id'] = user_id
share = share.new(res)
user = User.by_sid(user_id)
user.user_leaf += 10
user.save()
for i in tags.split(' '):
doc = {
'name': i,
'share_ids': share.id
}
Tag.new(doc)
self.redirect("/share/" + str(share.id))
def get(self, name=None):
if not name:
tags = Tag.find()
self.render("tag.html", tags=tags)
else:
tag = Tag.find_one({'name': name})
shares = []
for i in tag.share_ids.split(' '):
share = Share.by_sid(i)
user = User.by_sid(share.user_id)
share.name = user.user_name
share.published = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(share.published))
share.domain = user.user_domain
share.markdown = filter_tags(markdown2.markdown(
share.markdown))[:100]
shares.append(share)
self.render("tage.html", name=name, shares=shares)
def data_time(zone, score_key):
"""
Format of score_key should be:
"""
# Grab scores with dates, tag with MSA as we know that score is in the summary
scores = Tag.get_tag_dates(zone)
score_d = [model_to_dict(s)['date'] for s in scores]
return jsonify(score_d)
def get(self, name=None):
if not name:
tags = Tag.find()
self.render("tag.html", tags=tags)
else:
tag = Tag.find_one({'name': name})
shares = []
for i in tag.share_ids.split(' '):
share = Share.by_sid(i)
user = User.by_sid(share.user_id)
share.name = user.user_name
share.published = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(share.published))
share.domain = user.user_domain
share.markdown = filter_tags(
markdown2.markdown(share.markdown))[:100]
shares.append(share)
self.render("tage.html", name=name, shares=shares)
def submit_tag(type):
session = Session()
tag = None
if 'id' in request.form:
tag = session.query(Tag).filter(Tag.id == request.form['id']).first()
else:
tag = Tag()
tag.name = request.form['name']
tag.type = type
session.add(tag)
session.commit()
return redirect(url_for('index'))
def post(self, action):
entity_id = int(self.get_argument("entity_id", 0))
entity_type = self.get_argument("entity_type", None)
user_id = self.current_user["user_id"]
assert action in 'addlike dellike adddislike deldislike'.split()
assert entity_type in 'share comment viewpoint tag'.split()
_action = action[3:] + 'num'
doc = {
'user_id': user_id,
'entity_id': entity_id,
'entity_type': entity_type,
}
is_changed = Like.change_like(doc, _action, action[:3])
# 冗余储存 没有做成事件绑定，需要定期校验修复
if entity_type == 'share':
entity = Share.by_sid(entity_id)
# 如果是管理员，需要将status + 1
# 64=kp 65=kp email
# 63=lb # 60=xie
if is_changed and user_id in admin_ids:
if action == 'addlike':
if entity['status'] == 0:
entity['suggested'] = time.time()
entity['status'] += 1
elif action == 'adddislike':
entity['status'] -= 1
elif action == 'deldislike':
entity['status'] += 1
else:
entity['status'] -= 1
elif entity_type == 'comment':
entity = Comment.by_sid(entity_id)
elif entity_type == 'viewpoint':
entity = Viewpoint.by_sid(entity_id)
elif entity_type == 'tag':
entity = Tag.by_sid(entity_id)
user = User.by_sid(user_id)
if action == 'addlike' and entity.name not in user.user_tags:
user.user_tags.append(entity.name)
elif action == 'dellike' and entity.name in user.user_tags:
user.user_tags.pop(entity.name)
user.save()
if action[:3] == 'add':
entity[_action] += 1
else:
entity[_action] -= 1
entity.save()
if entity.dislikenum < 0:
entity.dislikenum = 0
if entity.likenum < 0:
entity.likenum = 0
self.res = {
'likenum': entity.likenum,
'dislikenum': entity.dislikenum,
}
self.write_json()
def tag_patch(tag_id):
body = request.json
tag = db.session.query(Tag).get(tag_id)
if body.get('description'):
tag.description = body['description']
db.session.commit()
return json.dumps(Tag.transform(tag))
def post(self):
share_id = self.get_argument("share_id", None)
tags = self.get_argument("tags", '')
tags = tags.strip()
if share_id:
share = Share.by_sid(share_id)
if share and tags not in share.tags:
tags = share.tags + ' ' + tags
res = {
'tags': tags,
'updated': time.time(),
}
share.update(res)
share.save()
tags = tags.split(' ')
tags = list(set(tags))
for i in tags:
doc = {'name': i, 'share_ids': share.id}
Tag.new(doc)
def check():
# share_num = Share.find().count()
# share_with_tag_num = share_num - Share.find({'tags': []}).count()
for i in adb.Share_Col.find().sort('_id', 1):
if i['status'] < 1:
continue
# if i['tags'] == []:
if i['tags']:
continue
# print(i['id'], i['title'])
print(i['user_id'])
# adb.Share_Col.update().sort('_id', 1):
tags = get_tags(i)
adb.Share_Col.update({'_id': i['_id']}, {'$set': {'tags': tags}})
for tag in tags:
doc = {'name': tag, 'share_ids': i['id']}
Tag.new(doc)
share_without_tag_num = Share.find({'tags': []}).count()
print(share_without_tag_num)
def updateTag():
tagIds=request.form.getlist('tags')
# print(type(tagIds))
# print(tagIds)
try:
current_user.tags=[]
session.commit()
for tagId in tagIds:
# if session.query(UserTag).filter(UserTag.userId==current_user.id,UserTag.tagId==tagId).count()==0:
# session.add(UserTag(userId=current_user.id,tagId=tagId))
current_user.tags.append(Tag.query_one(id=tagId))
session.commit()
return success({})
except Exception as e:
# print(e)
session.rollback()
return error({'message':"抱歉出现错误，请发送详细内容到[email protected]"})
def get_nearest_cafe(lat, lng):
geo_json = get_cafe_json(lat, lng)
if not geo_json:
return
cafes = []
tags_and_cafes = {}
for data in geo_json.get('results'):
cafe = Cafe(name=data['name'],
lat=data['geometry']['location']['lat'],
lng=data['geometry']['location']['lng'],
rating=data.get('rating'),
address=data['vicinity'])
saved_cafe = Cafe.query.filter(Cafe.lat == cafe.lat
and Cafe.lng == cafe.lng).first()
if saved_cafe:
cafe = saved_cafe
else:
db_session.add(cafe)
db_session.commit()
cafes.append(cafe)
for type_name in data.get('types'):
tags_and_cafes.setdefault(type_name, [])
tags_and_cafes[type_name].append(cafe.id)
for tag_name in tags_and_cafes.keys():
tag = Tag(tag_name=tag_name, localized_name='')
db_session.add(tag)
db_session.commit()
for cafe_id in tags_and_cafes[tag_name]:
tag_to_add = TagsForCafe(tag_id=tag.id, cafe_id=cafe_id)
db_session.add(tag_to_add)
db_session.commit()
return cafes
def create_tags():
for name in ['python', 'linux', 'java', 'mysql', 'lisp']:
tag = Tag(name=name)
session.add(tag)
def post(self):
# TODO
# print(self.request.arguments)
share_id = self.get_argument("id", None)
title = self.get_argument("title", '')
markdown = self.get_argument("markdown", '')
content = self.get_argument("content", '')
sharetype = self.get_argument("sharetype", '')
slug = self.get_argument("slug", '')
tags = self.get_argument("tags", '')
# upload_img = self.get_argument("uploadImg", '')
post_img = self.get_argument("post_Img", '')
link = self.get_argument("link", '')
user_id = self.current_user["user_id"]
vote_open = self.get_argument("vote_open", '')
vote_title = self.get_argument("vote_title", '')
img_url = self.get_argument("img_url", '')
tags = tags.split()
if link:
url = link
doc = Webcache.find_one({'url': url}, {'_id': 0})
if doc:
logger.info('already downloaded')
doc_title = doc.title
# markdown = doc.markdown
else:
sessions = requests.session()
sessions.headers[
'User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36'
try:
# response = sessions.get(url)
response = sessions.get(url, timeout=4)
# TODO: try to use a proxy
except (requests.ConnectionError, requests.Timeout) as e:
print(e)
self.write("GFW...")
return
# except requests.exceptions.HTTPError as e:
# if e.response.status_code == 400:
# error = e.response.json()
# code = error['code']
# message = error['message']
except Exception as e:
logger.info('e: {}'.format(e))
# self.redirect("/")
self.write("GFW")
return
# response.encoding = 'utf-8' # TODO
response.encoding = get_charset(response)
logger.info('response.encoding {}'.format(response.encoding))
doc = Document(response.text)
doc_title = doc.title()
summary = doc.summary()
_markdown = html2text.html2text(summary)
_markdown = _markdown.replace('-\n', '-').strip()
res_webcache = {}
res_webcache['url'] = url
res_webcache['title'] = doc_title
res_webcache['markdown'] = _markdown
if _markdown:
webcache = Webcache
webcache.new(res_webcache)
if vote_open.isdigit():
vote_open = int(vote_open)
else:
vote_open = 0
if not title:
title = doc_title
# 处理封面链接
if img_url and not post_img:
ext = img_url.split('?')[0].split('.')[-1]
ext = '.' + ext.lower()
print(ext)
assert ext in ['.jpg', '.jpeg', '.gif', '.png', '.bmp']
img_dir = 'static/upload/img'
now = datetime.datetime.now()
t = now.strftime('%Y%m%d_%H%M%S_%f')
img_name = '%s%s' % (t, ext)
img_path = '%s/%s' % (img_dir, img_name)
print(img_path)
r = requests.get(img_url, verify=False,
stream=True) # stream=True)
chunk_size = 100
with open(img_path, 'wb') as image:
for chunk in r.iter_content(chunk_size):
image.write(chunk)
im = Image.open(img_path)
width, height = im.size
if width / height > 5 or height / width > 5:
os.remove(img_path) # 判断比例 删除图片
print('请不要上传长宽比例过大的图片')
else:
# 创建1200x550 750x230 365x230缩略图
make_post_thumb(img_path,
sizes=[(1200, 550), (750, 230), (365, 230),
(260, 160)])
print('done')
post_img = img_path.split('/')[-1]
post_img = post_img.split('.')[0] + '_1200.jpg'
res = {
'title': title,
'markdown': markdown,
'content': content,
'sharetype': sharetype,
'slug': slug,
'tags': tags,
'post_img': post_img,
'link': link,
'vote_open': vote_open,
'vote_title': vote_title,
'updated': time.time(),
}
# if not markdown:
# self.redirect("/")
# return
if share_id:
share = Share.by_sid(share_id)
if not share:
self.redirect("/404")
share.update(res)
share.save()
else:
share = Share
res['user_id'] = user_id
share = share.new(res)
user = User.by_sid(user_id)
user.user_leaf += 10
user.save()
for i in tags:
doc = {'name': i, 'share_ids': share.id}
Tag.new(doc)
self.redirect("/share/" + str(share.id))
def get_tag_by_id(tag_id: str) -> ResultWithData[TagObject]:
tag = Tag.get(tag_id=tag_id)
if tag is None:
return get_result_with_error(TAG_ID_NOT_EXIST)
else:
return get_result_with_data(db_tag_to_tag_object(tag))
def get_tags() -> List[TagObject]:
tags = Tag.select(lambda t: True)
return [db_tag_to_tag_object(tag) for tag in tags]
def post(self):
data = request.get_json(force=True)
tag = Tag(**data)
print(to_dict(tag))
def get(self):
ver = self.get_argument("ver", 3)
name = self.get_argument("name", '')
sid = self.get_argument("id", 0)
ver = int(ver)
sid = int(sid)
# parents [0]
if name or sid:
# 具体某个标签
if not name and sid:
tag = Tag.by_sid(sid)
name = tag['name']
self.res = d_tags_v3.get(name, {})
if self.res:
parents = d_tags_parents.get(self.res['name'], {})
node = d_tags_v3.get(parents, {})
self.res['parents'] = copy.deepcopy(node)
self.res['parents'].pop('subs')
brothers = []
for sub in copy.deepcopy(node)['subs']:
sub.pop('subs')
brothers.append(sub)
self.res['brothers'] = brothers
if parents:
parents_p = d_tags_parents.get(parents, {})
if parents_p:
node = d_tags_v3.get(parents_p, {})
self.res['parents']['parents'] = copy.deepcopy(node)
self.res['parents']['parents'].pop('subs')
parents_pp = d_tags_parents.get(parents_p, {})
if parents_pp:
node = d_tags_v3.get(parents_pp, {})
self.res['parents']['parents'][
'parents'] = copy.deepcopy(node)
self.res['parents']['parents']['parents'].pop(
'subs')
self.res['articleNumber'] = Share.count_by_tag(
self.res['name'])
self.res['followerNumber'] = User.find({
'user_tags': {
'$in': [name]
}
}).count()
self.res['isFollowing'] = False
user_id = self.current_user[
"user_id"] if self.current_user else None
if user_id:
user = User.by_sid(user_id)
if user:
# model1
self.res['isFollowing'] = name in user['user_tags']
# model2 查看like 表
# self.res['isFollowing'] = name in user['user_tags']
print("self.res['followerNumber']",
self.res['followerNumber'])
tag = Tag.by_name(self.res['name'])
if tag:
self.res['id'] = tag['id']
else:
self.res['id'] = -1
else:
# 从根节点开始
if ver == 3:
self.res = d_tags_v2
self.res['parents'] = {} # root
self.res['articleNumber'] = Share.count_by_tag(
self.res['name'])
tag = Tag.by_name(self.res['name']) # 7491
self.res['id'] = -1
if tag:
self.res['id'] = tag['id']
elif ver == 2:
self.res = d_tags_v2
if self.current_user and 'user_id' in self.current_user:
user = User.by_sid(self.current_user['user_id'])
self.res['watched_tags'] = user['user_tags']
else:
self.res = d_tags
self.write_json()
import xmltodict
import sys
sys.path.append('../')
from db import Tag, Word
with open('wordbook.xml', encoding='utf8') as fd:
doc = xmltodict.parse(fd.read())
for item in doc['wordbook']['item']:
num = Tag.query_count(tag=item['tags'])
if num == 0:
tag = Tag(tag=item['tags'])
Tag.insert(tag)
else:
tag = Tag.query_one(tag=item['tags'])
wordNum = Word.query_count(word=item['word'])
if wordNum == 0:
word = Word(word=item['word'], explanation=item['trans'])
else:
continue
word.tags.append(tag)
Word.insert(word)
def get_tag(self, tag_name):
tag_name = Tag.fix_tag_name(tag_name)
return self.session.query(Tag).filter(
Tag.name == tag_name).one_or_none()
def zone_dates(zone):
dates = Tag.get_tag_dates(zone)
return jsonify([model_to_dict(d)['date'] for d in dates])
def tag_getcollection():
tags = Tag.get_collection(db.session)
return jsonify({'data': tags})
def create_tag(body):
tag = Tag(description=body.get("description"))
db.session.add(tag)
db.session.commit()
return tag.serialize()
def setting():
if current_user.task is None:
current_user.task=Task(user=current_user,wordNumPerDay=0)
session.commit()
return render_template('setting.html',tags=Tag.query_many(),wordNumPerDay=current_user.task.wordNumPerDay)
def add_from_file(rss_url, rss_hostname, rss_name):
# rss_file = 'content/gen/qdaily_2019-04-20 15:07:12.xml'
n = Share.find().count()
print(n)
print(rss_name)
feeds = feedparser.parse(rss_url)
for post in feeds.entries[::-1]:
# authors
# itunes_episodetype full
# itunes_episode
# itunes_explicit
# itunes_title
# itunes_duration
# published link subtitle id image title tags
# links title_detail author_detail summary_detail guidislink published_parsed summary content author
# subtitle_detail
# title title_detail
# published published_parsed
# summary summary_detail
# author
# link links guidislink
# authors
# 'itunes_title', 'itunes_episode'
# 'author_detail', 'id', 'itunes_duration'
# <itunes:duration>6957</itunes:duration>
# TODO
# 修正内容 目前暂时不支持
# <enclosure type="audio/mpeg" url="https://kernelpanic.fm/55/audio.mp3"/>
# <media:content url="https://cdn.flipboard.com/telegraph.co.uk/1356d637c7438f6fcffda0d5de177b6058904de6/original.jpg" medium="image" type="image/jpeg" width="480" height="300" />
# media_content
# print(post.keys())
if hasattr(post, 'summary'):
summary = post.summary
assert post.summary == post.description
else:
summary = ''
# 部分rss没有content
if hasattr(post, 'content'):
content = post.content[0]['value']
else:
if hasattr(post, 'summary'):
content = post.summary
else:
print('no content', rss_url, rss_hostname, rss_name)
continue
if content.startswith('<![CDATA[') and content.endswith(']]>'):
# m = rgx.search(content)
# content = m.group(1)
content = content[9:-3]
if summary.startswith('<![CDATA[') and summary.endswith(']]>'):
summary = summary[9:-3]
if hasattr(post, 'published'):
if 'GMT' == post.published[-3:]:
published = datetime.strptime(post.published,
"%a, %d %b %Y %H:%M:%S GMT")
elif ',' in post.published:
if post.published.endswith('2019'):
pass
# May 19, 2019
published = datetime.strptime(post.published, "%b %d, %Y")
else:
published = datetime.strptime(post.published,
"%a, %d %b %Y %H:%M:%S %z")
# Thu, 18 Apr 2019 19:32:58 +0800
elif '/' in post.published:
published = datetime.strptime(post.published,
"%Y/%m/%d %H:%M:%S %z")
elif 'Z' == post.published[-1]:
post.published = post.published.replace('.000Z', 'Z')
published = datetime.strptime(post.published,
"%Y-%m-%dT%H:%M:%SZ")
# <pubDate>15 Jun 2019 06:30:00 EST</pubDate>
elif 'EST' in post.published:
post.published = post.published[:-4]
published = datetime.strptime(post.published,
"%d %b %Y %H:%M:%S")
elif 'T' in post.published:
# 2019-05-24T15:05:50-04:00
post.published = post.published[:-6]
# tz = post.published[-6:].replace(':', '')
published = datetime.strptime(post.published,
"%Y-%m-%dT%H:%M:%S")
# published = published.replace(tzinfo=FixedOffset(tz))
elif post.published.count(' ') == 1:
published = datetime.strptime(post.published,
"%Y-%m-%d %H:%M:%S")
else:
published = datetime.strptime(post.published,
"%Y-%m-%d %H:%M:%S %z")
published = published.timestamp()
else:
if random.random() > 0.9:
print('no published time')
published = time.time()
title = post.title
link = post.link
author = ''
if hasattr(post, 'source'):
source_title = post.source.title
# print(source_title)
print(rss_name, source_title)
if rss_name == '虎嗅':
pass
author = source_title
else:
assert rss_name in source_title
# assert rss_name == source_title
source = rss_name
if hasattr(post, 'category_title'):
category = post.category_title
assert ' ' not in category
assert ',' not in category
tags = [category]
elif hasattr(post, 'tags'):
tags = post.tags
# print(tags)
# assert len(tags) == 1
# tags = tags[0]['term']
tags = ','.join([t['term'] for t in tags])
category = ''
if '-' in tags:
print(tags)
tags = tags.replace(' ', '-')
tags = tags.split(',')
for tag in tags:
if ' ' in tag:
print(tag)
else:
# print('no category')
category = ''
tags = []
sharetype = 'rss'
try:
markdown = html2text.html2text(content)
except Exception as e:
print('error in html-to-markdown: {}'.format(e))
continue
assert link
res = {
'title': title,
'link': link,
'source': source,
'category': category,
'content': content,
'summary': summary,
'sharetype': sharetype,
'tags': tags,
'markdown': markdown,
'published': published,
'updated': time.time(),
}
# print(post.keys())
if hasattr(post, 'author'):
# TODO
print('author: ', post.author)
res['author'] = post.author
else:
res['author'] = author
# 去重方案
# - 标题重复
found = Share.find({'title': title})
if found.count():
if found.count() > 1:
print('!! repeated article title: {}'.format(title))
elif found.count() == 1:
# continue
share = Share.by_sid(found[0].id)
if share and summary and not share.link and link:
print(res['link'])
print('title {} updated'.format(title))
share.update(res)
share.save()
else:
print('title {} adding'.format(title))
email = '{}@anwensf.com'.format(rss_hostname)
auser = User.by_email(email)
assert auser
share = Share
user_id = auser.id
res['user_id'] = user_id # just use 1 as default
# continue
assert res['link']
share = share.new(res)
user = User.by_sid(user_id)
user.user_leaf += 10
user.save()
for i in tags:
doc = {'name': i, 'share_ids': share.id}
Tag.new(doc)
