def publish():
if not session.get('is_admin'):
return redirect(url_for('admin.login'))
article_id = int(request.args.get('articleId', 0))
if not article_id:
return abort(404)
draft = db_session.query(Draft).filter(
Draft.article_id == article_id).first()
article = db_session.query(Article).filter(
Article.id == article_id).first()
if draft:
if not article:
article = Article(draft.user_id, draft.user_name, draft.title,
draft.content)
article.id = draft.article_id
db_session.add(article)
else:
article.title = draft.title
article.content = draft.content
db_session.delete(draft)
db_session.flush()
return redirect('/detail?articleId=' + str(article_id))
def index():
page = int(request.args.get('page', 1))
## 获取最新的文章
record_size = db_session.query(Article).count()
article_list = db_session.query(Article).filter(
Article.title != '').order_by(desc(Article.id)).offset(
PAGE_SIZE * (page - 1)).limit(PAGE_SIZE)
paging = Paging(page, record_size, request.args)
## 侧边栏显示的标签
tag_list = dao.aggregate_taged_article()
## 提取文章的摘要部分
article_dic = {}
for article in article_list:
article_dic[article.id] = htmlHelper.get_article_abs(article.content)
return render_template('index.html',
active='home',
tag_list=tag_list,
article_list=article_list,
article_dic=article_dic,
paging=paging)
def publish() :
if not session.get('is_admin'):
return redirect(url_for('admin.login'))
article_id = int(request.args.get('articleId', 0))
if not article_id :
return abort(404)
draft = db_session.query(Draft).filter(Draft.article_id == article_id).first()
article = db_session.query(Article).filter(Article.id == article_id).first()
if draft :
if not article :
article = Article(draft.user_id, draft.user_name, draft.title, draft.content)
article.id = draft.article_id
db_session.add(article)
else :
article.title = draft.title
article.content = draft.content
db_session.delete(draft)
db_session.flush()
return redirect('/detail?articleId=' + str(article_id))
def crawl_scrape(src_id):
source = db_session.query(Source).filter(Source.id == src_id).first()
xp = {}
f = {}
util = {}
util['src_id'] = src_id
util['dname'] = source.domain
xp['title'] = str(source.xp_title)
xp['lead'] = source.xp_lead
xp['content'] = source.xp_content
xp['date'] = source.xp_date
xp['author'] = source.xp_author
xp['keywords'] = source.xp_keywords
f['title'] = source.f_title
f['lead'] = source.f_lead
f['content'] = source.f_content
f['date'] = source.f_date
f['author'] = source.f_author
f['keywords'] = source.f_keywords
source.crawling = True
db_session.add(source)
db_session.commit()
proc = CrawlerProcess({
'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'CLOSESPIDER_PAGECOUNT': 20
})
proc.crawl(MainSpider, xp, f, util)
proc.start(stop_after_crawl=False)
#d.addBoth(lambda _: reactor.stop())
#reactor.run()
current_texts = db_session.query(Text).filter(Text.src_id == src_id).all()
c = 0
for t in stored:
duplicate = False
for at in current_texts:
if at.content == t['content']:
duplicate = True
if not duplicate:
c += 1
db_session.add(
Text(src_id=t['source'],
url=t['url'],
title=t['title'],
lead=t['lead'],
content=t['content'],
date=t['date'],
author=t['author'],
keywords=t['keywords'],
lang=t['lang'],
num_token=t['num_token']))
source.num_files_html += c
source.crawling = False
db_session.add(source)
db_session.commit()
return stored
def get_or_create(model, **kwargs):
"""
Get or create a model instance while preserving integrity.
:param model: model to get or create.
:return: tuple, Tuple of instance of the model and boolean reflecting if object was or was not created.
"""
try:
return db_session.query(model).filter_by(**kwargs).one(), False
except NoResultFound:
try:
with db_session.begin_nested():
instance = model(**kwargs)
db_session.add(instance)
return instance, True
except IntegrityError:
return db_session.query(model).filter_by(**kwargs).one(), False
def delete():
draft_id = 0
article_id = 0
try:
draft_id = int(request.args.get('draftId', 0))
article_id = int(request.args.get('articleId', 0))
except:
pass
if draft_id:
db_session.query(Draft).filter(Draft.id == draft_id).delete()
elif article_id:
db_session.query(Article).filter(Article.id == article_id).delete()
return redirect('/')
def comment():
article_id = request.form.get('articleId') or 0
if not article_id:
return jsonify(state=False, message='文章不存在')
user_id = session.get('userid')
user_name = session.get('username')
to_name = request.form.get('toName') or None
content = request.form.get('content') or None
if not content:
return jsonify(state=False, message='评论内容不能为空')
if not user_name or not user_id:
return jsonify(state=False, message='用户未登录')
user = db_session.query(User).filter(User.id == user_id).first()
## 向数据库添加一条评论
c = Comment(article_id, user.id, user.name, user.head_url, to_name,
content)
db_session.add(c)
db_session.flush()
article_url = 'http://hamilton.duapp.com/detail?articleId=' + str(
article_id)
bmq.send_mail(['*****@*****.**'], 'hamilton上的新回复',
'<!--HTML-->\n您有一条新回复需要处理<br/>点击查看：<a href="' + article_url +
'">' + article_url + '</a>')
return jsonify(state=True, message='success')
def comment() :
article_id = request.form.get('articleId') or 0
if not article_id :
return jsonify(state=False, message='文章不存在')
user_id = session.get('userid')
user_name = session.get('username')
to_name = request.form.get('toName') or None
content = request.form.get('content') or None
if not content :
return jsonify(state=False, message='评论内容不能为空')
if not user_name or not user_id :
return jsonify(state=False, message='用户未登录')
user = db_session.query(User).filter(User.id == user_id).first()
## 向数据库添加一条评论
c = Comment(article_id, user.id, user.name, user.head_url, to_name, content)
db_session.add(c)
db_session.flush()
article_url = 'http://hamilton.duapp.com/detail?articleId=' + str(article_id)
bmq.send_mail(['*****@*****.**'], 'hamilton上的新回复', '<!--HTML-->\n您有一条新回复需要处理<br/>点击查看：<a href="' + article_url + '">' + article_url + '</a>')
return jsonify(state=True, message='success')
def tags():
page = int(request.args.get('page', 1))
tag_id = int(request.args.get('tagid'))
if not tag_id:
return abort(404)
tag_name = db_session.query(Tags.name).filter(Tags.id == tag_id).first()[0]
## 侧边栏显示的标签
tag_list = dao.aggregate_taged_article()
## 查询标签tag_id下的全部文章，并提取摘要部分
record_size, article_list = dao.get_taged_articles(tag_id,
(page - 1) * PAGE_SIZE)
paging = Paging(page, record_size, request.args)
article_dic = {}
for article in article_list:
article_dic[article.id] = htmlHelper.get_article_abs(article.content)
return render_template('tags.html',
active='home',
tag_list=tag_list,
tag_name=tag_name,
article_list=article_list,
article_dic=article_dic,
paging=paging)
def products():
product_type = request.headers['X-API-KEY']
# confirm correct API key used.
if product_type not in INDUSTRY_KEYS:
return MESSAGES[UNKNOWN_API_KEY], HTTPStatus.FORBIDDEN
if request.method == 'POST':
json_data = request.get_json()
# Abort if no data supplied.
if json_data is None:
return MESSAGES[NO_JSON], HTTPStatus.BAD_REQUEST
product_name = json_data.get('name')
# Abort if product already exists.
if product_name and db_session.query(exists().where(Product.name == product_name)).scalar():
return MESSAGES[DUPLICATE_PRODUCT], HTTPStatus.BAD_REQUEST
product_creator = ProductCreator(data=json_data, product_type=product_type)
# confirm all fields present before doing any work.
# TODO: add info for customer - which fields are missing in the message.
json_fields = product_creator.get_json_fields()
if not product_creator.check_all_fields_present(json_fields):
return MESSAGES[INCORRECT_DATA], HTTPStatus.BAD_REQUEST
product_creator.create_product_from_data()
return MESSAGES[PRODUCT_CREATED], HTTPStatus.CREATED
else:
# Retrieve all products here
product_class = get_class_by_table_name(f'{product_type}_product')
if product_class:
retrieved_products = product_class.query.all()
return jsonify(retrieved_products)
return MESSAGES[UNKNOWN_API_KEY], HTTPStatus.NOT_FOUND
def delete() :
draft_id = 0
article_id = 0
try :
draft_id = int(request.args.get('draftId', 0))
article_id = int(request.args.get('articleId', 0))
except :
pass
if draft_id :
db_session.query(Draft).filter(Draft.id == draft_id).delete()
elif article_id :
db_session.query(Article).filter(Article.id == article_id).delete()
return redirect('/')
def get_user(id):
u = db_session.query(User).get(id)
return {
'id': str(u.id),
'username': str(u.username),
'name': '{} {}'.format(u.first, u.last),
'img_url': str(u.img_url)
}
def edit():
userid = session.get('userid')
username = session.get('username')
if not userid or not username or not session.get('is_admin'):
return redirect(url_for('admin.login'))
article_id = int(request.args.get('articleId', 0))
title = request.form.get('title')
content = request.form.get('content')
if not article_id:
return abort(404)
## 从数据库取出博客内容
draft = db_session.query(Draft).filter(
Draft.article_id == article_id).first()
if not draft:
article = db_session.query(Article).filter(
Article.id == article_id).first()
if not article:
return abort(404)
draft = Draft(article.id, article.user_id, article.user_name,
article.title, article.content)
db_session.add(draft)
db_session.flush()
## 浏览器发送的表单里没有数据，则把编辑页面发送给用户
if not title or not content:
tag_list = get_article_taglist(article_id)
return render_template('admin/publish.html',
active='publish',
article=draft,
tag_list=tag_list)
## 表单里有数据，需要更新数据库
draft.title = title
draft.content = htmlHelper.purge_background(content)
db_session.flush()
## 更新文章的标签表
tag_list = request.form.getlist('tags')
dao.update_tags(article_id, tag_list)
return redirect('/admin/draft?draftId=%d' % draft.id)
def save_tags(article_id, tag_list) :
if not tag_list :
return None
for tag_id in tag_list :
tag_name = db_session.query(Tags.name).filter(Tags.id == tag_id).first()[0]
db_session.add(TagedArticle(tag_id, article_id, tag_name))
db_session.flush()
def aggregate_taged_article() :
tag_list = []
tmp_list = db_session.query(TagedArticle.tag_id, TagedArticle.tag_name, func.count(TagedArticle.tag_id).label('count')).group_by(TagedArticle.tag_id, TagedArticle.tag_name).order_by(desc('count')).all()
for tup in tmp_list :
tag_list.append(TagAggr(tup[0], tup[1], tup[2]))
return tag_list
def clearOverride():
try:
if 'email' not in session:
return redirect(url_for('index'))
odel = db_session.query(overrides).delete()
db_session.commit()
except Exception:
return json.dumps({'status': 'clear failed'})
return json.dumps({'status': 'OK'})
def index() :
page = int(request.args.get('page', 1))
## 获取最新的文章
record_size = db_session.query(Article).count()
article_list = db_session.query(Article).filter(Article.title != '').order_by(desc(Article.id)).offset(PAGE_SIZE*(page-1)).limit(PAGE_SIZE)
paging = Paging(page, record_size, request.args)
## 侧边栏显示的标签
tag_list = dao.aggregate_taged_article()
## 提取文章的摘要部分
article_dic = {}
for article in article_list :
article_dic[article.id] = htmlHelper.get_article_abs(article.content)
return render_template('index.html', active='home', tag_list = tag_list, article_list = article_list, article_dic = article_dic, paging = paging)
def edit() :
userid = session.get('userid')
username = session.get('username')
if not userid or not username or not session.get('is_admin'):
return redirect(url_for('admin.login'))
article_id = int(request.args.get('articleId', 0))
title = request.form.get('title')
content = request.form.get('content')
if not article_id :
return abort(404)
## 从数据库取出博客内容
draft = db_session.query(Draft).filter(Draft.article_id == article_id).first()
if not draft :
article = db_session.query(Article).filter(Article.id == article_id).first()
if not article :
return abort(404)
draft = Draft(article.id, article.user_id, article.user_name, article.title, article.content)
db_session.add(draft)
db_session.flush();
## 浏览器发送的表单里没有数据，则把编辑页面发送给用户
if not title or not content :
tag_list = get_article_taglist(article_id)
return render_template('admin/publish.html', active = 'publish', article=draft, tag_list = tag_list)
## 表单里有数据，需要更新数据库
draft.title = title
draft.content = htmlHelper.purge_background(content)
db_session.flush()
## 更新文章的标签表
tag_list = request.form.getlist('tags')
dao.update_tags(article_id, tag_list)
return redirect('/admin/draft?draftId=%d' % draft.id)
def get_posts():
json = []
for post in db_session.query(Post.id, Post.user_id, Post.body, Post.artist,
Post.pub_date, Post.img_url):
json.append({
'id': post.id,
'user_id': post.user_id,
'body': post.body,
'artist': post.artist,
'pub_date': post.pub_date,
'img_url': post.img_url
})
return {'success': 1, 'locs': json, 'msg': 'success'}
def index() :
if not session.get('is_admin'):
return redirect(url_for('.login'))
draft_list = db_session.query(Draft).order_by(desc(Draft.id)).all()
## 提取文章的摘要部分
draft_dic = {}
for draft in draft_list :
draft_dic[draft.id] = htmlHelper.get_article_abs(draft.content)
return render_template('admin/index.html', active='home', draft_list = draft_list, draft_dic = draft_dic)
def sources():
ans = db_session.query(Source).all()
sources = []
for a in ans:
elem = {}
elem['name'] = a.name
elem['crawling'] = a.crawling
elem['dname'] = a.domain
elem['id'] = a.id
elem['num_tok_pdf'] = str(a.num_token_pdf)
elem['num_tok_html'] = str(a.num_token_html)
elem['num_files_pdf'] = str(a.num_files_pdf)
elem['num_files_html'] = str(a.num_files_html)
sources.append(elem)
return render_template('sources.html', sources=sources)
def index():
if not session.get('is_admin'):
return redirect(url_for('.login'))
draft_list = db_session.query(Draft).order_by(desc(Draft.id)).all()
## 提取文章的摘要部分
draft_dic = {}
for draft in draft_list:
draft_dic[draft.id] = htmlHelper.get_article_abs(draft.content)
return render_template('admin/index.html',
active='home',
draft_list=draft_list,
draft_dic=draft_dic)
def detail() :
article_id = int(request.args.get('articleId', 0))
if not article_id :
abort(404)
article = db_session.query(Article).filter(Article.id == article_id).first()
if not article :
return abort(404)
## 获取文章所属的标签
tag_list = dao.get_article_tags(article_id)
## 获取文章的全部评论
comments = dao.get_article_comments(article_id)
return render_template('article.html', active='home', article = article, tag_list = tag_list, comments=comments)
def create_user(username, password, first, last, img_url="null"):
if len(password) > 16:
return {
'success': 0,
'id': 'null',
'msg': 'password is too long, make less than 16'
}
u = User(username, password, first, last, img_url)
# make sure it is not duplicate
if db_session.query(User).filter(User.username == username).first():
return {'success': 0, 'id': 'null', 'msg': 'user already exist'}
db_session.add(u)
# make sure we update our database is updated with the id
db_session.flush()
db_session.commit()
return {'success': 1, 'id': u.id, 'msg': 'success'}
def draft() :
if not session.get('is_admin') :
return redirect(url_for('admin.login'))
draft_id = int(request.args.get('draftId', 0))
if not draft_id :
return abort(404)
draft = db_session.query(Draft).filter(Draft.id == draft_id).first()
if not draft_id :
return abort(404)
## 获取文章所属的标签
tag_list = dao.get_article_tags(draft.article_id)
return render_template('admin/draft.html', active='home', draft = draft, tag_list = tag_list)
def get_locs():
json = []
for post in db_session.query(Post.id, Post.artist, Post.lat, Post.lon):
json.append({
'id': post.id,
'artist': post.artist,
'loc': {
'lat': post.lat,
'lon': post.lon
}
})
return {'success': 1, 'locs': json, 'msg': 'success'}
#print(create_post(1, 1000, 100, "dsf", 'https://lh5.googleusercontent.com/p/AF1QipPwtpYhzEyqEa45JwsYzV8_CC7IHdKcTD1gnHdI=w408-h544-k-no'))
#print(create_post(1, 1000, 100, "dsf"))
#print(get_posts())
def draft():
if not session.get('is_admin'):
return redirect(url_for('admin.login'))
draft_id = int(request.args.get('draftId', 0))
if not draft_id:
return abort(404)
draft = db_session.query(Draft).filter(Draft.id == draft_id).first()
if not draft_id:
return abort(404)
## 获取文章所属的标签
tag_list = dao.get_article_tags(draft.article_id)
return render_template('admin/draft.html',
active='home',
draft=draft,
tag_list=tag_list)
def tags() :
page = int(request.args.get('page', 1))
tag_id = int(request.args.get('tagid'))
if not tag_id :
return abort(404)
tag_name = db_session.query(Tags.name).filter(Tags.id == tag_id).first()[0]
## 侧边栏显示的标签
tag_list = dao.aggregate_taged_article()
## 查询标签tag_id下的全部文章，并提取摘要部分
record_size, article_list = dao.get_taged_articles(tag_id, (page-1)*PAGE_SIZE)
paging = Paging(page, record_size, request.args)
article_dic = {}
for article in article_list :
article_dic[article.id] = htmlHelper.get_article_abs(article.content)
return render_template('tags.html', active='home', tag_list = tag_list, tag_name = tag_name, article_list = article_list, article_dic = article_dic, paging = paging)
def main():
overview = {
'num_html_files': 0,
'num_pdf_files': 0,
'num_html_tokens': 0,
'num_pdf_tokens': 0,
'srclogs': []
}
all_sources = db_session.query(Source).all()
for s in all_sources:
overview['num_html_files'] += s.num_files_html
overview['num_pdf_files'] += s.num_files_pdf
all_texts = db_session.query(Text).filter(Text.ftype == 'html').all()
for at in all_texts:
overview['num_html_tokens'] += at.num_token
all_texts = db_session.query(Text).filter(Text.ftype == 'pdf').all()
for at in all_texts:
overview['num_pdf_tokens'] += at.num_token
overview['num_de'] = len(
db_session.query(Text).filter(Text.lang == 'de').all())
overview['num_fr'] = len(
db_session.query(Text).filter(Text.lang == 'fr').all())
overview['num_it'] = len(
db_session.query(Text).filter(Text.lang == 'it').all())
overview['num_en'] = len(
db_session.query(Text).filter(Text.lang == 'en').all())
slogs = db_session.query(SourceLog).all()
for s in slogs:
overview['srclogs'].append({
"username": s.username,
"srcname": s.srcname,
"timestamp": s.timestamp
})
return render_template('index.html', overview=overview)
def detail():
article_id = int(request.args.get('articleId', 0))
if not article_id:
abort(404)
article = db_session.query(Article).filter(
Article.id == article_id).first()
if not article:
return abort(404)
## 获取文章所属的标签
tag_list = dao.get_article_tags(article_id)
## 获取文章的全部评论
comments = dao.get_article_comments(article_id)
return render_template('article.html',
active='home',
article=article,
tag_list=tag_list,
comments=comments)
def get_article_tagids(article_id) :
tag_ids = []
v_list = db_session.query(TagedArticle.tag_id).filter(TagedArticle.article_id == article_id).all()
for v in v_list :
tag_ids.append(v[0])
return tag_ids
def get_all_tags() : return db_session.query(Tags).all()
def get_taged_articles(tag_id, offset) : query = db_session.query(Article).join(TagedArticle, TagedArticle.article_id == Article.id).filter(and_(TagedArticle.tag_id == tag_id, Article.title != '')) return query.count(), query.order_by(desc(Article.id)).offset(offset).limit(PAGE_SIZE)
def get_user_info(id, key):
u = db_session.query(User).get(id)
value = getattr(u, key)
return {'id': str(id), key: str(value)}
def get_article_tags(article_id) :
tag_list = []
v_list = db_session.query(TagedArticle.tag_id, TagedArticle.tag_name).filter(TagedArticle.article_id == article_id).all()
for v in v_list :
tag_list.append(Tags(v[0], v[1]))
return tag_list
datetime_obj = datetime.strptime(post["metadata"]["Date"], '%d-%b-%Y_%H:%M:%S')
markdown_content = post["content"]
title = post["title"]
## Debug
'''
print ("Title: " + title)
print ("Date: " + str(datetime_obj))
print ("Content: " + markdown_content)
'''
new_post = Post(title = title, content = markdown_content, date = datetime_obj)
return new_post
# Delete current
db_session.query(Post).delete()
# Seeding code
# TODO grab all the posts
posts = get_formatted_posts()
# TODO Iterate through each and push it to the database
for post in posts:
parsed_post = parse_post(post)
db_session.add(parsed_post)
# Commit to db
db_session.commit()
# Show new Posts
def update_tags(article_id, tag_list) : db_session.query(TagedArticle).filter(TagedArticle.article_id == article_id).delete() save_tags(article_id, tag_list)
def get_article_comments(article_id) : return db_session.query(Comment).filter(Comment.article_id == article_id).order_by(desc(Comment.id)).all()
