def publish(): if not session.get('is_admin'): return redirect(url_for('admin.login')) article_id = int(request.args.get('articleId', 0)) if not article_id: return abort(404) draft = db_session.query(Draft).filter( Draft.article_id == article_id).first() article = db_session.query(Article).filter( Article.id == article_id).first() if draft: if not article: article = Article(draft.user_id, draft.user_name, draft.title, draft.content) article.id = draft.article_id db_session.add(article) else: article.title = draft.title article.content = draft.content db_session.delete(draft) db_session.flush() return redirect('/detail?articleId=' + str(article_id))
def index(): page = int(request.args.get('page', 1)) ## 获取最新的文章 record_size = db_session.query(Article).count() article_list = db_session.query(Article).filter( Article.title != '').order_by(desc(Article.id)).offset( PAGE_SIZE * (page - 1)).limit(PAGE_SIZE) paging = Paging(page, record_size, request.args) ## 侧边栏显示的标签 tag_list = dao.aggregate_taged_article() ## 提取文章的摘要部分 article_dic = {} for article in article_list: article_dic[article.id] = htmlHelper.get_article_abs(article.content) return render_template('index.html', active='home', tag_list=tag_list, article_list=article_list, article_dic=article_dic, paging=paging)
def publish() : if not session.get('is_admin'): return redirect(url_for('admin.login')) article_id = int(request.args.get('articleId', 0)) if not article_id : return abort(404) draft = db_session.query(Draft).filter(Draft.article_id == article_id).first() article = db_session.query(Article).filter(Article.id == article_id).first() if draft : if not article : article = Article(draft.user_id, draft.user_name, draft.title, draft.content) article.id = draft.article_id db_session.add(article) else : article.title = draft.title article.content = draft.content db_session.delete(draft) db_session.flush() return redirect('/detail?articleId=' + str(article_id))
def crawl_scrape(src_id): source = db_session.query(Source).filter(Source.id == src_id).first() xp = {} f = {} util = {} util['src_id'] = src_id util['dname'] = source.domain xp['title'] = str(source.xp_title) xp['lead'] = source.xp_lead xp['content'] = source.xp_content xp['date'] = source.xp_date xp['author'] = source.xp_author xp['keywords'] = source.xp_keywords f['title'] = source.f_title f['lead'] = source.f_lead f['content'] = source.f_content f['date'] = source.f_date f['author'] = source.f_author f['keywords'] = source.f_keywords source.crawling = True db_session.add(source) db_session.commit() proc = CrawlerProcess({ 'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', 'CLOSESPIDER_PAGECOUNT': 20 }) proc.crawl(MainSpider, xp, f, util) proc.start(stop_after_crawl=False) #d.addBoth(lambda _: reactor.stop()) #reactor.run() current_texts = db_session.query(Text).filter(Text.src_id == src_id).all() c = 0 for t in stored: duplicate = False for at in current_texts: if at.content == t['content']: duplicate = True if not duplicate: c += 1 db_session.add( Text(src_id=t['source'], url=t['url'], title=t['title'], lead=t['lead'], content=t['content'], date=t['date'], author=t['author'], keywords=t['keywords'], lang=t['lang'], num_token=t['num_token'])) source.num_files_html += c source.crawling = False db_session.add(source) db_session.commit() return stored
def get_or_create(model, **kwargs): """ Get or create a model instance while preserving integrity. :param model: model to get or create. :return: tuple, Tuple of instance of the model and boolean reflecting if object was or was not created. """ try: return db_session.query(model).filter_by(**kwargs).one(), False except NoResultFound: try: with db_session.begin_nested(): instance = model(**kwargs) db_session.add(instance) return instance, True except IntegrityError: return db_session.query(model).filter_by(**kwargs).one(), False
def delete(): draft_id = 0 article_id = 0 try: draft_id = int(request.args.get('draftId', 0)) article_id = int(request.args.get('articleId', 0)) except: pass if draft_id: db_session.query(Draft).filter(Draft.id == draft_id).delete() elif article_id: db_session.query(Article).filter(Article.id == article_id).delete() return redirect('/')
def comment(): article_id = request.form.get('articleId') or 0 if not article_id: return jsonify(state=False, message='文章不存在') user_id = session.get('userid') user_name = session.get('username') to_name = request.form.get('toName') or None content = request.form.get('content') or None if not content: return jsonify(state=False, message='评论内容不能为空') if not user_name or not user_id: return jsonify(state=False, message='用户未登录') user = db_session.query(User).filter(User.id == user_id).first() ## 向数据库添加一条评论 c = Comment(article_id, user.id, user.name, user.head_url, to_name, content) db_session.add(c) db_session.flush() article_url = 'http://hamilton.duapp.com/detail?articleId=' + str( article_id) bmq.send_mail(['*****@*****.**'], 'hamilton上的新回复', '<!--HTML-->\n您有一条新回复需要处理<br/>点击查看:<a href="' + article_url + '">' + article_url + '</a>') return jsonify(state=True, message='success')
def comment() : article_id = request.form.get('articleId') or 0 if not article_id : return jsonify(state=False, message='文章不存在') user_id = session.get('userid') user_name = session.get('username') to_name = request.form.get('toName') or None content = request.form.get('content') or None if not content : return jsonify(state=False, message='评论内容不能为空') if not user_name or not user_id : return jsonify(state=False, message='用户未登录') user = db_session.query(User).filter(User.id == user_id).first() ## 向数据库添加一条评论 c = Comment(article_id, user.id, user.name, user.head_url, to_name, content) db_session.add(c) db_session.flush() article_url = 'http://hamilton.duapp.com/detail?articleId=' + str(article_id) bmq.send_mail(['*****@*****.**'], 'hamilton上的新回复', '<!--HTML-->\n您有一条新回复需要处理<br/>点击查看:<a href="' + article_url + '">' + article_url + '</a>') return jsonify(state=True, message='success')
def tags(): page = int(request.args.get('page', 1)) tag_id = int(request.args.get('tagid')) if not tag_id: return abort(404) tag_name = db_session.query(Tags.name).filter(Tags.id == tag_id).first()[0] ## 侧边栏显示的标签 tag_list = dao.aggregate_taged_article() ## 查询标签tag_id下的全部文章,并提取摘要部分 record_size, article_list = dao.get_taged_articles(tag_id, (page - 1) * PAGE_SIZE) paging = Paging(page, record_size, request.args) article_dic = {} for article in article_list: article_dic[article.id] = htmlHelper.get_article_abs(article.content) return render_template('tags.html', active='home', tag_list=tag_list, tag_name=tag_name, article_list=article_list, article_dic=article_dic, paging=paging)
def products(): product_type = request.headers['X-API-KEY'] # confirm correct API key used. if product_type not in INDUSTRY_KEYS: return MESSAGES[UNKNOWN_API_KEY], HTTPStatus.FORBIDDEN if request.method == 'POST': json_data = request.get_json() # Abort if no data supplied. if json_data is None: return MESSAGES[NO_JSON], HTTPStatus.BAD_REQUEST product_name = json_data.get('name') # Abort if product already exists. if product_name and db_session.query(exists().where(Product.name == product_name)).scalar(): return MESSAGES[DUPLICATE_PRODUCT], HTTPStatus.BAD_REQUEST product_creator = ProductCreator(data=json_data, product_type=product_type) # confirm all fields present before doing any work. # TODO: add info for customer - which fields are missing in the message. json_fields = product_creator.get_json_fields() if not product_creator.check_all_fields_present(json_fields): return MESSAGES[INCORRECT_DATA], HTTPStatus.BAD_REQUEST product_creator.create_product_from_data() return MESSAGES[PRODUCT_CREATED], HTTPStatus.CREATED else: # Retrieve all products here product_class = get_class_by_table_name(f'{product_type}_product') if product_class: retrieved_products = product_class.query.all() return jsonify(retrieved_products) return MESSAGES[UNKNOWN_API_KEY], HTTPStatus.NOT_FOUND
def delete() : draft_id = 0 article_id = 0 try : draft_id = int(request.args.get('draftId', 0)) article_id = int(request.args.get('articleId', 0)) except : pass if draft_id : db_session.query(Draft).filter(Draft.id == draft_id).delete() elif article_id : db_session.query(Article).filter(Article.id == article_id).delete() return redirect('/')
def get_user(id): u = db_session.query(User).get(id) return { 'id': str(u.id), 'username': str(u.username), 'name': '{} {}'.format(u.first, u.last), 'img_url': str(u.img_url) }
def edit(): userid = session.get('userid') username = session.get('username') if not userid or not username or not session.get('is_admin'): return redirect(url_for('admin.login')) article_id = int(request.args.get('articleId', 0)) title = request.form.get('title') content = request.form.get('content') if not article_id: return abort(404) ## 从数据库取出博客内容 draft = db_session.query(Draft).filter( Draft.article_id == article_id).first() if not draft: article = db_session.query(Article).filter( Article.id == article_id).first() if not article: return abort(404) draft = Draft(article.id, article.user_id, article.user_name, article.title, article.content) db_session.add(draft) db_session.flush() ## 浏览器发送的表单里没有数据,则把编辑页面发送给用户 if not title or not content: tag_list = get_article_taglist(article_id) return render_template('admin/publish.html', active='publish', article=draft, tag_list=tag_list) ## 表单里有数据,需要更新数据库 draft.title = title draft.content = htmlHelper.purge_background(content) db_session.flush() ## 更新文章的标签表 tag_list = request.form.getlist('tags') dao.update_tags(article_id, tag_list) return redirect('/admin/draft?draftId=%d' % draft.id)
def save_tags(article_id, tag_list) : if not tag_list : return None for tag_id in tag_list : tag_name = db_session.query(Tags.name).filter(Tags.id == tag_id).first()[0] db_session.add(TagedArticle(tag_id, article_id, tag_name)) db_session.flush()
def aggregate_taged_article() : tag_list = [] tmp_list = db_session.query(TagedArticle.tag_id, TagedArticle.tag_name, func.count(TagedArticle.tag_id).label('count')).group_by(TagedArticle.tag_id, TagedArticle.tag_name).order_by(desc('count')).all() for tup in tmp_list : tag_list.append(TagAggr(tup[0], tup[1], tup[2])) return tag_list
def clearOverride(): try: if 'email' not in session: return redirect(url_for('index')) odel = db_session.query(overrides).delete() db_session.commit() except Exception: return json.dumps({'status': 'clear failed'}) return json.dumps({'status': 'OK'})
def index() : page = int(request.args.get('page', 1)) ## 获取最新的文章 record_size = db_session.query(Article).count() article_list = db_session.query(Article).filter(Article.title != '').order_by(desc(Article.id)).offset(PAGE_SIZE*(page-1)).limit(PAGE_SIZE) paging = Paging(page, record_size, request.args) ## 侧边栏显示的标签 tag_list = dao.aggregate_taged_article() ## 提取文章的摘要部分 article_dic = {} for article in article_list : article_dic[article.id] = htmlHelper.get_article_abs(article.content) return render_template('index.html', active='home', tag_list = tag_list, article_list = article_list, article_dic = article_dic, paging = paging)
def edit() : userid = session.get('userid') username = session.get('username') if not userid or not username or not session.get('is_admin'): return redirect(url_for('admin.login')) article_id = int(request.args.get('articleId', 0)) title = request.form.get('title') content = request.form.get('content') if not article_id : return abort(404) ## 从数据库取出博客内容 draft = db_session.query(Draft).filter(Draft.article_id == article_id).first() if not draft : article = db_session.query(Article).filter(Article.id == article_id).first() if not article : return abort(404) draft = Draft(article.id, article.user_id, article.user_name, article.title, article.content) db_session.add(draft) db_session.flush(); ## 浏览器发送的表单里没有数据,则把编辑页面发送给用户 if not title or not content : tag_list = get_article_taglist(article_id) return render_template('admin/publish.html', active = 'publish', article=draft, tag_list = tag_list) ## 表单里有数据,需要更新数据库 draft.title = title draft.content = htmlHelper.purge_background(content) db_session.flush() ## 更新文章的标签表 tag_list = request.form.getlist('tags') dao.update_tags(article_id, tag_list) return redirect('/admin/draft?draftId=%d' % draft.id)
def get_posts(): json = [] for post in db_session.query(Post.id, Post.user_id, Post.body, Post.artist, Post.pub_date, Post.img_url): json.append({ 'id': post.id, 'user_id': post.user_id, 'body': post.body, 'artist': post.artist, 'pub_date': post.pub_date, 'img_url': post.img_url }) return {'success': 1, 'locs': json, 'msg': 'success'}
def index() : if not session.get('is_admin'): return redirect(url_for('.login')) draft_list = db_session.query(Draft).order_by(desc(Draft.id)).all() ## 提取文章的摘要部分 draft_dic = {} for draft in draft_list : draft_dic[draft.id] = htmlHelper.get_article_abs(draft.content) return render_template('admin/index.html', active='home', draft_list = draft_list, draft_dic = draft_dic)
def sources(): ans = db_session.query(Source).all() sources = [] for a in ans: elem = {} elem['name'] = a.name elem['crawling'] = a.crawling elem['dname'] = a.domain elem['id'] = a.id elem['num_tok_pdf'] = str(a.num_token_pdf) elem['num_tok_html'] = str(a.num_token_html) elem['num_files_pdf'] = str(a.num_files_pdf) elem['num_files_html'] = str(a.num_files_html) sources.append(elem) return render_template('sources.html', sources=sources)
def index(): if not session.get('is_admin'): return redirect(url_for('.login')) draft_list = db_session.query(Draft).order_by(desc(Draft.id)).all() ## 提取文章的摘要部分 draft_dic = {} for draft in draft_list: draft_dic[draft.id] = htmlHelper.get_article_abs(draft.content) return render_template('admin/index.html', active='home', draft_list=draft_list, draft_dic=draft_dic)
def detail() : article_id = int(request.args.get('articleId', 0)) if not article_id : abort(404) article = db_session.query(Article).filter(Article.id == article_id).first() if not article : return abort(404) ## 获取文章所属的标签 tag_list = dao.get_article_tags(article_id) ## 获取文章的全部评论 comments = dao.get_article_comments(article_id) return render_template('article.html', active='home', article = article, tag_list = tag_list, comments=comments)
def create_user(username, password, first, last, img_url="null"): if len(password) > 16: return { 'success': 0, 'id': 'null', 'msg': 'password is too long, make less than 16' } u = User(username, password, first, last, img_url) # make sure it is not duplicate if db_session.query(User).filter(User.username == username).first(): return {'success': 0, 'id': 'null', 'msg': 'user already exist'} db_session.add(u) # make sure we update our database is updated with the id db_session.flush() db_session.commit() return {'success': 1, 'id': u.id, 'msg': 'success'}
def draft() : if not session.get('is_admin') : return redirect(url_for('admin.login')) draft_id = int(request.args.get('draftId', 0)) if not draft_id : return abort(404) draft = db_session.query(Draft).filter(Draft.id == draft_id).first() if not draft_id : return abort(404) ## 获取文章所属的标签 tag_list = dao.get_article_tags(draft.article_id) return render_template('admin/draft.html', active='home', draft = draft, tag_list = tag_list)
def get_locs(): json = [] for post in db_session.query(Post.id, Post.artist, Post.lat, Post.lon): json.append({ 'id': post.id, 'artist': post.artist, 'loc': { 'lat': post.lat, 'lon': post.lon } }) return {'success': 1, 'locs': json, 'msg': 'success'} #print(create_post(1, 1000, 100, "dsf", 'https://lh5.googleusercontent.com/p/AF1QipPwtpYhzEyqEa45JwsYzV8_CC7IHdKcTD1gnHdI=w408-h544-k-no')) #print(create_post(1, 1000, 100, "dsf")) #print(get_posts())
def draft(): if not session.get('is_admin'): return redirect(url_for('admin.login')) draft_id = int(request.args.get('draftId', 0)) if not draft_id: return abort(404) draft = db_session.query(Draft).filter(Draft.id == draft_id).first() if not draft_id: return abort(404) ## 获取文章所属的标签 tag_list = dao.get_article_tags(draft.article_id) return render_template('admin/draft.html', active='home', draft=draft, tag_list=tag_list)
def tags() : page = int(request.args.get('page', 1)) tag_id = int(request.args.get('tagid')) if not tag_id : return abort(404) tag_name = db_session.query(Tags.name).filter(Tags.id == tag_id).first()[0] ## 侧边栏显示的标签 tag_list = dao.aggregate_taged_article() ## 查询标签tag_id下的全部文章,并提取摘要部分 record_size, article_list = dao.get_taged_articles(tag_id, (page-1)*PAGE_SIZE) paging = Paging(page, record_size, request.args) article_dic = {} for article in article_list : article_dic[article.id] = htmlHelper.get_article_abs(article.content) return render_template('tags.html', active='home', tag_list = tag_list, tag_name = tag_name, article_list = article_list, article_dic = article_dic, paging = paging)
def main(): overview = { 'num_html_files': 0, 'num_pdf_files': 0, 'num_html_tokens': 0, 'num_pdf_tokens': 0, 'srclogs': [] } all_sources = db_session.query(Source).all() for s in all_sources: overview['num_html_files'] += s.num_files_html overview['num_pdf_files'] += s.num_files_pdf all_texts = db_session.query(Text).filter(Text.ftype == 'html').all() for at in all_texts: overview['num_html_tokens'] += at.num_token all_texts = db_session.query(Text).filter(Text.ftype == 'pdf').all() for at in all_texts: overview['num_pdf_tokens'] += at.num_token overview['num_de'] = len( db_session.query(Text).filter(Text.lang == 'de').all()) overview['num_fr'] = len( db_session.query(Text).filter(Text.lang == 'fr').all()) overview['num_it'] = len( db_session.query(Text).filter(Text.lang == 'it').all()) overview['num_en'] = len( db_session.query(Text).filter(Text.lang == 'en').all()) slogs = db_session.query(SourceLog).all() for s in slogs: overview['srclogs'].append({ "username": s.username, "srcname": s.srcname, "timestamp": s.timestamp }) return render_template('index.html', overview=overview)
def detail(): article_id = int(request.args.get('articleId', 0)) if not article_id: abort(404) article = db_session.query(Article).filter( Article.id == article_id).first() if not article: return abort(404) ## 获取文章所属的标签 tag_list = dao.get_article_tags(article_id) ## 获取文章的全部评论 comments = dao.get_article_comments(article_id) return render_template('article.html', active='home', article=article, tag_list=tag_list, comments=comments)
def get_article_tagids(article_id) : tag_ids = [] v_list = db_session.query(TagedArticle.tag_id).filter(TagedArticle.article_id == article_id).all() for v in v_list : tag_ids.append(v[0]) return tag_ids
def get_all_tags() : return db_session.query(Tags).all()
def get_taged_articles(tag_id, offset) : query = db_session.query(Article).join(TagedArticle, TagedArticle.article_id == Article.id).filter(and_(TagedArticle.tag_id == tag_id, Article.title != '')) return query.count(), query.order_by(desc(Article.id)).offset(offset).limit(PAGE_SIZE)
def get_user_info(id, key): u = db_session.query(User).get(id) value = getattr(u, key) return {'id': str(id), key: str(value)}
def get_article_tags(article_id) : tag_list = [] v_list = db_session.query(TagedArticle.tag_id, TagedArticle.tag_name).filter(TagedArticle.article_id == article_id).all() for v in v_list : tag_list.append(Tags(v[0], v[1])) return tag_list
datetime_obj = datetime.strptime(post["metadata"]["Date"], '%d-%b-%Y_%H:%M:%S') markdown_content = post["content"] title = post["title"] ## Debug ''' print ("Title: " + title) print ("Date: " + str(datetime_obj)) print ("Content: " + markdown_content) ''' new_post = Post(title = title, content = markdown_content, date = datetime_obj) return new_post # Delete current db_session.query(Post).delete() # Seeding code # TODO grab all the posts posts = get_formatted_posts() # TODO Iterate through each and push it to the database for post in posts: parsed_post = parse_post(post) db_session.add(parsed_post) # Commit to db db_session.commit() # Show new Posts
def update_tags(article_id, tag_list) : db_session.query(TagedArticle).filter(TagedArticle.article_id == article_id).delete() save_tags(article_id, tag_list)
def get_article_comments(article_id) : return db_session.query(Comment).filter(Comment.article_id == article_id).order_by(desc(Comment.id)).all()