def query(self):
pageNo = int(
request.args.get("pageNo") or request.form.get("pageNo") or 1)
pageSize = int(
request.args.get("pageSize") or request.form.get("pageSize") or 10)
# 查询数据库
rows = Paper.Query().offset(
(pageNo - 1) * pageSize).limit(pageSize).all()
count = Paper.Query().count()
pageCount = math.ceil(count / pageSize)
papers = []
for r in rows:
question_count = Question.Query().filter_by(paper_id=r.id).count()
paper = {
"id": r.id,
"name": r.name,
"question_count": question_count,
"subject": r.subject,
"created_time": datetime.timestamp(r.created_time)
}
papers.append(paper)
return error.success({
"list": papers,
"page": {
"total": count,
"pageCount": pageCount
}
})
def update_papers(title: str, sents: list, embeddings, ids: list):
"""
write to paper.db
Args:
title: title of doc
sents: a list of segments
ids: a list of ids
Returns:
none
"""
for s, id, e in zip(sents, ids, embeddings):
e = list(e)
e1 = e[0:int(len(e) * 0.25)]
e2 = e[int(len(e) * 0.25):int(len(e) * 0.5)]
e3 = e[int(len(e) * 0.5):int(len(e) * 0.75)]
e4 = e[int(len(e) * 0.75):]
link = gen_link(title, s)
p = Paper(title=title,
link=link,
seg=s,
e1=e1,
e2=e2,
e3=e3,
e4=e4,
id=id)
db.session.add(p)
db.session.commit()
def test_relation_author_paper(self):
author = Author()
paper = Paper()
paper.authors.append(author)
self.assertEqual(author, paper.authors[0])
self.assertEqual(paper, author.paper)
def fetch_and_save(self):
rss_text = self.fetch()
arxiv_xml = ArxivXml(rss_text)
date = arxiv_xml.get_date()
if not date:
raise RssParseError('Could not parse dc:date.')
if RssFetchHistory.exists(self._subject.id, date):
rss_fetch_history = self._subject.rss_fetch_histories.create(
date=date,
is_duplicated=True,
)
return []
rss_fetch_history = self._subject.rss_fetch_histories.create(date=date)
papers = []
for paper_item in arxiv_xml.get_paper_items():
# avoid Google Tranlate API limit (100,000chars/100sec)
if papers and len(papers) % 50 == 0:
time.sleep(100)
paper = Paper.from_xml(paper_item)
rss_fetch_history.papers.add(paper, bulk=False)
paper.add_authors(paper_item['authors'])
papers.append(paper)
return papers
def test_relation_review_paper(self):
review = Review()
paper = Paper()
paper.reviews.append(review)
self.assertEqual(review, paper.reviews[0])
self.assertEqual(paper, review.paper)
def test_write_paper(self):
paper = Paper(
title="How to conquer the World",
abstract="just conquer it!",
status=0
)
review = Review()
author = Author()
user_that_authored = User()
user_that_reviews = User()
user_that_authored.authors.append(author)
user_that_reviews.reviews.append(review)
paper.authors.append(author)
paper.reviews.append(review)
db.session.add(paper)
db.session.commit()
paper = Paper.query.all()[0]
self.assertEqual(review, paper.reviews[0])
self.assertEqual(author, paper.authors[0])
self.assertEqual(user_that_authored, paper.authors[0].user)
self.assertEqual(user_that_reviews, paper.reviews[0].user)
def test_handle(self, m_fetch_and_save, m_twitter, m_logger, m_slack):
twitter_mock = Mock()
slack_mock = Mock()
m_twitter.return_value = twitter_mock
m_fetch_and_save.return_value = [
Paper(title='xxx', abstract='yyy'),
Paper(title='xxx', abstract='yyy'),
]
m_slack.return_value = slack_mock
call_command('fetchrss')
twitter_mock.post_tweet.assert_called_with('4件の記事が追加されました。')
m_logger.info.assert_called_with(
'\x1b[32;1mSuccessfully fetch and save 4 papers (24 chars) from RSS.\x1b[0m'
)
slack_mock.notify_fetchrss.assert_called_with(
'Successfully fetch and save 4 papers (24 chars) from RSS.')
def test_relation_user_author_paper(self):
author = Author()
user = User()
paper = Paper()
user.authors.append(author)
paper.authors.append(author)
self.assertEqual(paper, user.authors[0].paper)
self.assertEqual(user, paper.authors[0].user)
def test_relation_user_review_paper(self):
review = Review()
user = User()
paper = Paper()
user.reviews.append(review)
paper.reviews.append(review)
self.assertEqual(paper, user.reviews[0].paper)
self.assertEqual(user, paper.reviews[0].user)
def post(self):
id = request.form.get("id")
name = request.form.get("name")
# 判断id和name是否为空
if not id or not name:
return error.error_1001()
# 查询数据库
paper = Paper.Query().filter_by(id=id).first()
if paper:
# 判断name是否重复
name_count = Paper.Query().filter(
and_(Paper.name == name, Paper.id != id)).count()
if name_count:
return error.error_1002("此考试名称已存在")
# 提交到数据库
paper.name = name
db.session.add(paper)
db.session.commit()
return error.success()
def create_papers(subs, vols, vots):
papers = []
params = ([[sub, vol, vot] for sub in subs
for vol in vols for vot in vots])
i = 0
for par in params:
p = Paper(subber=par[0], volunteer=par[1], voted=par[2],
title=f'Paper {i}')
papers.append(p)
i += 1
return papers
def query(self):
id = request.args.get("id") or request.form.get("id")
if not id:
return error.error_1001()
# 查询数据库
row = Paper.Query().filter_by(id=id).first()
paper = {}
if row:
paper = {"id": row.id, "name": row.name, "subject": row.subject}
return error.success(paper)
def add_paper():
data = request.get_json()
name = data['name']
description = data['description']
conference_name = data['conference_name']
paper = Paper(name=name,
description=description,
conference_id=Conference.query.filter_by(
name=conference_name).first().id)
db.session.add(paper)
db.session.commit()
return
def post(self):
form = request.form
name = form.get("name")
subject = form.get("subject")
# 判断name和subject是否为空
if not name or not subject:
return error.error_1001()
# 判断sucject是否合法
if subject not in config.subjects:
return error.error_1006()
# 判断name是否存在
paper_count = Paper.Query().filter_by(name=name).count()
if paper_count:
return error.error_1002("此考试名称已经存在")
# 提交到数据库
paper = Paper(form)
try:
db.session.add(paper)
db.session.commit()
except:
return error.error_1003()
return error.success()
def save_page():
paper_form = PaperForm()
if request.method == 'POST':
if paper_form.validate_on_submit():
# Get validated data from form
title = paper_form.title.data
abstract = paper_form.abstract.data
paper = Paper(title, abstract, author_id=current_user.id)
db.session.add(paper)
db.session.commit()
return redirect('mypage')
return render_template('newpaper.html', form=paper_form)
def create_paper(self, title, abstract, collaborators):
new_paper = Paper(title=title, abstract=abstract, status=0)
for collaborator in collaborators:
author = Author()
author.user = User.query.get(int(collaborator))
new_paper.authors.append(author)
if not collaborators:
author = Author()
author.user = User.query.get(session['user_id'])
new_paper.authors.append(author)
db.session.add(new_paper)
db.session.commit()
def test_from_xml(self, _):
arxiv_paper_item = {
'title': 'Some title.(arXiv:1611.07078v2 [cs.AI] UPDATED)',
'abstract': 'ABSTRACT',
'link': 'LINK',
}
result = Paper.from_xml(arxiv_paper_item)
self.assertIsInstance(result, Paper)
self.assertEqual(result.title,
'Some title.(arXiv:1611.07078v2 [cs.AI] UPDATED)')
self.assertEqual(result.abstract, 'ABSTRACT')
self.assertEqual(result.link, 'LINK')
self.assertEqual(result.subject, 'cs.AI')
self.assertEqual(result.submit_type, Paper.SUBMIT_TYPE_UPDATED)
def post(self):
id = request.form.get("id")
if not id:
return error.error_1001()
# 查询该考试中是否有题目
question_count = Question.Query().filter_by(paper_id=id).count()
if question_count:
return error.error_1008("此考试中有题目，不能被删除！")
# 查询数据库
paper = Paper.Query().filter_by(id=id).first()
if paper:
paper.is_valid = False
db.session.add(paper)
db.session.commit()
return error.success()
def pubs():
papers = Paper.query.order_by(Paper.date.desc()).all()
form = AddPubsForm()
# if request.method == 'POST' and 'file' in request.files:
# f = request.files.get('ile')
# filename = random_filename(f.filename)
# f.save(os.path.join(app.config['PUBS_UPLOAD_PATH'], filename))
if form.validate_on_submit():
# never alter the format when storing the data
title = form.title.data
author = str.title(form.author.data)
coauthor = str.title(form.coauthor.data)
citation = form.citation.data
issci = form.is_sci.data
isei = form.is_ei.data
if form.file.data:
filename = re.sub('\s', '_', str.capitalize(title[0:32]),
0) + '.pdf'
f = form.file.data
if os.path.splitext(f.filename)[1] != '.pdf':
flash('PDF only!', 'danger')
return redirect(url_for('pubs'))
f.save(os.path.join(app.config['PUBS_UPLOAD_PATH'], filename))
else:
filename = ''
paper = Paper(title=title,
author=author,
coauthor=coauthor,
filename=filename,
journal=form.journal.data,
date=form.date.data,
category=form.category.data,
abstract=form.abstract.data,
citation=citation,
issci=issci,
isei=isei)
db.session.add(paper)
db.session.commit()
flash('Paper cached!', 'success')
return redirect(url_for('pubs'))
return render_template('pubs.html',
title='Publication',
form=form,
papers=papers)
def submit_m():
form = ManualSubmissionForm()
if form.validate_on_submit():
p = Paper(
link=form.link.data,
subber=current_user,
authors=form.authors.data,
abstract=form.abstract.data,
title=form.title.data,
)
db.session.add(p)
# uploaded_file = request.files["attachment"]
# up = manage_upload(uploaded_file)
if form.comments.data:
c_text = form.comments.data
# elif up:
# c_text = ""
else:
c_text = None
if c_text is not None:
comment = Comment(
text=c_text,
commenter_id=current_user.id,
paper_id=p.id,
# upload=up,
)
db.session.add(comment)
# if up:
# up.comment_id = comment.id
db.session.commit()
if form.volunteering.data:
Paper.query.filter_by(
link=form.link.data).first().volunteer = current_user
db.session.commit()
flash('Paper submitted.')
return redirect(url_for('main.submit'))
papers = Paper.query.filter(Paper.timestamp >= last_month).all()
return render_template('main/submit_m.html',
papers=papers,
form=form,
title='Submit Paper',
showsub=True)
def query(self):
subject = request.args.get("subject") or request.form.get("subject")
# 判断subject是否为空
if not subject:
return error.error_1001()
# 判断subject值是否有效
if subject not in config.subjects:
return error.error_1006()
# 查询数据库
rows = Paper.Query().filter_by(subject=subject).all()
papers = []
for r in rows:
paper = {
"id": r.id,
"name": r.name,
"subject": r.subject,
"created_time": datetime.timestamp(r.created_time)
}
papers.append(paper)
return error.success(papers)
def test_volunteer(self):
u1 = User(username='******')
u2 = User(username='******')
db.session.add(u1)
db.session.add(u2)
db.session.commit()
self.assertEqual(u1.vols, [])
self.assertEqual(u2.vols, [])
p1 = Paper()
p2 = Paper()
db.session.add(p1, p2)
db.session.commit()
p1.volunteer = u2
p2.volunteer = u1
self.assertEqual(u1.vols[0], p2)
self.assertEqual(u2.vols[0], p1)
def test_submitter(self):
u1 = User(username='******')
u2 = User(username='******')
db.session.add(u1)
db.session.add(u2)
db.session.commit()
self.assertEqual(u1.subs, [])
self.assertEqual(u2.subs, [])
p1 = Paper()
p2 = Paper()
db.session.add(p1, p2)
db.session.commit()
p1.subber = u1
p2.subber = u2
self.assertEqual(u1.subs[0], p1)
self.assertEqual(u2.subs[0], p2)
def search():
"""
Searching the papers and uses pagination to render the results
Takes query from GET(query params) and POST (body params)
"""
if request.args.get('q'):
g.search_form.q.data = request.args.get('q')
q = g.search_form.q.data if g.search_form.q.data else None
if q is None:
return render_template('404.html'), 404
page = request.args.get('page', 1, type=int)
per_page = app.config['PER_PAGE']
paginated_papers, total = Paper.search(q, page, per_page)
href="search?q={}".format(q) + '&page={0}' ##customizing to include search query parameter
pagination = Pagination(href=href, page=page, per_page=per_page, total=total, record_name='papers',format_total=True, format_number=True)
# print(pagination.__dict__)
return render_template('papers.html', papers=paginated_papers, pagination=pagination, per_page=per_page)
def submit_m():
form = ManualSubmissionForm()
if form.validate_on_submit():
p = Paper(link=form.link.data,
subber=current_user,
authors=form.authors.data,
abstract=form.abstract.data,
title=form.title.data,
comment=form.comments.data)
db.session.add(p)
db.session.commit()
if form.volunteering.data:
Paper.query.filter_by(
link=form.link.data).first().volunteer = current_user
db.session.commit()
flash('Paper submitted.')
return redirect(url_for('main.submit'))
papers = Paper.query.filter(Paper.timestamp >= last_month).all()
return render_template('main/submit_m.html',
papers=papers,
form=form,
title='Submit Paper',
showsub=True)
def add_paper():
form = PaperForm()
if form.validate_on_submit():
paper = Paper(title=form.title.data,
first_author=form.first_author.data,
last_author=form.last_author.data,
doi=form.doi.data)
db.session.add(paper)
db.session.commit()
comment = Comment(paper_id=paper.id,
user_id=current_user.id,
rating=form.rating.data,
comment=form.comment.data)
db.session.add(comment)
db.session.commit()
flash('Your paper is now live!', 'success')
return redirect(url_for('main.see_paper_leaderboard'))
return render_template('insert_data.html', title='Add paper', form=form, header='Add paper')
def create_paper(self):
paper = Paper(paper_name='Test paper', author_email='*****@*****.**')
db.session.add(paper)
db.session.commit()
return paper
db.session.commit()
df = pd.read_excel(excel_file, sheet_name='Sheet1') ## Reading the excel file
dfs = df.where((pd.notnull(df)),
None) ## Changing nan to None in the dataframe
for i in range(len(dfs)):
# print(dfs['filenames'][i])
paper = Paper(paper_name=dfs['title'][i],
author_name=dfs['AUTHOR NAME'][i],
author_email=dfs['email-primary'][i],
tool_name=dfs['TOOL NAME'][i].split('-')[0],
link_to_pdf=dfs['ACM LINK'][i],
link_to_archive='Not provided',
link_to_tool_webpage=dfs['available(link)'][i],
link_to_demo=dfs['video LINK'][i],
bibtex=dfs['bibtext'][i],
description=dfs['DESCRIPTION'][i],
view_count=0,
conference=dfs['conference'][i],
year=dfs['year'][i])
tags = dfs['KEY WORDS'][i] or ''
filenames = dfs['File name for tool'][i]
add_temp(paper, tags, filenames, db, s3)
Paper.reindex()
def submit():
form = PaperSubmissionForm()
if form.submit.data and form.validate_on_submit():
link_str = form.link.data.split('?')[0].split('.pdf')[0]
scraper = Scraper()
scraper.get(link_str)
if scraper.failed:
flash('Scraping failed, submit manually.')
return redirect(url_for('main.submit_m'))
if scraper.error:
flash('Scraping error, check link or submit manually.')
return redirect(url_for('main.submit'))
authors = ", ".join(scraper.authors)
abstract = scraper.abstract
title = scraper.title
if form.comments.data:
comment_ = (str(current_user.firstname) + ': ' +
form.comments.data)
else:
comment_ = None
p = Paper(link=link_str,
subber=current_user,
authors=authors,
abstract=scraper.abstract,
title=scraper.title,
comment=comment_)
db.session.add(p)
db.session.commit()
if form.volunteering.data:
Paper.query.filter_by(
link=link_str).first().volunteer = current_user
db.session.commit()
flash('Paper submitted.')
return redirect(url_for('main.submit'))
papers = (Paper.query.filter(Paper.voted == None).order_by(
Paper.timestamp.desc()).all())
editform = FullEditForm(edits=range(len(papers)))
editforms = list(zip(papers, editform.edits))
for i in range(len(editform.data['edits'])):
paper = editforms[i][0]
button = editform.data['edits'][i]
if button['volunteer']:
paper.volunteer = current_user
elif button['vol_later']:
paper.vol_later = current_user
elif button['unvolunteer']:
if paper.volunteer:
paper.volunteer = None
elif paper.vol_later:
paper.vol_later = None
elif button['unsubmit']:
db.session.delete(paper)
elif button['comment']:
return redirect(url_for('main.comment', id=paper.id))
else:
continue
db.session.commit()
return redirect(url_for('main.submit'))
return render_template('main/submit.html',
form=form,
title='Submit Paper',
showsub=True,
editform=editform,
editforms=editforms,
extras=True)
from app import app, db
from app.models import Paper
import pandas as pd
excel_file = '/Users/lakshmanan/Downloads/april23_clean.xlsx'
def update_paper(paper_name, category):
if category:
paper = Paper.query.filter_by(paper_name=paper_name).first()
paper.category = category
print(paper)
db.session.commit()
df = pd.read_excel(excel_file, sheet_name='Sheet1') ## Reading the excel file
dfs = df.where((pd.notnull(df)),
None) ## Changing nan to None in the dataframe
for i in range(len(dfs)):
category_string = dfs['categories-Session name'][i].strip()
update_paper(dfs['title'][i], category_string)
Paper.reindex()
