def test_non_searchable_field(self, db):
with db.atomic():
wiki_page = WikiPage.create(
title='foo',
markdown='bar',
html='spam'
)
WikiPageIndex.create(
docid=wiki_page.id,
title=wiki_page.title,
markdown=wiki_page.markdown
)
query = (WikiPage
.select(WikiPage, WikiPageIndex.bm25(3.0, 2.0))
.join(
WikiPageIndex,
on=(WikiPage.id == WikiPageIndex.docid))
.where(WikiPageIndex.match('foo bar'))
.order_by(WikiPageIndex.bm25(3.0, 2.0)))
wiki_page = query.execute()[0]
count = query.count()
assert wiki_page.title == 'foo'
assert wiki_page.markdown == 'bar'
assert wiki_page.html == 'spam'
assert count == 1
def test_search_score(self, db):
with db.atomic():
wiki_page = WikiPage.create(
title='foo1',
markdown='bar'
)
WikiPageIndex.create(
docid=wiki_page.id,
title=wiki_page.title,
)
wiki_page = WikiPage.create(
title='bar',
markdown='foo1 '
)
WikiPageIndex.create(
docid=wiki_page.id,
title=wiki_page.title,
markdown=wiki_page.markdown
)
query = WikiPageIndex.search_bm25(
'foo1',
weights={'title': 3.0, 'markdown': 2.0},
with_score=True,
score_alias='search_score',
explicit_ordering=True
)
# assert False
assert isinstance(query.execute()[0], WikiPageIndex)
def test_search_when_fts_model_is_out_of_sync(self, db):
with db.atomic():
wiki_page = WikiPage.create(title='foo')
WikiPageIndex.create(rowid=wiki_page.id, title='bar')
wiki_pages_bar = WikiPageIndex.search('bar')
assert isinstance(wiki_pages_bar[0], WikiPageIndex)
assert wiki_pages_bar[0].title == 'foo'
def test_no_index_rebuild_optimize(self, db):
with db.atomic():
for i in range(3):
wiki_page = WikiPage.create(title='foo {0}'.format(i + 1),
markdown='bar {0}'.format(i + 1))
WikiPageIndex.create(rowid=wiki_page.id,
title=wiki_page.title,
markdown=wiki_page.markdown)
wiki_pages_foo = WikiPageIndex.search('foo')
assert [wp.title
for wp in wiki_pages_foo] == ['foo 1', 'foo 2', 'foo 3']
wiki_pages_bar = WikiPageIndex.search('bar')
assert [wp.markdown
for wp in wiki_pages_bar] == ['bar 1', 'bar 2', 'bar 3']
def rename(wiki_page_id):
wiki_page = get_object_or_404(WikiPage.select(WikiPage.id, WikiPage.title),
WikiPage.id == wiki_page_id)
if wiki_page.title == 'Home':
return redirect(url_for('.home'))
form = RenameForm(new_title=wiki_page.title)
if form.validate_on_submit():
new_title = form.new_title.data
if wiki_page.title == new_title:
flash('The page name is not changed.', 'warning')
elif WikiPage.select().where(WikiPage.title == new_title).count() > 0:
flash('The new page title has already been taken.', 'warning')
else:
with db.atomic():
old_markdown = '[[{}]]'.format(wiki_page.title)
new_markdown = '[[{}]]'.format(new_title)
old_html = render_wiki_page(wiki_page.id,
wiki_page.title,
tostring=True)
new_html = render_wiki_page(wiki_page.id,
new_title,
tostring=True)
# update the markdown of referencing wiki page
query = (WikiPage.select(
WikiPage.id, WikiPage.markdown, WikiPage.html).join(
WikiReference, on=WikiReference.referencing).where(
WikiReference.referenced == wiki_page))
wiki_referencing_pages = query.execute()
for ref in wiki_referencing_pages:
new_markdown_content = ref.markdown.replace(
old_markdown, new_markdown)
(WikiPageIndex.update(markdown=new_markdown_content).where(
WikiPageIndex.docid == ref.id).execute())
(WikiPage.update(
markdown=new_markdown_content,
html=ref.html.replace(
old_html,
new_html)).where(WikiPage.id == ref.id).execute())
# update the diff of related wiki page versions
query = (WikiPageVersion.select(
WikiPageVersion.id, WikiPageVersion.diff).where(
WikiPageVersion.diff.contains(old_markdown)))
wiki_page_versions = query.execute()
for pv in wiki_page_versions:
(WikiPageVersion.update(diff=pv.diff.replace(
old_markdown, new_markdown)).where(
WikiPageVersion.id == pv.id).execute())
(WikiPage.update(title=new_title).where(
WikiPage.id == wiki_page.id).execute())
return redirect(url_for('.page', wiki_page_id=wiki_page.id))
return render_template('wiki/rename.html', wiki_page=wiki_page, form=form)
def search():
keyword = request.args.get('keyword')
start_date = request.args.get('start')
end_date = request.args.get('end')
current_page_number=request.args.get('page', default=1, type=int)
number_per_page = 100
kwargs = dict()
form = SearchForm(keyword=keyword, start_date=start_date, end_date=end_date)
if keyword and not keyword.isspace():
filters = [WikiPageIndex.match(WikiPageIndex.clean_query(keyword))]
if start_date:
temp = datetime.strptime(start_date, '%m/%d/%Y')
temp = temp.replace(tzinfo=current_app.config['TIMEZONE'])
filters.append(WikiPage.modified_on >= temp.timestamp())
if end_date:
temp = datetime.strptime(end_date, '%m/%d/%Y')+timedelta(days=1)
temp = temp.replace(tzinfo=current_app.config['TIMEZONE'])
filters.append(WikiPage.modified_on <= temp.timestamp())
query = (WikiPage
.select(WikiPage.id, WikiPage.title, WikiPage.modified_on)
.join(
WikiPageIndex,
on=(WikiPage.id==WikiPageIndex.rowid))
.where(*filters)
.order_by(WikiPageIndex.rank(2.0, 1.0), WikiPage.modified_on.desc()))
# TODO: add title-only search
# query = query.where(WikiPage.title.contains(search_keyword))
kwargs = paginate(query)
if form.validate_on_submit():
return redirect(url_for(
'.search',
keyword=form.keyword.data,
start=form.start_date.data,
end=form.end_date.data
))
return render_template(
'wiki/search.html',
form=form,
**kwargs
)
def search():
keyword = request.args.get('keyword')
start_date = request.args.get('start')
end_date = request.args.get('end')
current_page_number = request.args.get('page', default=1, type=int)
number_per_page = 100
kwargs = dict()
form = SearchForm(search=keyword, start_date=start_date, end_date=end_date)
if keyword and not keyword.isspace():
filter = [WikiPageIndex.match(keyword)]
if start_date:
temp = datetime.strptime(start_date, '%m/%d/%Y')
filter.append(
WikiPage.modified_on > convert_utc_to_mdt(temp, reverse=True))
if end_date:
temp = datetime.strptime(end_date, '%m/%d/%Y') + timedelta(days=1)
filter.append(
WikiPage.modified_on < convert_utc_to_mdt(temp, reverse=True))
query = (WikiPage.select(
WikiPage.id, WikiPage.title, WikiPage.modified_on).join(
WikiPageIndex, on=(WikiPage.id == WikiPageIndex.docid)).where(
*filter).order_by(WikiPageIndex.rank(2.0, 1.0),
WikiPage.modified_on.desc()).paginate(
current_page_number,
paginate_by=100))
# TODO: add title-only search
# query = query.where(WikiPage.title.contains(search_keyword))
get_pagination_kwargs(kwargs, query, current_page_number,
number_per_page)
if form.validate_on_submit():
return redirect(
url_for('.search',
keyword=form.search.data,
start=form.start_date.data,
end=form.end_date.data))
return render_template('wiki/search.html', form=form, **kwargs)
def handle_upload():
form = request.form
wiki_page_id = int(form.get('wiki_page_id', None))
upload_from_upload_page = form.get('upload_page', None)
file_markdown, file_html = '', ''
with db.atomic():
for i, file in enumerate(request.files.getlist('wiki_file')):
# create a WikiFile in database and retrieve id
wiki_file = WikiFile.create(name=file.filename,
mime_type=file.mimetype)
# save uploaded file with WikiFile.id as filename
file.save(os.path.join(DB_PATH, g.wiki_group, str(wiki_file.id)))
wiki_file.size = file.tell()
wiki_file.save()
if 'image' in wiki_file.mime_type:
file_type = 'image'
else:
file_type = 'file'
file_markdown += '\n\n[{}:{}]'.format(file_type, wiki_file.id)
file_html += '<p>{}</p>'.format(
render_wiki_file(wiki_file.id,
wiki_file.name,
file_type,
tostring=True))
if upload_from_upload_page:
wiki_page = get_object_or_404(
WikiPage.select(WikiPage.id, WikiPage.markdown,
WikiPage.current_version,
WikiPage.modified_on),
WikiPage.id == wiki_page_id)
diff = make_patch(xstr(wiki_page.markdown),
xstr(wiki_page.markdown) + file_markdown)
WikiPageVersion.create(wiki_page=wiki_page,
diff=diff,
version=wiki_page.current_version,
modified_on=wiki_page.modified_on)
(WikiPageIndex.update(
markdown=wiki_page.markdown + file_markdown).where(
WikiPageIndex.docid == wiki_page.id).execute())
(WikiPage.update(markdown=WikiPage.markdown + file_markdown,
html=WikiPage.html + file_html,
current_version=WikiPage.current_version + 1,
modified_on=datetime.utcnow()).where(
WikiPage.id == wiki_page.id).execute())
return ''
return file_markdown
def edit(wiki_page_id):
wiki_page = get_object_or_404(
WikiPage.select(WikiPage.id, WikiPage.title, WikiPage.markdown,
WikiPage.current_version, WikiPage.modified_on),
WikiPage.id == wiki_page_id)
form = WikiEditForm()
upload_form = UploadForm()
if form.validate_on_submit():
if form.current_version.data == wiki_page.current_version:
g.wiki_page = wiki_page
g.wiki_refs = list(
WikiPage.select(WikiPage.id).join(
WikiReference, on=WikiReference.referenced).where(
WikiReference.referencing == wiki_page).execute())
diff = make_patch(wiki_page.markdown, form.textArea.data)
if diff:
with db.atomic():
toc, html = markdown(form.textArea.data)
WikiPageVersion.create(wiki_page=wiki_page,
diff=diff,
version=wiki_page.current_version,
modified_on=wiki_page.modified_on)
(WikiPageIndex.update(markdown=form.textArea.data).where(
WikiPageIndex.docid == wiki_page.id).execute())
(WikiPage.update(
markdown=form.textArea.data,
html=html,
toc=toc,
current_version=WikiPage.current_version,
modified_on=datetime.utcnow()).where(
WikiPage.id == wiki_page.id).execute())
# remove unused WikiReference
(WikiReference.delete().where(
WikiReference.referenced.in_(g.wiki_refs)).execute())
return redirect(url_for('.page', wiki_page_id=wiki_page.id))
else:
flash('Other changes have been made to this '
'wiki page since you started editing it.')
return render_template('wiki/edit.html',
wiki_page=wiki_page,
form=form,
upload_form=upload_form)
def test_search_markdown(self, db):
with db.atomic():
for i in range(3):
WikiPage.create(title='title {0}'.format(i), markdown='foo {0}'.format(i+1))
WikiPage.create(title='title {0}'.format(i+3), markdown='bar {0}'.format(i+1))
WikiPageIndex.rebuild()
WikiPageIndex.optimize()
wiki_pages_foo = WikiPageIndex.search('foo')
assert [wp.markdown for wp in wiki_pages_foo] == ['foo 1', 'foo 2', 'foo 3']
wiki_pages_bar = WikiPageIndex.search('bar')
assert [wp.markdown for wp in wiki_pages_bar] == ['bar 1', 'bar 2', 'bar 3']
def test_search_after_update(self, db):
with db.atomic():
wiki_page = WikiPage.create(title='foo')
WikiPageIndex.create(rowid=wiki_page.id, title=wiki_page.title)
wiki_pages_bar = WikiPageIndex.search('bar')
assert len(wiki_pages_bar) == 0
WikiPageIndex.update(title='bar').where(
WikiPageIndex.rowid == 1).execute()
WikiPage.update(title='bar').where(WikiPage.id == 1).execute()
wiki_pages_bar = WikiPageIndex.search('bar')
assert wiki_pages_bar[0].title == 'bar'
def handleMatch(self, m):
wiki_page_title = m.group(2)
try:
wiki_page = (WikiPage.select(
WikiPage.id).where(WikiPage.title == wiki_page_title).get())
except WikiPage.DoesNotExist:
wiki_page = WikiPage.create(title=wiki_page_title)
wiki_page_index = WikiPageIndex.create(docid=wiki_page.id,
title=wiki_page_title)
(WikiReference.insert(
referencing=g.wiki_page,
referenced=wiki_page).on_conflict_ignore().execute())
try:
g.wiki_refs.remove(wiki_page)
except ValueError:
pass
return render_wiki_page(wiki_page.id, wiki_page_title)
def wiki_page(self, title):
wiki_page_title = title
try:
wiki_page = (WikiPage.select(
WikiPage.id).where(WikiPage.title == wiki_page_title).get())
except WikiPage.DoesNotExist:
wiki_page = WikiPage.create(title=wiki_page_title)
wiki_page_index = WikiPageIndex.create(rowid=wiki_page.id,
title=wiki_page_title)
(WikiReference.insert(
referencing=g.wiki_page,
referenced=wiki_page).on_conflict_ignore().execute())
try:
g.wiki_refs.remove(wiki_page)
# AttributeError: g.wiki_refs not exist
# ValueError: wiki_page not in g.wiki_refs
except (AttributeError, ValueError):
pass
return render_wiki_page(wiki_page.id, wiki_page_title)