def main():
import connection
labels = Label.objects(name='_testing')
if labels.count():
label = labels.first()
else:
label = labels.upsert_one(name='_testing')
post1 = Page(title="Foo",
content="<div>Foo bar</div>",
url='https://foo.bar.com',
label=label)
print(repr(post1))
post1.save()
print()
post2 = Page(title="Bar",
content="<div>Fizz</div>",
url='https://fub.buzz.com',
label=label)
print(repr(post2))
post2.save()
print()
posts = Page.objects(label=label)
print(posts.count())
print(posts)
print()
print("Deleting")
post1.delete()
post2.delete()
def add_data():
data = request.get_json('page')
if data['page']:
for post in data['page']:
record = Page.objects(keyword = post['keyword']).update_one(
set__status = post['status'],
set__url = post['url'],
upsert = True)
return render_template('page.html')
def show_pages():
if request.method == "POST" and g.site.domain == g.user:
name = request.form['name']
if name:
p = Page()
p.name = name
p.slug = slugify(name)
p.site = g.site.domain
p.save()
return redirect(url_for(".edit_page", slug=p.slug))
pages = Page.objects(site=g.site.domain)
return render_template('pages.html', pages=pages)
def show_pages():
if request.method == "POST" and g.site.domain == g.user:
name = request.form['name']
if name:
p = Page()
p.name = name
p.slug = slugify(name)
p.site = g.site.domain
p.save()
return redirect(url_for(".edit_page", slug=p.slug))
pages = Page.objects(site=g.site.domain)
return render_template('pages.html', pages=pages)
def rebuild_url_lookup():
''' Rebuld URL lookup data'''
print "Rebuilding URL"
env = __open_env()
urls = env.open_db('urls')
with env.begin(db=urls, write=True) as txn:
cursor = txn.cursor()
for k, v in cursor:
txn.delete(k)
for item in Page.objects():
t = item.url.encode('utf-8')
txn.put(t, '1')
print "Done"
env.close()
def search(query, field, start=0, size=100, recommend=False, fuzzy=False, highlight=None):
'''
Perform Elasticsearch index lookup for documents with given parameters
:param query: text to be searched for
:type query: str
:param field: field to be searched
:type field: str
:param start: offset for results presentation
:type start: int
:param size: how many documents to present
:type size: int
:param recommend: fill in document's recommendations
:type recommend: bool
:param fuzzy: perform fuzzy search
:type fuzzy: bool
:param highlight: highlight query in result document
:type highlight: bool
:returns: number of results and results
:rtype: tuple (nrOfResults, results)
'''
body = {
"from": start,
'query': {}
}
if size > 0:
body['size'] = size
query = re.sub(r'[/:]', '\\/', query)
body['query']['query_string'] = {"fields": [field], "query": query, "default_operator": "AND"}
if fuzzy:
body['query']['query_string']['fuzziness'] = 'AUTO' # is default - can be ommited
else:
body['query']['query_string']['fuzziness'] = 0 # or 1 - both should be ok
if highlight and len(highlight) is 2:
body['highlight'] = {
"pre_tags": [highlight[0]],
"post_tags": [highlight[1]],
"fields": {
field: {}
}
}
# print body
res = Elasticsearch().search(index=INDEX_NAME, body=body)
# print res
pres = []
for r in res['hits']['hits']:
hit = Page.objects(id=r['_source']['_id']).first()
if hit:
if highlight and len(highlight) is 2:
hit.btext = '<br><br>'.join(r['highlight']['btext'])
pres.append(hit)
fin = []
for page in pres:
tmp = []
if recommend:
if page.parent:
for r in page.parent.recs:
x = [r.url, r.title]
tmp.append(x)
else:
for r in page.recs:
x = [r.url, r.url]
tmp.append(x)
fin.append([page, tmp])
return (res['hits']['total'], fin)
def delete_record():
page = request.get_json('row')['row']
record = Page.objects(keyword = page['keyword']).update_one(set__archived = True, upsert = True)
return render_template('page.html')