def main(): import connection labels = Label.objects(name='_testing') if labels.count(): label = labels.first() else: label = labels.upsert_one(name='_testing') post1 = Page(title="Foo", content="<div>Foo bar</div>", url='https://foo.bar.com', label=label) print(repr(post1)) post1.save() print() post2 = Page(title="Bar", content="<div>Fizz</div>", url='https://fub.buzz.com', label=label) print(repr(post2)) post2.save() print() posts = Page.objects(label=label) print(posts.count()) print(posts) print() print("Deleting") post1.delete() post2.delete()
def add_data(): data = request.get_json('page') if data['page']: for post in data['page']: record = Page.objects(keyword = post['keyword']).update_one( set__status = post['status'], set__url = post['url'], upsert = True) return render_template('page.html')
def show_pages(): if request.method == "POST" and g.site.domain == g.user: name = request.form['name'] if name: p = Page() p.name = name p.slug = slugify(name) p.site = g.site.domain p.save() return redirect(url_for(".edit_page", slug=p.slug)) pages = Page.objects(site=g.site.domain) return render_template('pages.html', pages=pages)
def rebuild_url_lookup(): ''' Rebuld URL lookup data''' print "Rebuilding URL" env = __open_env() urls = env.open_db('urls') with env.begin(db=urls, write=True) as txn: cursor = txn.cursor() for k, v in cursor: txn.delete(k) for item in Page.objects(): t = item.url.encode('utf-8') txn.put(t, '1') print "Done" env.close()
def search(query, field, start=0, size=100, recommend=False, fuzzy=False, highlight=None): ''' Perform Elasticsearch index lookup for documents with given parameters :param query: text to be searched for :type query: str :param field: field to be searched :type field: str :param start: offset for results presentation :type start: int :param size: how many documents to present :type size: int :param recommend: fill in document's recommendations :type recommend: bool :param fuzzy: perform fuzzy search :type fuzzy: bool :param highlight: highlight query in result document :type highlight: bool :returns: number of results and results :rtype: tuple (nrOfResults, results) ''' body = { "from": start, 'query': {} } if size > 0: body['size'] = size query = re.sub(r'[/:]', '\\/', query) body['query']['query_string'] = {"fields": [field], "query": query, "default_operator": "AND"} if fuzzy: body['query']['query_string']['fuzziness'] = 'AUTO' # is default - can be ommited else: body['query']['query_string']['fuzziness'] = 0 # or 1 - both should be ok if highlight and len(highlight) is 2: body['highlight'] = { "pre_tags": [highlight[0]], "post_tags": [highlight[1]], "fields": { field: {} } } # print body res = Elasticsearch().search(index=INDEX_NAME, body=body) # print res pres = [] for r in res['hits']['hits']: hit = Page.objects(id=r['_source']['_id']).first() if hit: if highlight and len(highlight) is 2: hit.btext = '<br><br>'.join(r['highlight']['btext']) pres.append(hit) fin = [] for page in pres: tmp = [] if recommend: if page.parent: for r in page.parent.recs: x = [r.url, r.title] tmp.append(x) else: for r in page.recs: x = [r.url, r.url] tmp.append(x) fin.append([page, tmp]) return (res['hits']['total'], fin)
def delete_record(): page = request.get_json('row')['row'] record = Page.objects(keyword = page['keyword']).update_one(set__archived = True, upsert = True) return render_template('page.html')