def get_s(cls, mapping_type=None): if mapping_type is not None: s = S(mapping_type) else: s = S() return (s.es(**cls.es_settings).indexes(cls.index_name).doctypes( cls.mapping_type_name))
def test_filter_or_3(self): eq_( len( S(FakeModel).filter( F(tag='awesome') | F(tag='boat') | F(tag='boring'))), 5) eq_( len( S(FakeModel).filter(or_={ 'foo': 'bar', 'or_': { 'tag': 'boat', 'width': '5' } })), 3)
def order_by(self, *args, **kwargs): """ Doesn't allow the user to call order_by on an empty set. """ if not self.count(): return self return _S.order_by(self, *args, **kwargs)
def test_index(self): o = Opinion.objects.create( product=1, description='Get me a chocolate milk, with extra salt.') self.es.refresh() eq_(len(S('chocolate')), 1) return o
def get_s(cls, mapping_type=None): """Returns an S for the settings on this class Uses ``cls.es_settings`` to configure the Elasticsearch object. Uses ``cls.index_name`` for the index and ``cls.mapping_type_name`` for the MappingType to search. :arg mapping_type: The MappingType class to use to create the S """ if mapping_type is not None: s = S(mapping_type) else: s = S() return (s.es(**cls.es_settings).indexes(cls.index_name).doctypes( cls.mapping_type_name))
def __init__(self, tenant_id, model_type=None, sort=None, page=0, size=10): """ Setup of search. Arguments: tenant_id (int): ID of the tenant model_type (string): limit the search to a model sort (string): sort option for results page (int): page number of pagination size (int): max number of returned results """ search_request = S().es(**get_es_client_kwargs()).indexes(settings.ES_INDEXES['default']) self.search = search_request.all() # Always filter on Tenant. self.raw_filters = [{ 'term': { 'tenant': tenant_id } }] # Filter on model type. self.model_type = model_type # Add sorting. if sort: self.search = self.search.order_by(sort) # Pagination. from_hits = page * size to_hits = (page + 1) * size self.search = self.search[from_hits:to_hits]
def esAlertsSummary(begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = datetime.now() - timedelta(hours=12) begindateUTC = toUTC(begindateUTC) if enddateUTC is None: enddateUTC = datetime.now() enddateUTC = toUTC(enddateUTC) try: #q=S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert').filter(utctimestamp__range=[begindateUTC.isoformat(),enddateUTC.isoformat()]) #f=q.facet_raw(alerttype={"terms" : {"script_field" : "_source.type","size" : 500}}) #get all alerts #q= S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert') q = S().es(urls=list('{0}'.format(s) for s in options.esservers)).query(_type='alert') #create a facet field using the entire 'category' field (not the sub terms) and filter it by date. f=q.facet_raw(\ alerttype={"terms" : {"script_field" : "_source.category"},\ "facet_filter":{'range': {'utctimestamp': \ {'gte': begindateUTC.isoformat(), 'lte': enddateUTC.isoformat()}}}\ }) return (json.dumps(f.facet_counts()['alerttype'])) except Exception as e: sys.stderr.write('%r' % e)
def test_values_list_no_args(self): """Calling values() with no args fetches only id.""" eq_(S().query(fld1=2).values_list().build_search(), {'query': { "term": { "fld1": 2 } }})
def test_unindex(self): obj1 = FakeModel(id=1, title='First post!', tags=['blog', 'post']) FakeMappingType.index(FakeMappingType.extract_document(obj_id=obj1.id, obj=obj1), id_=obj1.id) FakeMappingType.refresh_index() s = S(FakeMappingType) eq_(s.count(), 1) eq_(list(s.execute())[0].title, obj1.title) FakeMappingType.unindex(id_=obj1.id) FakeMappingType.refresh_index() s = S(FakeMappingType) eq_(s.count(), 0)
def test_values_dict_no_args(self): """Calling values_dict() with no args fetches all fields.""" eq_(S().query(fld1=2).values_dict().build_search(), {"query": { "term": { "fld1": 2 } }})
def search(cls, query, vouched=None, photo=None): """Sensible default search for UserProfiles.""" query = query.lower().strip() fields = ('first_name__text', 'last_name__text', 'display_name__text', 'username__text', 'bio__text', 'website__text', 'email__text', 'groups__text', 'first_name__startswith', 'last_name__startswith', 'ircname') if query: q = dict((field, query) for field in fields) s = S(cls).query(or_=q) else: s = S(cls) if vouched is not None: s = s.filter(is_vouched=vouched) if photo is not None: s = s.filter(has_photo=photo) return s
def init_app(self, app): app.config.setdefault( 'ELASTICSEARCH_URL', os.environ.get('ELASTICSEARCH_URL', 'localhost:9200')) if not hasattr(app, 'extensions'): app.extensions = {} app.extensions['elasticutils'] = S().es( urls=[app.config['ELASTICSEARCH_URL']])
def __init__(self, params): super(ElasticSearch, self).__init__(params) # Get settings self.es_urls = params.get('URLS', ['http://localhost:9200']) self.es_index = params.get('INDEX', 'wagtail') # Get ElasticSearch interface self.es = get_es(urls=self.es_urls) self.s = S().es(urls=self.es_urls).indexes(self.es_index)
def test_to_python(self): person_es_data = { 'age': 1, 'name': u'2020-01-01T06:00:00', 'uuid': u'foo', } person_es_data2 = person_es_data.copy() S_eg = S() S_original = SBase() eg_person = S_eg.to_python(person_es_data) original_person = S_original.to_python(person_es_data2) # check that both methods operate in place self.assertIs(person_es_data, eg_person) self.assertIs(original_person, person_es_data2) # check equality aside from datetime conversion self.assertIsInstance(eg_person.pop('name'), basestring) self.assertIsInstance(original_person.pop('name'), datetime) self.assertEqual(eg_person, original_person)
def test_search_results_to_python(self): person_es_data = { 'age': 1, 'name': u'2020-01-01T06:00:00', 'uuid': u'foo', } person_es_data2 = person_es_data.copy() S_eg = S() S_original = SBase() eg_person = S_eg.to_python(person_es_data) original_person = S_original.to_python(person_es_data2) # check that both methods operate in place self.assertIs(person_es_data, eg_person) self.assertIs(original_person, person_es_data2) # check equality aside from datetime conversion self.assertIsInstance(eg_person.pop('name'), basestring) self.assertIsInstance(original_person.pop('name'), datetime) self.assertEqual(eg_person, original_person)
def getTopN(self, site_id, args): keywords = args.get("keywords") if not keywords: return [] try: amount = int(args.get("amount", "5")) except ValueError: raise ArgumentError("amount should be an integer.") s = S().indexes( es_search_functions.getESItemIndexName(site_id)).doctypes("item") query = es_search_functions.construct_or_query(keywords, delimiter=",") s = s.query_raw(query) s = s.filter(available=True) topn = [(item["item_id"], 1) for item in s[:amount]] return topn
def test_bulk_index(self): FakeModel(id=1, title='First post!', tags=['blog', 'post']) FakeModel(id=2, title='Second post!', tags=['blog', 'post']) documents = [] for obj in FakeModel.get_objects(): documents.append( FakeMappingType.extract_document(obj_id=obj.id, obj=obj)) FakeMappingType.bulk_index(documents, id_field='id') FakeMappingType.refresh_index() s = S(FakeMappingType) eq_(s.count(), 2) eq_(sorted([res.title for res in s.execute()]), ['First post!', 'Second post!'])
def es_utils_get(doctypes=None): # basic_s = S().es(urls=[URL]).indexes(INDEX).doctypes(DOCTYPE) from django.conf import settings connection = get_thread_var('es_utils_connetion') if not connection: from elasticutils import S # s = S().indexes('dbvestlitecms').doctypes('product') connection = S().es().indexes(settings.ES['default']['NAME']) # ElasticSearch('%s:%s/' % (settings.ES['default']['HOST'], settings.ES['default']['PORT'])) set_thread_var('es_utils_connetion', connection) if doctypes: return connection.doctypes(doctypes) return connection
def __init__(self, params): super(ElasticSearch, self).__init__(params) # Get settings self.es_urls = params.pop('URLS', ['http://localhost:9200']) self.es_index = params.pop('INDEX', 'wagtail') self.es_timeout = params.pop('TIMEOUT', 5) self.es_force_new = params.pop('FORCE_NEW', False) # Get ElasticSearch interface # Any remaining params are passed into the ElasticSearch constructor self.es = get_es( urls=self.es_urls, timeout=self.es_timeout, force_new=self.es_force_new, **params) self.s = S().es( urls=self.es_urls, timeout=self.es_timeout, force_new=self.es_force_new, **params).indexes(self.es_index)
def run(): s = S().indexes("item-index").doctypes("item") query_str = u"碧呵 婴儿" query = {"multi_match": {"query": query_str, "operator": "or", "fields": ["item_name"]}} s = s.query_raw(query) s = s.filter(categories=u"13") #for result in s: # print result["item_id"], result["item_name"] p = Paginator(s, 5) print "Total", p.count page_range = p.page_range[:20] print "Pages:", page_range print "======================" for page_num in page_range: print "Page:", page_num page = p.page(page_num) for item in page.object_list: #print item["item_name_suggest"] print item._score, item["item_id"], item["item_name"] break print "------------------"
def search_result( request): #TODO: change result div name, change hard coded url. query = request.GET.get('query') profile_type_filter = request.GET.get('profile_type_filter') lat = float(request.GET.get('lat')) if request.GET.get('lat') else None lng = float(request.GET.get('lng')) if request.GET.get('lng') else None distance = request.GET.get('distance') if request.user.is_authenticated(): user_profile_obj = UserProfile() user_profile = user_profile_obj.get_profile_by_id(str(request.user.id)) if user_profile.get('latlng') and user_profile.get('latlng').get( 'coordinates'): lat = user_profile.get('latlng').get('coordinates')[1] lng = user_profile.get('latlng').get('coordinates')[0] distance = '500km' sqs = S().indexes( settings.ES_INDEX).query(_all__fuzzy=query).facet('sign_up_as') if lat and lng: sqs = sqs.filter(latlng__distance=(distance, lat, lng)) if profile_type_filter: sqs = sqs.filter(sign_up_as=profile_type_filter) sqs = sqs[:50] html = render_to_string('_partials/search_result.html', {'sqs': sqs}, context_instance=RequestContext(request)) objs = list( sqs.values_dict('id', 'name', 'profile_img', 'description', 'latlng', 'username')) facets = {} #TODO: check why this iteration is required for k, v in sqs.facet_counts().items(): facets[k] = {'total': v['total'], 'terms': v['terms']} return HttpResponse( json.dumps({ 'html': html, 'objs': objs, 'facets': facets }))
def test_object(self): s = S(FakeMappingType).query(name__prefix='odin') eq_(len(s), 1) eq_(s[0].object.id, 1)
def test_values_list_no_args(self): """Calling values_list() with no args fetches all fields.""" eq_(S().query(fld1=2) .values_list() .build_search(), {'query': {"term": {"fld1": 2}}, 'fields': ['*']})
def test_typed_s_get_doctypes(self): eq_(S(FakeMappingType).get_doctypes(), ['doctype123'])
def test_bad_search(self): with self.assertRaises(BadSearch): len(S().doctypes('abc'))
def test_untyped_s_get_doctypes(self): eq_(S().get_doctypes(), DEFAULT_DOCTYPES) eq_(S().doctypes('abc').get_doctypes(), ['abc'])
def check_to_python(obj, expected): eq_(S().to_python(obj), expected)
def test_delete(self): a = self.test_index() a.delete() self.es.refresh() eq_(len(S('chocolate')), 0, 'We deleted this... WTF?')
def test_untyped_s_get_indexes(self): eq_(S().get_indexes(), DEFAULT_INDEXES) eq_(S().indexes('abc').get_indexes(), ['abc'])
}] es.bulk_index(INDEX, DOCTYPE, documents, id_field='id') # ElasticSearch will refresh the indexes and make those documents # available for querying in a second or so (it's configurable in # ElasticSearch), but we want them available right now, so we refresh # the index. es.refresh(INDEX) # Ok. We've created an index and tossed some stuff in it. Let's # do some basic queries. # Let's build a basic S that looks at the right instance of # ElasticSearch, index, and doctype. basic_s = S().es(urls=[URL]).indexes(INDEX).doctypes(DOCTYPE) # How many documents are in our index? print basic_s.count() # Prints: # 5 # Let's get all the cookie articles. print[item['title'] for item in basic_s.query(title__text='cookie')] # Prints: # [u'Deleting cookies', u'What is a cookie?', # u'Websites say cookies are blocked - Unblock them'] # Let's see cookie articles for websites. print[ item['title']
def test_typed_s_get_indexes(self): eq_(S(FakeMappingType).get_indexes(), ['index123'])