def get_s(cls, mapping_type=None):
if mapping_type is not None:
s = S(mapping_type)
else:
s = S()
return (s.es(**cls.es_settings).indexes(cls.index_name).doctypes(
cls.mapping_type_name))
def test_filter_or_3(self):
eq_(
len(
S(FakeModel).filter(
F(tag='awesome') | F(tag='boat') | F(tag='boring'))), 5)
eq_(
len(
S(FakeModel).filter(or_={
'foo': 'bar',
'or_': {
'tag': 'boat',
'width': '5'
}
})), 3)
def order_by(self, *args, **kwargs):
"""
Doesn't allow the user to call order_by on an empty set.
"""
if not self.count():
return self
return _S.order_by(self, *args, **kwargs)
def order_by(self, *args, **kwargs):
"""
Doesn't allow the user to call order_by on an empty set.
"""
if not self.count():
return self
return _S.order_by(self, *args, **kwargs)
def test_index(self):
o = Opinion.objects.create(
product=1,
description='Get me a chocolate milk, with extra salt.')
self.es.refresh()
eq_(len(S('chocolate')), 1)
return o
def get_s(cls, mapping_type=None):
"""Returns an S for the settings on this class
Uses ``cls.es_settings`` to configure the Elasticsearch
object. Uses ``cls.index_name`` for the index and
``cls.mapping_type_name`` for the MappingType to search.
:arg mapping_type: The MappingType class to use to create the S
"""
if mapping_type is not None:
s = S(mapping_type)
else:
s = S()
return (s.es(**cls.es_settings).indexes(cls.index_name).doctypes(
cls.mapping_type_name))
def __init__(self, tenant_id, model_type=None, sort=None, page=0, size=10):
"""
Setup of search.
Arguments:
tenant_id (int): ID of the tenant
model_type (string): limit the search to a model
sort (string): sort option for results
page (int): page number of pagination
size (int): max number of returned results
"""
search_request = S().es(**get_es_client_kwargs()).indexes(settings.ES_INDEXES['default'])
self.search = search_request.all()
# Always filter on Tenant.
self.raw_filters = [{
'term': {
'tenant': tenant_id
}
}]
# Filter on model type.
self.model_type = model_type
# Add sorting.
if sort:
self.search = self.search.order_by(sort)
# Pagination.
from_hits = page * size
to_hits = (page + 1) * size
self.search = self.search[from_hits:to_hits]
def esAlertsSummary(begindateUTC=None, enddateUTC=None):
resultsList = list()
if begindateUTC is None:
begindateUTC = datetime.now() - timedelta(hours=12)
begindateUTC = toUTC(begindateUTC)
if enddateUTC is None:
enddateUTC = datetime.now()
enddateUTC = toUTC(enddateUTC)
try:
#q=S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert').filter(utctimestamp__range=[begindateUTC.isoformat(),enddateUTC.isoformat()])
#f=q.facet_raw(alerttype={"terms" : {"script_field" : "_source.type","size" : 500}})
#get all alerts
#q= S().es(urls=['http://{0}:{1}'.format(options.esserver,options.esport)]).query(_type='alert')
q = S().es(urls=list('{0}'.format(s)
for s in options.esservers)).query(_type='alert')
#create a facet field using the entire 'category' field (not the sub terms) and filter it by date.
f=q.facet_raw(\
alerttype={"terms" : {"script_field" : "_source.category"},\
"facet_filter":{'range': {'utctimestamp': \
{'gte': begindateUTC.isoformat(), 'lte': enddateUTC.isoformat()}}}\
})
return (json.dumps(f.facet_counts()['alerttype']))
except Exception as e:
sys.stderr.write('%r' % e)
def test_values_list_no_args(self):
"""Calling values() with no args fetches only id."""
eq_(S().query(fld1=2).values_list().build_search(),
{'query': {
"term": {
"fld1": 2
}
}})
def test_unindex(self):
obj1 = FakeModel(id=1, title='First post!', tags=['blog', 'post'])
FakeMappingType.index(FakeMappingType.extract_document(obj_id=obj1.id,
obj=obj1),
id_=obj1.id)
FakeMappingType.refresh_index()
s = S(FakeMappingType)
eq_(s.count(), 1)
eq_(list(s.execute())[0].title, obj1.title)
FakeMappingType.unindex(id_=obj1.id)
FakeMappingType.refresh_index()
s = S(FakeMappingType)
eq_(s.count(), 0)
def test_values_dict_no_args(self):
"""Calling values_dict() with no args fetches all fields."""
eq_(S().query(fld1=2).values_dict().build_search(),
{"query": {
"term": {
"fld1": 2
}
}})
def search(cls, query, vouched=None, photo=None):
"""Sensible default search for UserProfiles."""
query = query.lower().strip()
fields = ('first_name__text', 'last_name__text', 'display_name__text',
'username__text', 'bio__text', 'website__text',
'email__text', 'groups__text', 'first_name__startswith',
'last_name__startswith', 'ircname')
if query:
q = dict((field, query) for field in fields)
s = S(cls).query(or_=q)
else:
s = S(cls)
if vouched is not None:
s = s.filter(is_vouched=vouched)
if photo is not None:
s = s.filter(has_photo=photo)
return s
def init_app(self, app):
app.config.setdefault(
'ELASTICSEARCH_URL',
os.environ.get('ELASTICSEARCH_URL', 'localhost:9200'))
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['elasticutils'] = S().es(
urls=[app.config['ELASTICSEARCH_URL']])
def __init__(self, params):
super(ElasticSearch, self).__init__(params)
# Get settings
self.es_urls = params.get('URLS', ['http://localhost:9200'])
self.es_index = params.get('INDEX', 'wagtail')
# Get ElasticSearch interface
self.es = get_es(urls=self.es_urls)
self.s = S().es(urls=self.es_urls).indexes(self.es_index)
def test_to_python(self):
person_es_data = {
'age': 1,
'name': u'2020-01-01T06:00:00',
'uuid': u'foo',
}
person_es_data2 = person_es_data.copy()
S_eg = S()
S_original = SBase()
eg_person = S_eg.to_python(person_es_data)
original_person = S_original.to_python(person_es_data2)
# check that both methods operate in place
self.assertIs(person_es_data, eg_person)
self.assertIs(original_person, person_es_data2)
# check equality aside from datetime conversion
self.assertIsInstance(eg_person.pop('name'), basestring)
self.assertIsInstance(original_person.pop('name'), datetime)
self.assertEqual(eg_person, original_person)
def test_search_results_to_python(self):
person_es_data = {
'age': 1,
'name': u'2020-01-01T06:00:00',
'uuid': u'foo',
}
person_es_data2 = person_es_data.copy()
S_eg = S()
S_original = SBase()
eg_person = S_eg.to_python(person_es_data)
original_person = S_original.to_python(person_es_data2)
# check that both methods operate in place
self.assertIs(person_es_data, eg_person)
self.assertIs(original_person, person_es_data2)
# check equality aside from datetime conversion
self.assertIsInstance(eg_person.pop('name'), basestring)
self.assertIsInstance(original_person.pop('name'), datetime)
self.assertEqual(eg_person, original_person)
def getTopN(self, site_id, args):
keywords = args.get("keywords")
if not keywords:
return []
try:
amount = int(args.get("amount", "5"))
except ValueError:
raise ArgumentError("amount should be an integer.")
s = S().indexes(
es_search_functions.getESItemIndexName(site_id)).doctypes("item")
query = es_search_functions.construct_or_query(keywords, delimiter=",")
s = s.query_raw(query)
s = s.filter(available=True)
topn = [(item["item_id"], 1) for item in s[:amount]]
return topn
def test_bulk_index(self):
FakeModel(id=1, title='First post!', tags=['blog', 'post'])
FakeModel(id=2, title='Second post!', tags=['blog', 'post'])
documents = []
for obj in FakeModel.get_objects():
documents.append(
FakeMappingType.extract_document(obj_id=obj.id, obj=obj))
FakeMappingType.bulk_index(documents, id_field='id')
FakeMappingType.refresh_index()
s = S(FakeMappingType)
eq_(s.count(), 2)
eq_(sorted([res.title for res in s.execute()]),
['First post!', 'Second post!'])
def es_utils_get(doctypes=None):
# basic_s = S().es(urls=[URL]).indexes(INDEX).doctypes(DOCTYPE)
from django.conf import settings
connection = get_thread_var('es_utils_connetion')
if not connection:
from elasticutils import S
# s = S().indexes('dbvestlitecms').doctypes('product')
connection = S().es().indexes(settings.ES['default']['NAME'])
# ElasticSearch('%s:%s/' % (settings.ES['default']['HOST'], settings.ES['default']['PORT']))
set_thread_var('es_utils_connetion', connection)
if doctypes:
return connection.doctypes(doctypes)
return connection
def __init__(self, params):
super(ElasticSearch, self).__init__(params)
# Get settings
self.es_urls = params.pop('URLS', ['http://localhost:9200'])
self.es_index = params.pop('INDEX', 'wagtail')
self.es_timeout = params.pop('TIMEOUT', 5)
self.es_force_new = params.pop('FORCE_NEW', False)
# Get ElasticSearch interface
# Any remaining params are passed into the ElasticSearch constructor
self.es = get_es(
urls=self.es_urls,
timeout=self.es_timeout,
force_new=self.es_force_new,
**params)
self.s = S().es(
urls=self.es_urls,
timeout=self.es_timeout,
force_new=self.es_force_new,
**params).indexes(self.es_index)
def run():
s = S().indexes("item-index").doctypes("item")
query_str = u"碧呵 婴儿"
query = {"multi_match": {"query": query_str, "operator": "or",
"fields": ["item_name"]}}
s = s.query_raw(query)
s = s.filter(categories=u"13")
#for result in s:
# print result["item_id"], result["item_name"]
p = Paginator(s, 5)
print "Total", p.count
page_range = p.page_range[:20]
print "Pages:", page_range
print "======================"
for page_num in page_range:
print "Page:", page_num
page = p.page(page_num)
for item in page.object_list:
#print item["item_name_suggest"]
print item._score, item["item_id"], item["item_name"]
break
print "------------------"
def search_result(
request): #TODO: change result div name, change hard coded url.
query = request.GET.get('query')
profile_type_filter = request.GET.get('profile_type_filter')
lat = float(request.GET.get('lat')) if request.GET.get('lat') else None
lng = float(request.GET.get('lng')) if request.GET.get('lng') else None
distance = request.GET.get('distance')
if request.user.is_authenticated():
user_profile_obj = UserProfile()
user_profile = user_profile_obj.get_profile_by_id(str(request.user.id))
if user_profile.get('latlng') and user_profile.get('latlng').get(
'coordinates'):
lat = user_profile.get('latlng').get('coordinates')[1]
lng = user_profile.get('latlng').get('coordinates')[0]
distance = '500km'
sqs = S().indexes(
settings.ES_INDEX).query(_all__fuzzy=query).facet('sign_up_as')
if lat and lng:
sqs = sqs.filter(latlng__distance=(distance, lat, lng))
if profile_type_filter:
sqs = sqs.filter(sign_up_as=profile_type_filter)
sqs = sqs[:50]
html = render_to_string('_partials/search_result.html', {'sqs': sqs},
context_instance=RequestContext(request))
objs = list(
sqs.values_dict('id', 'name', 'profile_img', 'description', 'latlng',
'username'))
facets = {} #TODO: check why this iteration is required
for k, v in sqs.facet_counts().items():
facets[k] = {'total': v['total'], 'terms': v['terms']}
return HttpResponse(
json.dumps({
'html': html,
'objs': objs,
'facets': facets
}))
def test_object(self):
s = S(FakeMappingType).query(name__prefix='odin')
eq_(len(s), 1)
eq_(s[0].object.id, 1)
def test_values_list_no_args(self):
"""Calling values_list() with no args fetches all fields."""
eq_(S().query(fld1=2)
.values_list()
.build_search(),
{'query': {"term": {"fld1": 2}}, 'fields': ['*']})
def test_typed_s_get_doctypes(self):
eq_(S(FakeMappingType).get_doctypes(), ['doctype123'])
def test_bad_search(self):
with self.assertRaises(BadSearch):
len(S().doctypes('abc'))
def test_untyped_s_get_doctypes(self):
eq_(S().get_doctypes(), DEFAULT_DOCTYPES)
eq_(S().doctypes('abc').get_doctypes(), ['abc'])
def check_to_python(obj, expected):
eq_(S().to_python(obj), expected)
def test_delete(self):
a = self.test_index()
a.delete()
self.es.refresh()
eq_(len(S('chocolate')), 0, 'We deleted this... WTF?')
def test_untyped_s_get_indexes(self):
eq_(S().get_indexes(), DEFAULT_INDEXES)
eq_(S().indexes('abc').get_indexes(), ['abc'])
}]
es.bulk_index(INDEX, DOCTYPE, documents, id_field='id')
# ElasticSearch will refresh the indexes and make those documents
# available for querying in a second or so (it's configurable in
# ElasticSearch), but we want them available right now, so we refresh
# the index.
es.refresh(INDEX)
# Ok. We've created an index and tossed some stuff in it. Let's
# do some basic queries.
# Let's build a basic S that looks at the right instance of
# ElasticSearch, index, and doctype.
basic_s = S().es(urls=[URL]).indexes(INDEX).doctypes(DOCTYPE)
# How many documents are in our index?
print basic_s.count()
# Prints:
# 5
# Let's get all the cookie articles.
print[item['title'] for item in basic_s.query(title__text='cookie')]
# Prints:
# [u'Deleting cookies', u'What is a cookie?',
# u'Websites say cookies are blocked - Unblock them']
# Let's see cookie articles for websites.
print[
item['title']
def test_typed_s_get_indexes(self):
eq_(S(FakeMappingType).get_indexes(), ['index123'])
