def tearDown(self):
# Cleanup to remove these from the index.
self.app1.delete()
self.app2.delete()
unindex_webapps([self.app1.id, self.app2.id])
# Required to purge the suggestions data structure. In Lucene, a
# document is not deleted from a segment, just marked as deleted.
WebappIndexer.get_es().optimize(WebappIndexer.get_index(), only_expunge_deletes=True)
def tearDown(self):
# Cleanup to remove these from the index.
self.app1.delete()
self.app2.delete()
unindex_webapps([self.app1.id, self.app2.id])
# Required to purge the suggestions data structure. In Lucene, a
# document is not deleted from a segment, just marked as deleted.
WebappIndexer.get_es().optimize(WebappIndexer.get_index(),
only_expunge_deletes=True)
def index_webapps(ids, **kw):
task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))
index = kw.pop('index', WebappIndexer.get_index())
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = get_indices(index)
es = WebappIndexer.get_es(urls=settings.ES_URLS)
qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))
for obj in qs:
doc = WebappIndexer.extract_document(obj.id, obj)
for idx in indices:
WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)
def index_webapps(ids, **kw):
task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))
index = kw.pop('index', WebappIndexer.get_index())
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = get_indices(index)
es = WebappIndexer.get_es(urls=settings.ES_URLS)
qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))
for obj in qs:
doc = WebappIndexer.extract_document(obj.id, obj)
for idx in indices:
WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)
def unindex_webapps(ids, **kw):
task_log.info("Un-indexing apps %s-%s. [%s]" % (ids[0], ids[-1], len(ids)))
index = kw.pop("index", WebappIndexer.get_index())
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = get_indices(index)
es = WebappIndexer.get_es(urls=settings.ES_URLS)
for id_ in ids:
for idx in indices:
try:
WebappIndexer.unindex(id_=id_, es=es, index=idx)
except ElasticHttpNotFoundError:
# Ignore if it's not there.
task_log.info(u"[Webapp:%s] Unindexing app but not found in index" % id_)
def unindex_webapps(ids, **kw):
task_log.info('Un-indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))
index = kw.pop('index', WebappIndexer.get_index())
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = get_indices(index)
es = WebappIndexer.get_es(urls=settings.ES_URLS)
for id_ in ids:
for idx in indices:
try:
WebappIndexer.unindex(id_=id_, es=es, index=idx)
except ElasticHttpNotFoundError:
# Ignore if it's not there.
task_log.info(
u'[Webapp:%s] Unindexing app but not found in index' % id_)
def test_q_num_requests_no_results(self):
es = WebappIndexer.get_es()
orig_search = es.search
es.counter = 0
def monkey_search(*args, **kwargs):
es.counter += 1
return orig_search(*args, **kwargs)
es.search = monkey_search
res = self.client.get(self.url, data={'q': 'noresults'})
eq_(res.status_code, 200)
eq_(res.json['meta']['total_count'], 0)
eq_(len(res.json['objects']), 0)
# Verify only one search call was made.
eq_(es.counter, 1)
es.search = orig_search
def test_q_num_requests_no_results(self):
es = WebappIndexer.get_es()
orig_search = es.search
es.counter = 0
def monkey_search(*args, **kwargs):
es.counter += 1
return orig_search(*args, **kwargs)
es.search = monkey_search
res = self.client.get(self.url, data={'q': 'noresults'})
eq_(res.status_code, 200)
eq_(res.json['meta']['total_count'], 0)
eq_(len(res.json['objects']), 0)
# Verify only one search call was made.
eq_(es.counter, 1)
es.search = orig_search
def test_q_num_requests(self):
es = WebappIndexer.get_es()
orig_search = es.search
es.counter = 0
def monkey_search(*args, **kwargs):
es.counter += 1
return orig_search(*args, **kwargs)
es.search = monkey_search
res = self.client.get(self.url, data={'q': 'something'})
eq_(res.status_code, 200)
obj = res.json['objects'][0]
eq_(obj['slug'], self.webapp.app_slug)
# Verify only one search call was made.
eq_(es.counter, 1)
es.search = orig_search
def test_q_num_requests(self):
es = WebappIndexer.get_es()
orig_search = es.search
es.counter = 0
def monkey_search(*args, **kwargs):
es.counter += 1
return orig_search(*args, **kwargs)
es.search = monkey_search
res = self.client.get(self.url, data={'q': 'something'})
eq_(res.status_code, 200)
obj = res.json['objects'][0]
eq_(obj['slug'], self.webapp.app_slug)
# Verify only one search call was made.
eq_(es.counter, 1)
es.search = orig_search
def test_q_num_requests(self):
es = WebappIndexer.get_es()
orig_search = es.search
es.counter = 0
def monkey_search(*args, **kwargs):
es.counter += 1
return orig_search(*args, **kwargs)
es.search = monkey_search
res = self.client.get(self.url, data={"q": "something"})
eq_(res.status_code, 200)
eq_(res.json["meta"]["total_count"], 1)
eq_(len(res.json["objects"]), 1)
obj = res.json["objects"][0]
eq_(obj["slug"], self.webapp.app_slug)
# Verify only one search call was made.
eq_(es.counter, 1)
es.search = orig_search
def handle(self, *args, **kwargs):
index = WebappIndexer.get_index()
doctype = WebappIndexer.get_mapping_type_name()
es = WebappIndexer.get_es()
apps = Webapp.objects.values_list('id', flat=True)
missing_ids = []
for app in apps:
try:
res = es.get(index, doctype, app, fields='id')
except ElasticHttpNotFoundError:
# App doesn't exist in our index, add it to `missing_ids`.
missing_ids.append(app)
if missing_ids:
sys.stdout.write('Adding %s doc(s) to the index.'
% len(missing_ids))
index_webapps.delay(missing_ids)
else:
sys.stdout.write('No docs missing from index.')
def handle(self, *args, **kwargs):
index = WebappIndexer.get_index()
doctype = WebappIndexer.get_mapping_type_name()
es = WebappIndexer.get_es()
apps = Webapp.objects.values_list('id', flat=True)
missing_ids = []
for app in apps:
try:
res = es.get(index, doctype, app, fields='id')
except ElasticHttpNotFoundError:
# App doesn't exist in our index, add it to `missing_ids`.
missing_ids.append(app)
if missing_ids:
sys.stdout.write('Adding %s doc(s) to the index.' %
len(missing_ids))
index_webapps.delay(missing_ids)
else:
sys.stdout.write('No docs missing from index.')
def setUp(self):
super(TestFixupCommand, self).setUp()
self.index = WebappIndexer.get_index()
self.doctype = WebappIndexer.get_mapping_type_name()
self.es = WebappIndexer.get_es()
self.app = Webapp.objects.get(pk=337141)