def search(self):
self.reindex(Addon)
qs = Search(using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name())
return qs.filter('term', id=self.addon.pk).execute()[0]
def find_inconsistencies_between_es_and_db(ids, **kw):
length = len(ids)
log.info(
'Searching for inconsistencies between db and es %d-%d [%d].',
ids[0], ids[-1], length)
db_addons = Addon.unfiltered.in_bulk(ids)
es_addons = Search(
doc_type=AddonIndexer.get_doctype_name(),
index=AddonIndexer.get_index_alias(),
using=amo.search.get_es()).filter('ids', values=ids)[:length].execute()
es_addons = es_addons
db_len = len(db_addons)
es_len = len(es_addons)
if db_len != es_len:
log.info('Inconsistency found: %d in db vs %d in es.',
db_len, es_len)
for result in es_addons.hits.hits:
pk = result['_source']['id']
db_modified = db_addons[pk].modified.isoformat()
es_modified = result['_source']['modified']
if db_modified != es_modified:
log.info('Inconsistency found for addon %d: '
'modified is %s in db vs %s in es.',
pk, db_modified, es_modified)
db_status = db_addons[pk].status
es_status = result['_source']['status']
if db_status != es_status:
log.info('Inconsistency found for addon %d: '
'status is %s in db vs %s in es.',
pk, db_status, es_status)
def find_inconsistencies_between_es_and_db(ids, **kw):
length = len(ids)
log.info(
'Searching for inconsistencies between db and es %d-%d [%d].',
ids[0], ids[-1], length)
db_addons = Addon.unfiltered.in_bulk(ids)
es_addons = Search(
doc_type=AddonIndexer.get_doctype_name(),
index=AddonIndexer.get_index_alias(),
using=amo.search.get_es()).filter('ids', values=ids)[:length].execute()
es_addons = es_addons
db_len = len(db_addons)
es_len = len(es_addons)
if db_len != es_len:
log.info('Inconsistency found: %d in db vs %d in es.',
db_len, es_len)
for result in es_addons.hits.hits:
pk = result['_source']['id']
db_modified = db_addons[pk].modified.isoformat()
es_modified = result['_source']['modified']
if db_modified != es_modified:
log.info('Inconsistency found for addon %d: '
'modified is %s in db vs %s in es.',
pk, db_modified, es_modified)
db_status = db_addons[pk].status
es_status = result['_source']['status']
if db_status != es_status:
log.info('Inconsistency found for addon %d: '
'status is %s in db vs %s in es.',
pk, db_status, es_status)
def search(self):
self.reindex(Addon)
qs = Search(using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name())
return qs.filter('term', id=self.addon.pk).execute()[0]
def serialize(self):
self.reindex(Addon)
qs = Search(using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name())
obj = qs.filter('term', id=self.addon.pk).execute()[0]
with self.assertNumQueries(0):
serializer = ESAddonSerializer(context={'request': self.request})
result = serializer.to_representation(obj)
return result
def serialize(self):
self.reindex(Addon)
qs = Search(using=amo.search.get_es(),
index=AddonIndexer.get_index_alias(),
doc_type=AddonIndexer.get_doctype_name())
obj = qs.filter('term', id=self.addon.pk).execute()[0]
with self.assertNumQueries(0):
serializer = ESAddonSerializer(context={'request': self.request})
result = serializer.to_representation(obj)
return result
def unindex_addons(ids, **kw):
es = get_es()
for addon_id in ids:
log.info('Removing addon [%s] from search index.' % addon_id)
try:
es.delete(
AddonIndexer.get_index_alias(),
AddonIndexer.get_doctype_name(),
addon_id,
)
except TransportError:
# We ignore already deleted add-ons.
pass
def test_mapping(self):
"""Compare actual mapping in ES with the one the indexer returns, once
an object has been indexed.
We don't want dynamic mapping for addons (too risky), so the two
mappings should be equal."""
self.reindex(Addon)
indexer = AddonIndexer()
doc_name = indexer.get_doctype_name()
real_index_name = self.get_index_name(SearchMixin.ES_ALIAS_KEY)
mappings = self.es.indices.get_mapping(
indexer.get_index_alias())[real_index_name]['mappings']
actual_properties = mappings[doc_name]['properties']
indexer_properties = indexer.get_mapping()[doc_name]['properties']
assert set(actual_properties.keys()) == set(indexer_properties.keys())
def test_mapping(self):
"""Compare actual mapping in ES with the one the indexer returns, once
an object has been indexed.
We don't want dynamic mapping for addons (too risky), so the two
mappings should be equal."""
self.reindex(Addon)
indexer = AddonIndexer()
doc_name = indexer.get_doctype_name()
real_index_name = self.index_names[SearchMixin.ES_ALIAS_KEY]
mappings = self.es.indices.get_mapping(
indexer.get_index_alias())[real_index_name]['mappings']
actual_properties = mappings[doc_name]['properties']
indexer_properties = indexer.get_mapping()[doc_name]['properties']
assert set(actual_properties.keys()) == set(indexer_properties.keys())
def get_bayesian_rating(self):
qs = Search(using=get_es(),
index=AddonIndexer.get_index_alias()).filter(
Q('term', id=self.addon.pk))
return qs.execute()[0]['bayesian_rating']
