Beispiel #1
0
 def tearDown(self):
     # Cleanup to remove these from the index.
     self.app1.delete()
     self.app2.delete()
     unindex_webapps([self.app1.id, self.app2.id])
     # Required to purge the suggestions data structure. In Lucene, a
     # document is not deleted from a segment, just marked as deleted.
     WebappIndexer.get_es().optimize(WebappIndexer.get_index(), only_expunge_deletes=True)
Beispiel #2
0
 def tearDown(self):
     # Cleanup to remove these from the index.
     self.app1.delete()
     self.app2.delete()
     unindex_webapps([self.app1.id, self.app2.id])
     # Required to purge the suggestions data structure. In Lucene, a
     # document is not deleted from a segment, just marked as deleted.
     WebappIndexer.get_es().optimize(WebappIndexer.get_index(),
                                     only_expunge_deletes=True)
Beispiel #3
0
def index_webapps(ids, **kw):
    task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))

    index = kw.pop('index', WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))
    for obj in qs:
        doc = WebappIndexer.extract_document(obj.id, obj)
        for idx in indices:
            WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)
Beispiel #4
0
def index_webapps(ids, **kw):
    task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))

    index = kw.pop('index', WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))
    for obj in qs:
        doc = WebappIndexer.extract_document(obj.id, obj)
        for idx in indices:
            WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)
Beispiel #5
0
def unindex_webapps(ids, **kw):
    task_log.info("Un-indexing apps %s-%s. [%s]" % (ids[0], ids[-1], len(ids)))

    index = kw.pop("index", WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    for id_ in ids:
        for idx in indices:
            try:
                WebappIndexer.unindex(id_=id_, es=es, index=idx)
            except ElasticHttpNotFoundError:
                # Ignore if it's not there.
                task_log.info(u"[Webapp:%s] Unindexing app but not found in index" % id_)
Beispiel #6
0
def unindex_webapps(ids, **kw):
    task_log.info('Un-indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))

    index = kw.pop('index', WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    for id_ in ids:
        for idx in indices:
            try:
                WebappIndexer.unindex(id_=id_, es=es, index=idx)
            except ElasticHttpNotFoundError:
                # Ignore if it's not there.
                task_log.info(
                    u'[Webapp:%s] Unindexing app but not found in index' % id_)
Beispiel #7
0
    def test_q_num_requests_no_results(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={'q': 'noresults'})
        eq_(res.status_code, 200)
        eq_(res.json['meta']['total_count'], 0)
        eq_(len(res.json['objects']), 0)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search
Beispiel #8
0
    def test_q_num_requests_no_results(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={'q': 'noresults'})
        eq_(res.status_code, 200)
        eq_(res.json['meta']['total_count'], 0)
        eq_(len(res.json['objects']), 0)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search
Beispiel #9
0
    def test_q_num_requests(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={'q': 'something'})
        eq_(res.status_code, 200)
        obj = res.json['objects'][0]
        eq_(obj['slug'], self.webapp.app_slug)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search
Beispiel #10
0
    def test_q_num_requests(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={'q': 'something'})
        eq_(res.status_code, 200)
        obj = res.json['objects'][0]
        eq_(obj['slug'], self.webapp.app_slug)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search
Beispiel #11
0
    def test_q_num_requests(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={"q": "something"})
        eq_(res.status_code, 200)
        eq_(res.json["meta"]["total_count"], 1)
        eq_(len(res.json["objects"]), 1)
        obj = res.json["objects"][0]
        eq_(obj["slug"], self.webapp.app_slug)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search
Beispiel #12
0
    def handle(self, *args, **kwargs):
        index = WebappIndexer.get_index()
        doctype = WebappIndexer.get_mapping_type_name()
        es = WebappIndexer.get_es()

        apps = Webapp.objects.values_list('id', flat=True)

        missing_ids = []

        for app in apps:
            try:
                res = es.get(index, doctype, app, fields='id')
            except ElasticHttpNotFoundError:
                # App doesn't exist in our index, add it to `missing_ids`.
                missing_ids.append(app)

        if missing_ids:
            sys.stdout.write('Adding %s doc(s) to the index.'
                             % len(missing_ids))
            index_webapps.delay(missing_ids)
        else:
            sys.stdout.write('No docs missing from index.')
Beispiel #13
0
    def handle(self, *args, **kwargs):
        index = WebappIndexer.get_index()
        doctype = WebappIndexer.get_mapping_type_name()
        es = WebappIndexer.get_es()

        apps = Webapp.objects.values_list('id', flat=True)

        missing_ids = []

        for app in apps:
            try:
                res = es.get(index, doctype, app, fields='id')
            except ElasticHttpNotFoundError:
                # App doesn't exist in our index, add it to `missing_ids`.
                missing_ids.append(app)

        if missing_ids:
            sys.stdout.write('Adding %s doc(s) to the index.' %
                             len(missing_ids))
            index_webapps.delay(missing_ids)
        else:
            sys.stdout.write('No docs missing from index.')
Beispiel #14
0
 def setUp(self):
     super(TestFixupCommand, self).setUp()
     self.index = WebappIndexer.get_index()
     self.doctype = WebappIndexer.get_mapping_type_name()
     self.es = WebappIndexer.get_es()
     self.app = Webapp.objects.get(pk=337141)