Exemple #1
0
    def build(self):
        index_col = \
            IndexProfileCollection(
                self._database[IndexProfileCollection.get_collection_name()])
        query_col = \
            QueryProfileCollection(
                self._database[QueryProfileCollection.get_collection_name()])
        query = {} if self._session is None else {'session': self._session}
        for doc in index_col.find_iter(query):
            if any([r.match(doc['index']) for r in _INDEXES_TO_SKIP]):
                continue
            logging.debug('working on index %s in collection %s' %
                          (doc['index'], doc['collection']))
            index_name = doc['index'].split()[1]
            try:
                stats = self._current_indexes[doc['collection']].get('__stats',
                                                                     None)
                index = self._current_indexes[doc['collection']][index_name]
                index['query_count'] = len(doc['queries'])
                index['used_count'] = sum([q['count'] for q in doc['queries']])
                index['queries'] = dict([(q['query'], {}) for q in doc['queries']])
                if stats is not None:
                    index['total_size'] = stats['indexSizes'][index_name]
                    try:
                        index['index_size_ratio'] = (float(index['total_size']) /
                                                     stats['totalIndexSize'])
                    except ZeroDivisionError:
                        index['index_size_ratio'] = float(index['total_size'])
                    try:
                        index['collection_size_ratio'] = \
                            float(index['total_size']) / stats['size']
                    except ZeroDivisionError:
                        index['collection_size_ratio'] = float(index['total_size'])
                    # FIXME: make this more meaningful
                    index['removal_score'] = index['index_size_ratio'] * \
                                             index['collection_size_ratio']

                for q in doc['queries']:
                    logging.debug('gathering query informatin for index %s, ' \
                                  'query %s' % (doc['index'], q['query']))
                    query = {'query': q['query'],
                             'collection': doc['collection'],
                             'explain.cursor': doc['index']}
                    if self._session is not None:
                        query.update({'session': self._session})
                    for query_doc in query_col.find_iter(query):
                        queries = \
                            self._current_indexes[doc['collection']][index_name]['queries']
                        if query_doc['source'] not in queries[q['query']]:
                            queries[q['query']][query_doc['source']] = 0
                        queries[q['query']][query_doc['source']] += 1

            except KeyError:
                logging.warning('skipping index %s on collection %s:\n%s' %
                                (index_name, doc['collection'],
                                 traceback.format_exc()))
Exemple #2
0
    def build(self):
        index_col = \
            IndexProfileCollection(
                self._database[IndexProfileCollection.get_collection_name()])
        query_col = \
            QueryProfileCollection(
                self._database[QueryProfileCollection.get_collection_name()])
        query = {} if self._session is None else {'session': self._session}
        for doc in index_col.find_iter(query):
            if any([r.match(doc['index']) for r in _INDEXES_TO_SKIP]):
                continue
            logging.debug('working on index %s in collection %s' %
                          (doc['index'], doc['collection']))
            index_name = doc['index'].split()[1]
            try:
                stats = self._current_indexes[doc['collection']].get('__stats',
                                                                     None)
                index = self._current_indexes[doc['collection']][index_name]
                index['query_count'] = len(doc['queries'])
                index['used_count'] = sum([q['count'] for q in doc['queries']])
                index['queries'] = dict([(q['query'], {}) for q in doc['queries']])
                if stats is not None:
                    index['total_size'] = stats['indexSizes'][index_name]
                    try:
                        index['index_size_ratio'] = (float(index['total_size']) /
                                                     stats['totalIndexSize'])
                    except ZeroDivisionError:
                        index['index_size_ratio'] = float(index['total_size'])
                    try:
                        index['collection_size_ratio'] = \
                            float(index['total_size']) / stats['size']
                    except ZeroDivisionError:
                        index['collection_size_ratio'] = float(index['total_size'])
                    # FIXME: make this more meaningful
                    index['removal_score'] = index['index_size_ratio'] * \
                                             index['collection_size_ratio']

                for q in doc['queries']:
                    logging.debug('gathering query informatin for index %s, ' \
                                  'query %s' % (doc['index'], q['query']))
                    query = {'query': q['query'],
                             'collection': doc['collection'],
                             'explain.cursor': doc['index']}
                    if self._session is not None:
                        query.update({'session': self._session})
                    for query_doc in query_col.find_iter(query):
                        queries = \
                            self._current_indexes[doc['collection']][index_name]['queries']
                        if query_doc['source'] not in queries[q['query']]:
                            queries[q['query']][query_doc['source']] = 0
                        queries[q['query']][query_doc['source']] += 1

            except KeyError:
                logging.warning('skipping index %s on collection %s:\n%s' %
                                (index, doc['collection'],
                                 traceback.format_exc()))
Exemple #3
0
 def test_update_explain_logging(self):
     with instrument():
         ret = self.db.foo.update({'store': 'store_0'},
                                   {'$inc': {'sold': 1},
                                    '$inc': {'in_stock': -1}},
                                   multi=True)
         self.assertEqual(ret['n'], 100)
         self.assertEqual(len(self._msgs), 1)
     for msg in self._msgs:
         self._index_profile_sink.handle(msg, ('127.0.0.1', 65535))
         self._query_profile_sink.handle(msg, ('127.0.0.1', 65535))
     query_profile_col = QueryProfileCollection.get_collection_name()
     index_profile_col = IndexProfileCollection.get_collection_name()
     self.assertEqual(self.sink_db[query_profile_col].find().count(), 1)
     self.assertEqual(self.sink_db[index_profile_col].find().count(), 1)
Exemple #4
0
 def test_find_explain_logging(self):
     with instrument():
         count = self.db.foo.find({'sold': {'$gt': 100}}).count()
         self.assertEqual(count, 899)
         self.assertEqual(len(self._msgs), 2)
         count = self.db.foo.find().count()
         self.assertEqual(count, 1000)
         self.assertEqual(len(self._msgs), 4)
     for msg in self._msgs:
         self._index_profile_sink.handle(msg, ('127.0.0.1', 65535))
         self._query_profile_sink.handle(msg, ('127.0.0.1', 65535))
     query_profile_col = QueryProfileCollection.get_collection_name()
     index_profile_col = IndexProfileCollection.get_collection_name()
     self.assertEqual(self.sink_db[query_profile_col].find().count(), 2)
     self.assertEqual(self.sink_db[index_profile_col].find().count(), 1)