def build(self): index_col = \ IndexProfileCollection( self._database[IndexProfileCollection.get_collection_name()]) query_col = \ QueryProfileCollection( self._database[QueryProfileCollection.get_collection_name()]) query = {} if self._session is None else {'session': self._session} for doc in index_col.find_iter(query): if any([r.match(doc['index']) for r in _INDEXES_TO_SKIP]): continue logging.debug('working on index %s in collection %s' % (doc['index'], doc['collection'])) index_name = doc['index'].split()[1] try: stats = self._current_indexes[doc['collection']].get('__stats', None) index = self._current_indexes[doc['collection']][index_name] index['query_count'] = len(doc['queries']) index['used_count'] = sum([q['count'] for q in doc['queries']]) index['queries'] = dict([(q['query'], {}) for q in doc['queries']]) if stats is not None: index['total_size'] = stats['indexSizes'][index_name] try: index['index_size_ratio'] = (float(index['total_size']) / stats['totalIndexSize']) except ZeroDivisionError: index['index_size_ratio'] = float(index['total_size']) try: index['collection_size_ratio'] = \ float(index['total_size']) / stats['size'] except ZeroDivisionError: index['collection_size_ratio'] = float(index['total_size']) # FIXME: make this more meaningful index['removal_score'] = index['index_size_ratio'] * \ index['collection_size_ratio'] for q in doc['queries']: logging.debug('gathering query informatin for index %s, ' \ 'query %s' % (doc['index'], q['query'])) query = {'query': q['query'], 'collection': doc['collection'], 'explain.cursor': doc['index']} if self._session is not None: query.update({'session': self._session}) for query_doc in query_col.find_iter(query): queries = \ self._current_indexes[doc['collection']][index_name]['queries'] if query_doc['source'] not in queries[q['query']]: queries[q['query']][query_doc['source']] = 0 queries[q['query']][query_doc['source']] += 1 except KeyError: logging.warning('skipping index %s on collection %s:\n%s' % (index_name, doc['collection'], traceback.format_exc()))
def build(self): index_col = \ IndexProfileCollection( self._database[IndexProfileCollection.get_collection_name()]) query_col = \ QueryProfileCollection( self._database[QueryProfileCollection.get_collection_name()]) query = {} if self._session is None else {'session': self._session} for doc in index_col.find_iter(query): if any([r.match(doc['index']) for r in _INDEXES_TO_SKIP]): continue logging.debug('working on index %s in collection %s' % (doc['index'], doc['collection'])) index_name = doc['index'].split()[1] try: stats = self._current_indexes[doc['collection']].get('__stats', None) index = self._current_indexes[doc['collection']][index_name] index['query_count'] = len(doc['queries']) index['used_count'] = sum([q['count'] for q in doc['queries']]) index['queries'] = dict([(q['query'], {}) for q in doc['queries']]) if stats is not None: index['total_size'] = stats['indexSizes'][index_name] try: index['index_size_ratio'] = (float(index['total_size']) / stats['totalIndexSize']) except ZeroDivisionError: index['index_size_ratio'] = float(index['total_size']) try: index['collection_size_ratio'] = \ float(index['total_size']) / stats['size'] except ZeroDivisionError: index['collection_size_ratio'] = float(index['total_size']) # FIXME: make this more meaningful index['removal_score'] = index['index_size_ratio'] * \ index['collection_size_ratio'] for q in doc['queries']: logging.debug('gathering query informatin for index %s, ' \ 'query %s' % (doc['index'], q['query'])) query = {'query': q['query'], 'collection': doc['collection'], 'explain.cursor': doc['index']} if self._session is not None: query.update({'session': self._session}) for query_doc in query_col.find_iter(query): queries = \ self._current_indexes[doc['collection']][index_name]['queries'] if query_doc['source'] not in queries[q['query']]: queries[q['query']][query_doc['source']] = 0 queries[q['query']][query_doc['source']] += 1 except KeyError: logging.warning('skipping index %s on collection %s:\n%s' % (index, doc['collection'], traceback.format_exc()))
def test_update_explain_logging(self): with instrument(): ret = self.db.foo.update({'store': 'store_0'}, {'$inc': {'sold': 1}, '$inc': {'in_stock': -1}}, multi=True) self.assertEqual(ret['n'], 100) self.assertEqual(len(self._msgs), 1) for msg in self._msgs: self._index_profile_sink.handle(msg, ('127.0.0.1', 65535)) self._query_profile_sink.handle(msg, ('127.0.0.1', 65535)) query_profile_col = QueryProfileCollection.get_collection_name() index_profile_col = IndexProfileCollection.get_collection_name() self.assertEqual(self.sink_db[query_profile_col].find().count(), 1) self.assertEqual(self.sink_db[index_profile_col].find().count(), 1)
def test_find_explain_logging(self): with instrument(): count = self.db.foo.find({'sold': {'$gt': 100}}).count() self.assertEqual(count, 899) self.assertEqual(len(self._msgs), 2) count = self.db.foo.find().count() self.assertEqual(count, 1000) self.assertEqual(len(self._msgs), 4) for msg in self._msgs: self._index_profile_sink.handle(msg, ('127.0.0.1', 65535)) self._query_profile_sink.handle(msg, ('127.0.0.1', 65535)) query_profile_col = QueryProfileCollection.get_collection_name() index_profile_col = IndexProfileCollection.get_collection_name() self.assertEqual(self.sink_db[query_profile_col].find().count(), 2) self.assertEqual(self.sink_db[index_profile_col].find().count(), 1)