def build(self):
index_col = \
IndexProfileCollection(
self._database[IndexProfileCollection.get_collection_name()])
query_col = \
QueryProfileCollection(
self._database[QueryProfileCollection.get_collection_name()])
query = {} if self._session is None else {'session': self._session}
for doc in index_col.find_iter(query):
if any([r.match(doc['index']) for r in _INDEXES_TO_SKIP]):
continue
logging.debug('working on index %s in collection %s' %
(doc['index'], doc['collection']))
index_name = doc['index'].split()[1]
try:
stats = self._current_indexes[doc['collection']].get('__stats',
None)
index = self._current_indexes[doc['collection']][index_name]
index['query_count'] = len(doc['queries'])
index['used_count'] = sum([q['count'] for q in doc['queries']])
index['queries'] = dict([(q['query'], {}) for q in doc['queries']])
if stats is not None:
index['total_size'] = stats['indexSizes'][index_name]
try:
index['index_size_ratio'] = (float(index['total_size']) /
stats['totalIndexSize'])
except ZeroDivisionError:
index['index_size_ratio'] = float(index['total_size'])
try:
index['collection_size_ratio'] = \
float(index['total_size']) / stats['size']
except ZeroDivisionError:
index['collection_size_ratio'] = float(index['total_size'])
# FIXME: make this more meaningful
index['removal_score'] = index['index_size_ratio'] * \
index['collection_size_ratio']
for q in doc['queries']:
logging.debug('gathering query informatin for index %s, ' \
'query %s' % (doc['index'], q['query']))
query = {'query': q['query'],
'collection': doc['collection'],
'explain.cursor': doc['index']}
if self._session is not None:
query.update({'session': self._session})
for query_doc in query_col.find_iter(query):
queries = \
self._current_indexes[doc['collection']][index_name]['queries']
if query_doc['source'] not in queries[q['query']]:
queries[q['query']][query_doc['source']] = 0
queries[q['query']][query_doc['source']] += 1
except KeyError:
logging.warning('skipping index %s on collection %s:\n%s' %
(index_name, doc['collection'],
traceback.format_exc()))
def build(self):
index_col = \
IndexProfileCollection(
self._database[IndexProfileCollection.get_collection_name()])
query_col = \
QueryProfileCollection(
self._database[QueryProfileCollection.get_collection_name()])
query = {} if self._session is None else {'session': self._session}
for doc in index_col.find_iter(query):
if any([r.match(doc['index']) for r in _INDEXES_TO_SKIP]):
continue
logging.debug('working on index %s in collection %s' %
(doc['index'], doc['collection']))
index_name = doc['index'].split()[1]
try:
stats = self._current_indexes[doc['collection']].get('__stats',
None)
index = self._current_indexes[doc['collection']][index_name]
index['query_count'] = len(doc['queries'])
index['used_count'] = sum([q['count'] for q in doc['queries']])
index['queries'] = dict([(q['query'], {}) for q in doc['queries']])
if stats is not None:
index['total_size'] = stats['indexSizes'][index_name]
try:
index['index_size_ratio'] = (float(index['total_size']) /
stats['totalIndexSize'])
except ZeroDivisionError:
index['index_size_ratio'] = float(index['total_size'])
try:
index['collection_size_ratio'] = \
float(index['total_size']) / stats['size']
except ZeroDivisionError:
index['collection_size_ratio'] = float(index['total_size'])
# FIXME: make this more meaningful
index['removal_score'] = index['index_size_ratio'] * \
index['collection_size_ratio']
for q in doc['queries']:
logging.debug('gathering query informatin for index %s, ' \
'query %s' % (doc['index'], q['query']))
query = {'query': q['query'],
'collection': doc['collection'],
'explain.cursor': doc['index']}
if self._session is not None:
query.update({'session': self._session})
for query_doc in query_col.find_iter(query):
queries = \
self._current_indexes[doc['collection']][index_name]['queries']
if query_doc['source'] not in queries[q['query']]:
queries[q['query']][query_doc['source']] = 0
queries[q['query']][query_doc['source']] += 1
except KeyError:
logging.warning('skipping index %s on collection %s:\n%s' %
(index, doc['collection'],
traceback.format_exc()))
def test_update_explain_logging(self):
with instrument():
ret = self.db.foo.update({'store': 'store_0'},
{'$inc': {'sold': 1},
'$inc': {'in_stock': -1}},
multi=True)
self.assertEqual(ret['n'], 100)
self.assertEqual(len(self._msgs), 1)
for msg in self._msgs:
self._index_profile_sink.handle(msg, ('127.0.0.1', 65535))
self._query_profile_sink.handle(msg, ('127.0.0.1', 65535))
query_profile_col = QueryProfileCollection.get_collection_name()
index_profile_col = IndexProfileCollection.get_collection_name()
self.assertEqual(self.sink_db[query_profile_col].find().count(), 1)
self.assertEqual(self.sink_db[index_profile_col].find().count(), 1)
def test_find_explain_logging(self):
with instrument():
count = self.db.foo.find({'sold': {'$gt': 100}}).count()
self.assertEqual(count, 899)
self.assertEqual(len(self._msgs), 2)
count = self.db.foo.find().count()
self.assertEqual(count, 1000)
self.assertEqual(len(self._msgs), 4)
for msg in self._msgs:
self._index_profile_sink.handle(msg, ('127.0.0.1', 65535))
self._query_profile_sink.handle(msg, ('127.0.0.1', 65535))
query_profile_col = QueryProfileCollection.get_collection_name()
index_profile_col = IndexProfileCollection.get_collection_name()
self.assertEqual(self.sink_db[query_profile_col].find().count(), 2)
self.assertEqual(self.sink_db[index_profile_col].find().count(), 1)
