def test_generate_query_analysis(self): analyzer = dex.Dex(TEST_URI, False, [], 0, True, 0)._query_analyzer analysis = analyzer._generate_query_analysis(self.parser.parse('{ a: null }'), 'db', 'collection') self.assertEqual(analysis['fieldCount'], 1) self.assertTrue(analysis['supported']) self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'a') self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'EQUIV') analysis = analyzer._generate_query_analysis(self.parser.parse('{ a: null , b: { $lt: 4 }}'), 'db', 'collection') self.assertEqual(analysis['fieldCount'], 2) self.assertTrue(analysis['supported']) self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'a') self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'EQUIV') self.assertEqual(analysis['analyzedFields'][1]['fieldName'], 'b') self.assertEqual(analysis['analyzedFields'][1]['fieldType'], 'RANGE') analysis = analyzer._generate_query_analysis(self.parser.parse('{$query: { a: null , b: { $lt: 4 }}, $orderby: {c: 1}}'), 'db', 'collection') self.assertEqual(analysis['fieldCount'], 3) self.assertTrue(analysis['supported']) self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'c') self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'SORT') self.assertEqual(analysis['analyzedFields'][1]['fieldName'], 'a') self.assertEqual(analysis['analyzedFields'][1]['fieldType'], 'EQUIV') self.assertEqual(analysis['analyzedFields'][2]['fieldName'], 'b') self.assertEqual(analysis['analyzedFields'][2]['fieldType'], 'RANGE') analysis = analyzer._generate_query_analysis(self.parser.parse('{$query: { a: null , b: { $lt: 4 }, d: {$near: [50, 50]}}, $orderby: {c: 1}}'), 'db', 'collection') self.assertEqual(analysis['fieldCount'], 4) self.assertFalse(analysis['supported']) self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'c') self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'SORT') self.assertEqual(analysis['analyzedFields'][1]['fieldName'], 'a') self.assertEqual(analysis['analyzedFields'][1]['fieldType'], 'EQUIV') self.assertEqual(analysis['analyzedFields'][2]['fieldName'], 'b') self.assertEqual(analysis['analyzedFields'][2]['fieldType'], 'RANGE') self.assertEqual(analysis['analyzedFields'][3]['fieldName'], 'd') self.assertEqual(analysis['analyzedFields'][3]['fieldType'], 'UNSUPPORTED')
def test_report_aggregation(self): test_dex = dex.Dex(TEST_URI, True, [], 0, True, 0) report = test_dex._report._reports test_query = "{ simpleUnindexedField: null }" result = test_dex.generate_query_report(None, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) test_dex._report.add_query_occurrence(result) self.assertEqual(len(report), 1) self.assertEqual(report[0]['stats']['count'], 1) test_query = "{ $query: {}, $orderby: { simpleUnindexedField: null }}" result = test_dex.generate_query_report(None, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) test_dex._report.add_query_occurrence(result) self.assertEqual(len(report), 2) test_query = "{ anotherUnindexedField: null }" result = test_dex.generate_query_report(None, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) #adding twice for a double query test_dex._report.add_query_occurrence(result) test_dex._report.add_query_occurrence(result) self.assertEqual(len(report), 3)
def test_sort_ordering(self): test_dex = dex.Dex(TEST_URI, True, [], 0, True, 0) report = test_dex._report._reports test_query = "{ $query: {}, $orderby: { simpleUnindexedField: null, simpleUnindexedFieldTwo: null }}" result = test_dex.generate_query_report(None, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1, "simpleUnindexedFieldTwo": 1}') test_query = "{ $query: {}, $orderby: { simpleUnindexedFieldTwo: null, simpleUnindexedFieldOne: null }}" result = test_dex.generate_query_report(None, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedFieldTwo": 1, "simpleUnindexedFieldOne": 1}')
def database_dex_analyze_view(self, request, database_id): database = Database.objects.get(id=database_id) uri = 'mongodb://{}:{}@{}:{}/admin'.format(database.databaseinfra.user, database.databaseinfra.password, database.databaseinfra.instances.all()[0].address, #'10.236.1.15', database.databaseinfra.instances.all()[0].port) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() md = dex.Dex(db_uri = uri, verbose=False, namespaces_list = [], slowms=0, check_indexes=True, timeout=0) md.analyze_profile() sys.stdout = old_stdout dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace("'", "\"").replace("&&","'")) import ast final_mask = """<div>""" for result in dexanalyzer['results']: final_mask += "<h3> Collection: " + result['namespace']+ "</h3>" final_mask += \ """<li> Query: """ +\ str(ast.literal_eval(result['queryMask'])['$query']) +\ """</li>""" +\ """<li> Index: """+\ result['recommendation']['index']+\ """</li>"""+\ """<li> Command: """+\ result['recommendation']['shellCommand']+\ """</li>""" final_mask += """<br>""" final_mask += """</ul> </div>""" return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))
def test_report_aggregation(self): test_dex = dex.Dex(TEST_URI, True, []) report = test_dex._full_report._reports test_query = "{ query: { simpleUnindexedField: null }, " \ "ns: 'dex_test.test_collection', 'millis': 150}" result = test_dex.analyze_query(None, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) test_dex._full_report.add_report(result) self.assertEqual(len(report), 1) self.assertEqual(report[0]['queryCount'], 1) self.assertEqual(len(report[0]['queriesCovered']), 1) test_query = "{ query: {}, orderby: { simpleUnindexedField: null }," \ "ns: 'dex_test.test_collection', 'millis': 50}" result = test_dex.analyze_query(None, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) test_dex._full_report.add_report(result) self.assertEqual(len(report), 1) self.assertEqual(report[0]['queryCount'], 2) self.assertEqual(len(report[0]['queriesCovered']), 2) totalAvg = report[0]['avgTimeMillis'] self.assertEqual(totalAvg, 100) test_query = "{ query: { anotherUnindexedField: null }, "\ "ns: 'dex_test.test_collection', 'millis': 100}" result = test_dex.analyze_query(None, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) #adding twice for a double query test_dex._full_report.add_report(result) test_dex._full_report.add_report(result) self.assertEqual(len(report), 2) self.assertEqual(report[0]['queryCount'], 2) self.assertEqual(len(report[0]['queriesCovered']), 2) self.assertEqual(report[1]['queryCount'], 2) self.assertEqual(len(report[1]['queriesCovered']), 1) totalAvg = report[0]['avgTimeMillis'] self.assertEqual(totalAvg, 100)
def database_dex_analyze_view(self, request, database_id): import os import string def generate_random_string(length, stringset=string.ascii_letters + string.digits): return ''.join([stringset[i % len(stringset)] for i in [ord(x) for x in os.urandom(length)]]) database = Database.objects.get(id=database_id) if database.status != Database.ALIVE or not database.database_status.is_alive: self.message_user( request, "Database is not alive cannot be analyzed", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) if database.is_being_used_elsewhere(): self.message_user( request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) parsed_logs = '' arq_path = Configuration.get_by_name( 'database_clone_dir') + '/' + database.name + generate_random_string(20) + '.txt' arq = open(arq_path, 'w') arq.write(parsed_logs) arq.close() uri = 'mongodb://{}:{}@{}:{}/admin'.format(database.databaseinfra.user, database.databaseinfra.password, database.databaseinfra.instances.all()[ 0].address, database.databaseinfra.instances.all()[0].port) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() md = dex.Dex(db_uri=uri, verbose=False, namespaces_list=[], slowms=0, check_indexes=True, timeout=0) md.analyze_logfile(arq_path) sys.stdout = old_stdout dexanalyzer = loads( mystdout.getvalue().replace("\"", "&&").replace("'", "\"").replace("&&", "'")) os.remove(arq_path) import ast final_mask = """<div>""" for result in dexanalyzer['results']: final_mask += "<h3> Collection: " + result['namespace'] + "</h3>" final_mask += \ """<li> Query: """ +\ str(ast.literal_eval(result['queryMask'])['$query']) +\ """</li>""" +\ """<li> Index: """ +\ result['recommendation']['index'] +\ """</li>""" +\ """<li> Command: """ +\ result['recommendation']['shellCommand'] +\ """</li>""" final_mask += """<br>""" final_mask += """</ul> </div>""" return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))
arq = open(arq_path, 'w') arq.write(parsed_logs) arq.close() uri = 'mongodb://{}:{}@{}:{}/admin'.format( database.databaseinfra.user, database.databaseinfra.password, database.databaseinfra.instances.all()[0].address, database.databaseinfra.instances.all()[0].port) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() md = dex.Dex(db_uri=uri, verbose=False, namespaces_list=[], slowms=0, check_indexes=True, timeout=0) md.analyze_logfile(arq_path) sys.stdout = old_stdout dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace( "'", "\"").replace("&&", "'")) os.remove(arq_path) import ast final_mask = """<div>"""
def database_dex_analyze_view(self, request, database_id): if database.status != Database.ALIVE or not database.database_status.is_alive: self.message_user(request, "Database is not alive cannot be analyzed", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) if database.is_beeing_used_elsewhere(): self.message_user( request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) database = Database.objects.get(id=database_id) uri = 'mongodb://{}:{}@{}:{}/admin'.format( database.databaseinfra.user, database.databaseinfra.password, database.databaseinfra.instances.all()[0].address, database.databaseinfra.instances.all()[0].port) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() md = dex.Dex(db_uri=uri, verbose=False, namespaces_list=[], slowms=0, check_indexes=True, timeout=0) md.analyze_profile() sys.stdout = old_stdout dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace( "'", "\"").replace("&&", "'")) import ast final_mask = """<div>""" for result in dexanalyzer['results']: final_mask += "<h3> Collection: " + result['namespace'] + "</h3>" final_mask += \ """<li> Query: """ +\ str(ast.literal_eval(result['queryMask'])['$query']) +\ """</li>""" +\ """<li> Index: """ +\ result['recommendation']['index'] +\ """</li>""" +\ """<li> Command: """ +\ result['recommendation']['shellCommand'] +\ """</li>""" final_mask += """<br>""" final_mask += """</ul> </div>""" return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))
def test_analyze_query(self): test_dex = dex.Dex(TEST_URI, False, [], 0, True, 0) test_query = "{ simpleUnindexedField: null }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1}') test_query = "{ simpleIndexedField: null }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None, pretty_json(result)) test_query = "{ simpleUnindexedField: {$lt: 4}}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1}') test_query = "{ simpleIndexedField: { $lt: 4 }}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ $query: {}, $orderby: { simpleUnindexedField }}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1}') test_query = "{ $query: {}, $orderby: { simpleIndexedField: 1 }}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{complexUnindexedFieldOne: null, complexUnindexedFieldTwo: null }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"complexUnindexedFieldOne": 1, "complexUnindexedFieldTwo": 1}') test_query = "{ complexIndexedFieldOne: null, complexIndexedFieldTwo: null }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) self.assertEqual(result['indexAnalysis']['fullIndexes'][0]['index']['key'], [('complexIndexedFieldOne', -1), ('complexIndexedFieldTwo', -1), ('complexIndexedFieldThree', -1)]) test_query = "{ complexUnindexedFieldOne: null, complexUnindexedFieldTwo: { $lt: 4 }}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"complexUnindexedFieldOne": 1, "complexUnindexedFieldTwo": 1}') test_query = "{ complexIndexedFieldOne: null, complexIndexedFieldTwo: { $lt: 4 } }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None, pretty_json(result)) test_query = "{ complexIndexedFieldNine: null, complexIndexedFieldTen: { $lt: 4 } }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"complexIndexedFieldNine": 1, "complexIndexedFieldTen": 1}') test_query = "{ $query: {complexUnindexedFieldOne: null}, $orderby: { complexUnindexedFieldTwo: 1 } }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"complexUnindexedFieldOne": 1, "complexUnindexedFieldTwo": 1}') test_query = "{ $query: {complexIndexedFieldOne: null}, $orderby: { complexIndexedFieldTwo: 1 } }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ $query: {complexIndexedFieldTen: {$lt: 4}}, $orderby: { complexIndexedFieldNine: 1 } }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"complexIndexedFieldNine": 1, "complexIndexedFieldTen": 1}') test_query = "{ $query: {complexIndexedFieldThree: null, complexIndexedFieldTwo: {$lt: 4}}, $orderby: { complexIndexedFieldOne: 1 }}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], '{"complexIndexedFieldThree": 1, "complexIndexedFieldOne": 1, "complexIndexedFieldTwo": 1}') test_query = "{ $query: {complexIndexedFieldOne: null, complexIndexedFieldThree: {$lt: 4}}, $orderby: { complexIndexedFieldTwo: 1 } }" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ $query: { $or: [ { orFieldOne: { $lt: 4 } }, {orFieldTwo: { $gt: 5 } }], complexUnindexedFieldOne: 'A'}, $orderby: { _id: 1 }}" result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ geoIndexedFieldOne: { $near: [50, 50] } } " result = test_dex.generate_query_report(TEST_URI, self.parser.parse(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None)
def test_generate_index_report(self): analyzer = dex.Dex(TEST_URI, False, [], 0, True, 0)._query_analyzer index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)], "v": 1} analysis = {'supported': True, 'analyzedFields': [{'fieldName': 'simpleUnindexedField', 'fieldType': 'EQUIV'}], 'fieldCount': 1} report = analyzer._generate_index_report(index, analysis) self.assertEqual(report['queryFieldsCovered'], 0) self.assertEqual(report['index'], index) self.assertEqual(report['coverage'], 'none') index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)], "v": 1} analysis = {'supported': True, 'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'EQUIV'}], 'fieldCount': 1} report = analyzer._generate_index_report(index, analysis) self.assertEqual(report['queryFieldsCovered'], 0) self.assertEqual(report['index'], index) self.assertEqual(report['coverage'], 'none') index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)], "v": 1} analysis = {'supported': True, 'analyzedFields': [{'fieldName': 'complexIndexedFieldOne', 'fieldType': 'EQUIV'}], 'fieldCount': 1} report = analyzer._generate_index_report(index, analysis) self.assertEqual(report['queryFieldsCovered'], 1) self.assertEqual(report['index'], index) self.assertEqual(report['coverage'], 'full') index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)], "v": 1} analysis = {'supported': True, 'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'EQUIV'}, {'fieldName': 'complexIndexedFieldOne', 'fieldType': 'RANGE'}], 'fieldCount': 2} report = analyzer._generate_index_report(index, analysis) self.assertEqual(report['queryFieldsCovered'], 2) self.assertEqual(report['index'], index) self.assertEqual(report['coverage'], 'full') self.assertFalse(report['idealOrder']) index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)], "v": 1} analysis = {'supported': True, 'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'RANGE'}, {'fieldName': 'complexIndexedFieldOne', 'fieldType': 'EQUIV'}], 'fieldCount': 2} report = analyzer._generate_index_report(index, analysis) self.assertEqual(report['queryFieldsCovered'], 2) self.assertEqual(report['index'], index) self.assertEqual(report['coverage'], 'full') self.assertTrue(report['idealOrder']) index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)], "v": 1} analysis = {'supported': True, 'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'RANGE'}, {'fieldName': 'complexIndexedFieldOne', 'fieldType': 'SORT'}], 'fieldCount': 2} report = analyzer._generate_index_report(index, analysis) self.assertEqual(report['queryFieldsCovered'], 2) self.assertEqual(report['index'], index) self.assertEqual(report['coverage'], 'full') self.assertTrue(report['idealOrder'])
def test_analyze_query(self): test_dex = dex.Dex(TEST_URI, False, []) test_query = "{ query: { simpleUnindexedField: null }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], "{'simpleUnindexedField': 1}") test_query = "{ query: { simpleIndexedField: null }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None, dex.pretty_json(result)) test_query = "{ query: { simpleUnindexedField: {$lt: 4}}, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], "{'simpleUnindexedField': 1}") test_query = "{ query: { simpleIndexedField: { $lt: 4 } }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ query: {}, orderby: { simpleUnindexedField }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation']['index'], "{'simpleUnindexedField': 1}") test_query = "{ query: {}, orderby: { simpleIndexedField: 1 }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ query: {complexUnindexedFieldOne: null, complexUnindexedFieldTwo: null }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual( result['recommendation']['index'], "{'complexUnindexedFieldOne': 1, 'complexUnindexedFieldTwo': 1}") test_query = "{ query: { complexIndexedFieldOne: null, complexIndexedFieldTwo: null }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) self.assertEqual( result['indexAnalysis']['fullIndexes'][0]['index']['key'], [('complexIndexedFieldOne', -1), ('complexIndexedFieldTwo', -1), ('complexIndexedFieldThree', -1)]) test_query = "{ query: { complexUnindexedFieldOne: null, complexUnindexedFieldTwo: { $lt: 4 }}, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual( result['recommendation']['index'], "{'complexUnindexedFieldOne': 1, 'complexUnindexedFieldTwo': 1}") test_query = "{ query: { complexIndexedFieldOne: null, complexIndexedFieldTwo: { $lt: 4 } }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None, dex.pretty_json(result)) test_query = "{ query: { complexIndexedFieldNine: null, complexIndexedFieldTen: { $lt: 4 } }, ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual( result['recommendation']['index'], "{'complexIndexedFieldNine': 1, 'complexIndexedFieldTen': 1}") test_query = "{ query: {complexUnindexedFieldOne: null}, orderby: { complexUnindexedFieldTwo: 1 } , ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual( result['recommendation']['index'], "{'complexUnindexedFieldOne': 1, 'complexUnindexedFieldTwo': 1}") test_query = "{ query: {complexIndexedFieldOne: null}, orderby: { complexIndexedFieldTwo: 1 } , ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ query: {complexIndexedFieldTen: {$lt: 4}}, orderby: { complexIndexedFieldNine: 1 } , ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual( result['recommendation']['index'], "{'complexIndexedFieldNine': 1, 'complexIndexedFieldTen': 1}") test_query = "{ query: {complexIndexedFieldThree: null, complexIndexedFieldTwo: {$lt: 4}}, orderby: { complexIndexedFieldOne: 1 } , ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual( result['recommendation']['index'], "{'complexIndexedFieldThree': 1, 'complexIndexedFieldOne': 1, 'complexIndexedFieldTwo': 1}" ) test_query = "{ query: {complexIndexedFieldOne: null, complexIndexedFieldThree: {$lt: 4}}, orderby: { complexIndexedFieldTwo: 1 } , ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ query: { $or: [ { orFieldOne: { $lt: 4 } }, {orFieldTwo: { $gt: 5 } }], complexUnindexedFieldOne: 'A'}, orderby: { _id: 1 }, ns: 'dex_test.test_collection' }" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None) test_query = "{ query: { geoIndexedFieldOne: { $near: [50, 50] } } , ns: 'dex_test.test_collection'}" result = test_dex.analyze_query(TEST_URI, yaml.load(test_query), TEST_DBNAME, TEST_COLLECTION) self.assertEqual(result['recommendation'], None)
def database_dex_analyze_view(self, request, database_id): import json import random from dbaas_laas.provider import LaaSProvider from util import get_credentials_for from util.laas import get_group_name from dbaas_credentials.models import CredentialType import os import string from datetime import datetime, timedelta def generate_random_string(length, stringset=string.ascii_letters + string.digits): return ''.join([ stringset[i % len(stringset)] for i in [ord(x) for x in os.urandom(length)] ]) database = Database.objects.get(id=database_id) if database.status != Database.ALIVE or not database.database_status.is_alive: self.message_user(request, "Database is not alive cannot be analyzed", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) if database.is_beeing_used_elsewhere(): self.message_user( request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) credential = get_credentials_for(environment=database.environment, credential_type=CredentialType.LAAS) db_name = database.name environment = database.environment endpoint = credential.endpoint username = credential.user password = credential.password lognit_environment = credential.get_parameter_by_name( 'lognit_environment') provider = LaaSProvider() group_name = get_group_name(database) today = (datetime.now()).strftime('%Y%m%d') yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d') uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format( group_name, yesterday, today) parsed_logs = '' database_logs = provider.get_logs_for_group(environment, lognit_environment, uri) try: database_logs = json.loads(database_logs) except Exception as e: pass else: for database_log in database_logs: try: items = database_log['items'] except KeyError as e: pass else: parsed_logs = "\n".join( (item['message'] for item in items)) arq_path = Configuration.get_by_name( 'database_clone_dir' ) + '/' + database.name + generate_random_string(20) + '.txt' arq = open(arq_path, 'w') arq.write(parsed_logs) arq.close() uri = 'mongodb://{}:{}@{}:{}/admin'.format( database.databaseinfra.user, database.databaseinfra.password, database.databaseinfra.instances.all()[0].address, database.databaseinfra.instances.all()[0].port) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() md = dex.Dex(db_uri=uri, verbose=False, namespaces_list=[], slowms=0, check_indexes=True, timeout=0) md.analyze_logfile(arq_path) sys.stdout = old_stdout dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace( "'", "\"").replace("&&", "'")) os.remove(arq_path) import ast final_mask = """<div>""" print dexanalyzer['results'] for result in dexanalyzer['results']: final_mask += "<h3> Collection: " + result['namespace'] + "</h3>" final_mask += \ """<li> Query: """ +\ str(ast.literal_eval(result['queryMask'])['$query']) +\ """</li>""" +\ """<li> Index: """ +\ result['recommendation']['index'] +\ """</li>""" +\ """<li> Command: """ +\ result['recommendation']['shellCommand'] +\ """</li>""" final_mask += """<br>""" final_mask += """</ul> </div>""" return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))