def test_generate_query_analysis(self):
analyzer = dex.Dex(TEST_URI, False, [], 0, True, 0)._query_analyzer
analysis = analyzer._generate_query_analysis(self.parser.parse('{ a: null }'), 'db', 'collection')
self.assertEqual(analysis['fieldCount'], 1)
self.assertTrue(analysis['supported'])
self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'a')
self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'EQUIV')
analysis = analyzer._generate_query_analysis(self.parser.parse('{ a: null , b: { $lt: 4 }}'), 'db', 'collection')
self.assertEqual(analysis['fieldCount'], 2)
self.assertTrue(analysis['supported'])
self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'a')
self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'EQUIV')
self.assertEqual(analysis['analyzedFields'][1]['fieldName'], 'b')
self.assertEqual(analysis['analyzedFields'][1]['fieldType'], 'RANGE')
analysis = analyzer._generate_query_analysis(self.parser.parse('{$query: { a: null , b: { $lt: 4 }}, $orderby: {c: 1}}'), 'db', 'collection')
self.assertEqual(analysis['fieldCount'], 3)
self.assertTrue(analysis['supported'])
self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'c')
self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'SORT')
self.assertEqual(analysis['analyzedFields'][1]['fieldName'], 'a')
self.assertEqual(analysis['analyzedFields'][1]['fieldType'], 'EQUIV')
self.assertEqual(analysis['analyzedFields'][2]['fieldName'], 'b')
self.assertEqual(analysis['analyzedFields'][2]['fieldType'], 'RANGE')
analysis = analyzer._generate_query_analysis(self.parser.parse('{$query: { a: null , b: { $lt: 4 }, d: {$near: [50, 50]}}, $orderby: {c: 1}}'), 'db', 'collection')
self.assertEqual(analysis['fieldCount'], 4)
self.assertFalse(analysis['supported'])
self.assertEqual(analysis['analyzedFields'][0]['fieldName'], 'c')
self.assertEqual(analysis['analyzedFields'][0]['fieldType'], 'SORT')
self.assertEqual(analysis['analyzedFields'][1]['fieldName'], 'a')
self.assertEqual(analysis['analyzedFields'][1]['fieldType'], 'EQUIV')
self.assertEqual(analysis['analyzedFields'][2]['fieldName'], 'b')
self.assertEqual(analysis['analyzedFields'][2]['fieldType'], 'RANGE')
self.assertEqual(analysis['analyzedFields'][3]['fieldName'], 'd')
self.assertEqual(analysis['analyzedFields'][3]['fieldType'], 'UNSUPPORTED')
def test_report_aggregation(self):
test_dex = dex.Dex(TEST_URI, True, [], 0, True, 0)
report = test_dex._report._reports
test_query = "{ simpleUnindexedField: null }"
result = test_dex.generate_query_report(None,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
test_dex._report.add_query_occurrence(result)
self.assertEqual(len(report), 1)
self.assertEqual(report[0]['stats']['count'], 1)
test_query = "{ $query: {}, $orderby: { simpleUnindexedField: null }}"
result = test_dex.generate_query_report(None,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
test_dex._report.add_query_occurrence(result)
self.assertEqual(len(report), 2)
test_query = "{ anotherUnindexedField: null }"
result = test_dex.generate_query_report(None,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
#adding twice for a double query
test_dex._report.add_query_occurrence(result)
test_dex._report.add_query_occurrence(result)
self.assertEqual(len(report), 3)
def test_sort_ordering(self):
test_dex = dex.Dex(TEST_URI, True, [], 0, True, 0)
report = test_dex._report._reports
test_query = "{ $query: {}, $orderby: { simpleUnindexedField: null, simpleUnindexedFieldTwo: null }}"
result = test_dex.generate_query_report(None,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'],
'{"simpleUnindexedField": 1, "simpleUnindexedFieldTwo": 1}')
test_query = "{ $query: {}, $orderby: { simpleUnindexedFieldTwo: null, simpleUnindexedFieldOne: null }}"
result = test_dex.generate_query_report(None,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'],
'{"simpleUnindexedFieldTwo": 1, "simpleUnindexedFieldOne": 1}')
def database_dex_analyze_view(self, request, database_id):
database = Database.objects.get(id=database_id)
uri = 'mongodb://{}:{}@{}:{}/admin'.format(database.databaseinfra.user,
database.databaseinfra.password,
database.databaseinfra.instances.all()[0].address,
#'10.236.1.15',
database.databaseinfra.instances.all()[0].port)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
md = dex.Dex(db_uri = uri, verbose=False, namespaces_list = [], slowms=0, check_indexes=True, timeout=0)
md.analyze_profile()
sys.stdout = old_stdout
dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace("'", "\"").replace("&&","'"))
import ast
final_mask = """<div>"""
for result in dexanalyzer['results']:
final_mask += "<h3> Collection: " + result['namespace']+ "</h3>"
final_mask += \
"""<li> Query: """ +\
str(ast.literal_eval(result['queryMask'])['$query']) +\
"""</li>""" +\
"""<li> Index: """+\
result['recommendation']['index']+\
"""</li>"""+\
"""<li> Command: """+\
result['recommendation']['shellCommand']+\
"""</li>"""
final_mask += """<br>"""
final_mask += """</ul> </div>"""
return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))
def test_report_aggregation(self):
test_dex = dex.Dex(TEST_URI, True, [])
report = test_dex._full_report._reports
test_query = "{ query: { simpleUnindexedField: null }, " \
"ns: 'dex_test.test_collection', 'millis': 150}"
result = test_dex.analyze_query(None, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
test_dex._full_report.add_report(result)
self.assertEqual(len(report), 1)
self.assertEqual(report[0]['queryCount'], 1)
self.assertEqual(len(report[0]['queriesCovered']), 1)
test_query = "{ query: {}, orderby: { simpleUnindexedField: null }," \
"ns: 'dex_test.test_collection', 'millis': 50}"
result = test_dex.analyze_query(None, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
test_dex._full_report.add_report(result)
self.assertEqual(len(report), 1)
self.assertEqual(report[0]['queryCount'], 2)
self.assertEqual(len(report[0]['queriesCovered']), 2)
totalAvg = report[0]['avgTimeMillis']
self.assertEqual(totalAvg, 100)
test_query = "{ query: { anotherUnindexedField: null }, "\
"ns: 'dex_test.test_collection', 'millis': 100}"
result = test_dex.analyze_query(None, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
#adding twice for a double query
test_dex._full_report.add_report(result)
test_dex._full_report.add_report(result)
self.assertEqual(len(report), 2)
self.assertEqual(report[0]['queryCount'], 2)
self.assertEqual(len(report[0]['queriesCovered']), 2)
self.assertEqual(report[1]['queryCount'], 2)
self.assertEqual(len(report[1]['queriesCovered']), 1)
totalAvg = report[0]['avgTimeMillis']
self.assertEqual(totalAvg, 100)
def database_dex_analyze_view(self, request, database_id):
import os
import string
def generate_random_string(length, stringset=string.ascii_letters + string.digits):
return ''.join([stringset[i % len(stringset)]
for i in [ord(x) for x in os.urandom(length)]])
database = Database.objects.get(id=database_id)
if database.status != Database.ALIVE or not database.database_status.is_alive:
self.message_user(
request, "Database is not alive cannot be analyzed", level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
if database.is_being_used_elsewhere():
self.message_user(
request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
parsed_logs = ''
arq_path = Configuration.get_by_name(
'database_clone_dir') + '/' + database.name + generate_random_string(20) + '.txt'
arq = open(arq_path, 'w')
arq.write(parsed_logs)
arq.close()
uri = 'mongodb://{}:{}@{}:{}/admin'.format(database.databaseinfra.user,
database.databaseinfra.password,
database.databaseinfra.instances.all()[
0].address,
database.databaseinfra.instances.all()[0].port)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
md = dex.Dex(db_uri=uri, verbose=False, namespaces_list=[],
slowms=0, check_indexes=True, timeout=0)
md.analyze_logfile(arq_path)
sys.stdout = old_stdout
dexanalyzer = loads(
mystdout.getvalue().replace("\"", "&&").replace("'", "\"").replace("&&", "'"))
os.remove(arq_path)
import ast
final_mask = """<div>"""
for result in dexanalyzer['results']:
final_mask += "<h3> Collection: " + result['namespace'] + "</h3>"
final_mask += \
"""<li> Query: """ +\
str(ast.literal_eval(result['queryMask'])['$query']) +\
"""</li>""" +\
"""<li> Index: """ +\
result['recommendation']['index'] +\
"""</li>""" +\
"""<li> Command: """ +\
result['recommendation']['shellCommand'] +\
"""</li>"""
final_mask += """<br>"""
final_mask += """</ul> </div>"""
return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))
arq = open(arq_path, 'w')
arq.write(parsed_logs)
arq.close()
uri = 'mongodb://{}:{}@{}:{}/admin'.format(
database.databaseinfra.user, database.databaseinfra.password,
database.databaseinfra.instances.all()[0].address,
database.databaseinfra.instances.all()[0].port)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
md = dex.Dex(db_uri=uri,
verbose=False,
namespaces_list=[],
slowms=0,
check_indexes=True,
timeout=0)
md.analyze_logfile(arq_path)
sys.stdout = old_stdout
dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace(
"'", "\"").replace("&&", "'"))
os.remove(arq_path)
import ast
final_mask = """<div>"""
def database_dex_analyze_view(self, request, database_id):
if database.status != Database.ALIVE or not database.database_status.is_alive:
self.message_user(request,
"Database is not alive cannot be analyzed",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
if database.is_beeing_used_elsewhere():
self.message_user(
request,
"Database cannot be analyzed because it is in use by another task.",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
database = Database.objects.get(id=database_id)
uri = 'mongodb://{}:{}@{}:{}/admin'.format(
database.databaseinfra.user, database.databaseinfra.password,
database.databaseinfra.instances.all()[0].address,
database.databaseinfra.instances.all()[0].port)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
md = dex.Dex(db_uri=uri,
verbose=False,
namespaces_list=[],
slowms=0,
check_indexes=True,
timeout=0)
md.analyze_profile()
sys.stdout = old_stdout
dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace(
"'", "\"").replace("&&", "'"))
import ast
final_mask = """<div>"""
for result in dexanalyzer['results']:
final_mask += "<h3> Collection: " + result['namespace'] + "</h3>"
final_mask += \
"""<li> Query: """ +\
str(ast.literal_eval(result['queryMask'])['$query']) +\
"""</li>""" +\
"""<li> Index: """ +\
result['recommendation']['index'] +\
"""</li>""" +\
"""<li> Command: """ +\
result['recommendation']['shellCommand'] +\
"""</li>"""
final_mask += """<br>"""
final_mask += """</ul> </div>"""
return render_to_response("logical/database/dex_analyze.html",
locals(),
context_instance=RequestContext(request))
def test_analyze_query(self):
test_dex = dex.Dex(TEST_URI, False, [], 0, True, 0)
test_query = "{ simpleUnindexedField: null }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1}')
test_query = "{ simpleIndexedField: null }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None, pretty_json(result))
test_query = "{ simpleUnindexedField: {$lt: 4}}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1}')
test_query = "{ simpleIndexedField: { $lt: 4 }}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ $query: {}, $orderby: { simpleUnindexedField }}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"simpleUnindexedField": 1}')
test_query = "{ $query: {}, $orderby: { simpleIndexedField: 1 }}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{complexUnindexedFieldOne: null, complexUnindexedFieldTwo: null }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"complexUnindexedFieldOne": 1, "complexUnindexedFieldTwo": 1}')
test_query = "{ complexIndexedFieldOne: null, complexIndexedFieldTwo: null }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
self.assertEqual(result['indexAnalysis']['fullIndexes'][0]['index']['key'], [('complexIndexedFieldOne', -1), ('complexIndexedFieldTwo', -1), ('complexIndexedFieldThree', -1)])
test_query = "{ complexUnindexedFieldOne: null, complexUnindexedFieldTwo: { $lt: 4 }}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"complexUnindexedFieldOne": 1, "complexUnindexedFieldTwo": 1}')
test_query = "{ complexIndexedFieldOne: null, complexIndexedFieldTwo: { $lt: 4 } }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None, pretty_json(result))
test_query = "{ complexIndexedFieldNine: null, complexIndexedFieldTen: { $lt: 4 } }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"complexIndexedFieldNine": 1, "complexIndexedFieldTen": 1}')
test_query = "{ $query: {complexUnindexedFieldOne: null}, $orderby: { complexUnindexedFieldTwo: 1 } }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"complexUnindexedFieldOne": 1, "complexUnindexedFieldTwo": 1}')
test_query = "{ $query: {complexIndexedFieldOne: null}, $orderby: { complexIndexedFieldTwo: 1 } }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ $query: {complexIndexedFieldTen: {$lt: 4}}, $orderby: { complexIndexedFieldNine: 1 } }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"complexIndexedFieldNine": 1, "complexIndexedFieldTen": 1}')
test_query = "{ $query: {complexIndexedFieldThree: null, complexIndexedFieldTwo: {$lt: 4}}, $orderby: { complexIndexedFieldOne: 1 }}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'], '{"complexIndexedFieldThree": 1, "complexIndexedFieldOne": 1, "complexIndexedFieldTwo": 1}')
test_query = "{ $query: {complexIndexedFieldOne: null, complexIndexedFieldThree: {$lt: 4}}, $orderby: { complexIndexedFieldTwo: 1 } }"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ $query: { $or: [ { orFieldOne: { $lt: 4 } }, {orFieldTwo: { $gt: 5 } }], complexUnindexedFieldOne: 'A'}, $orderby: { _id: 1 }}"
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ geoIndexedFieldOne: { $near: [50, 50] } } "
result = test_dex.generate_query_report(TEST_URI,
self.parser.parse(test_query),
TEST_DBNAME,
TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
def test_generate_index_report(self):
analyzer = dex.Dex(TEST_URI, False, [], 0, True, 0)._query_analyzer
index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)],
"v": 1}
analysis = {'supported': True,
'analyzedFields': [{'fieldName': 'simpleUnindexedField', 'fieldType': 'EQUIV'}],
'fieldCount': 1}
report = analyzer._generate_index_report(index, analysis)
self.assertEqual(report['queryFieldsCovered'], 0)
self.assertEqual(report['index'], index)
self.assertEqual(report['coverage'], 'none')
index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)],
"v": 1}
analysis = {'supported': True,
'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'EQUIV'}],
'fieldCount': 1}
report = analyzer._generate_index_report(index, analysis)
self.assertEqual(report['queryFieldsCovered'], 0)
self.assertEqual(report['index'], index)
self.assertEqual(report['coverage'], 'none')
index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)],
"v": 1}
analysis = {'supported': True,
'analyzedFields': [{'fieldName': 'complexIndexedFieldOne', 'fieldType': 'EQUIV'}],
'fieldCount': 1}
report = analyzer._generate_index_report(index, analysis)
self.assertEqual(report['queryFieldsCovered'], 1)
self.assertEqual(report['index'], index)
self.assertEqual(report['coverage'], 'full')
index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)],
"v": 1}
analysis = {'supported': True,
'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'EQUIV'},
{'fieldName': 'complexIndexedFieldOne', 'fieldType': 'RANGE'}],
'fieldCount': 2}
report = analyzer._generate_index_report(index, analysis)
self.assertEqual(report['queryFieldsCovered'], 2)
self.assertEqual(report['index'], index)
self.assertEqual(report['coverage'], 'full')
self.assertFalse(report['idealOrder'])
index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)],
"v": 1}
analysis = {'supported': True,
'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'RANGE'},
{'fieldName': 'complexIndexedFieldOne', 'fieldType': 'EQUIV'}],
'fieldCount': 2}
report = analyzer._generate_index_report(index, analysis)
self.assertEqual(report['queryFieldsCovered'], 2)
self.assertEqual(report['index'], index)
self.assertEqual(report['coverage'], 'full')
self.assertTrue(report['idealOrder'])
index = {"key": [ ("complexIndexedFieldOne", -1), ("complexIndexedFieldTwo", -1)],
"v": 1}
analysis = {'supported': True,
'analyzedFields': [{'fieldName': 'complexIndexedFieldTwo', 'fieldType': 'RANGE'},
{'fieldName': 'complexIndexedFieldOne', 'fieldType': 'SORT'}],
'fieldCount': 2}
report = analyzer._generate_index_report(index, analysis)
self.assertEqual(report['queryFieldsCovered'], 2)
self.assertEqual(report['index'], index)
self.assertEqual(report['coverage'], 'full')
self.assertTrue(report['idealOrder'])
def test_analyze_query(self):
test_dex = dex.Dex(TEST_URI, False, [])
test_query = "{ query: { simpleUnindexedField: null }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'],
"{'simpleUnindexedField': 1}")
test_query = "{ query: { simpleIndexedField: null }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None,
dex.pretty_json(result))
test_query = "{ query: { simpleUnindexedField: {$lt: 4}}, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'],
"{'simpleUnindexedField': 1}")
test_query = "{ query: { simpleIndexedField: { $lt: 4 } }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ query: {}, orderby: { simpleUnindexedField }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation']['index'],
"{'simpleUnindexedField': 1}")
test_query = "{ query: {}, orderby: { simpleIndexedField: 1 }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ query: {complexUnindexedFieldOne: null, complexUnindexedFieldTwo: null }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(
result['recommendation']['index'],
"{'complexUnindexedFieldOne': 1, 'complexUnindexedFieldTwo': 1}")
test_query = "{ query: { complexIndexedFieldOne: null, complexIndexedFieldTwo: null }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
self.assertEqual(
result['indexAnalysis']['fullIndexes'][0]['index']['key'],
[('complexIndexedFieldOne', -1), ('complexIndexedFieldTwo', -1),
('complexIndexedFieldThree', -1)])
test_query = "{ query: { complexUnindexedFieldOne: null, complexUnindexedFieldTwo: { $lt: 4 }}, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(
result['recommendation']['index'],
"{'complexUnindexedFieldOne': 1, 'complexUnindexedFieldTwo': 1}")
test_query = "{ query: { complexIndexedFieldOne: null, complexIndexedFieldTwo: { $lt: 4 } }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None,
dex.pretty_json(result))
test_query = "{ query: { complexIndexedFieldNine: null, complexIndexedFieldTen: { $lt: 4 } }, ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(
result['recommendation']['index'],
"{'complexIndexedFieldNine': 1, 'complexIndexedFieldTen': 1}")
test_query = "{ query: {complexUnindexedFieldOne: null}, orderby: { complexUnindexedFieldTwo: 1 } , ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(
result['recommendation']['index'],
"{'complexUnindexedFieldOne': 1, 'complexUnindexedFieldTwo': 1}")
test_query = "{ query: {complexIndexedFieldOne: null}, orderby: { complexIndexedFieldTwo: 1 } , ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ query: {complexIndexedFieldTen: {$lt: 4}}, orderby: { complexIndexedFieldNine: 1 } , ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(
result['recommendation']['index'],
"{'complexIndexedFieldNine': 1, 'complexIndexedFieldTen': 1}")
test_query = "{ query: {complexIndexedFieldThree: null, complexIndexedFieldTwo: {$lt: 4}}, orderby: { complexIndexedFieldOne: 1 } , ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(
result['recommendation']['index'],
"{'complexIndexedFieldThree': 1, 'complexIndexedFieldOne': 1, 'complexIndexedFieldTwo': 1}"
)
test_query = "{ query: {complexIndexedFieldOne: null, complexIndexedFieldThree: {$lt: 4}}, orderby: { complexIndexedFieldTwo: 1 } , ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ query: { $or: [ { orFieldOne: { $lt: 4 } }, {orFieldTwo: { $gt: 5 } }], complexUnindexedFieldOne: 'A'}, orderby: { _id: 1 }, ns: 'dex_test.test_collection' }"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
test_query = "{ query: { geoIndexedFieldOne: { $near: [50, 50] } } , ns: 'dex_test.test_collection'}"
result = test_dex.analyze_query(TEST_URI, yaml.load(test_query),
TEST_DBNAME, TEST_COLLECTION)
self.assertEqual(result['recommendation'], None)
def database_dex_analyze_view(self, request, database_id):
import json
import random
from dbaas_laas.provider import LaaSProvider
from util import get_credentials_for
from util.laas import get_group_name
from dbaas_credentials.models import CredentialType
import os
import string
from datetime import datetime, timedelta
def generate_random_string(length,
stringset=string.ascii_letters +
string.digits):
return ''.join([
stringset[i % len(stringset)]
for i in [ord(x) for x in os.urandom(length)]
])
database = Database.objects.get(id=database_id)
if database.status != Database.ALIVE or not database.database_status.is_alive:
self.message_user(request,
"Database is not alive cannot be analyzed",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
if database.is_beeing_used_elsewhere():
self.message_user(
request,
"Database cannot be analyzed because it is in use by another task.",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
credential = get_credentials_for(environment=database.environment,
credential_type=CredentialType.LAAS)
db_name = database.name
environment = database.environment
endpoint = credential.endpoint
username = credential.user
password = credential.password
lognit_environment = credential.get_parameter_by_name(
'lognit_environment')
provider = LaaSProvider()
group_name = get_group_name(database)
today = (datetime.now()).strftime('%Y%m%d')
yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d')
uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format(
group_name, yesterday, today)
parsed_logs = ''
database_logs = provider.get_logs_for_group(environment,
lognit_environment, uri)
try:
database_logs = json.loads(database_logs)
except Exception as e:
pass
else:
for database_log in database_logs:
try:
items = database_log['items']
except KeyError as e:
pass
else:
parsed_logs = "\n".join(
(item['message'] for item in items))
arq_path = Configuration.get_by_name(
'database_clone_dir'
) + '/' + database.name + generate_random_string(20) + '.txt'
arq = open(arq_path, 'w')
arq.write(parsed_logs)
arq.close()
uri = 'mongodb://{}:{}@{}:{}/admin'.format(
database.databaseinfra.user, database.databaseinfra.password,
database.databaseinfra.instances.all()[0].address,
database.databaseinfra.instances.all()[0].port)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
md = dex.Dex(db_uri=uri,
verbose=False,
namespaces_list=[],
slowms=0,
check_indexes=True,
timeout=0)
md.analyze_logfile(arq_path)
sys.stdout = old_stdout
dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace(
"'", "\"").replace("&&", "'"))
os.remove(arq_path)
import ast
final_mask = """<div>"""
print dexanalyzer['results']
for result in dexanalyzer['results']:
final_mask += "<h3> Collection: " + result['namespace'] + "</h3>"
final_mask += \
"""<li> Query: """ +\
str(ast.literal_eval(result['queryMask'])['$query']) +\
"""</li>""" +\
"""<li> Index: """ +\
result['recommendation']['index'] +\
"""</li>""" +\
"""<li> Command: """ +\
result['recommendation']['shellCommand'] +\
"""</li>"""
final_mask += """<br>"""
final_mask += """</ul> </div>"""
return render_to_response("logical/database/dex_analyze.html",
locals(),
context_instance=RequestContext(request))
