def register_database_laas(database):
workspace_json = {}
workspace_json["team"] = {
"name" : database.team.name,
"users" : get_users_for_team(database.team),
}
if re.match(r'^mongo.*', database.engine_type):
app = ["mongod.27017"]
elif re.match(r'^mysql.*', database.engine_type):
app = ["mysqld", "mysql-slow"]
elif re.match(r'^redis.*', database.engine_type):
app = ["redis", "sentinel"]
hosts = get_hosts_for_database(database)
groups = [{
"filter_name" : "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)),
"name" : get_group_name(database)
}]
workspace_json["workspace"] = {
"description" : "%s Workspace" % database.team.name,
"name" : database.team.name,
"groups" : groups
}
workspace_json = json.dumps(workspace_json)
LOG.info("Register workspace on LaaS. Workspace info: %s" % (workspace_json))
try:
LaaSProvider.update_laas_workspace(environment=database.environment, laas_workspace=workspace_json)
except Exception, e:
LOG.error("Ops... something went wrong: %s" % e)
def register_database_laas(database):
workspace_json = {}
workspace_json["team"] = {
"name": database.team.name,
"users": get_users_for_team(database.team),
}
if re.match(r'^mongo.*', database.engine_type):
app = ["mongod.27017"]
elif re.match(r'^mysql.*', database.engine_type):
app = ["mysqld", "mysql-slow"]
elif re.match(r'^redis.*', database.engine_type):
app = ["redis", "sentinel"]
hosts = get_hosts_for_database(database)
groups = [{
"filter_name": "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)),
"name": get_group_name(database)
}]
workspace_json["workspace"] = {
"description": "%s Workspace" % database.team.name,
"name": database.team.name,
"groups": groups
}
workspace_json = json.dumps(workspace_json)
LOG.info("Register workspace on LaaS. Workspace info: %s" %
(workspace_json))
try:
LaaSProvider.update_laas_workspace(
environment=database.environment, laas_workspace=workspace_json)
except Exception, e:
LOG.error("Ops... something went wrong: %s" % e)
def register_database_laas(database):
workspace_json = {}
workspace_json["team"] = {
"name" : database.team.name,
"users" : get_users_for_team(database.team),
}
from util.providers import MYSQL, MONGODB, get_engine
engine = get_engine(database.engine_type)
if engine == MYSQL:
app = ["mysqld", "mysql-slow"]
elif engine == MONGODB:
app = ["mongod.27017"]
else:
app =[]
hosts = get_hosts_for_database(database)
groups = [{
"filter_name" : "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)),
"name" : get_group_name(database)
}]
workspace_json["workspace"] = {
"description" : "%s Workspace" % database.team.name,
"name" : database.team.name,
"groups" : groups
}
workspace_json = json.dumps(workspace_json)
LOG.info("Register workspace on LaaS. Workspace info: %s" % (workspace_json))
try:
LaaSProvider.update_laas_workspace(environment=database.environment, laas_workspace=workspace_json)
except Exception, e:
LOG.error("Ops... something went wrong: %s" % e)
def database_dex_analyze_view(self, request, database_id):
import json
import random
from dbaas_laas.provider import LaaSProvider
from util import get_credentials_for
from util.laas import get_group_name
from dbaas_credentials.models import CredentialType
import os
import string
from datetime import datetime, timedelta
def generate_random_string(length, stringset=string.ascii_letters+string.digits):
return ''.join([stringset[i%len(stringset)] \
for i in [ord(x) for x in os.urandom(length)]])
database = Database.objects.get(id=database_id)
if database.status != Database.ALIVE or not database.database_status.is_alive:
self.message_user(
request, "Database is not alive cannot be analyzed", level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
if database.is_beeing_used_elsewhere():
self.message_user(
request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
credential =get_credentials_for(environment=database.environment,
credential_type=CredentialType.LAAS)
db_name = database.name
environment = database.environment
endpoint = credential.endpoint
username = credential.user
password = credential.password
lognit_environment = credential.get_parameter_by_name('lognit_environment')
provider = LaaSProvider()
group_name = get_group_name(database)
today = (datetime.now()).strftime('%Y%m%d')
yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d')
uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format(group_name,yesterday,today)
parsed_logs = ''
database_logs = provider.get_logs_for_group(environment, lognit_environment, uri)
try:
database_logs = json.loads(database_logs)
except Exception, e:
pass
def register_team_laas(team):
for environment in Environment.objects.all():
if team.databases_in_use_for(environment=environment):
team_json = {}
team_json["team"] = {
"name" : team.name,
"users" : get_users_for_team(team),
}
team_json = json.dumps(team_json)
LOG.info("Register team on LaaS. Team info: %s" % (team_json))
try:
LaaSProvider.update_laas_team(environment=environment, laas_team=team_json)
except Exception, e:
LOG.error("Ops... something went wrong: %s" % e)
def register_team_laas(team):
for environment in Environment.objects.all():
if team.databases_in_use_for(environment=environment):
team_json = {}
team_json["team"] = {
"name" : team.name,
"users" : get_users_for_team(team),
}
team_json = json.dumps(team_json)
LOG.info("Register team on LaaS. Team info: %s" % (team_json))
try:
LaaSProvider.update_laas_team(environment=environment, laas_team=team_json)
except Exception, e:
LOG.error("Ops... something went wrong: %s" % e)
def register_database_laas(database):
workspace_json = {}
workspace_json["team"] = {
"name": database.team.name,
"users": get_users_for_team(database.team),
}
from util.providers import MYSQL, MONGODB, REDIS, get_engine
engine = get_engine(database.engine_type)
if engine == MYSQL:
app = ["mysqld", "mysql-slow"]
elif engine == MONGODB:
app = ["mongod.27017"]
elif engine == REDIS:
app = ["redis"]
else:
app = []
hosts = get_hosts_for_database(database)
groups = [{
"filter_name":
"host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)),
"name":
get_group_name(database)
}]
workspace_json["workspace"] = {
"description": "%s Workspace" % database.team.name,
"name": database.team.name,
"groups": groups
}
workspace_json = json.dumps(workspace_json)
LOG.info("Register workspace on LaaS. Workspace info: %s" %
(workspace_json))
try:
LaaSProvider.update_laas_workspace(environment=database.environment,
laas_workspace=workspace_json)
except Exception, e:
LOG.error("Ops... something went wrong: %s" % e)
def database_dex_analyze_view(self, request, database_id):
import json
import random
from dbaas_laas.provider import LaaSProvider
from util import get_credentials_for
from util.laas import get_group_name
from dbaas_credentials.models import CredentialType
import os
import string
from datetime import datetime, timedelta
def generate_random_string(length,
stringset=string.ascii_letters +
string.digits):
return ''.join([
stringset[i % len(stringset)]
for i in [ord(x) for x in os.urandom(length)]
])
database = Database.objects.get(id=database_id)
if database.status != Database.ALIVE or not database.database_status.is_alive:
self.message_user(request,
"Database is not alive cannot be analyzed",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
if database.is_beeing_used_elsewhere():
self.message_user(
request,
"Database cannot be analyzed because it is in use by another task.",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
credential = get_credentials_for(environment=database.environment,
credential_type=CredentialType.LAAS)
db_name = database.name
environment = database.environment
endpoint = credential.endpoint
username = credential.user
password = credential.password
lognit_environment = credential.get_parameter_by_name(
'lognit_environment')
provider = LaaSProvider()
group_name = get_group_name(database)
today = (datetime.now()).strftime('%Y%m%d')
yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d')
uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format(
group_name, yesterday, today)
parsed_logs = ''
database_logs = provider.get_logs_for_group(environment,
lognit_environment, uri)
try:
database_logs = json.loads(database_logs)
except Exception, e:
pass
def database_dex_analyze_view(self, request, database_id):
import json
import random
from dbaas_laas.provider import LaaSProvider
from util import get_credentials_for
from util.laas import get_group_name
from dbaas_credentials.models import CredentialType
import os
import string
from datetime import datetime, timedelta
def generate_random_string(length,
stringset=string.ascii_letters +
string.digits):
return ''.join([
stringset[i % len(stringset)]
for i in [ord(x) for x in os.urandom(length)]
])
database = Database.objects.get(id=database_id)
if database.status != Database.ALIVE or not database.database_status.is_alive:
self.message_user(request,
"Database is not alive cannot be analyzed",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
if database.is_beeing_used_elsewhere():
self.message_user(
request,
"Database cannot be analyzed because it is in use by another task.",
level=messages.ERROR)
url = reverse('admin:logical_database_changelist')
return HttpResponseRedirect(url)
credential = get_credentials_for(environment=database.environment,
credential_type=CredentialType.LAAS)
db_name = database.name
environment = database.environment
endpoint = credential.endpoint
username = credential.user
password = credential.password
lognit_environment = credential.get_parameter_by_name(
'lognit_environment')
provider = LaaSProvider()
group_name = get_group_name(database)
today = (datetime.now()).strftime('%Y%m%d')
yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d')
uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format(
group_name, yesterday, today)
parsed_logs = ''
database_logs = provider.get_logs_for_group(environment,
lognit_environment, uri)
try:
database_logs = json.loads(database_logs)
except Exception as e:
pass
else:
for database_log in database_logs:
try:
items = database_log['items']
except KeyError as e:
pass
else:
parsed_logs = "\n".join(
(item['message'] for item in items))
arq_path = Configuration.get_by_name(
'database_clone_dir'
) + '/' + database.name + generate_random_string(20) + '.txt'
arq = open(arq_path, 'w')
arq.write(parsed_logs)
arq.close()
uri = 'mongodb://{}:{}@{}:{}/admin'.format(
database.databaseinfra.user, database.databaseinfra.password,
database.databaseinfra.instances.all()[0].address,
database.databaseinfra.instances.all()[0].port)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
md = dex.Dex(db_uri=uri,
verbose=False,
namespaces_list=[],
slowms=0,
check_indexes=True,
timeout=0)
md.analyze_logfile(arq_path)
sys.stdout = old_stdout
dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace(
"'", "\"").replace("&&", "'"))
os.remove(arq_path)
import ast
final_mask = """<div>"""
print dexanalyzer['results']
for result in dexanalyzer['results']:
final_mask += "<h3> Collection: " + result['namespace'] + "</h3>"
final_mask += \
"""<li> Query: """ +\
str(ast.literal_eval(result['queryMask'])['$query']) +\
"""</li>""" +\
"""<li> Index: """ +\
result['recommendation']['index'] +\
"""</li>""" +\
"""<li> Command: """ +\
result['recommendation']['shellCommand'] +\
"""</li>"""
final_mask += """<br>"""
final_mask += """</ul> </div>"""
return render_to_response("logical/database/dex_analyze.html",
locals(),
context_instance=RequestContext(request))
