def register_database_laas(database): workspace_json = {} workspace_json["team"] = { "name" : database.team.name, "users" : get_users_for_team(database.team), } if re.match(r'^mongo.*', database.engine_type): app = ["mongod.27017"] elif re.match(r'^mysql.*', database.engine_type): app = ["mysqld", "mysql-slow"] elif re.match(r'^redis.*', database.engine_type): app = ["redis", "sentinel"] hosts = get_hosts_for_database(database) groups = [{ "filter_name" : "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)), "name" : get_group_name(database) }] workspace_json["workspace"] = { "description" : "%s Workspace" % database.team.name, "name" : database.team.name, "groups" : groups } workspace_json = json.dumps(workspace_json) LOG.info("Register workspace on LaaS. Workspace info: %s" % (workspace_json)) try: LaaSProvider.update_laas_workspace(environment=database.environment, laas_workspace=workspace_json) except Exception, e: LOG.error("Ops... something went wrong: %s" % e)
def register_database_laas(database): workspace_json = {} workspace_json["team"] = { "name": database.team.name, "users": get_users_for_team(database.team), } if re.match(r'^mongo.*', database.engine_type): app = ["mongod.27017"] elif re.match(r'^mysql.*', database.engine_type): app = ["mysqld", "mysql-slow"] elif re.match(r'^redis.*', database.engine_type): app = ["redis", "sentinel"] hosts = get_hosts_for_database(database) groups = [{ "filter_name": "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)), "name": get_group_name(database) }] workspace_json["workspace"] = { "description": "%s Workspace" % database.team.name, "name": database.team.name, "groups": groups } workspace_json = json.dumps(workspace_json) LOG.info("Register workspace on LaaS. Workspace info: %s" % (workspace_json)) try: LaaSProvider.update_laas_workspace( environment=database.environment, laas_workspace=workspace_json) except Exception, e: LOG.error("Ops... something went wrong: %s" % e)
def register_database_laas(database): workspace_json = {} workspace_json["team"] = { "name" : database.team.name, "users" : get_users_for_team(database.team), } from util.providers import MYSQL, MONGODB, get_engine engine = get_engine(database.engine_type) if engine == MYSQL: app = ["mysqld", "mysql-slow"] elif engine == MONGODB: app = ["mongod.27017"] else: app =[] hosts = get_hosts_for_database(database) groups = [{ "filter_name" : "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)), "name" : get_group_name(database) }] workspace_json["workspace"] = { "description" : "%s Workspace" % database.team.name, "name" : database.team.name, "groups" : groups } workspace_json = json.dumps(workspace_json) LOG.info("Register workspace on LaaS. Workspace info: %s" % (workspace_json)) try: LaaSProvider.update_laas_workspace(environment=database.environment, laas_workspace=workspace_json) except Exception, e: LOG.error("Ops... something went wrong: %s" % e)
def database_dex_analyze_view(self, request, database_id): import json import random from dbaas_laas.provider import LaaSProvider from util import get_credentials_for from util.laas import get_group_name from dbaas_credentials.models import CredentialType import os import string from datetime import datetime, timedelta def generate_random_string(length, stringset=string.ascii_letters+string.digits): return ''.join([stringset[i%len(stringset)] \ for i in [ord(x) for x in os.urandom(length)]]) database = Database.objects.get(id=database_id) if database.status != Database.ALIVE or not database.database_status.is_alive: self.message_user( request, "Database is not alive cannot be analyzed", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) if database.is_beeing_used_elsewhere(): self.message_user( request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) credential =get_credentials_for(environment=database.environment, credential_type=CredentialType.LAAS) db_name = database.name environment = database.environment endpoint = credential.endpoint username = credential.user password = credential.password lognit_environment = credential.get_parameter_by_name('lognit_environment') provider = LaaSProvider() group_name = get_group_name(database) today = (datetime.now()).strftime('%Y%m%d') yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d') uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format(group_name,yesterday,today) parsed_logs = '' database_logs = provider.get_logs_for_group(environment, lognit_environment, uri) try: database_logs = json.loads(database_logs) except Exception, e: pass
def register_team_laas(team): for environment in Environment.objects.all(): if team.databases_in_use_for(environment=environment): team_json = {} team_json["team"] = { "name" : team.name, "users" : get_users_for_team(team), } team_json = json.dumps(team_json) LOG.info("Register team on LaaS. Team info: %s" % (team_json)) try: LaaSProvider.update_laas_team(environment=environment, laas_team=team_json) except Exception, e: LOG.error("Ops... something went wrong: %s" % e)
def register_database_laas(database): workspace_json = {} workspace_json["team"] = { "name": database.team.name, "users": get_users_for_team(database.team), } from util.providers import MYSQL, MONGODB, REDIS, get_engine engine = get_engine(database.engine_type) if engine == MYSQL: app = ["mysqld", "mysql-slow"] elif engine == MONGODB: app = ["mongod.27017"] elif engine == REDIS: app = ["redis"] else: app = [] hosts = get_hosts_for_database(database) groups = [{ "filter_name": "host:(%s) AND app:(%s)" % (" OR ".join(hosts), " OR ".join(app)), "name": get_group_name(database) }] workspace_json["workspace"] = { "description": "%s Workspace" % database.team.name, "name": database.team.name, "groups": groups } workspace_json = json.dumps(workspace_json) LOG.info("Register workspace on LaaS. Workspace info: %s" % (workspace_json)) try: LaaSProvider.update_laas_workspace(environment=database.environment, laas_workspace=workspace_json) except Exception, e: LOG.error("Ops... something went wrong: %s" % e)
def database_dex_analyze_view(self, request, database_id): import json import random from dbaas_laas.provider import LaaSProvider from util import get_credentials_for from util.laas import get_group_name from dbaas_credentials.models import CredentialType import os import string from datetime import datetime, timedelta def generate_random_string(length, stringset=string.ascii_letters + string.digits): return ''.join([ stringset[i % len(stringset)] for i in [ord(x) for x in os.urandom(length)] ]) database = Database.objects.get(id=database_id) if database.status != Database.ALIVE or not database.database_status.is_alive: self.message_user(request, "Database is not alive cannot be analyzed", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) if database.is_beeing_used_elsewhere(): self.message_user( request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) credential = get_credentials_for(environment=database.environment, credential_type=CredentialType.LAAS) db_name = database.name environment = database.environment endpoint = credential.endpoint username = credential.user password = credential.password lognit_environment = credential.get_parameter_by_name( 'lognit_environment') provider = LaaSProvider() group_name = get_group_name(database) today = (datetime.now()).strftime('%Y%m%d') yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d') uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format( group_name, yesterday, today) parsed_logs = '' database_logs = provider.get_logs_for_group(environment, lognit_environment, uri) try: database_logs = json.loads(database_logs) except Exception, e: pass
def database_dex_analyze_view(self, request, database_id): import json import random from dbaas_laas.provider import LaaSProvider from util import get_credentials_for from util.laas import get_group_name from dbaas_credentials.models import CredentialType import os import string from datetime import datetime, timedelta def generate_random_string(length, stringset=string.ascii_letters + string.digits): return ''.join([ stringset[i % len(stringset)] for i in [ord(x) for x in os.urandom(length)] ]) database = Database.objects.get(id=database_id) if database.status != Database.ALIVE or not database.database_status.is_alive: self.message_user(request, "Database is not alive cannot be analyzed", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) if database.is_beeing_used_elsewhere(): self.message_user( request, "Database cannot be analyzed because it is in use by another task.", level=messages.ERROR) url = reverse('admin:logical_database_changelist') return HttpResponseRedirect(url) credential = get_credentials_for(environment=database.environment, credential_type=CredentialType.LAAS) db_name = database.name environment = database.environment endpoint = credential.endpoint username = credential.user password = credential.password lognit_environment = credential.get_parameter_by_name( 'lognit_environment') provider = LaaSProvider() group_name = get_group_name(database) today = (datetime.now()).strftime('%Y%m%d') yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y%m%d') uri = "group:{} text:query date:[{} TO {}] time:[000000 TO 235959]".format( group_name, yesterday, today) parsed_logs = '' database_logs = provider.get_logs_for_group(environment, lognit_environment, uri) try: database_logs = json.loads(database_logs) except Exception as e: pass else: for database_log in database_logs: try: items = database_log['items'] except KeyError as e: pass else: parsed_logs = "\n".join( (item['message'] for item in items)) arq_path = Configuration.get_by_name( 'database_clone_dir' ) + '/' + database.name + generate_random_string(20) + '.txt' arq = open(arq_path, 'w') arq.write(parsed_logs) arq.close() uri = 'mongodb://{}:{}@{}:{}/admin'.format( database.databaseinfra.user, database.databaseinfra.password, database.databaseinfra.instances.all()[0].address, database.databaseinfra.instances.all()[0].port) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() md = dex.Dex(db_uri=uri, verbose=False, namespaces_list=[], slowms=0, check_indexes=True, timeout=0) md.analyze_logfile(arq_path) sys.stdout = old_stdout dexanalyzer = loads(mystdout.getvalue().replace("\"", "&&").replace( "'", "\"").replace("&&", "'")) os.remove(arq_path) import ast final_mask = """<div>""" print dexanalyzer['results'] for result in dexanalyzer['results']: final_mask += "<h3> Collection: " + result['namespace'] + "</h3>" final_mask += \ """<li> Query: """ +\ str(ast.literal_eval(result['queryMask'])['$query']) +\ """</li>""" +\ """<li> Index: """ +\ result['recommendation']['index'] +\ """</li>""" +\ """<li> Command: """ +\ result['recommendation']['shellCommand'] +\ """</li>""" final_mask += """<br>""" final_mask += """</ul> </div>""" return render_to_response("logical/database/dex_analyze.html", locals(), context_instance=RequestContext(request))