def kill_con(dbname, dbuser, dbuserpass, username): """ stop and remove container """ # get state info (running container) state_info = admin_db.get_container_state(dbname) data = admin_db.get_container_data('', state_info.c_id) dbengine = data['Info']['dbengine'] port = data['Info']['Port'] print('DEBUG: kill_con: con: %s user: %s pass: %s port: %s' % (dbname, dbuser, dbuserpass, port)) auth = False if dbengine == 'Postgres': auth = postgres_util.auth_check(dbuser, dbuserpass, port) elif dbengine == 'MongoDB': auth = mongodb_util.auth_mongodb(dbuser, dbuserpass, port) elif dbengine == 'MariaDB': auth = mariadb_util.auth_mariadb(dbuser, dbuserpass, port) elif dbengine == 'Neo4j': auth = True else: return "Error: Container not found" if auth: print("auth is true; deleting: %s" % dbname) admin_kill(dbname, username) result = "Container %s successfully deleted. " % dbname result += "Container type: %s" % dbengine else: result = "Authentication failed; You must be super user to remove" return result
def backup_audit(outfp): """ inspect the backup logs for every container that is running. get list of all "running" containers inspect backup logs based on backup policy for each container """ check_list = [] containers = admin_db.list_active_containers() day_of_week = datetime.datetime.today().weekday() for (c_id, con_name) in containers: data = admin_db.get_container_data('', c_id) if 'BACKUP_FREQ' in data['Info']: policy = data['Info']['BACKUP_FREQ'] else: check_list.append({ 'Name': con_name, 'Status': 'failure', 'Message': 'Policy not defined', 'Policy': 'None' }) continue if policy == 'Daily' or policy == 'Weekly': if policy == 'Weekly' and day_of_week != day_for_weekly: continue (status, start, duration, msg) = check_backup_logs(c_id) print("%s" % start) audit = dict({ 'Name': con_name, 'Status': status, 'Start': start, 'Duration': duration, 'Policy': policy, 'Message': msg }) check_list.append(audit) output_prometheus(outfp, check_list)
def restart_con(dbname, dbuser, dbuserpass, username, admin_log=True): """ restart container - determin which container type - check authentication with db - restart docker container """ state_info = admin_db.get_container_state(dbname) data = admin_db.get_container_data('', state_info.c_id) dbengine = data['Info']['dbengine'] port = data['Info']['Port'] auth = False if dbengine == 'Postgres': auth = postgres_util.auth_check(dbuser, dbuserpass, port) elif dbengine == 'MongoDB': auth = mongodb_util.auth_mongodb(dbuser, dbuserpass, port) elif dbengine == 'MariaDB': auth = mariadb_util.auth_mariadb(dbuser, dbuserpass, port) elif dbengine == 'Neo4j': auth = neo4j_util.auth_check(dbuser, dbuserpass, port) else: return "Error: container type not found" if auth: try: client.restart(dbname) result = "%s successfully restarted" % dbname except Exception, e: result = "An error occured: %s" % e if admin_log: state_info = admin_db.get_container_state(dbname) message = 'Restarted by %s' % username admin_db.add_container_log(state_info.c_id, dbname, message, description='')
def backup_mariadb(params): """ MariaDB mysqldump - full database dump. Backups are PIPED to AWS S3 backups are run from dbaas container. Require connection to container db object. Requires AWS S3 tools on dbaas. """ con_name = params['dbname'] passwd = Config.accounts['MariaDB']['admin_pass'] (c_id, dbengine) = admin_db.get_container_type(con_name) data = admin_db.get_container_data('', c_id) port = data['Info']['Port'] (backupdir, backup_id) = container_util.create_backupdir(con_name) if 'backup_type' in params: backup_type = params['backup_type'] else: backup_type = 'NA' bucket = os.environ.get('AWS_BUCKET') s3_url = bucket + '/' + con_name + backupdir s3_filename = s3_url + '/dump' + backup_id + '.sql' s3_infopath = s3_url + '/infoschema' + backup_id + '.sql' # mysqldump to S3 Backups template = "/usr/bin/mysqldump --host %s --port %s --user=root " % ( Config.container_ip, port) template += "--password=%s " dump_cmd = template + "--all-databases" cmd = dump_cmd % passwd aws_cmd = "aws --only-show-errors s3 cp - %s --sse" % s3_filename admin_db.backup_log(c_id, con_name, 'start', backup_id, backup_type, url='', command=cmd, err_msg='') print("DEBUG: mysqdump command: %s" % cmd) p1 = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) p2 = subprocess.Popen(aws_cmd.split(), stdin=p1.stdout, stdout=subprocess.PIPE) (processed, err) = p2.communicate() if processed and len(processed) > 0: message = "MariaDB Backup error. Container: %s" % con_name message += "Error message: %s" % processed print(message) send_mail("DB4SCI: MariaDB backup", message) else: message = '\nExecuted MariaDB dump comand:\n ' message += dump_cmd % 'xxxxx' message += "\nResult:\n" + processed message += "Dump file: %s\n\n" % s3_filename admin_db.backup_log(c_id, con_name, 'end', backup_id, backup_type, url=s3_filename, command=cmd, err_msg=processed) return "command", message
def display_info(con_name): '''display ['Info'] data for a container''' data = admin_db.get_container_data(con_name) info = data['Info'] print(con_name) for k in info.keys(): print('%-20s: %s' % (k, info[k]))
def add_info_field(key, value): """ Add key value to all existing DB4SCI containers """ containers = admin_db.list_containers() for [cid, name] in containers: print('%3d Update: %s' % (cid, name)) data = admin_db.get_container_data('', cid) info = data['Info'] if key in info: continue info[key] = value admin_db.update_container_info(cid, info, 'admUser')
def backup(params, tag=None): """Backup all databases for a given Postgres container pg_dump is run from the database container """ error = '' dbtype = 'Postgres' con_name = params['dbname'] (c_id, dbengine) = admin_db.get_container_type(con_name) (dirname, backup_id) = container_util.create_backupdir(con_name) state_info = admin_db.get_container_state(con_name) data = admin_db.get_container_data(con_name, state_info.c_id) if 'backup_type' in params: backup_type = params['backup_type'] else: backup_type = data['Info']['BACKUP_FREQ'] port = data['Info']['Port'] db_vol = data['Info']['DBVOL'] # Dump postgres account information localpath = Config.info[dbtype]['backupdir'] + dirname command = "pg_dumpall -g -w -U %s " % Config.accounts[dbtype]['admin'] command += "-f %s/%s.sql" % (localpath, con_name) start = time.time() result = container_util.exec_command(con_name, command) if len(result) > 0: error += result # write start infor to backup log admin_db.backup_log(c_id, con_name, 'start', backup_id, backup_type, url='', command=command, err_msg=result) # copy alter system commands dest = '/' + con_name + '/' src_file = db_vol + dest + 'postgresql.auto.conf' dest_file = Config.backup_voll + dest + dirname + '/postgresql.auto.conf' print('DEBUG: cp ' + src_file + ' ' + dest_file) copyfile(src_file, dest_file) # Get list of databases to be backed up connect = "dbname='postgres' user='******' " % Config.accounts[dbtype]['admin'] connect += "host=%s " % Config.container_host connect += "password='******' " % Config.accounts[dbtype]['admin_pass'] connect += "port=%s" % port message = '\nExecuted Postgres dump_all commands:\n' try: connection = psycopg2.connect(connect) except Exception as e: message = "Error: DB4SCI Postgres Backup; " message += "psycopg2 connect: container: %s, " % con_name message += "message: %s," % e message += "connect string: %s" % connect print('ERROR: ' + message) return command, message cur = connection.cursor() select = "SELECT datname from pg_database where datname " select += "<> 'postgres' and datistemplate=false" cur.execute(select) dbs = cur.fetchall() # back up each database for db in dbs: dbname = (db[0]) if tag: dumpfile = '/' + con_name + "_" + dbname + "_" + tag + ".dump" else: dumpfile = '/' + con_name + "_" + dbname + ".dump" dumppath = localpath + dumpfile command = "pg_dump --dbname %s " % dbname command += "--username %s -F c -f %s" % ( Config.accounts[dbtype]['admin'], dumppath) print("DEBUG: dump_now command: %s" % (command)) result = container_util.exec_command(con_name, command) message += command message += '\nResult: ' + result + '\n' admin_db.add_container_log(state_info.c_id, con_name, 'GUI backup', 'user: '******'username']) bucket = os.environ.get('AWS_BUCKET') url = bucket + '/' + con_name + dirname admin_db.backup_log(c_id, con_name, 'end', backup_id, backup_type, url=url, command=command, err_msg=message) return command, message
def admin(cmd): if not session['logged_in']: return redirect(url_for('login')) elif session['username'] not in Config.admins: title = 'User ' + session['username'] title += ' not in list of admins. Update Config.admins' return render_template('dblist.html', Error=True, title=title, dbheader='', dbs='') username = session['username'] if cmd == 'help': body = admin_help() title = 'DB4SCI Administrative Features\n' return render_template('dblist.html', title=title, dbheader='', dbs=body) elif cmd == 'state': (header, body) = admin_db.display_container_state() return render_template('dblist.html', title='Admin DB State Table', dbheader=header, dbs=body) elif cmd == 'list': (header, body) = admin_db.display_active_containers() return render_template('dblist.html', title='Active Containers', dbheader=header, dbs=body) elif cmd == 'containers': (header, body) = admin_db.display_containers() return render_template('dblist.html', title='Containers Summary', dbheader=header, dbs=body) elif cmd == 'du': body = admin_db.du_all() return render_template('dblist.html', title='File System du', dbheader='', dbs=body) elif cmd == 'log': (header, body) = admin_db.display_container_log() return render_template('dblist.html', title='Admin DB Log', dbheader=header, dbs=body) elif cmd == 'data': if 'cid' in request.args: data = admin_db.get_container_data('', c_id=request.args['cid']) body = json.dumps(data, indent=4) title = 'Container Inspect from admindb' return render_template('dblist.html', title=title, dbheader='', dbs=body) elif cmd == 'info': cid = None if 'name' in request.args: body = admin_db.display_container_info(request.args['name']) elif 'cid' in request.args: cid = request.args['cid'] body = admin_db.display_container_info('', c_id=cid) else: return 'DEBUG: admin-info: No URL arguments' title = 'Container Info ' # for %s' % body['Name'] return render_template('dblist.html', title=title, dbheader='', dbs=body) elif cmd == 'update': info = {} for item in request.args.keys(): if 'cid' != item: info[item] = request.args[item] if 'cid' in request.args and len(info.keys()) > 0: admin_db.update_container_info(request.args['cid'], info) return 'Updated Info\n' + json.dumps(info, indent=4) else: return 'DEBUG: admin-update: No URL arguments' elif cmd == 'delete': if 'name' in request.args: name = request.args['name'] username = session['username'] result = container_util.admin_kill(name, username) return render_template('deleted.html', result=result) elif cmd == 'setMaintenance': new_state = 'maintenance' message = 'All containers set to "maintenance"' state_info = admin_db.get_container_state() if 'done' in request.args: new_state = 'running' message = 'All containers set to "running"' for state in state_info: admin_db.update_container_state(state.c_id, new_state, who=session['username']) return message else: return 'incorect admin URL'