def restart_con(dbname, dbuser, dbuserpass, username, admin_log=True):
""" restart container
- determin which container type
- check authentication with db
- restart docker container
"""
state_info = admin_db.get_container_state(dbname)
data = admin_db.get_container_data('', state_info.c_id)
dbengine = data['Info']['dbengine']
port = data['Info']['Port']
auth = False
if dbengine == 'Postgres':
auth = postgres_util.auth_check(dbuser, dbuserpass, port)
elif dbengine == 'MongoDB':
auth = mongodb_util.auth_mongodb(dbuser, dbuserpass, port)
elif dbengine == 'MariaDB':
auth = mariadb_util.auth_mariadb(dbuser, dbuserpass, port)
elif dbengine == 'Neo4j':
auth = neo4j_util.auth_check(dbuser, dbuserpass, port)
else:
return "Error: container type not found"
if auth:
try:
client.restart(dbname)
result = "%s successfully restarted" % dbname
except Exception, e:
result = "An error occured: %s" % e
if admin_log:
state_info = admin_db.get_container_state(dbname)
message = 'Restarted by %s' % username
admin_db.add_container_log(state_info.c_id,
dbname,
message,
description='')
def mydb_setup():
"""Create mydb_admin database if it does not exist.
DB4SCI depends on mydb_admin database.
"""
if container_util.container_exists('mydb_admin'):
print('Administrative DB is running.\nStarting DB4Sci')
return
print('Create Administrative DB')
params = setup_data()
dbtype = params['dbtype']
con_name = params['dbname']
result = postgres_util.create(params)
# wait for container to startup
print('Container Id: %s' % params['con']['Id'])
print('Waiting for mydb_admin to start')
time.sleep(20)
badness = 0
status = False
while (not status) and (badness < 6):
badness += 1
status = postgres_util.auth_check(params['dbuser'],
params['dbuserpass'], params['port'])
print('mydb_admin setup status: %s count: %d' % (status, badness))
time.sleep(5)
if not status:
print('mydb_admin restart error. Could not setup db')
return
print('Setup mydb_admin tables')
admin_db.init_db()
inspect = container_util.inspect_con(params['con']['Id'])
c_id = admin_db.add_container(inspect, params)
state_info = admin_db.get_container_state(con_name)
description = 'created %s by user %s' % (con_name, params['username'])
admin_db.add_container_log(c_id, con_name, 'created', description)
def kill_con(dbname, dbuser, dbuserpass, username):
""" stop and remove container
"""
# get state info (running container)
state_info = admin_db.get_container_state(dbname)
data = admin_db.get_container_data('', state_info.c_id)
dbengine = data['Info']['dbengine']
port = data['Info']['Port']
print('DEBUG: kill_con: con: %s user: %s pass: %s port: %s' %
(dbname, dbuser, dbuserpass, port))
auth = False
if dbengine == 'Postgres':
auth = postgres_util.auth_check(dbuser, dbuserpass, port)
elif dbengine == 'MongoDB':
auth = mongodb_util.auth_mongodb(dbuser, dbuserpass, port)
elif dbengine == 'MariaDB':
auth = mariadb_util.auth_mariadb(dbuser, dbuserpass, port)
elif dbengine == 'Neo4j':
auth = True
else:
return "Error: Container not found"
if auth:
print("auth is true; deleting: %s" % dbname)
admin_kill(dbname, username)
result = "Container %s successfully deleted. " % dbname
result += "Container type: %s" % dbengine
else:
result = "Authentication failed; You must be super user to remove"
return result
def create_con(params, env, args=None):
"""Create new container
<params> dict of specific parameters for a given container:
dbname, username, userpass, lables and environment
args: Arguments to "run" command. (Example: run mongo --auth)
params['port_bindings'] is a dict of ports that need to be mapped.
{'3306': ('140.107.117.18', '30020')}
"""
dbtype = params['dbtype']
con_name = params['dbname']
if not args:
args = ''
volumes = []
bindings = {}
con_labels = {}
build_arguments(params, con_labels, volumes, bindings)
env['TZ'] = Config.TZ
msg = json.dumps(env, indent=4)
print("DEBUG: container_util; create_con: env=%s" % msg)
msg = json.dumps(params, indent=4)
print("DEBUG: container_util; params: %s" % msg)
print("DEBUG: container_util; create_con: ports=%s" %
params['port_bindings'])
host_conf = client.create_host_config(
mem_limit=0,
restart_policy={'Name': 'on-failure'},
port_bindings=params['port_bindings'],
binds=bindings)
print("con_labels: ", con_labels)
print("volumes: (%s)" % type(volumes))
print("volumes: ", volumes)
print("disk bind: ", bindings)
con = client.create_container(
image=params['image'],
name=con_name,
detach=True,
command=args,
environment=env,
labels=con_labels,
ports=Config.info[dbtype]['pub_ports'],
volumes=volumes,
host_config=host_conf,
)
msg = json.dumps(con, indent=4)
print("DEBUG: container_util; create_con: result=%s" % msg)
client.start(con['Id'])
# Admin_db - add container, add state, log creation
if con_name == 'mydb_admin':
return -1, con
inspect = inspect_con(con['Id'])
c_id = admin_db.add_container(inspect, params)
state_info = admin_db.get_container_state(con_name)
description = 'created %s by user %s' % (con_name, params['username'])
admin_db.add_container_log(c_id, con_name, 'created', description)
return c_id, con
def full_test(params):
admin_db.init_db()
con_name = params['dbname']
dbtype = params['dbtype']
print('Starting %s Test; Container Name: %s' % (dbtype, con_name))
if container_util.container_exists(con_name):
print(' Duplicate container: KILLING')
result = container_util.kill_con(con_name,
Config.accounts[dbtype]['admin'],
Config.accounts[dbtype]['admin_pass'],
params['username'])
time.sleep(5)
print(result)
print(' removing old directories')
volumes.cleanup_dirs(con_name)
print(' Create container')
result = postgres_util.create(params)
print(' Create result: %s' % result)
port = params['port']
#
# Admin DB checking
#
print(' Check Admin DB log for "create"')
admin_db.display_container_log(limit=1)
print(' Check Admin DB for State entry')
info = admin_db.get_container_state(con_name)
print(' Name: %s ' % info.name),
print('State: %s ' % info.state),
print('TS: %s ' % info.ts),
print('CID: %d' % info.c_id)
print(' Check Admin DB for Container Info')
info = admin_db.display_container_info(con_name)
print('Info: %s' % info)
print(' Postgres Show All')
postgres_util.showall(params)
print("\n=========")
print(" - Test Accounts\n")
print("=========")
admin_user = Config.accounts[dbtype]['admin']
admin_pass = Config.accounts[dbtype]['admin_pass']
test_user = Config.accounts['test_user']['admin']
test_pass = Config.accounts['test_user']['admin_pass']
for dbuser, dbuserpass in [[test_user, test_pass],
['svc_' + test_user, params['longpass']],
[admin_user, admin_pass]]:
auth = postgres_util.auth_check(dbuser, dbuserpass, port)
if auth:
print('User %s verified!' % dbuser)
else:
print('user account not valid: %s' % dbuser)
print(" - Test Complete")
def admin_kill(dbname, username):
""" Stop and remove container without using authtication.
Called from the admin command console Only. Clean up
admin_db. Implemented becuase I forget passwords.
"""
if stop_remove(dbname):
result = "Container %s successfully deleted " % dbname
result += "from admin console by %s" % username
send_mail("DBaaS: container removed", result)
state_info = admin_db.get_container_state(dbname)
if state_info:
admin_db.delete_container_state(state_info.c_id)
description = 'deleted %s by user %s' % (dbname, username)
admin_db.add_container_log(state_info.c_id, dbname, 'deleted',
description)
def backup(params, tag=None):
"""Backup all databases for a given Postgres container
pg_dump is run from the database container
"""
error = ''
dbtype = 'Postgres'
con_name = params['dbname']
(c_id, dbengine) = admin_db.get_container_type(con_name)
(dirname, backup_id) = container_util.create_backupdir(con_name)
state_info = admin_db.get_container_state(con_name)
data = admin_db.get_container_data(con_name, state_info.c_id)
if 'backup_type' in params:
backup_type = params['backup_type']
else:
backup_type = data['Info']['BACKUP_FREQ']
port = data['Info']['Port']
db_vol = data['Info']['DBVOL']
# Dump postgres account information
localpath = Config.info[dbtype]['backupdir'] + dirname
command = "pg_dumpall -g -w -U %s " % Config.accounts[dbtype]['admin']
command += "-f %s/%s.sql" % (localpath, con_name)
start = time.time()
result = container_util.exec_command(con_name, command)
if len(result) > 0:
error += result
# write start infor to backup log
admin_db.backup_log(c_id,
con_name,
'start',
backup_id,
backup_type,
url='',
command=command,
err_msg=result)
# copy alter system commands
dest = '/' + con_name + '/'
src_file = db_vol + dest + 'postgresql.auto.conf'
dest_file = Config.backup_voll + dest + dirname + '/postgresql.auto.conf'
print('DEBUG: cp ' + src_file + ' ' + dest_file)
copyfile(src_file, dest_file)
# Get list of databases to be backed up
connect = "dbname='postgres' user='******' " % Config.accounts[dbtype]['admin']
connect += "host=%s " % Config.container_host
connect += "password='******' " % Config.accounts[dbtype]['admin_pass']
connect += "port=%s" % port
message = '\nExecuted Postgres dump_all commands:\n'
try:
connection = psycopg2.connect(connect)
except Exception as e:
message = "Error: DB4SCI Postgres Backup; "
message += "psycopg2 connect: container: %s, " % con_name
message += "message: %s," % e
message += "connect string: %s" % connect
print('ERROR: ' + message)
return command, message
cur = connection.cursor()
select = "SELECT datname from pg_database where datname "
select += "<> 'postgres' and datistemplate=false"
cur.execute(select)
dbs = cur.fetchall()
# back up each database
for db in dbs:
dbname = (db[0])
if tag:
dumpfile = '/' + con_name + "_" + dbname + "_" + tag + ".dump"
else:
dumpfile = '/' + con_name + "_" + dbname + ".dump"
dumppath = localpath + dumpfile
command = "pg_dump --dbname %s " % dbname
command += "--username %s -F c -f %s" % (
Config.accounts[dbtype]['admin'], dumppath)
print("DEBUG: dump_now command: %s" % (command))
result = container_util.exec_command(con_name, command)
message += command
message += '\nResult: ' + result + '\n'
admin_db.add_container_log(state_info.c_id, con_name, 'GUI backup',
'user: '******'username'])
bucket = os.environ.get('AWS_BUCKET')
url = bucket + '/' + con_name + dirname
admin_db.backup_log(c_id,
con_name,
'end',
backup_id,
backup_type,
url=url,
command=command,
err_msg=message)
return command, message
def admin(cmd):
if not session['logged_in']:
return redirect(url_for('login'))
elif session['username'] not in Config.admins:
title = 'User ' + session['username']
title += ' not in list of admins. Update Config.admins'
return render_template('dblist.html',
Error=True,
title=title,
dbheader='',
dbs='')
username = session['username']
if cmd == 'help':
body = admin_help()
title = 'DB4SCI Administrative Features\n'
return render_template('dblist.html',
title=title,
dbheader='',
dbs=body)
elif cmd == 'state':
(header, body) = admin_db.display_container_state()
return render_template('dblist.html',
title='Admin DB State Table',
dbheader=header,
dbs=body)
elif cmd == 'list':
(header, body) = admin_db.display_active_containers()
return render_template('dblist.html',
title='Active Containers',
dbheader=header,
dbs=body)
elif cmd == 'containers':
(header, body) = admin_db.display_containers()
return render_template('dblist.html',
title='Containers Summary',
dbheader=header,
dbs=body)
elif cmd == 'du':
body = admin_db.du_all()
return render_template('dblist.html',
title='File System du',
dbheader='',
dbs=body)
elif cmd == 'log':
(header, body) = admin_db.display_container_log()
return render_template('dblist.html',
title='Admin DB Log',
dbheader=header,
dbs=body)
elif cmd == 'data':
if 'cid' in request.args:
data = admin_db.get_container_data('', c_id=request.args['cid'])
body = json.dumps(data, indent=4)
title = 'Container Inspect from admindb'
return render_template('dblist.html',
title=title,
dbheader='',
dbs=body)
elif cmd == 'info':
cid = None
if 'name' in request.args:
body = admin_db.display_container_info(request.args['name'])
elif 'cid' in request.args:
cid = request.args['cid']
body = admin_db.display_container_info('', c_id=cid)
else:
return 'DEBUG: admin-info: No URL arguments'
title = 'Container Info ' # for %s' % body['Name']
return render_template('dblist.html',
title=title,
dbheader='',
dbs=body)
elif cmd == 'update':
info = {}
for item in request.args.keys():
if 'cid' != item:
info[item] = request.args[item]
if 'cid' in request.args and len(info.keys()) > 0:
admin_db.update_container_info(request.args['cid'], info)
return 'Updated Info\n' + json.dumps(info, indent=4)
else:
return 'DEBUG: admin-update: No URL arguments'
elif cmd == 'delete':
if 'name' in request.args:
name = request.args['name']
username = session['username']
result = container_util.admin_kill(name, username)
return render_template('deleted.html', result=result)
elif cmd == 'setMaintenance':
new_state = 'maintenance'
message = 'All containers set to "maintenance"'
state_info = admin_db.get_container_state()
if 'done' in request.args:
new_state = 'running'
message = 'All containers set to "running"'
for state in state_info:
admin_db.update_container_state(state.c_id,
new_state,
who=session['username'])
return message
else:
return 'incorect admin URL'