def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def remove_dependency(key):
etcd_conn = Etcd()
for record in etcd_conn.search('/', search_key=key):
record_key = record[1].key.decode('utf-8')
record_split = re.split(key, record_key)
if not record_split[1] or record_split[1][:1] == '/':
etcd_conn.delete(record_key)
pass
class TestEtcd(unittest.TestCase):
def setUp(self):
logger = logging.getLogger()
stream = logging.StreamHandler(sys.stdout)
stream.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
stream.setFormatter(formatter)
logger.addHandler(stream)
self.etcd = Etcd(logger)
def test_a_setkey(self):
ret = self.etcd.set_key('message', 'Hello World')
self.assertTrue(ret)
def test_b_getkey(self):
self.etcd.set_key('message', 'Hello World')
text = self.etcd.get_key('message')
self.assertEqual(text, 'Hello World')
def test_c_deletekey(self):
#Set the key first before deleting it
self.etcd.set_key('message', 'Hello World')
text = self.etcd.delete_key('message')
regex = re.compile(r'{"action":"delete","node":{"key":"/message",'
'"modifiedIndex":\d+,"createdIndex":\d+},"prevNode":{"key":"/message"'
',"value":"Hello World","modifiedIndex":\d+,"createdIndex":\d+}}')
self.assertRegexpMatches(text, regex)
def test_d_directorylist(self):
#List a directory in Etcd
dir_list = self.etcd.list_directory('formations/cholcomb')
self.assertIsInstance(dir_list, list)
def __init__(self):
self.etcd = Etcd()
self.etcdLock = threading.Lock()
self.kubeletList = []
self.requestWaiting = threading.Event()
# self.controller = PIDController(0.0001,0.01,0) #PI
self.controller = PIDController(0.01, 0.001, 0.01) #PID
def setUp(self):
logger = logging.getLogger()
stream = logging.StreamHandler(sys.stdout)
stream.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
stream.setFormatter(formatter)
logger.addHandler(stream)
self.etcd = Etcd(logger)
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def external_services(url_type=None):
etcd_conn = Etcd()
headers = [(_('Name'), 'name'), (_('Enabled'), 'enabled')]
button_list = [{'name': _('New'), 'href': '/external_services/change'},
{'name': _('List'), 'href': '/external_services'}]
if url_type == 'change':
form = forms.ServicesForm()
if form.validate_on_submit():
status = 'false'
if form.enabled.data is True:
status = 'true'
row = {"enabled": status, "name": form.name.data, "role_service_url": form.role_service_url.data,
"role_service_param": form.role_service_param.data, "role_service_key": form.role_service_key.data,
"role_service_value": form.role_service_value.data, "user_service_url": form.user_service_url.data,
"user_service_key": form.user_service_key.data, "user_service_param": form.user_service_param.data,
"username": form.username.data, "password": form.password.data
}
etcd_conn.put('/services/' + form.name.data.lower().replace(' ', '_'),
json.dumps(row))
flash(_('Service') + ' ' + _('Added') + ' / ' + _('Updated'), 'info')
return flask.redirect(flask.url_for('external_services'))
elif flask.request.args.get('key'):
form_data = etcd_conn.get_list(flask.request.args.get('key'))
form.enabled.data = False
if form_data.get('enabled') == 'true':
form.enabled.data = True
form.role_service_url.data = form_data.get('role_service_url')
form.role_service_param.data = form_data.get('role_service_param')
form.role_service_key.data = form_data.get('role_service_key')
form.role_service_value.data = form_data.get('role_service_value')
form.user_service_url.data = form_data.get('user_service_url')
form.user_service_key.data = form_data.get('user_service_key')
form.user_service_param.data = form_data.get('user_service_param')
form.username.data = form_data.get('username')
form.password.data = form_data.get('password')
# form.name.data = flask.request.args.get('key').replace('/services/', '')
form.name.data = form_data.get('name')
form.name.render_kw = {'readonly': True}
return flask.render_template('list.html', main_header=_('Register Services'), form=form,
button_list=button_list)
elif url_type == 'delete':
etcd_conn.delete(flask.request.args.get('key'))
flash(_('Service') + ' ' + _('Deleted'), 'error')
return flask.redirect(flask.url_for('external_services'))
group_list = etcd_conn.search('/services/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/external_services/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/external_services/change'}]
return flask.render_template('list.html', main_header=_('Services'),
list=get_calculated_page(group_list, page), pagination=page,
button_list=button_list, links=links, headers=headers)
def system_users(url_type=None):
etcd_conn = Etcd()
headers = [(_('Username'), 'username'), (_('Role'), 'role'), (_('Enabled'), 'enabled')]
button_list = [{'name': _('New'), 'href': '/system_users/change'}, {'name': _('List'), 'href': '/system_users'}]
if url_type == 'change':
form = forms.UserForm()
if form.validate_on_submit():
status = False
if form.enabled.data is True:
status = True
row = {"enabled": status, "locale": form.locale.data, "email": form.email.data,
"username": form.username.data, "role": form.role.data}
if flask.request.args.get('key'):
if form.password.data:
row['password'] = g.user.hash_password(form.password.data)
else:
form_data = etcd_conn.get_list(flask.request.args.get('key'))
row['password'] = form_data.get('password')
else:
row['password'] = g.user.hash_password(form.password.data)
etcd_conn.put('/appuser/' + form.username.data, json.dumps(row))
flash(_('System User') + ' ' + _('Added') + ' / ' + _('Updated'), 'info')
return flask.redirect(flask.url_for('system_users'))
elif flask.request.args.get('key'):
form_data = etcd_conn.get_list(flask.request.args.get('key'))
form.enabled.data = False
if form_data.get('enabled') is True:
form.enabled.data = True
form.username.render_kw = {'readonly': True}
form.email.data = form_data.get('email')
form.locale.data = form_data.get('locale')
form.username.data = form_data.get('username')
form.role.data = form_data.get('role')
return flask.render_template('list.html', main_header=_('Users'), form=form, button_list=button_list)
elif url_type == 'delete':
etcd_conn.delete(flask.request.args.get('key'))
flash(_('SQL User') + ' ' + _('Deleted'), 'error')
return flask.redirect(flask.url_for('system_users'))
group_list = etcd_conn.search('/appuser/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/system_users/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/system_users/change'}]
return flask.render_template('list.html', main_header=_('SQL Users'),
list=get_calculated_page(group_list, page), headers=headers,
button_list=button_list, links=links, pagination=page)
def main(template, client_url, haproxy_binary, haproxy_pid, haproxy_service, haproxy_config, interval_check):
"""Generate a haproxy config file based on etcd then gracefully reload haproxy"""
# TODO: if haproxy_binary, make sure haproxy_pid is provided
etcd = Etcd(client_url)
haproxy = Haproxy(haproxy_service, haproxy_binary, haproxy_pid)
if template:
haproxy.set_template(template)
if haproxy_config:
haproxy.set_config_file(haproxy_config)
while True:
data = []
services = etcd.fetch_services()
for service in services:
instances = etcd.fetch_instances_of(service)
data.append({'service': service, 'instances': instances})
haproxy.reload(data)
time.sleep(interval_check)
def main():
"""Generate a haproxy config file based on etcd then gracefully reload haproxy"""
# This is designed to work with containers, we expect to receive params via the environment
template = os.environ.get('TEMPLATE', '/app/templates/haproxy.tpl')
client_url = os.environ.get('CLIENT_URL', None)
haproxy_binary = os.environ.get('HAPROXY_BINARY', '/usr/sbin/haproxy')
haproxy_pid = os.environ.get('HAPROXY_PID', '/var/run/haproxy.pid')
haproxy_service = None
haproxy_config = os.environ.get('HAPROXY_CONFIG',
'/etc/haproxy/haproxy.cfg')
interval_check = os.environ.get('INTERVAL_CHECK', 5)
cluster_name = os.environ.get('CLUSTER_NAME', 'cluster1')
# Cant continue if we don't have an etcd endpoint
if not client_url:
sys.exit('CLIENT_URL has not been defined')
etcd = Etcd(client_url)
haproxy = Haproxy(haproxy_service, haproxy_binary, haproxy_pid)
if template:
haproxy.set_template(template)
if haproxy_config:
haproxy.set_config_file(haproxy_config)
while True:
data = []
services = etcd.fetch_services()
for service in services:
# Rename the service to match the template, if the cluster name in etcd matches our galera cluster.
if service['name'] != cluster_name:
continue
service['name'] = 'galera'
instances = etcd.fetch_instances_of(service)
data.append({'service': service, 'instances': instances})
# TODO - if we don't have any cluster data, should we die?
haproxy.reload(data)
time.sleep(interval_check)
def autocomplete(url_type=None, key=None):
etcd_conn = Etcd()
autocomplete_list = []
if url_type == 'autocomplete_table' or url_type == 'autocomplete_table_without_columns':
if key.count('.') == 0:
fields = etcd_conn.search_keys('/' + key.replace('.', '/'))
for i in fields:
if i.split('.')[0] + '.' not in autocomplete_list:
autocomplete_list.append(i.split('.')[0] + '.')
elif key.count('.') == 1:
fields = etcd_conn.search_keys('/' + key.replace('.', '/'))
for i in fields:
if '.'.join(i.split('.')[0:2]) + '.' not in autocomplete_list:
autocomplete_list.append('.'.join(i.split('.')[0:2]) + '.')
elif key.count('.') == 2:
fields = etcd_conn.search_keys('/' + key.replace('.', '/'))
for i in fields:
if '.'.join(i.split('.')[0:3]) + '.' not in autocomplete_list:
dot = ''
if url_type == 'autocomplete_table':
dot = '.'
autocomplete_list.append('.'.join(i.split('.')[0:3]) + dot)
elif key.count('.') == 3 and url_type == 'autocomplete_table':
search_key = key.split('.')[-1]
fields = etcd_conn.search('/' + '/'.join(key.split('.')[:-1]))
for i in fields[0][0]:
if i.get('column_name').find(search_key) > -1:
autocomplete_list.append('.'.join(key.split('.')[:-1]) + '.' + i.get('column_name'))
elif url_type == 'autocomplete_role':
service_key = request.args.get("service")
if service_key is not None:
service = etcd_conn.get_list(service_key)
import requests
r = requests.post(service.get('role_service_url'), json={service.get('role_service_param'): key},
auth=(service.get('username'), service.get('password')))
for row in r.json():
autocomplete_list.append({"value": row.get(service.get('role_service_key')),
"label": row.get(service.get('role_service_value'))})
else:
autocomplete_list = etcd_conn.search_keys(
'/' + url_type.replace('autocomplete_', '') + '/' + key.replace('.', '/'))
return Response(json.dumps(autocomplete_list), mimetype='application/json')
def sqlfilter(url_type=None):
etcd_conn = Etcd()
button_list = [{'name': _('New'), 'href': '/sqlfilter/change'}, {'name': _('List'), 'href': '/sqlfilter'}]
headers = [(_('SQL Filter'), 'filter'), (_('Group Name'), 'group_name'), (_('Enabled'), 'enabled')]
if url_type == 'change':
form = forms.SQLFilterForm()
if form.validate_on_submit():
enabled = 'false'
if form.enabled.data:
enabled = 'true'
if form.group_name.data == '':
form.group_name.data = '*'
row = {"filter": form.filter.data, "group_name": form.group_name.data, "enabled": enabled}
etcd_conn.put(
'/sqlfilter/' + form.table.data.replace('.', '/') + '/' + form.group_name.data.replace('.', '/'),
json.dumps(row))
flash(_('SQL Filter') + ' ' + _('Added'), 'info')
return flask.redirect(flask.url_for('sqlfilter'))
elif flask.request.args.get('key'):
form_data = etcd_conn.get_list(flask.request.args.get('key'))
form.enabled.data = False
if form_data.get('enabled') == 'true':
form.enabled.data = True
form.filter.data = form_data.get('filter')
form.group_name.data = form_data.get('group_name')
form.table.data = flask.request.args.get('key').replace('/sqlfilter/', '').replace('/', '.').replace(
'.' + form_data.get('group_name'), '')
form.group_name.render_kw = {'readonly': True}
form.table.render_kw = {'readonly': True}
return flask.render_template('list.html', main_header=_('SQL Filter'), form=form, button_list=button_list)
elif url_type == 'delete':
etcd_conn.delete(flask.request.args.get('key'))
flash(_('SQL Filter') + ' ' + _('Deleted'), 'error')
return flask.redirect(flask.url_for('sqlfilter'))
group_list = etcd_conn.search('/sqlfilter/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/sqlfilter/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/sqlfilter/change'}]
return flask.render_template('list.html', main_header=_('SQL Filter'), list=get_calculated_page(group_list, page),
headers=headers,
button_list=button_list, links=links, pagination=page)
def dbusers(url_type=None):
etcd_conn = Etcd()
headers = [(_('Enabled'), 'enabled')]
button_list = [{'name': _('New'), 'href': '/dbusers/change'}, {'name': _('List'), 'href': '/dbusers'}]
if url_type == 'change':
form = forms.DBorCommentUsersForm()
if form.validate_on_submit():
status = 'false'
if form.enabled.data is True:
status = 'true'
row = {"enabled": status}
etcd_conn.put('/dbuser/' + form.user.data + '/' + form.group_name.data.replace('.', '/'),
json.dumps(row))
flash(_('DB User') + ' ' + _('Added') + ' / ' + _('Updated'), 'info')
return flask.redirect(flask.url_for('dbusers'))
elif flask.request.args.get('key'):
form_data = etcd_conn.get_list(flask.request.args.get('key'))
form.enabled.data = False
if form_data.get('enabled') == 'true':
form.enabled.data = True
parse_name = flask.request.args.get('key').split('/')
form.user.data = parse_name[2]
form.user.render_kw = {'readonly': True}
form.group_name.data = parse_name[3] + '.' + parse_name[4]
form.group_name.render_kw = {'readonly': True}
return flask.render_template('list.html', main_header=_('Users'), form=form, button_list=button_list)
elif url_type == 'delete':
etcd_conn.delete(flask.request.args.get('key'))
flash(_('DB User') + ' ' + _('Deleted'), 'error')
return flask.redirect(flask.url_for('dbusers'))
group_list = etcd_conn.search('/dbuser/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/dbusers/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/dbusers/change'}]
return flask.render_template('list.html', main_header=_('DB Users'),
list=get_calculated_page(group_list, page), headers=headers,
button_list=button_list, links=links, pagination=page)
def groups(url_type=None):
etcd_conn = Etcd()
headers = [(_('Description'), 'desc'), (_('Enabled'), 'enabled')]
button_list = [{'name': _('New'), 'href': '/groups/change'}, {'name': _('List'), 'href': '/groups'}]
if url_type == 'change':
form = forms.GroupsForm()
if form.validate_on_submit():
status = 'false'
if form.enabled.data is True:
status = 'true'
row = {"enabled": status, "desc": form.desc.data}
etcd_conn.put('/groups/' + form.name.data.replace(' ', '_'),
json.dumps(row))
flash(_('Group') + ' ' + _('Added') + ' / ' + _('Updated'), 'info')
return flask.redirect(flask.url_for('groups'))
elif flask.request.args.get('key'):
form_data = etcd_conn.get_list(flask.request.args.get('key'))
form.enabled.data = False
if form_data.get('enabled') == 'true':
form.enabled.data = True
form.desc.data = form_data.get('desc')
form.name.data = flask.request.args.get('key').replace('/groups/', '')
form.name.render_kw = {'readonly': True}
return flask.render_template('list.html', main_header=_('Groups'), form=form, button_list=button_list)
elif url_type == 'delete':
remove_dependency(flask.request.args.get('key'))
flash(_('Group') + ' ' + _('Deleted'), 'error')
return flask.redirect(flask.url_for('groups'))
group_list = etcd_conn.search('/groups/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/groups/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/groups/change'}]
return flask.render_template('list.html', main_header=_('Groups'),
list=get_calculated_page(group_list, page), pagination=page,
headers=headers, button_list=button_list, links=links)
def __init__(self, manager, logger):
self.logger = logger
self.manager = manager
self.etcd = Etcd(logger)
def __init__(self, manager, logger):
self.logger = logger
self.salt_client = salt.client.LocalClient()
self.manager = manager
self.etcd = Etcd(logger)
class VerifyFormations(object):
def __init__(self, manager, logger):
self.logger = logger
self.salt_client = salt.client.LocalClient()
self.manager = manager
self.etcd = Etcd(logger)
def start_verifying(self):
# Parse out the username and formation name
# from the ETCD directory string
formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
# call out to ETCD and load all the formations
formation_list = []
user_list = self.etcd.list_directory('formations')
if user_list:
for user in user_list:
formations = self.etcd.list_directory(user)
for formation in formations:
parse_results = formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
username = parse_results['username']
self.logger.info(
'Attempting to load formation: {formation_name} '
'with username: {username}'.format(
formation_name=formation_name,
username=username))
f = self.manager.load_formation_from_etcd(
username, formation_name)
formation_list.append(f)
else:
self.logger.error("Could not parse the ETCD string")
if formation_list:
# TODO Use background salt jobs
# Start verifying things
# Ask salt to do these things for me and give me back an job_id
# results = self.salt_client.cmd_async(host, 'cmd.run',
# ['netstat -an | grep %s | grep tcp | grep -i listen' % port],
# expr_form='list')
#
# salt-run jobs.lookup_jid <job id number>
for f in formation_list:
for app in f.application_list:
# Check to make sure it's up and running
self.logger.info(
"Running verification on app: "
"{app_name}".format(app_name=app.hostname))
self.logger.info(
'{server} docker ps | grep {container_id}'.format(
server=app.host_server,
container_id=app.container_id))
results = self.salt_client.cmd(
app.host_server,
'cmd.run', [
'docker ps | grep {container_id}'.format(
container_id=app.container_id)
],
expr_form='list')
if results:
self.logger.debug(
"Salt return: {docker_results}".format(
docker_results=results[app.host_server]))
if results[app.host_server] == "":
self.logger.error(
"App {app} is not running!".format(
app=app.hostname))
# Start the app back up and run start.sh on there
self.start_application(app)
else:
self.logger.info(
"App {app} is running. Checking if "
"cron is running also".format(
app=app.hostname))
# Check if cron is running on the container and bring it back
# up if needed
# Log in with ssh and check if cron is up and running
self.logger.info(
"Sleeping 2 seconds while the container starts"
)
time.sleep(2)
self.check_running_application(app)
else:
self.logger.error(
"Call out to server {server} failed. Moving it"
.format(server=app.host_server))
# move the container
self.move_application(app)
# Start an application that isn't running
def start_application(self, app):
# Start the application and run start.sh to kick off cron
self.logger.info(
"Starting app {app} with docker id: {app_id} up".format(
app=app.hostname, app_id=app.container_id))
results = self.salt_client.cmd(
app.host_server,
'cmd.run', [
'docker start {container_id}'.format(
container_id=app.container_id)
],
expr_form='list')
self.logger.debug(results)
if results:
if "Error: No such container" in results[app.host_server]:
# We need to recreate the container
self.logger.error("Container is missing on the host!. "
"Trying to recreate")
self.manager.start_application(app)
self.logger.info(
"Sleeping 2 seconds while the container starts")
time.sleep(2)
self.manager.bootstrap_application(app)
elif "Error: start: No such container:" in results[
app.host_server]:
# Seems the container already exists but won't start. Bug?
self.logger.error("Container failed to start")
self.move_application(app)
else:
self.logger.info(
"Waiting 2 seconds for docker to start the container")
time.sleep(2)
self.check_running_application(app)
else:
# Move the container to another host, this host is messed up
self.logger.error(
"Failed to start {container_id} on host {host}".format(
container_id=app.container_id, host=app.host_server))
self.move_application(app)
# Move an application to another host and record the change in etcd
def move_application(self, app):
old_host = app.host_server
cluster_list = self.manager.get_docker_cluster()
circular_cluster_list = CircularList(
self.manager.order_cluster_by_load(cluster_list))
if app.host_server in circular_cluster_list:
index = circular_cluster_list.index(app.host_server)
app.host_server = circular_cluster_list[index + 1].hostname
else:
# Assign the first one in the list if not found above
app.host_server = circular_cluster_list[0].hostname
self.logger.info(
"Moving app {app_name} from {old_host} to {new_host}".format(
app_name=app.hostname,
old_host=old_host,
new_host=app.host_server))
self.logger.info("Bootstrapping the application on the new host")
self.start_application(app)
# Log into the application via ssh and check everything
def check_running_application(self, app):
# TODO
# Use the docker top command to see if cron is running instead of using ssh
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Move this user/pass into a config file
self.logger.info('SSHing into host {hostname}:{port}'.format(
hostname=app.host_server, port=app.ssh_port))
ssh.connect(hostname=app.host_server,
port=app.ssh_port,
username='******',
password='******')
# Is cron running?
# If not run start.sh
stdin, stdout, stderr = ssh.exec_command("pgrep cron")
output = stdout.readlines()
self.logger.debug(output)
if len(output) == 0:
# cron isn't running
self.logger.info("Cron is not running. Starting it back up")
stdin, stdout, stderr = ssh.exec_command("/root/start.sh")
else:
self.logger.info("Cron is running.")
ssh.close()
except SSHException:
self.logger.error("Failed to log into server.")
def __init__(self, logger): self.salt_client = salt.client.LocalClient() self.etcd = Etcd(logger) self.logger = logger
class Manager(object):
'''
A manager to orchestrate the creation and
deletion of container clusters
'''
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
def fqdn_to_shortname(self, fqdn):
if '.' in fqdn:
return fqdn.split('.')[0]
else:
return fqdn
def check_port_used(self, host, port):
self.logger.info("Checking if {port} on {host} is open with salt-client".format(
host=host, port=port))
results = self.salt_client.cmd(host, 'cmd.run',
['netstat -an | grep %s | grep tcp | grep -i listen' % port],
expr_form='list')
self.logger.debug("Salt return: {lsof}".format(lsof=results[host]))
if results[host] is not '':
return True
else:
return False
# TODO
def check_for_existing_formation(self, formation_name):
# If the user passed in an existing formation name lets append to it
pass
def get_docker_cluster(self):
# Return a list of docker hosts
cluster = self.etcd.get_key('docker_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def get_load_balancer_cluster(self):
# Return a list of nginx hosts
cluster = self.etcd.get_key('nginx_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def order_cluster_by_load(self, cluster_list):
# Sample salt output
# {'dlceph01.drwg.local': '0.27 0.16 0.15 1/1200 26234'}
# define grammar
point = Literal('.')
number = Word(nums)
floatnumber = Combine( number + point + number)
float_list = OneOrMore(floatnumber)
results = self.salt_client.cmd(','.join(cluster_list), 'cmd.run', ['cat /proc/loadavg'], expr_form='list')
load_list = []
self.logger.debug("Salt load return: {load}".format(load=results))
for host in results:
host_load = results[host]
match = float_list.parseString(host_load)
if match:
one_min = match[0]
five_min = match[1]
fifteen_min = match[2]
self.logger.debug("Adding Load({host}, {one_min}, {five_min}, {fifteen_min}".format(
host=host, one_min=one_min, five_min=five_min, fifteen_min=fifteen_min))
load_list.append(Load(host, one_min, five_min, fifteen_min))
else:
self.logger.error("Could not parse host load output")
# Sort the list by fifteen min load
load_list = sorted(load_list, key=lambda x: x.fifteen_min_load)
for load in load_list:
self.logger.debug("Sorted load list: " + str(load))
return load_list
# Load the formation and return a Formation object
def load_formation_from_etcd(self, username, formation_name):
f = Formation(username,formation_name)
app_list = json.loads(json.loads(
self.etcd.get_key('/formations/{username}/{formation_name}'.format(
username=username, formation_name=formation_name))))
for app in app_list:
# If our host doesn't support swapping we're going to get some garbage
# message in here
if "WARNING" in app['container_id']:
app['container_id'] = app['container_id'].replace("WARNING: Your "\
"kernel does not support memory swap capabilities. Limitation discarded.\n","")
# Set volumes if needed
volumes = None
if app['volumes']:
self.logger.info("Setting volumes to: " + ''.join(app['volumes']))
volumes = app['volumes']
f.add_app(app['container_id'], app['hostname'], app['cpu_shares'],
app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes)
# Return fully parsed and populated formation object
return f
def save_formation_to_etcd(self, formation):
name = formation.name
username = formation.username
self.etcd.set_key('formations/{username}/{formation_name}'.format(
username=username, formation_name=name), formation)
# TODO write code to add new apps to load balancer
def add_app_to_nginx(self, app):
pass
# TODO write code to add new apps to the load balancer
def add_app_to_apache(self, app):
pass
def start_application(self, app):
# Run a salt cmd to startup the formation
docker_command = "docker run -c={cpu_shares} -d -h=\"{hostname}\" -m={ram} "\
"-name=\"{hostname}\" {port_list} {volume_list} {image} /usr/sbin/sshd -D"
self.logger.info("Port list %s" % app.port_list)
port_list = ' '.join(map(lambda x: '-p ' + x, app.port_list))
# Only create this list if needed
volume_list = ''
if app.volume_list:
volume_list = ' '.join(map(lambda x: '-v ' + x, app.volume_list))
d = docker_command.format(cpu_shares=app.cpu_shares,
hostname=app.hostname, ram=app.ram, image='dlcephgw01:5000/sshd',
port_list=port_list, volume_list=volume_list)
self.logger.info("Starting up docker container on {host_server} with cmd: {docker_cmd}".format(
host_server=app.host_server, docker_cmd=d))
salt_process = self.salt_client.cmd(app.host_server,'cmd.run', [d], expr_form='list')
container_id = salt_process[app.host_server]
if container_id:
app.change_container_id(container_id)
def bootstrap_application(self, app):
# Log into the host with paramiko and run the salt bootstrap script
host_server = self.fqdn_to_shortname(app.host_server)
self.logger.info("Bootstrapping {hostname} on server: {host_server} port: {port}".format(
hostname=app.hostname,
host_server=host_server,
port=app.ssh_port))
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host_server, port=app.ssh_port,
username='******', password='******')
transport = paramiko.Transport((host_server, app.ssh_port))
transport.connect(username = '******', password = '******')
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.put('bootstrap.sh', '/root/bootstrap.sh')
sftp.put('start.sh', '/root/start.sh')
ssh.exec_command("chmod +x /root/bootstrap.sh")
ssh.exec_command("chmod +x /root/start.sh")
stdin, stdout, stderr = ssh.exec_command("bash /root/start.sh")
self.logger.debug(''.join(stdout.readlines()))
ssh.close()
except SSHException:
self.logger.error("Failed to log into server. Shutting it down and cleaning up the mess.")
self.delete_container(app.host_server, app.container_id)
# Stops and deletes a container
def delete_container(self, host_server, container_id):
results = self.salt_client.cmd(host_server, 'cmd.run',
['docker stop {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug("Salt return: {stop_cmd}".format(stop_cmd=results[host_server]))
results = self.salt_client.cmd(host_server, 'cmd.run',
['docker rm {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug("Salt return: {rm_cmd}".format(rm_cmd=results[host_server]))
def create_containers(self, user, number, formation_name,
cpu_shares, ram, port_list, hostname_scheme, volume_list,
force_host_server=None):
f = Formation(user, formation_name)
# Convert ram to bytes from MB
ram = ram * 1024 * 1024
# Get the cluster machines on each creation
cluster_list = self.get_docker_cluster()
circular_cluster_list = CircularList(self.order_cluster_by_load(cluster_list))
# Loop for the requested amount of containers to be created
for i in range(1, number+1):
# [{"host_port":ssh_host_port, "container_port":ssh_container_port}]
ssh_host_port = 9022 + i
ssh_container_port = 22
host_server = circular_cluster_list[i].hostname
# We are being asked to overwrite this
if force_host_server:
host_server = force_host_server
validated_ports = []
while self.check_port_used(host_server, ssh_host_port):
ssh_host_port = ssh_host_port +1
for port in port_list:
self.logger.info("Checking if port {port} on {host} is in use".format(
port=port, host=host_server))
if ':' in port:
ports = port.split(':')
# Only check if the host port is free. The container port should be free
while self.check_port_used(host_server, ports[0]):
ports[0] = int(ports[0]) + 1
# Add this to the validated port list
validated_ports.append('{host_port}:{container_port}'.format(
host_port = str(ports[0]),
container_port = str(ports[1])))
else:
while self.check_port_used(host_server, port):
port = int(port) + 1
validated_ports.append(str(port))
self.logger.info('Adding app to formation {formation_name}: {hostname}{number} cpu_shares={cpu} '
'ram={ram} ports={ports} host_server={host_server}'.format(formation_name=formation_name,
hostname=hostname_scheme, number=str(i).zfill(3), cpu=cpu_shares, ram=ram,
ports=validated_ports, host_server=host_server))
f.add_app(None, '{hostname}{number}'.format(hostname=hostname_scheme,
number=str(i).zfill(3)), cpu_shares, ram, validated_ports, ssh_host_port,
ssh_container_port, circular_cluster_list[i].hostname, volume_list)
# Lets get this party started
for app in f.application_list:
self.start_application(app)
self.logger.info("Sleeping 2 seconds while the container starts")
time.sleep(2)
self.bootstrap_application(app)
self.logger.info("Saving the formation to ETCD")
self.save_formation_to_etcd(f)
class User(UserMixin):
etcd = None
password = None
username = None
password_hash = None
secret = "89660c74da48ddd4efbe4d3c8f8150de"
locale = 'tr'
enabled = False
email = None
role = 'viewer'
def __init__(self, username, password=None, etcd=None):
self.etcd = Etcd()
self.username = username
self.password = self.hash_password(password)
self.get(username)
def hash_password(self, password):
return hashlib.sha256(str(password).encode('utf-8')).hexdigest()
# return password
def verify_password(self):
if self.password_hash == self.password:
return True
else:
return False
def get(self, username):
user = json.loads(self.etcd.get('/appuser/' + username)[0])
if user is not None:
if user['enabled'] is True:
self.password_hash = user['password']
self.username = user['username']
self.locale = user['locale']
self.enabled = user['enabled']
self.email = user['email']
self.role = user['role']
return True
return False
def set(self):
user = {}
user['password'] = self.password_hash
user['username'] = self.username
user['locale'] = self.locale
user['enabled'] = self.enabled
user['email'] = self.email
user['role'] = self.role
self.etcd.put('/appuser/' + self.username, json.dumps(user))
def is_authenticated(self):
return self.enabled
def is_active(self):
return self.enabled
def is_anonymous(self):
return False
def get_id(self):
return unicode(self.username)
def __repr__(self):
return '<User %r>' % self.username
def __init__(self, username, password=None, etcd=None):
self.etcd = Etcd()
self.username = username
self.password = self.hash_password(password)
self.get(username)
class Manager(object):
'''
A manager to orchestrate the creation and
deletion of container clusters
'''
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def fqdn_to_shortname(self, fqdn):
if '.' in fqdn:
return fqdn.split('.')[0]
else:
return fqdn
def check_salt_key_used(self, hostname):
self.logger.info(
"Checking if the key for {host} is already used".format(
host=hostname))
s = subprocess.Popen('salt-key', shell=True, stdout=PIPE)
salt_list = s.communicate()[0]
if hostname in salt_list:
return True
else:
return False
def check_port_used(self, host, port):
self.logger.info(
"Checking if {port} on {host} is open with salt-client".format(
host=host, port=port))
results = self.salt_client.cmd(
host,
'cmd.run',
['netstat -an | grep %s | grep tcp | grep -i listen' % port],
expr_form='list')
self.logger.debug("Salt return: {lsof}".format(lsof=results[host]))
if results[host] is not '':
return True
else:
return False
# TODO
def check_for_existing_formation(self, formation_name):
# If the user passed in an existing formation name lets append to it
pass
def get_docker_cluster(self):
# Return a list of docker hosts
cluster = self.etcd.get_key('docker_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def get_load_balancer_cluster(self):
# Return a list of nginx hosts
cluster = self.etcd.get_key('nginx_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def order_cluster_by_load(self, cluster_list):
# Sample salt output
# {'dlceph01.drwg.local': '0.27 0.16 0.15 1/1200 26234'}
# define grammar
point = Literal('.')
number = Word(nums)
floatnumber = Combine(number + point + number)
float_list = OneOrMore(floatnumber)
results = self.salt_client.cmd(','.join(cluster_list),
'cmd.run', ['cat /proc/loadavg'],
expr_form='list')
load_list = []
self.logger.debug("Salt load return: {load}".format(load=results))
for host in results:
host_load = results[host]
match = float_list.parseString(host_load)
if match:
one_min = match[0]
five_min = match[1]
fifteen_min = match[2]
self.logger.debug(
"Adding Load({host}, {one_min}, {five_min}, {fifteen_min}".
format(host=host,
one_min=one_min,
five_min=five_min,
fifteen_min=fifteen_min))
load_list.append(Load(host, one_min, five_min, fifteen_min))
else:
self.logger.error("Could not parse host load output")
# Sort the list by fifteen min load
load_list = sorted(load_list, key=lambda x: x.fifteen_min_load)
for load in load_list:
self.logger.debug("Sorted load list: " + str(load))
return load_list
# Retun a list of formations the user owns
def list_formations(self, username):
formation_list = []
formations = self.etcd.list_directory('formations/' + username)
for formation in formations:
parse_results = self.formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
formation_list.append(formation_name)
else:
self.logger.error("Could not parse the ETCD string")
self.logger.info('Formation list {formations} for user {user}'.format(
formations=formation_list, user=username))
return formation_list
# Load the formation and return a Formation object
def load_formation_from_etcd(self, username, formation_name):
f = Formation(username, formation_name)
app_list = json.loads(
json.loads(
self.etcd.get_key(
'/formations/{username}/{formation_name}'.format(
username=username, formation_name=formation_name))))
for app in app_list:
# If our host doesn't support swapping we're going to get some garbage
# message in here
if "WARNING" in app['container_id']:
app['container_id'] = app['container_id'].replace("WARNING: Your "\
"kernel does not support memory swap capabilities. Limitation discarded.\n","")
#Message changed in docker 0.8.0
app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
app['container_id'].strip('\n')
# Set volumes if needed
volumes = None
if app['volumes']:
self.logger.info("Setting volumes to: " +
''.join(app['volumes']))
volumes = app['volumes']
f.add_app(app['container_id'], app['hostname'], app['cpu_shares'],
app['ram'], app['port_list'], app['ssh_port'], 22,
app['host_server'], volumes)
# Return fully parsed and populated formation object
return f
def save_formation_to_etcd(self, formation):
name = formation.name
username = formation.username
self.etcd.set_key(
'formations/{username}/{formation_name}'.format(
username=username, formation_name=name), formation)
# TODO write code to add new apps to load balancer
def add_app_to_nginx(self, app):
pass
# TODO write code to add new apps to the load balancer
def add_app_to_apache(self, app):
pass
def start_application(self, app):
# Run a salt cmd to startup the formation
docker_command = "docker run -c={cpu_shares} -d -i -t -h=\"{hostname}\" -m={ram}m "\
"--name={hostname} {port_list} {volume_list} {image} /sbin/my_init -- bash"
self.logger.info("Port list %s" % app.port_list)
port_list = ' '.join(map(lambda x: '-p ' + x, app.port_list))
# Only create this list if needed
volume_list = ''
if app.volume_list:
volume_list = ' '.join(map(lambda x: '-v ' + x, app.volume_list))
d = docker_command.format(cpu_shares=app.cpu_shares,
hostname=app.hostname,
ram=app.ram,
image=app.docker_image,
port_list=port_list,
volume_list=volume_list)
self.logger.info(
"Starting up docker container on {host_server} with cmd: {docker_cmd}"
.format(host_server=app.host_server, docker_cmd=d))
salt_process = self.salt_client.cmd(app.host_server,
'cmd.run', [d],
expr_form='list')
container_id = salt_process[app.host_server]
if container_id:
if "WARNING" in container_id:
container_id = container_id.replace("WARNING: "\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
container_id.strip("\n")
#Docker only uses the first 12 chars to identify a container
app.change_container_id(container_id[0:12])
def bootstrap_application(self, app):
# Log into the host with paramiko and run the salt bootstrap script
host_server = self.fqdn_to_shortname(app.host_server)
self.logger.info(
"Bootstrapping {hostname} on server: {host_server} port: {port}".
format(hostname=app.hostname,
host_server=host_server,
port=app.ssh_port))
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host_server,
port=app.ssh_port,
username='******',
password='******')
transport = paramiko.Transport((host_server, app.ssh_port))
transport.connect(username='******', password='******')
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.put('bootstrap.sh', '/root/bootstrap.sh')
sftp.put('start.sh', '/root/start.sh')
ssh.exec_command("chmod +x /root/bootstrap.sh")
ssh.exec_command("chmod +x /root/start.sh")
stdin, stdout, stderr = ssh.exec_command("bash /root/start.sh")
self.logger.debug(''.join(stdout.readlines()))
ssh.close()
except SSHException:
self.logger.error(
"Failed to log into server. Shutting it down and cleaning up the mess."
)
self.delete_container(app.host_server, app.container_id)
# Stops and deletes a container
def delete_container(self, host_server, container_id):
results = self.salt_client.cmd(
host_server,
'cmd.run',
['docker stop {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug(
"Salt return: {stop_cmd}".format(stop_cmd=results[host_server]))
results = self.salt_client.cmd(
host_server,
'cmd.run',
['docker rm {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug(
"Salt return: {rm_cmd}".format(rm_cmd=results[host_server]))
# Stops and deletes a formation. Use with caution
def delete_formation(self, user, formation_name):
formation_list = self.list_formations(user)
if formation_name in formation_list:
pass
else:
self.logger.error("Formation name not found!")
def list_containers(self, user, formation_name):
pass
def create_containers(self,
user,
number,
formation_name,
cpu_shares,
ram,
port_list,
hostname_scheme,
volume_list,
docker_image,
force_host_server=None):
f = Formation(user, formation_name)
# Convert ram to bytes from MB
ram = ram * 1024 * 1024
# Get the cluster machines on each creation
cluster_list = self.get_docker_cluster()
circular_cluster_list = CircularList(
self.order_cluster_by_load(cluster_list))
# Loop for the requested amount of containers to be created
for i in range(1, number + 1):
# [{"host_port":ssh_host_port, "container_port":ssh_container_port}]
ssh_host_port = 9022 + i
ssh_container_port = 22
host_server = circular_cluster_list[i].hostname
hostname = '{hostname}{number}'.format(hostname=hostname_scheme,
number=str(i).zfill(3))
# First check if we can add this host to salt. If not exit with -1
if self.check_salt_key_used(hostname):
self.logger.error(
'Salt key is already taken for {hostname}'.format(
hostname=hostname))
sys.exit(-1)
# We are being asked to overwrite this
if force_host_server:
host_server = force_host_server
validated_ports = []
while self.check_port_used(host_server, ssh_host_port):
ssh_host_port = ssh_host_port + 1
for port in port_list:
self.logger.info(
"Checking if port {port} on {host} is in use".format(
port=port, host=host_server))
if ':' in port:
ports = port.split(':')
# Only check if the host port is free. The container port should be free
while self.check_port_used(host_server, ports[0]):
ports[0] = int(ports[0]) + 1
# Add this to the validated port list
validated_ports.append(
'{host_port}:{container_port}'.format(
host_port=str(ports[0]),
container_port=str(ports[1])))
else:
while self.check_port_used(host_server, port):
port = int(port) + 1
validated_ports.append(str(port))
self.logger.info(
'Adding app to formation {formation_name}: {hostname} cpu_shares={cpu} '
'ram={ram} ports={ports} host_server={host_server} docker_image={docker_image}'
.format(formation_name=formation_name,
hostname=hostname,
cpu=cpu_shares,
ram=ram,
ports=validated_ports,
host_server=host_server,
docker_image=docker_image))
f.add_app(None, '{hostname}'.format(hostname=hostname), cpu_shares,
ram, validated_ports, ssh_host_port, ssh_container_port,
host_server, docker_image, volume_list)
# Lets get this party started
for app in f.application_list:
self.start_application(app)
#self.logger.info("Sleeping 2 seconds while the container starts")
#time.sleep(2)
#self.bootstrap_application(app)
self.logger.info("Saving the formation to ETCD")
self.save_formation_to_etcd(f)
def __init__(self):
self.etcd = Etcd()
self.etcdLock = threading.Lock()
self.requestWaiting = threading.Event()
def role_to_group(url_type=None):
etcd_conn = Etcd()
button_list = [{'name': _('New'), 'href': '/role_to_group/change'}, {'name': _('List'), 'href': '/role_to_group'}]
header = [(_('Services'), 'service_key'), (_('Group Name'), 'group'), (_('Enabled'), 'enabled')]
if url_type == 'change':
form = forms.RoleForm()
service_list = [(None, _('Select'))]
for x in etcd_conn.search('/services'):
service_list.append((str(x[1].key.decode("utf-8")), str(x[0].get('name'))))
form.service.choices = service_list
if form.validate_on_submit():
try:
status = 'false'
if form.enabled.data:
status = 'true'
key = '/role_to_group/' + form.group_name.data.replace('.', '/') + "/" + str(form.role_id.data)
etcd_conn.put(key, json.dumps(
{"enabled": status, "group": form.role.data, "service_key": form.service.data}))
except Exception as error:
print(error)
flash(_('Role to Group') + ' ' + _('Added') + ' / ' + _('Updated'), 'success')
return flask.redirect('/role_to_group/refresh?key={}'.format(key))
elif flask.request.args.get('key'):
parse_key = flask.request.args.get('key').split('/')
form_data = etcd_conn.get_list(flask.request.args.get('key'))
form.enabled.data = False
if form_data.get('enabled') == 'true':
form.enabled.data = True
form.service.data = form_data.get('service_key')
form.group_name.data = parse_key[2] + '.' + parse_key[3]
form.role.data = form_data.get('group')
form.role_id.data = parse_key[4]
form.service.render_kw = {'readonly': True}
form.group_name.render_kw = {'readonly': True}
form.role.render_kw = {'readonly': True}
return flask.render_template('role.html', main_header=_('Role to Group'), form=form, button_list=button_list)
elif url_type == 'delete':
parse_key = flask.request.args.get('key').split('/')
for x in etcd_conn.get_prefix('/users/', sort_order="ascend", sort_target="key"):
parse_user_key = str(x[1].key.decode("utf-8")).split('/')
# print(parse_user_key)
if parse_user_key[3] == parse_key[2] and parse_user_key[4] == parse_key[3]:
etcd_conn.delete(x[1].key.decode("utf-8"))
etcd_conn.delete(flask.request.args.get('key'))
flash(_('Role to Group') + ' ' + _('Deleted'), 'error')
return flask.redirect('/role_to_group')
elif url_type == 'refresh':
key = flask.request.args.get('key')
parse_key = key.split('/')
role_to_group = etcd_conn.get_list(key)
status = role_to_group.get('enabled')
service = etcd_conn.get_list(role_to_group.get('service_key'))
import requests
r = requests.post(service.get('user_service_url'),
json={service.get('user_service_param'): parse_key[4]},
auth=(service.get('username'), service.get('password')))
for row in r.json():
id = row.get(service.get('user_service_key'))
key = '/users/{}/{}/{}'.format(str(id), parse_key[2], parse_key[3])
try:
user = etcd_conn.get_list(key)
if user.get('enabled') != status:
etcd_conn.put(key, json.dumps({"enabled": status}))
except:
etcd_conn.put(key, json.dumps({"enabled": status}))
flash(_('Role to Group') + ' ' + _('Refreshed'), 'success')
return flask.redirect('/role_to_group')
group_list = etcd_conn.search('/role_to_group/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/role_to_group/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/role_to_group/change'},
{'name': _('Refresh'), 'type': 'success', 'link': '/role_to_group/refresh'}]
return flask.render_template('list.html', main_header=_('Role to Group'), button_list=button_list,
list=get_calculated_page(group_list, page), headers=header,
links=links, pagination=page)
class VerifyFormations(object):
def __init__(self, manager, logger):
self.logger = logger
self.salt_client = salt.client.LocalClient()
self.manager = manager
self.etcd = Etcd(logger)
def start_verifying(self):
# Parse out the username and formation name
# from the ETCD directory string
formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
# call out to ETCD and load all the formations
formation_list = []
user_list = self.etcd.list_directory('formations')
if user_list:
for user in user_list:
formations = self.etcd.list_directory(user)
for formation in formations:
parse_results = formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
username = parse_results['username']
self.logger.info('Attempting to load formation: {formation_name} '
'with username: {username}'.format(formation_name=formation_name,
username=username))
f = self.manager.load_formation_from_etcd(username, formation_name)
formation_list.append(f)
else:
self.logger.error("Could not parse the ETCD string")
if formation_list:
# TODO Use background salt jobs
# Start verifying things
# Ask salt to do these things for me and give me back an job_id
# results = self.salt_client.cmd_async(host, 'cmd.run',
# ['netstat -an | grep %s | grep tcp | grep -i listen' % port],
# expr_form='list')
#
# salt-run jobs.lookup_jid <job id number>
for f in formation_list:
for app in f.application_list:
# Check to make sure it's up and running
self.logger.info("Running verification on app: "
"{app_name}".format(app_name=app.hostname))
self.logger.info('{server} docker ps | grep {container_id}'.format(
server=app.host_server,
container_id=app.container_id))
results = self.salt_client.cmd(app.host_server, 'cmd.run',
['docker ps | grep {container_id}'.format(container_id=app.container_id)],
expr_form='list')
if results:
self.logger.debug("Salt return: {docker_results}".format(
docker_results=results[app.host_server]))
if results[app.host_server] == "":
self.logger.error("App {app} is not running!".format(
app=app.hostname))
# Start the app back up and run start.sh on there
self.start_application(app)
else:
self.logger.info("App {app} is running. Checking if "
"cron is running also".format(app=app.hostname))
# Check if cron is running on the container and bring it back
# up if needed
# Log in with ssh and check if cron is up and running
self.logger.info("Sleeping 2 seconds while the container starts")
time.sleep(2)
self.check_running_application(app)
else:
self.logger.error("Call out to server {server} failed. Moving it".format(
server=app.host_server))
# move the container
self.move_application(app)
# Start an application that isn't running
def start_application(self, app):
# Start the application and run start.sh to kick off cron
self.logger.info("Starting app {app} with docker id: {app_id} up".format(
app=app.hostname, app_id=app.container_id))
results = self.salt_client.cmd(app.host_server, 'cmd.run',
['docker start {container_id}'.format(container_id=app.container_id)],
expr_form='list')
self.logger.debug(results)
if results:
if "Error: No such container" in results[app.host_server]:
# We need to recreate the container
self.logger.error("Container is missing on the host!. "
"Trying to recreate")
self.manager.start_application(app)
self.logger.info("Sleeping 2 seconds while the container starts")
time.sleep(2)
self.manager.bootstrap_application(app)
elif "Error: start: No such container:" in results[app.host_server]:
# Seems the container already exists but won't start. Bug?
self.logger.error("Container failed to start")
self.move_application(app)
else:
self.logger.info("Waiting 2 seconds for docker to start the container")
time.sleep(2)
self.check_running_application(app)
else:
# Move the container to another host, this host is messed up
self.logger.error("Failed to start {container_id} on host {host}".format(
container_id=app.container_id, host=app.host_server))
self.move_application(app)
# Move an application to another host and record the change in etcd
def move_application(self, app):
old_host = app.host_server
cluster_list = self.manager.get_docker_cluster()
circular_cluster_list = CircularList(
self.manager.order_cluster_by_load(cluster_list))
if app.host_server in circular_cluster_list:
index = circular_cluster_list.index(app.host_server)
app.host_server = circular_cluster_list[index+1].hostname
else:
# Assign the first one in the list if not found above
app.host_server = circular_cluster_list[0].hostname
self.logger.info("Moving app {app_name} from {old_host} to {new_host}".format(
app_name=app.hostname, old_host=old_host, new_host=app.host_server))
self.logger.info("Bootstrapping the application on the new host")
self.start_application(app)
# Log into the application via ssh and check everything
def check_running_application(self, app):
# TODO
# Use the docker top command to see if cron is running instead of using ssh
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Move this user/pass into a config file
self.logger.info('SSHing into host {hostname}:{port}'.format(
hostname=app.host_server, port=app.ssh_port))
ssh.connect(hostname=app.host_server, port=app.ssh_port,
username='******', password='******')
# Is cron running?
# If not run start.sh
stdin, stdout, stderr = ssh.exec_command("pgrep cron")
output = stdout.readlines()
self.logger.debug(output)
if len(output) == 0:
# cron isn't running
self.logger.info("Cron is not running. Starting it back up")
stdin, stdout, stderr = ssh.exec_command("/root/start.sh")
else:
self.logger.info("Cron is running.")
ssh.close()
except SSHException:
self.logger.error("Failed to log into server.")
def dbmeta(url_type=None):
etcd_conn = Etcd()
try:
pg_ddm_config = configparser.RawConfigParser(allow_no_value=True)
if str(config['general']['get_db_info_in_pg_ddm_config_file']).lower() == 'true':
pg_ddm_config.read(config['general']['pg_ddm_config_file_path'])
databases = pg_ddm_config['databases']
else:
databases = config['database']
db_list = []
conn_info = {}
for db in databases:
db_list.append((db, db))
key_list = {}
tmp_dsn = []
for i in databases[db].split(" "):
key = i.split("=")
if key[0] != 'search_path' and key[0] != 'route':
tmp_dsn.append(key[0] + '=' + key[1])
key_list[key[0]] = key[1]
conn_info[db] = key_list
databases[db] = ' '.join(tmp_dsn)
form = forms.TableSelectForm()
form.db.choices = db_list
button_list = [{'name': _('Refresh'), 'href': '/dbmeta/change'}, {'name': _('List'), 'href': '/dbmeta'}]
# if url_type == 'update':
# print('update')
if url_type == 'change':
if form.validate_on_submit():
form_cred = forms.TablesForm()
if form.db.data is not None:
if 'user' in conn_info[form.db.data]:
form_cred.username.data = conn_info[form.db.data]['user']
if 'password' in conn_info[form.db.data]:
form_cred.password.data = conn_info[form.db.data]['password']
form_cred.db.data = form.db.data
if form_cred.validate_on_submit():
conn = None
try:
db = databases[form.db.data]
conn = psycopg2.connect(db, user=form_cred.username.data, password=form_cred.password.data)
cur = conn.cursor()
cur.execute("""SELECT i.table_catalog,i.table_schema,i.table_name,(
SELECT array_to_json(array_agg(col_array)) FROM (
SELECT i2.column_name,i2.data_type
FROM information_schema.columns i2
WHERE i2.table_catalog = i.table_catalog
AND i2.table_schema = i.table_schema
AND i2.table_name = i.table_name
) col_array
) FROM information_schema.columns i
WHERE i.table_schema NOT IN ('pg_catalog','information_schema','mask')
GROUP BY 1,2,3""")
for row in cur.fetchall():
key = '/{}/{}/{}'.format(str(form.db.data), str(row[1]), str(row[2]))
etcd_conn.put(key, json.dumps(row[3]))
cur.close()
except (Exception, psycopg2.DatabaseError) as e:
flash(_('DB Error'), e)
except (Exception, psycopg2.DatabaseError) as e:
flash(_('General Error'), e)
finally:
if conn is not None:
conn.close()
flash(_('Database Metadata') + ' ' + _('Updated'), 'info')
return flask.redirect(flask.url_for('dbmeta'))
return flask.render_template('list.html', main_header=_('Database connection'), form=form_cred,
button_list=button_list)
return flask.render_template('list.html', main_header=_('Database select'), form=form,
button_list=button_list)
else:
if form.validate_on_submit() or request.args.get('db'):
if request.args.get('db') is not None:
db = request.args.get('db')
else:
db = str(form.db.data)
group_list = etcd_conn.search('/{}/'.format(db), json_field=False)
headers = [(_('Columns'), '')]
# links = [{'name': _('Update'), 'type': 'info', 'link': '/dbmeta/update'}]
links = []
extra_param = '&db=' + str(db)
if request.args.get('search_key') is not None and request.args.get('search_key'):
extra_param += '&search_key=' + str(request.args.get('search_key')) + '&search_type=' + str(
request.args.get('search_type'))
page = pagination(len(group_list), extra_param)
return flask.render_template('list.html', main_header=_('Database Metadata'),
list=get_calculated_page(group_list, page), pagination=page,
headers=headers,
links=links, button_list=button_list)
return flask.render_template('list.html', main_header=_('Database Metadata'), form=form,
button_list=button_list)
except FileNotFoundError:
flash(_('pg_ddm config file not found'), 'error')
except KeyError:
flash(_('pg_ddm_config_file_path key not found in settings.cfg'), 'warning')
return flask.render_template('list.html', main_header=_('Database Metadata'))
def rules(url_type=None):
etcd_conn = Etcd()
button_list = [{'name': _('New'), 'href': '/rules/change'}, {'name': _('List'), 'href': '/rules'}]
headers = [(_('Description'), 'description'), (_('Group Name'), 'group_name'), (_('Table'), 'table_column'),
(_('Enabled'), 'enabled')]
if url_type == 'change':
form = forms.RulesForm()
if form.validate_on_submit():
status = 'false'
if form.enabled.data is True:
status = 'true'
prop = ''
if form.rule.data in form.test.keys():
if len(form.test[form.rule.data]) > 0:
prop = "["
for i in form.test[form.rule.data]:
x = str(i).replace('open_close_', '')
if x == 'col':
prop += '%col%,'
elif str(flask.request.form[x]).isdigit():
prop += '{"A_Const": {"val": ' + flask.request.form[x] + '}},'
else:
prop += '{"A_Const": {"val": {"String": {"str": "' + flask.request.form[x] + '"}}}},'
prop = prop[:-1] + ']'
else:
prop = '[]'
row = {"name": form.name.data, "description": form.description.data,
"table_column": form.table_column.data,
"filter": form.filter.data, "enabled": status,
"group_name": form.group_name.data,
"prop": prop,
"rule": form.rule.data}
etcd_conn.put('/rules/' + form.table_column.data.replace('.', '/') + '/' + form.group_name.data.replace('.',
'/') + '/' + form.name.data,
json.dumps(row))
flash(_('Rule') + ' ' + _('Added') + ' / ' + _('Updated'), 'info')
return flask.redirect(flask.url_for('rules'))
elif flask.request.args.get('key'):
form_data = etcd_conn.get_list(flask.request.args.get('key'))
i = 0
prop = form_data.get('prop').replace('%col%', '"%col%"')
for x in json.loads(prop):
if x != "%col%":
obj = getattr(form, form.test.get(form_data.get('rule'))[i].replace('open_close_', ''))
try:
obj.data = x.get('A_Const').get('val').get("String").get("str")
except:
obj.data = x.get('A_Const').get('val')
i = i + 1
form.enabled.data = False
if form_data.get('enabled') == 'true':
form.enabled.data = True
form.rule.data = form_data.get('rule')
form.description.data = form_data.get('description')
form.group_name.data = form_data.get('group_name')
form.filter.data = form_data.get('filter')
form.table_column.data = form_data.get('table_column')
form.name.data = form_data.get('name')
form.name.render_kw = {'readonly': True}
form.table_column.render_kw = {'readonly': True}
form.group_name.render_kw = {'readonly': True}
return flask.render_template('rules.html', main_header=_('Rules'), form=form, button_list=button_list)
elif url_type == 'delete':
etcd_conn.delete(flask.request.args.get('key'))
flash(_('Rule') + ' ' + _('Deleted'), 'error')
return flask.redirect(flask.url_for('rules'))
group_list = etcd_conn.search('/rules/')
page = pagination(len(group_list))
links = [{'name': _('Delete'), 'type': 'danger', 'link': '/rules/delete'},
{'name': _('Update'), 'type': 'info', 'link': '/rules/change'}]
return flask.render_template('list.html', main_header=_('Rules'),
list=get_calculated_page(group_list, page),
pagination=page, headers=headers,
button_list=button_list, links=links)
