def __init__(self, logger): self.salt_client = salt.client.LocalClient() self.etcd = Etcd(logger) self.logger = logger # Parse out the username and formation name # from the ETCD directory string self.formation_parser = Literal('/formations/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def remove_dependency(key): etcd_conn = Etcd() for record in etcd_conn.search('/', search_key=key): record_key = record[1].key.decode('utf-8') record_split = re.split(key, record_key) if not record_split[1] or record_split[1][:1] == '/': etcd_conn.delete(record_key) pass
class TestEtcd(unittest.TestCase): def setUp(self): logger = logging.getLogger() stream = logging.StreamHandler(sys.stdout) stream.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') stream.setFormatter(formatter) logger.addHandler(stream) self.etcd = Etcd(logger) def test_a_setkey(self): ret = self.etcd.set_key('message', 'Hello World') self.assertTrue(ret) def test_b_getkey(self): self.etcd.set_key('message', 'Hello World') text = self.etcd.get_key('message') self.assertEqual(text, 'Hello World') def test_c_deletekey(self): #Set the key first before deleting it self.etcd.set_key('message', 'Hello World') text = self.etcd.delete_key('message') regex = re.compile(r'{"action":"delete","node":{"key":"/message",' '"modifiedIndex":\d+,"createdIndex":\d+},"prevNode":{"key":"/message"' ',"value":"Hello World","modifiedIndex":\d+,"createdIndex":\d+}}') self.assertRegexpMatches(text, regex) def test_d_directorylist(self): #List a directory in Etcd dir_list = self.etcd.list_directory('formations/cholcomb') self.assertIsInstance(dir_list, list)
def __init__(self): self.etcd = Etcd() self.etcdLock = threading.Lock() self.kubeletList = [] self.requestWaiting = threading.Event() # self.controller = PIDController(0.0001,0.01,0) #PI self.controller = PIDController(0.01, 0.001, 0.01) #PID
def setUp(self): logger = logging.getLogger() stream = logging.StreamHandler(sys.stdout) stream.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') stream.setFormatter(formatter) logger.addHandler(stream) self.etcd = Etcd(logger)
def external_services(url_type=None): etcd_conn = Etcd() headers = [(_('Name'), 'name'), (_('Enabled'), 'enabled')] button_list = [{'name': _('New'), 'href': '/external_services/change'}, {'name': _('List'), 'href': '/external_services'}] if url_type == 'change': form = forms.ServicesForm() if form.validate_on_submit(): status = 'false' if form.enabled.data is True: status = 'true' row = {"enabled": status, "name": form.name.data, "role_service_url": form.role_service_url.data, "role_service_param": form.role_service_param.data, "role_service_key": form.role_service_key.data, "role_service_value": form.role_service_value.data, "user_service_url": form.user_service_url.data, "user_service_key": form.user_service_key.data, "user_service_param": form.user_service_param.data, "username": form.username.data, "password": form.password.data } etcd_conn.put('/services/' + form.name.data.lower().replace(' ', '_'), json.dumps(row)) flash(_('Service') + ' ' + _('Added') + ' / ' + _('Updated'), 'info') return flask.redirect(flask.url_for('external_services')) elif flask.request.args.get('key'): form_data = etcd_conn.get_list(flask.request.args.get('key')) form.enabled.data = False if form_data.get('enabled') == 'true': form.enabled.data = True form.role_service_url.data = form_data.get('role_service_url') form.role_service_param.data = form_data.get('role_service_param') form.role_service_key.data = form_data.get('role_service_key') form.role_service_value.data = form_data.get('role_service_value') form.user_service_url.data = form_data.get('user_service_url') form.user_service_key.data = form_data.get('user_service_key') form.user_service_param.data = form_data.get('user_service_param') form.username.data = form_data.get('username') form.password.data = form_data.get('password') # form.name.data = flask.request.args.get('key').replace('/services/', '') form.name.data = form_data.get('name') form.name.render_kw = {'readonly': True} return flask.render_template('list.html', main_header=_('Register Services'), form=form, button_list=button_list) elif url_type == 'delete': etcd_conn.delete(flask.request.args.get('key')) flash(_('Service') + ' ' + _('Deleted'), 'error') return flask.redirect(flask.url_for('external_services')) group_list = etcd_conn.search('/services/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/external_services/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/external_services/change'}] return flask.render_template('list.html', main_header=_('Services'), list=get_calculated_page(group_list, page), pagination=page, button_list=button_list, links=links, headers=headers)
def system_users(url_type=None): etcd_conn = Etcd() headers = [(_('Username'), 'username'), (_('Role'), 'role'), (_('Enabled'), 'enabled')] button_list = [{'name': _('New'), 'href': '/system_users/change'}, {'name': _('List'), 'href': '/system_users'}] if url_type == 'change': form = forms.UserForm() if form.validate_on_submit(): status = False if form.enabled.data is True: status = True row = {"enabled": status, "locale": form.locale.data, "email": form.email.data, "username": form.username.data, "role": form.role.data} if flask.request.args.get('key'): if form.password.data: row['password'] = g.user.hash_password(form.password.data) else: form_data = etcd_conn.get_list(flask.request.args.get('key')) row['password'] = form_data.get('password') else: row['password'] = g.user.hash_password(form.password.data) etcd_conn.put('/appuser/' + form.username.data, json.dumps(row)) flash(_('System User') + ' ' + _('Added') + ' / ' + _('Updated'), 'info') return flask.redirect(flask.url_for('system_users')) elif flask.request.args.get('key'): form_data = etcd_conn.get_list(flask.request.args.get('key')) form.enabled.data = False if form_data.get('enabled') is True: form.enabled.data = True form.username.render_kw = {'readonly': True} form.email.data = form_data.get('email') form.locale.data = form_data.get('locale') form.username.data = form_data.get('username') form.role.data = form_data.get('role') return flask.render_template('list.html', main_header=_('Users'), form=form, button_list=button_list) elif url_type == 'delete': etcd_conn.delete(flask.request.args.get('key')) flash(_('SQL User') + ' ' + _('Deleted'), 'error') return flask.redirect(flask.url_for('system_users')) group_list = etcd_conn.search('/appuser/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/system_users/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/system_users/change'}] return flask.render_template('list.html', main_header=_('SQL Users'), list=get_calculated_page(group_list, page), headers=headers, button_list=button_list, links=links, pagination=page)
def main(template, client_url, haproxy_binary, haproxy_pid, haproxy_service, haproxy_config, interval_check): """Generate a haproxy config file based on etcd then gracefully reload haproxy""" # TODO: if haproxy_binary, make sure haproxy_pid is provided etcd = Etcd(client_url) haproxy = Haproxy(haproxy_service, haproxy_binary, haproxy_pid) if template: haproxy.set_template(template) if haproxy_config: haproxy.set_config_file(haproxy_config) while True: data = [] services = etcd.fetch_services() for service in services: instances = etcd.fetch_instances_of(service) data.append({'service': service, 'instances': instances}) haproxy.reload(data) time.sleep(interval_check)
def main(): """Generate a haproxy config file based on etcd then gracefully reload haproxy""" # This is designed to work with containers, we expect to receive params via the environment template = os.environ.get('TEMPLATE', '/app/templates/haproxy.tpl') client_url = os.environ.get('CLIENT_URL', None) haproxy_binary = os.environ.get('HAPROXY_BINARY', '/usr/sbin/haproxy') haproxy_pid = os.environ.get('HAPROXY_PID', '/var/run/haproxy.pid') haproxy_service = None haproxy_config = os.environ.get('HAPROXY_CONFIG', '/etc/haproxy/haproxy.cfg') interval_check = os.environ.get('INTERVAL_CHECK', 5) cluster_name = os.environ.get('CLUSTER_NAME', 'cluster1') # Cant continue if we don't have an etcd endpoint if not client_url: sys.exit('CLIENT_URL has not been defined') etcd = Etcd(client_url) haproxy = Haproxy(haproxy_service, haproxy_binary, haproxy_pid) if template: haproxy.set_template(template) if haproxy_config: haproxy.set_config_file(haproxy_config) while True: data = [] services = etcd.fetch_services() for service in services: # Rename the service to match the template, if the cluster name in etcd matches our galera cluster. if service['name'] != cluster_name: continue service['name'] = 'galera' instances = etcd.fetch_instances_of(service) data.append({'service': service, 'instances': instances}) # TODO - if we don't have any cluster data, should we die? haproxy.reload(data) time.sleep(interval_check)
def autocomplete(url_type=None, key=None): etcd_conn = Etcd() autocomplete_list = [] if url_type == 'autocomplete_table' or url_type == 'autocomplete_table_without_columns': if key.count('.') == 0: fields = etcd_conn.search_keys('/' + key.replace('.', '/')) for i in fields: if i.split('.')[0] + '.' not in autocomplete_list: autocomplete_list.append(i.split('.')[0] + '.') elif key.count('.') == 1: fields = etcd_conn.search_keys('/' + key.replace('.', '/')) for i in fields: if '.'.join(i.split('.')[0:2]) + '.' not in autocomplete_list: autocomplete_list.append('.'.join(i.split('.')[0:2]) + '.') elif key.count('.') == 2: fields = etcd_conn.search_keys('/' + key.replace('.', '/')) for i in fields: if '.'.join(i.split('.')[0:3]) + '.' not in autocomplete_list: dot = '' if url_type == 'autocomplete_table': dot = '.' autocomplete_list.append('.'.join(i.split('.')[0:3]) + dot) elif key.count('.') == 3 and url_type == 'autocomplete_table': search_key = key.split('.')[-1] fields = etcd_conn.search('/' + '/'.join(key.split('.')[:-1])) for i in fields[0][0]: if i.get('column_name').find(search_key) > -1: autocomplete_list.append('.'.join(key.split('.')[:-1]) + '.' + i.get('column_name')) elif url_type == 'autocomplete_role': service_key = request.args.get("service") if service_key is not None: service = etcd_conn.get_list(service_key) import requests r = requests.post(service.get('role_service_url'), json={service.get('role_service_param'): key}, auth=(service.get('username'), service.get('password'))) for row in r.json(): autocomplete_list.append({"value": row.get(service.get('role_service_key')), "label": row.get(service.get('role_service_value'))}) else: autocomplete_list = etcd_conn.search_keys( '/' + url_type.replace('autocomplete_', '') + '/' + key.replace('.', '/')) return Response(json.dumps(autocomplete_list), mimetype='application/json')
def sqlfilter(url_type=None): etcd_conn = Etcd() button_list = [{'name': _('New'), 'href': '/sqlfilter/change'}, {'name': _('List'), 'href': '/sqlfilter'}] headers = [(_('SQL Filter'), 'filter'), (_('Group Name'), 'group_name'), (_('Enabled'), 'enabled')] if url_type == 'change': form = forms.SQLFilterForm() if form.validate_on_submit(): enabled = 'false' if form.enabled.data: enabled = 'true' if form.group_name.data == '': form.group_name.data = '*' row = {"filter": form.filter.data, "group_name": form.group_name.data, "enabled": enabled} etcd_conn.put( '/sqlfilter/' + form.table.data.replace('.', '/') + '/' + form.group_name.data.replace('.', '/'), json.dumps(row)) flash(_('SQL Filter') + ' ' + _('Added'), 'info') return flask.redirect(flask.url_for('sqlfilter')) elif flask.request.args.get('key'): form_data = etcd_conn.get_list(flask.request.args.get('key')) form.enabled.data = False if form_data.get('enabled') == 'true': form.enabled.data = True form.filter.data = form_data.get('filter') form.group_name.data = form_data.get('group_name') form.table.data = flask.request.args.get('key').replace('/sqlfilter/', '').replace('/', '.').replace( '.' + form_data.get('group_name'), '') form.group_name.render_kw = {'readonly': True} form.table.render_kw = {'readonly': True} return flask.render_template('list.html', main_header=_('SQL Filter'), form=form, button_list=button_list) elif url_type == 'delete': etcd_conn.delete(flask.request.args.get('key')) flash(_('SQL Filter') + ' ' + _('Deleted'), 'error') return flask.redirect(flask.url_for('sqlfilter')) group_list = etcd_conn.search('/sqlfilter/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/sqlfilter/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/sqlfilter/change'}] return flask.render_template('list.html', main_header=_('SQL Filter'), list=get_calculated_page(group_list, page), headers=headers, button_list=button_list, links=links, pagination=page)
def dbusers(url_type=None): etcd_conn = Etcd() headers = [(_('Enabled'), 'enabled')] button_list = [{'name': _('New'), 'href': '/dbusers/change'}, {'name': _('List'), 'href': '/dbusers'}] if url_type == 'change': form = forms.DBorCommentUsersForm() if form.validate_on_submit(): status = 'false' if form.enabled.data is True: status = 'true' row = {"enabled": status} etcd_conn.put('/dbuser/' + form.user.data + '/' + form.group_name.data.replace('.', '/'), json.dumps(row)) flash(_('DB User') + ' ' + _('Added') + ' / ' + _('Updated'), 'info') return flask.redirect(flask.url_for('dbusers')) elif flask.request.args.get('key'): form_data = etcd_conn.get_list(flask.request.args.get('key')) form.enabled.data = False if form_data.get('enabled') == 'true': form.enabled.data = True parse_name = flask.request.args.get('key').split('/') form.user.data = parse_name[2] form.user.render_kw = {'readonly': True} form.group_name.data = parse_name[3] + '.' + parse_name[4] form.group_name.render_kw = {'readonly': True} return flask.render_template('list.html', main_header=_('Users'), form=form, button_list=button_list) elif url_type == 'delete': etcd_conn.delete(flask.request.args.get('key')) flash(_('DB User') + ' ' + _('Deleted'), 'error') return flask.redirect(flask.url_for('dbusers')) group_list = etcd_conn.search('/dbuser/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/dbusers/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/dbusers/change'}] return flask.render_template('list.html', main_header=_('DB Users'), list=get_calculated_page(group_list, page), headers=headers, button_list=button_list, links=links, pagination=page)
def groups(url_type=None): etcd_conn = Etcd() headers = [(_('Description'), 'desc'), (_('Enabled'), 'enabled')] button_list = [{'name': _('New'), 'href': '/groups/change'}, {'name': _('List'), 'href': '/groups'}] if url_type == 'change': form = forms.GroupsForm() if form.validate_on_submit(): status = 'false' if form.enabled.data is True: status = 'true' row = {"enabled": status, "desc": form.desc.data} etcd_conn.put('/groups/' + form.name.data.replace(' ', '_'), json.dumps(row)) flash(_('Group') + ' ' + _('Added') + ' / ' + _('Updated'), 'info') return flask.redirect(flask.url_for('groups')) elif flask.request.args.get('key'): form_data = etcd_conn.get_list(flask.request.args.get('key')) form.enabled.data = False if form_data.get('enabled') == 'true': form.enabled.data = True form.desc.data = form_data.get('desc') form.name.data = flask.request.args.get('key').replace('/groups/', '') form.name.render_kw = {'readonly': True} return flask.render_template('list.html', main_header=_('Groups'), form=form, button_list=button_list) elif url_type == 'delete': remove_dependency(flask.request.args.get('key')) flash(_('Group') + ' ' + _('Deleted'), 'error') return flask.redirect(flask.url_for('groups')) group_list = etcd_conn.search('/groups/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/groups/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/groups/change'}] return flask.render_template('list.html', main_header=_('Groups'), list=get_calculated_page(group_list, page), pagination=page, headers=headers, button_list=button_list, links=links)
def __init__(self, manager, logger): self.logger = logger self.manager = manager self.etcd = Etcd(logger)
def __init__(self, manager, logger): self.logger = logger self.salt_client = salt.client.LocalClient() self.manager = manager self.etcd = Etcd(logger)
class VerifyFormations(object): def __init__(self, manager, logger): self.logger = logger self.salt_client = salt.client.LocalClient() self.manager = manager self.etcd = Etcd(logger) def start_verifying(self): # Parse out the username and formation name # from the ETCD directory string formation_parser = Literal('/formations/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name') # call out to ETCD and load all the formations formation_list = [] user_list = self.etcd.list_directory('formations') if user_list: for user in user_list: formations = self.etcd.list_directory(user) for formation in formations: parse_results = formation_parser.parseString(formation) if parse_results: formation_name = parse_results['formation_name'] username = parse_results['username'] self.logger.info( 'Attempting to load formation: {formation_name} ' 'with username: {username}'.format( formation_name=formation_name, username=username)) f = self.manager.load_formation_from_etcd( username, formation_name) formation_list.append(f) else: self.logger.error("Could not parse the ETCD string") if formation_list: # TODO Use background salt jobs # Start verifying things # Ask salt to do these things for me and give me back an job_id # results = self.salt_client.cmd_async(host, 'cmd.run', # ['netstat -an | grep %s | grep tcp | grep -i listen' % port], # expr_form='list') # # salt-run jobs.lookup_jid <job id number> for f in formation_list: for app in f.application_list: # Check to make sure it's up and running self.logger.info( "Running verification on app: " "{app_name}".format(app_name=app.hostname)) self.logger.info( '{server} docker ps | grep {container_id}'.format( server=app.host_server, container_id=app.container_id)) results = self.salt_client.cmd( app.host_server, 'cmd.run', [ 'docker ps | grep {container_id}'.format( container_id=app.container_id) ], expr_form='list') if results: self.logger.debug( "Salt return: {docker_results}".format( docker_results=results[app.host_server])) if results[app.host_server] == "": self.logger.error( "App {app} is not running!".format( app=app.hostname)) # Start the app back up and run start.sh on there self.start_application(app) else: self.logger.info( "App {app} is running. Checking if " "cron is running also".format( app=app.hostname)) # Check if cron is running on the container and bring it back # up if needed # Log in with ssh and check if cron is up and running self.logger.info( "Sleeping 2 seconds while the container starts" ) time.sleep(2) self.check_running_application(app) else: self.logger.error( "Call out to server {server} failed. Moving it" .format(server=app.host_server)) # move the container self.move_application(app) # Start an application that isn't running def start_application(self, app): # Start the application and run start.sh to kick off cron self.logger.info( "Starting app {app} with docker id: {app_id} up".format( app=app.hostname, app_id=app.container_id)) results = self.salt_client.cmd( app.host_server, 'cmd.run', [ 'docker start {container_id}'.format( container_id=app.container_id) ], expr_form='list') self.logger.debug(results) if results: if "Error: No such container" in results[app.host_server]: # We need to recreate the container self.logger.error("Container is missing on the host!. " "Trying to recreate") self.manager.start_application(app) self.logger.info( "Sleeping 2 seconds while the container starts") time.sleep(2) self.manager.bootstrap_application(app) elif "Error: start: No such container:" in results[ app.host_server]: # Seems the container already exists but won't start. Bug? self.logger.error("Container failed to start") self.move_application(app) else: self.logger.info( "Waiting 2 seconds for docker to start the container") time.sleep(2) self.check_running_application(app) else: # Move the container to another host, this host is messed up self.logger.error( "Failed to start {container_id} on host {host}".format( container_id=app.container_id, host=app.host_server)) self.move_application(app) # Move an application to another host and record the change in etcd def move_application(self, app): old_host = app.host_server cluster_list = self.manager.get_docker_cluster() circular_cluster_list = CircularList( self.manager.order_cluster_by_load(cluster_list)) if app.host_server in circular_cluster_list: index = circular_cluster_list.index(app.host_server) app.host_server = circular_cluster_list[index + 1].hostname else: # Assign the first one in the list if not found above app.host_server = circular_cluster_list[0].hostname self.logger.info( "Moving app {app_name} from {old_host} to {new_host}".format( app_name=app.hostname, old_host=old_host, new_host=app.host_server)) self.logger.info("Bootstrapping the application on the new host") self.start_application(app) # Log into the application via ssh and check everything def check_running_application(self, app): # TODO # Use the docker top command to see if cron is running instead of using ssh try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Move this user/pass into a config file self.logger.info('SSHing into host {hostname}:{port}'.format( hostname=app.host_server, port=app.ssh_port)) ssh.connect(hostname=app.host_server, port=app.ssh_port, username='******', password='******') # Is cron running? # If not run start.sh stdin, stdout, stderr = ssh.exec_command("pgrep cron") output = stdout.readlines() self.logger.debug(output) if len(output) == 0: # cron isn't running self.logger.info("Cron is not running. Starting it back up") stdin, stdout, stderr = ssh.exec_command("/root/start.sh") else: self.logger.info("Cron is running.") ssh.close() except SSHException: self.logger.error("Failed to log into server.")
def __init__(self, logger): self.salt_client = salt.client.LocalClient() self.etcd = Etcd(logger) self.logger = logger
class Manager(object): ''' A manager to orchestrate the creation and deletion of container clusters ''' def __init__(self, logger): self.salt_client = salt.client.LocalClient() self.etcd = Etcd(logger) self.logger = logger def fqdn_to_shortname(self, fqdn): if '.' in fqdn: return fqdn.split('.')[0] else: return fqdn def check_port_used(self, host, port): self.logger.info("Checking if {port} on {host} is open with salt-client".format( host=host, port=port)) results = self.salt_client.cmd(host, 'cmd.run', ['netstat -an | grep %s | grep tcp | grep -i listen' % port], expr_form='list') self.logger.debug("Salt return: {lsof}".format(lsof=results[host])) if results[host] is not '': return True else: return False # TODO def check_for_existing_formation(self, formation_name): # If the user passed in an existing formation name lets append to it pass def get_docker_cluster(self): # Return a list of docker hosts cluster = self.etcd.get_key('docker_cluster') if cluster is not None: return cluster.split(',') else: return None def get_load_balancer_cluster(self): # Return a list of nginx hosts cluster = self.etcd.get_key('nginx_cluster') if cluster is not None: return cluster.split(',') else: return None def order_cluster_by_load(self, cluster_list): # Sample salt output # {'dlceph01.drwg.local': '0.27 0.16 0.15 1/1200 26234'} # define grammar point = Literal('.') number = Word(nums) floatnumber = Combine( number + point + number) float_list = OneOrMore(floatnumber) results = self.salt_client.cmd(','.join(cluster_list), 'cmd.run', ['cat /proc/loadavg'], expr_form='list') load_list = [] self.logger.debug("Salt load return: {load}".format(load=results)) for host in results: host_load = results[host] match = float_list.parseString(host_load) if match: one_min = match[0] five_min = match[1] fifteen_min = match[2] self.logger.debug("Adding Load({host}, {one_min}, {five_min}, {fifteen_min}".format( host=host, one_min=one_min, five_min=five_min, fifteen_min=fifteen_min)) load_list.append(Load(host, one_min, five_min, fifteen_min)) else: self.logger.error("Could not parse host load output") # Sort the list by fifteen min load load_list = sorted(load_list, key=lambda x: x.fifteen_min_load) for load in load_list: self.logger.debug("Sorted load list: " + str(load)) return load_list # Load the formation and return a Formation object def load_formation_from_etcd(self, username, formation_name): f = Formation(username,formation_name) app_list = json.loads(json.loads( self.etcd.get_key('/formations/{username}/{formation_name}'.format( username=username, formation_name=formation_name)))) for app in app_list: # If our host doesn't support swapping we're going to get some garbage # message in here if "WARNING" in app['container_id']: app['container_id'] = app['container_id'].replace("WARNING: Your "\ "kernel does not support memory swap capabilities. Limitation discarded.\n","") # Set volumes if needed volumes = None if app['volumes']: self.logger.info("Setting volumes to: " + ''.join(app['volumes'])) volumes = app['volumes'] f.add_app(app['container_id'], app['hostname'], app['cpu_shares'], app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes) # Return fully parsed and populated formation object return f def save_formation_to_etcd(self, formation): name = formation.name username = formation.username self.etcd.set_key('formations/{username}/{formation_name}'.format( username=username, formation_name=name), formation) # TODO write code to add new apps to load balancer def add_app_to_nginx(self, app): pass # TODO write code to add new apps to the load balancer def add_app_to_apache(self, app): pass def start_application(self, app): # Run a salt cmd to startup the formation docker_command = "docker run -c={cpu_shares} -d -h=\"{hostname}\" -m={ram} "\ "-name=\"{hostname}\" {port_list} {volume_list} {image} /usr/sbin/sshd -D" self.logger.info("Port list %s" % app.port_list) port_list = ' '.join(map(lambda x: '-p ' + x, app.port_list)) # Only create this list if needed volume_list = '' if app.volume_list: volume_list = ' '.join(map(lambda x: '-v ' + x, app.volume_list)) d = docker_command.format(cpu_shares=app.cpu_shares, hostname=app.hostname, ram=app.ram, image='dlcephgw01:5000/sshd', port_list=port_list, volume_list=volume_list) self.logger.info("Starting up docker container on {host_server} with cmd: {docker_cmd}".format( host_server=app.host_server, docker_cmd=d)) salt_process = self.salt_client.cmd(app.host_server,'cmd.run', [d], expr_form='list') container_id = salt_process[app.host_server] if container_id: app.change_container_id(container_id) def bootstrap_application(self, app): # Log into the host with paramiko and run the salt bootstrap script host_server = self.fqdn_to_shortname(app.host_server) self.logger.info("Bootstrapping {hostname} on server: {host_server} port: {port}".format( hostname=app.hostname, host_server=host_server, port=app.ssh_port)) try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=host_server, port=app.ssh_port, username='******', password='******') transport = paramiko.Transport((host_server, app.ssh_port)) transport.connect(username = '******', password = '******') sftp = paramiko.SFTPClient.from_transport(transport) sftp.put('bootstrap.sh', '/root/bootstrap.sh') sftp.put('start.sh', '/root/start.sh') ssh.exec_command("chmod +x /root/bootstrap.sh") ssh.exec_command("chmod +x /root/start.sh") stdin, stdout, stderr = ssh.exec_command("bash /root/start.sh") self.logger.debug(''.join(stdout.readlines())) ssh.close() except SSHException: self.logger.error("Failed to log into server. Shutting it down and cleaning up the mess.") self.delete_container(app.host_server, app.container_id) # Stops and deletes a container def delete_container(self, host_server, container_id): results = self.salt_client.cmd(host_server, 'cmd.run', ['docker stop {container_id}'.format(container_id=container_id)], expr_form='list') self.logger.debug("Salt return: {stop_cmd}".format(stop_cmd=results[host_server])) results = self.salt_client.cmd(host_server, 'cmd.run', ['docker rm {container_id}'.format(container_id=container_id)], expr_form='list') self.logger.debug("Salt return: {rm_cmd}".format(rm_cmd=results[host_server])) def create_containers(self, user, number, formation_name, cpu_shares, ram, port_list, hostname_scheme, volume_list, force_host_server=None): f = Formation(user, formation_name) # Convert ram to bytes from MB ram = ram * 1024 * 1024 # Get the cluster machines on each creation cluster_list = self.get_docker_cluster() circular_cluster_list = CircularList(self.order_cluster_by_load(cluster_list)) # Loop for the requested amount of containers to be created for i in range(1, number+1): # [{"host_port":ssh_host_port, "container_port":ssh_container_port}] ssh_host_port = 9022 + i ssh_container_port = 22 host_server = circular_cluster_list[i].hostname # We are being asked to overwrite this if force_host_server: host_server = force_host_server validated_ports = [] while self.check_port_used(host_server, ssh_host_port): ssh_host_port = ssh_host_port +1 for port in port_list: self.logger.info("Checking if port {port} on {host} is in use".format( port=port, host=host_server)) if ':' in port: ports = port.split(':') # Only check if the host port is free. The container port should be free while self.check_port_used(host_server, ports[0]): ports[0] = int(ports[0]) + 1 # Add this to the validated port list validated_ports.append('{host_port}:{container_port}'.format( host_port = str(ports[0]), container_port = str(ports[1]))) else: while self.check_port_used(host_server, port): port = int(port) + 1 validated_ports.append(str(port)) self.logger.info('Adding app to formation {formation_name}: {hostname}{number} cpu_shares={cpu} ' 'ram={ram} ports={ports} host_server={host_server}'.format(formation_name=formation_name, hostname=hostname_scheme, number=str(i).zfill(3), cpu=cpu_shares, ram=ram, ports=validated_ports, host_server=host_server)) f.add_app(None, '{hostname}{number}'.format(hostname=hostname_scheme, number=str(i).zfill(3)), cpu_shares, ram, validated_ports, ssh_host_port, ssh_container_port, circular_cluster_list[i].hostname, volume_list) # Lets get this party started for app in f.application_list: self.start_application(app) self.logger.info("Sleeping 2 seconds while the container starts") time.sleep(2) self.bootstrap_application(app) self.logger.info("Saving the formation to ETCD") self.save_formation_to_etcd(f)
class User(UserMixin): etcd = None password = None username = None password_hash = None secret = "89660c74da48ddd4efbe4d3c8f8150de" locale = 'tr' enabled = False email = None role = 'viewer' def __init__(self, username, password=None, etcd=None): self.etcd = Etcd() self.username = username self.password = self.hash_password(password) self.get(username) def hash_password(self, password): return hashlib.sha256(str(password).encode('utf-8')).hexdigest() # return password def verify_password(self): if self.password_hash == self.password: return True else: return False def get(self, username): user = json.loads(self.etcd.get('/appuser/' + username)[0]) if user is not None: if user['enabled'] is True: self.password_hash = user['password'] self.username = user['username'] self.locale = user['locale'] self.enabled = user['enabled'] self.email = user['email'] self.role = user['role'] return True return False def set(self): user = {} user['password'] = self.password_hash user['username'] = self.username user['locale'] = self.locale user['enabled'] = self.enabled user['email'] = self.email user['role'] = self.role self.etcd.put('/appuser/' + self.username, json.dumps(user)) def is_authenticated(self): return self.enabled def is_active(self): return self.enabled def is_anonymous(self): return False def get_id(self): return unicode(self.username) def __repr__(self): return '<User %r>' % self.username
def __init__(self, username, password=None, etcd=None): self.etcd = Etcd() self.username = username self.password = self.hash_password(password) self.get(username)
class Manager(object): ''' A manager to orchestrate the creation and deletion of container clusters ''' def __init__(self, logger): self.salt_client = salt.client.LocalClient() self.etcd = Etcd(logger) self.logger = logger # Parse out the username and formation name # from the ETCD directory string self.formation_parser = Literal('/formations/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name') def fqdn_to_shortname(self, fqdn): if '.' in fqdn: return fqdn.split('.')[0] else: return fqdn def check_salt_key_used(self, hostname): self.logger.info( "Checking if the key for {host} is already used".format( host=hostname)) s = subprocess.Popen('salt-key', shell=True, stdout=PIPE) salt_list = s.communicate()[0] if hostname in salt_list: return True else: return False def check_port_used(self, host, port): self.logger.info( "Checking if {port} on {host} is open with salt-client".format( host=host, port=port)) results = self.salt_client.cmd( host, 'cmd.run', ['netstat -an | grep %s | grep tcp | grep -i listen' % port], expr_form='list') self.logger.debug("Salt return: {lsof}".format(lsof=results[host])) if results[host] is not '': return True else: return False # TODO def check_for_existing_formation(self, formation_name): # If the user passed in an existing formation name lets append to it pass def get_docker_cluster(self): # Return a list of docker hosts cluster = self.etcd.get_key('docker_cluster') if cluster is not None: return cluster.split(',') else: return None def get_load_balancer_cluster(self): # Return a list of nginx hosts cluster = self.etcd.get_key('nginx_cluster') if cluster is not None: return cluster.split(',') else: return None def order_cluster_by_load(self, cluster_list): # Sample salt output # {'dlceph01.drwg.local': '0.27 0.16 0.15 1/1200 26234'} # define grammar point = Literal('.') number = Word(nums) floatnumber = Combine(number + point + number) float_list = OneOrMore(floatnumber) results = self.salt_client.cmd(','.join(cluster_list), 'cmd.run', ['cat /proc/loadavg'], expr_form='list') load_list = [] self.logger.debug("Salt load return: {load}".format(load=results)) for host in results: host_load = results[host] match = float_list.parseString(host_load) if match: one_min = match[0] five_min = match[1] fifteen_min = match[2] self.logger.debug( "Adding Load({host}, {one_min}, {five_min}, {fifteen_min}". format(host=host, one_min=one_min, five_min=five_min, fifteen_min=fifteen_min)) load_list.append(Load(host, one_min, five_min, fifteen_min)) else: self.logger.error("Could not parse host load output") # Sort the list by fifteen min load load_list = sorted(load_list, key=lambda x: x.fifteen_min_load) for load in load_list: self.logger.debug("Sorted load list: " + str(load)) return load_list # Retun a list of formations the user owns def list_formations(self, username): formation_list = [] formations = self.etcd.list_directory('formations/' + username) for formation in formations: parse_results = self.formation_parser.parseString(formation) if parse_results: formation_name = parse_results['formation_name'] formation_list.append(formation_name) else: self.logger.error("Could not parse the ETCD string") self.logger.info('Formation list {formations} for user {user}'.format( formations=formation_list, user=username)) return formation_list # Load the formation and return a Formation object def load_formation_from_etcd(self, username, formation_name): f = Formation(username, formation_name) app_list = json.loads( json.loads( self.etcd.get_key( '/formations/{username}/{formation_name}'.format( username=username, formation_name=formation_name)))) for app in app_list: # If our host doesn't support swapping we're going to get some garbage # message in here if "WARNING" in app['container_id']: app['container_id'] = app['container_id'].replace("WARNING: Your "\ "kernel does not support memory swap capabilities. Limitation discarded.\n","") #Message changed in docker 0.8.0 app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\ "Your kernel does not support swap limit capabilities. Limitation "\ "discarded.\n","") app['container_id'].strip('\n') # Set volumes if needed volumes = None if app['volumes']: self.logger.info("Setting volumes to: " + ''.join(app['volumes'])) volumes = app['volumes'] f.add_app(app['container_id'], app['hostname'], app['cpu_shares'], app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes) # Return fully parsed and populated formation object return f def save_formation_to_etcd(self, formation): name = formation.name username = formation.username self.etcd.set_key( 'formations/{username}/{formation_name}'.format( username=username, formation_name=name), formation) # TODO write code to add new apps to load balancer def add_app_to_nginx(self, app): pass # TODO write code to add new apps to the load balancer def add_app_to_apache(self, app): pass def start_application(self, app): # Run a salt cmd to startup the formation docker_command = "docker run -c={cpu_shares} -d -i -t -h=\"{hostname}\" -m={ram}m "\ "--name={hostname} {port_list} {volume_list} {image} /sbin/my_init -- bash" self.logger.info("Port list %s" % app.port_list) port_list = ' '.join(map(lambda x: '-p ' + x, app.port_list)) # Only create this list if needed volume_list = '' if app.volume_list: volume_list = ' '.join(map(lambda x: '-v ' + x, app.volume_list)) d = docker_command.format(cpu_shares=app.cpu_shares, hostname=app.hostname, ram=app.ram, image=app.docker_image, port_list=port_list, volume_list=volume_list) self.logger.info( "Starting up docker container on {host_server} with cmd: {docker_cmd}" .format(host_server=app.host_server, docker_cmd=d)) salt_process = self.salt_client.cmd(app.host_server, 'cmd.run', [d], expr_form='list') container_id = salt_process[app.host_server] if container_id: if "WARNING" in container_id: container_id = container_id.replace("WARNING: "\ "Your kernel does not support swap limit capabilities. Limitation "\ "discarded.\n","") container_id.strip("\n") #Docker only uses the first 12 chars to identify a container app.change_container_id(container_id[0:12]) def bootstrap_application(self, app): # Log into the host with paramiko and run the salt bootstrap script host_server = self.fqdn_to_shortname(app.host_server) self.logger.info( "Bootstrapping {hostname} on server: {host_server} port: {port}". format(hostname=app.hostname, host_server=host_server, port=app.ssh_port)) try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=host_server, port=app.ssh_port, username='******', password='******') transport = paramiko.Transport((host_server, app.ssh_port)) transport.connect(username='******', password='******') sftp = paramiko.SFTPClient.from_transport(transport) sftp.put('bootstrap.sh', '/root/bootstrap.sh') sftp.put('start.sh', '/root/start.sh') ssh.exec_command("chmod +x /root/bootstrap.sh") ssh.exec_command("chmod +x /root/start.sh") stdin, stdout, stderr = ssh.exec_command("bash /root/start.sh") self.logger.debug(''.join(stdout.readlines())) ssh.close() except SSHException: self.logger.error( "Failed to log into server. Shutting it down and cleaning up the mess." ) self.delete_container(app.host_server, app.container_id) # Stops and deletes a container def delete_container(self, host_server, container_id): results = self.salt_client.cmd( host_server, 'cmd.run', ['docker stop {container_id}'.format(container_id=container_id)], expr_form='list') self.logger.debug( "Salt return: {stop_cmd}".format(stop_cmd=results[host_server])) results = self.salt_client.cmd( host_server, 'cmd.run', ['docker rm {container_id}'.format(container_id=container_id)], expr_form='list') self.logger.debug( "Salt return: {rm_cmd}".format(rm_cmd=results[host_server])) # Stops and deletes a formation. Use with caution def delete_formation(self, user, formation_name): formation_list = self.list_formations(user) if formation_name in formation_list: pass else: self.logger.error("Formation name not found!") def list_containers(self, user, formation_name): pass def create_containers(self, user, number, formation_name, cpu_shares, ram, port_list, hostname_scheme, volume_list, docker_image, force_host_server=None): f = Formation(user, formation_name) # Convert ram to bytes from MB ram = ram * 1024 * 1024 # Get the cluster machines on each creation cluster_list = self.get_docker_cluster() circular_cluster_list = CircularList( self.order_cluster_by_load(cluster_list)) # Loop for the requested amount of containers to be created for i in range(1, number + 1): # [{"host_port":ssh_host_port, "container_port":ssh_container_port}] ssh_host_port = 9022 + i ssh_container_port = 22 host_server = circular_cluster_list[i].hostname hostname = '{hostname}{number}'.format(hostname=hostname_scheme, number=str(i).zfill(3)) # First check if we can add this host to salt. If not exit with -1 if self.check_salt_key_used(hostname): self.logger.error( 'Salt key is already taken for {hostname}'.format( hostname=hostname)) sys.exit(-1) # We are being asked to overwrite this if force_host_server: host_server = force_host_server validated_ports = [] while self.check_port_used(host_server, ssh_host_port): ssh_host_port = ssh_host_port + 1 for port in port_list: self.logger.info( "Checking if port {port} on {host} is in use".format( port=port, host=host_server)) if ':' in port: ports = port.split(':') # Only check if the host port is free. The container port should be free while self.check_port_used(host_server, ports[0]): ports[0] = int(ports[0]) + 1 # Add this to the validated port list validated_ports.append( '{host_port}:{container_port}'.format( host_port=str(ports[0]), container_port=str(ports[1]))) else: while self.check_port_used(host_server, port): port = int(port) + 1 validated_ports.append(str(port)) self.logger.info( 'Adding app to formation {formation_name}: {hostname} cpu_shares={cpu} ' 'ram={ram} ports={ports} host_server={host_server} docker_image={docker_image}' .format(formation_name=formation_name, hostname=hostname, cpu=cpu_shares, ram=ram, ports=validated_ports, host_server=host_server, docker_image=docker_image)) f.add_app(None, '{hostname}'.format(hostname=hostname), cpu_shares, ram, validated_ports, ssh_host_port, ssh_container_port, host_server, docker_image, volume_list) # Lets get this party started for app in f.application_list: self.start_application(app) #self.logger.info("Sleeping 2 seconds while the container starts") #time.sleep(2) #self.bootstrap_application(app) self.logger.info("Saving the formation to ETCD") self.save_formation_to_etcd(f)
def __init__(self): self.etcd = Etcd() self.etcdLock = threading.Lock() self.requestWaiting = threading.Event()
def role_to_group(url_type=None): etcd_conn = Etcd() button_list = [{'name': _('New'), 'href': '/role_to_group/change'}, {'name': _('List'), 'href': '/role_to_group'}] header = [(_('Services'), 'service_key'), (_('Group Name'), 'group'), (_('Enabled'), 'enabled')] if url_type == 'change': form = forms.RoleForm() service_list = [(None, _('Select'))] for x in etcd_conn.search('/services'): service_list.append((str(x[1].key.decode("utf-8")), str(x[0].get('name')))) form.service.choices = service_list if form.validate_on_submit(): try: status = 'false' if form.enabled.data: status = 'true' key = '/role_to_group/' + form.group_name.data.replace('.', '/') + "/" + str(form.role_id.data) etcd_conn.put(key, json.dumps( {"enabled": status, "group": form.role.data, "service_key": form.service.data})) except Exception as error: print(error) flash(_('Role to Group') + ' ' + _('Added') + ' / ' + _('Updated'), 'success') return flask.redirect('/role_to_group/refresh?key={}'.format(key)) elif flask.request.args.get('key'): parse_key = flask.request.args.get('key').split('/') form_data = etcd_conn.get_list(flask.request.args.get('key')) form.enabled.data = False if form_data.get('enabled') == 'true': form.enabled.data = True form.service.data = form_data.get('service_key') form.group_name.data = parse_key[2] + '.' + parse_key[3] form.role.data = form_data.get('group') form.role_id.data = parse_key[4] form.service.render_kw = {'readonly': True} form.group_name.render_kw = {'readonly': True} form.role.render_kw = {'readonly': True} return flask.render_template('role.html', main_header=_('Role to Group'), form=form, button_list=button_list) elif url_type == 'delete': parse_key = flask.request.args.get('key').split('/') for x in etcd_conn.get_prefix('/users/', sort_order="ascend", sort_target="key"): parse_user_key = str(x[1].key.decode("utf-8")).split('/') # print(parse_user_key) if parse_user_key[3] == parse_key[2] and parse_user_key[4] == parse_key[3]: etcd_conn.delete(x[1].key.decode("utf-8")) etcd_conn.delete(flask.request.args.get('key')) flash(_('Role to Group') + ' ' + _('Deleted'), 'error') return flask.redirect('/role_to_group') elif url_type == 'refresh': key = flask.request.args.get('key') parse_key = key.split('/') role_to_group = etcd_conn.get_list(key) status = role_to_group.get('enabled') service = etcd_conn.get_list(role_to_group.get('service_key')) import requests r = requests.post(service.get('user_service_url'), json={service.get('user_service_param'): parse_key[4]}, auth=(service.get('username'), service.get('password'))) for row in r.json(): id = row.get(service.get('user_service_key')) key = '/users/{}/{}/{}'.format(str(id), parse_key[2], parse_key[3]) try: user = etcd_conn.get_list(key) if user.get('enabled') != status: etcd_conn.put(key, json.dumps({"enabled": status})) except: etcd_conn.put(key, json.dumps({"enabled": status})) flash(_('Role to Group') + ' ' + _('Refreshed'), 'success') return flask.redirect('/role_to_group') group_list = etcd_conn.search('/role_to_group/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/role_to_group/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/role_to_group/change'}, {'name': _('Refresh'), 'type': 'success', 'link': '/role_to_group/refresh'}] return flask.render_template('list.html', main_header=_('Role to Group'), button_list=button_list, list=get_calculated_page(group_list, page), headers=header, links=links, pagination=page)
class VerifyFormations(object): def __init__(self, manager, logger): self.logger = logger self.salt_client = salt.client.LocalClient() self.manager = manager self.etcd = Etcd(logger) def start_verifying(self): # Parse out the username and formation name # from the ETCD directory string formation_parser = Literal('/formations/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \ Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name') # call out to ETCD and load all the formations formation_list = [] user_list = self.etcd.list_directory('formations') if user_list: for user in user_list: formations = self.etcd.list_directory(user) for formation in formations: parse_results = formation_parser.parseString(formation) if parse_results: formation_name = parse_results['formation_name'] username = parse_results['username'] self.logger.info('Attempting to load formation: {formation_name} ' 'with username: {username}'.format(formation_name=formation_name, username=username)) f = self.manager.load_formation_from_etcd(username, formation_name) formation_list.append(f) else: self.logger.error("Could not parse the ETCD string") if formation_list: # TODO Use background salt jobs # Start verifying things # Ask salt to do these things for me and give me back an job_id # results = self.salt_client.cmd_async(host, 'cmd.run', # ['netstat -an | grep %s | grep tcp | grep -i listen' % port], # expr_form='list') # # salt-run jobs.lookup_jid <job id number> for f in formation_list: for app in f.application_list: # Check to make sure it's up and running self.logger.info("Running verification on app: " "{app_name}".format(app_name=app.hostname)) self.logger.info('{server} docker ps | grep {container_id}'.format( server=app.host_server, container_id=app.container_id)) results = self.salt_client.cmd(app.host_server, 'cmd.run', ['docker ps | grep {container_id}'.format(container_id=app.container_id)], expr_form='list') if results: self.logger.debug("Salt return: {docker_results}".format( docker_results=results[app.host_server])) if results[app.host_server] == "": self.logger.error("App {app} is not running!".format( app=app.hostname)) # Start the app back up and run start.sh on there self.start_application(app) else: self.logger.info("App {app} is running. Checking if " "cron is running also".format(app=app.hostname)) # Check if cron is running on the container and bring it back # up if needed # Log in with ssh and check if cron is up and running self.logger.info("Sleeping 2 seconds while the container starts") time.sleep(2) self.check_running_application(app) else: self.logger.error("Call out to server {server} failed. Moving it".format( server=app.host_server)) # move the container self.move_application(app) # Start an application that isn't running def start_application(self, app): # Start the application and run start.sh to kick off cron self.logger.info("Starting app {app} with docker id: {app_id} up".format( app=app.hostname, app_id=app.container_id)) results = self.salt_client.cmd(app.host_server, 'cmd.run', ['docker start {container_id}'.format(container_id=app.container_id)], expr_form='list') self.logger.debug(results) if results: if "Error: No such container" in results[app.host_server]: # We need to recreate the container self.logger.error("Container is missing on the host!. " "Trying to recreate") self.manager.start_application(app) self.logger.info("Sleeping 2 seconds while the container starts") time.sleep(2) self.manager.bootstrap_application(app) elif "Error: start: No such container:" in results[app.host_server]: # Seems the container already exists but won't start. Bug? self.logger.error("Container failed to start") self.move_application(app) else: self.logger.info("Waiting 2 seconds for docker to start the container") time.sleep(2) self.check_running_application(app) else: # Move the container to another host, this host is messed up self.logger.error("Failed to start {container_id} on host {host}".format( container_id=app.container_id, host=app.host_server)) self.move_application(app) # Move an application to another host and record the change in etcd def move_application(self, app): old_host = app.host_server cluster_list = self.manager.get_docker_cluster() circular_cluster_list = CircularList( self.manager.order_cluster_by_load(cluster_list)) if app.host_server in circular_cluster_list: index = circular_cluster_list.index(app.host_server) app.host_server = circular_cluster_list[index+1].hostname else: # Assign the first one in the list if not found above app.host_server = circular_cluster_list[0].hostname self.logger.info("Moving app {app_name} from {old_host} to {new_host}".format( app_name=app.hostname, old_host=old_host, new_host=app.host_server)) self.logger.info("Bootstrapping the application on the new host") self.start_application(app) # Log into the application via ssh and check everything def check_running_application(self, app): # TODO # Use the docker top command to see if cron is running instead of using ssh try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Move this user/pass into a config file self.logger.info('SSHing into host {hostname}:{port}'.format( hostname=app.host_server, port=app.ssh_port)) ssh.connect(hostname=app.host_server, port=app.ssh_port, username='******', password='******') # Is cron running? # If not run start.sh stdin, stdout, stderr = ssh.exec_command("pgrep cron") output = stdout.readlines() self.logger.debug(output) if len(output) == 0: # cron isn't running self.logger.info("Cron is not running. Starting it back up") stdin, stdout, stderr = ssh.exec_command("/root/start.sh") else: self.logger.info("Cron is running.") ssh.close() except SSHException: self.logger.error("Failed to log into server.")
def dbmeta(url_type=None): etcd_conn = Etcd() try: pg_ddm_config = configparser.RawConfigParser(allow_no_value=True) if str(config['general']['get_db_info_in_pg_ddm_config_file']).lower() == 'true': pg_ddm_config.read(config['general']['pg_ddm_config_file_path']) databases = pg_ddm_config['databases'] else: databases = config['database'] db_list = [] conn_info = {} for db in databases: db_list.append((db, db)) key_list = {} tmp_dsn = [] for i in databases[db].split(" "): key = i.split("=") if key[0] != 'search_path' and key[0] != 'route': tmp_dsn.append(key[0] + '=' + key[1]) key_list[key[0]] = key[1] conn_info[db] = key_list databases[db] = ' '.join(tmp_dsn) form = forms.TableSelectForm() form.db.choices = db_list button_list = [{'name': _('Refresh'), 'href': '/dbmeta/change'}, {'name': _('List'), 'href': '/dbmeta'}] # if url_type == 'update': # print('update') if url_type == 'change': if form.validate_on_submit(): form_cred = forms.TablesForm() if form.db.data is not None: if 'user' in conn_info[form.db.data]: form_cred.username.data = conn_info[form.db.data]['user'] if 'password' in conn_info[form.db.data]: form_cred.password.data = conn_info[form.db.data]['password'] form_cred.db.data = form.db.data if form_cred.validate_on_submit(): conn = None try: db = databases[form.db.data] conn = psycopg2.connect(db, user=form_cred.username.data, password=form_cred.password.data) cur = conn.cursor() cur.execute("""SELECT i.table_catalog,i.table_schema,i.table_name,( SELECT array_to_json(array_agg(col_array)) FROM ( SELECT i2.column_name,i2.data_type FROM information_schema.columns i2 WHERE i2.table_catalog = i.table_catalog AND i2.table_schema = i.table_schema AND i2.table_name = i.table_name ) col_array ) FROM information_schema.columns i WHERE i.table_schema NOT IN ('pg_catalog','information_schema','mask') GROUP BY 1,2,3""") for row in cur.fetchall(): key = '/{}/{}/{}'.format(str(form.db.data), str(row[1]), str(row[2])) etcd_conn.put(key, json.dumps(row[3])) cur.close() except (Exception, psycopg2.DatabaseError) as e: flash(_('DB Error'), e) except (Exception, psycopg2.DatabaseError) as e: flash(_('General Error'), e) finally: if conn is not None: conn.close() flash(_('Database Metadata') + ' ' + _('Updated'), 'info') return flask.redirect(flask.url_for('dbmeta')) return flask.render_template('list.html', main_header=_('Database connection'), form=form_cred, button_list=button_list) return flask.render_template('list.html', main_header=_('Database select'), form=form, button_list=button_list) else: if form.validate_on_submit() or request.args.get('db'): if request.args.get('db') is not None: db = request.args.get('db') else: db = str(form.db.data) group_list = etcd_conn.search('/{}/'.format(db), json_field=False) headers = [(_('Columns'), '')] # links = [{'name': _('Update'), 'type': 'info', 'link': '/dbmeta/update'}] links = [] extra_param = '&db=' + str(db) if request.args.get('search_key') is not None and request.args.get('search_key'): extra_param += '&search_key=' + str(request.args.get('search_key')) + '&search_type=' + str( request.args.get('search_type')) page = pagination(len(group_list), extra_param) return flask.render_template('list.html', main_header=_('Database Metadata'), list=get_calculated_page(group_list, page), pagination=page, headers=headers, links=links, button_list=button_list) return flask.render_template('list.html', main_header=_('Database Metadata'), form=form, button_list=button_list) except FileNotFoundError: flash(_('pg_ddm config file not found'), 'error') except KeyError: flash(_('pg_ddm_config_file_path key not found in settings.cfg'), 'warning') return flask.render_template('list.html', main_header=_('Database Metadata'))
def rules(url_type=None): etcd_conn = Etcd() button_list = [{'name': _('New'), 'href': '/rules/change'}, {'name': _('List'), 'href': '/rules'}] headers = [(_('Description'), 'description'), (_('Group Name'), 'group_name'), (_('Table'), 'table_column'), (_('Enabled'), 'enabled')] if url_type == 'change': form = forms.RulesForm() if form.validate_on_submit(): status = 'false' if form.enabled.data is True: status = 'true' prop = '' if form.rule.data in form.test.keys(): if len(form.test[form.rule.data]) > 0: prop = "[" for i in form.test[form.rule.data]: x = str(i).replace('open_close_', '') if x == 'col': prop += '%col%,' elif str(flask.request.form[x]).isdigit(): prop += '{"A_Const": {"val": ' + flask.request.form[x] + '}},' else: prop += '{"A_Const": {"val": {"String": {"str": "' + flask.request.form[x] + '"}}}},' prop = prop[:-1] + ']' else: prop = '[]' row = {"name": form.name.data, "description": form.description.data, "table_column": form.table_column.data, "filter": form.filter.data, "enabled": status, "group_name": form.group_name.data, "prop": prop, "rule": form.rule.data} etcd_conn.put('/rules/' + form.table_column.data.replace('.', '/') + '/' + form.group_name.data.replace('.', '/') + '/' + form.name.data, json.dumps(row)) flash(_('Rule') + ' ' + _('Added') + ' / ' + _('Updated'), 'info') return flask.redirect(flask.url_for('rules')) elif flask.request.args.get('key'): form_data = etcd_conn.get_list(flask.request.args.get('key')) i = 0 prop = form_data.get('prop').replace('%col%', '"%col%"') for x in json.loads(prop): if x != "%col%": obj = getattr(form, form.test.get(form_data.get('rule'))[i].replace('open_close_', '')) try: obj.data = x.get('A_Const').get('val').get("String").get("str") except: obj.data = x.get('A_Const').get('val') i = i + 1 form.enabled.data = False if form_data.get('enabled') == 'true': form.enabled.data = True form.rule.data = form_data.get('rule') form.description.data = form_data.get('description') form.group_name.data = form_data.get('group_name') form.filter.data = form_data.get('filter') form.table_column.data = form_data.get('table_column') form.name.data = form_data.get('name') form.name.render_kw = {'readonly': True} form.table_column.render_kw = {'readonly': True} form.group_name.render_kw = {'readonly': True} return flask.render_template('rules.html', main_header=_('Rules'), form=form, button_list=button_list) elif url_type == 'delete': etcd_conn.delete(flask.request.args.get('key')) flash(_('Rule') + ' ' + _('Deleted'), 'error') return flask.redirect(flask.url_for('rules')) group_list = etcd_conn.search('/rules/') page = pagination(len(group_list)) links = [{'name': _('Delete'), 'type': 'danger', 'link': '/rules/delete'}, {'name': _('Update'), 'type': 'info', 'link': '/rules/change'}] return flask.render_template('list.html', main_header=_('Rules'), list=get_calculated_page(group_list, page), pagination=page, headers=headers, button_list=button_list, links=links)