def setup_redis_cluster(tid): servers = Server.query.filter(Server.redis.is_(True)).filter( Server.stunnel.is_(True)).all() appconf = AppConfiguration.query.first() master_conf = [ "port 7000", "cluster-enabled yes", "cluster-config-file nodes_7000.conf", "cluster-node-timeout 5000", "appendonly yes" ] slave_conf = [ "port 7001", "cluster-enabled yes", "cluster-config-file nodes_7001.conf", "cluster-node-timeout 5000", "appendonly yes" ] for server in servers: rc = RemoteClient(server.hostname, ip=server.ip) try: rc.startup() wlogger.log(tid, "Connecting to server ...", "success", server_id=server.id) except Exception as e: wlogger.log(tid, "Could not connect to the server over SSH. Error:" "{0}\nRedis configuration failed.".format(e), "error", server_id=server.id) continue chdir = '/' if server.gluu_server: chdir = "/opt/gluu-server-{0}".format(appconf.gluu_version) # upload the conf files wlogger.log(tid, "Uploading redis conf files...", "debug", server_id=server.id) rc.put_file(os.path.join(chdir, "etc/redis/redis_7000.conf"), "\n".join(master_conf)) rc.put_file(os.path.join(chdir, "etc/redis/redis_7001.conf"), "\n".join(slave_conf)) # upload the modified init.d file rc.upload( os.path.join(app.root_path, "templates", "redis", "redis-server"), os.path.join(chdir, "etc/init.d/redis-server")) wlogger.log(tid, "Configuration upload complete.", "success", server_id=server.id) return True
def edit_slapd_conf(server_id): """This view provides editing of slapd.conf file before depoloyments.""" server = Server.query.get(server_id) appconf = AppConfiguration.query.first() # If there is no server with server_id return to home if not server: flash("No such server.", "warning") return redirect(url_for('index.home')) if not server.gluu_server: chroot = '/' else: chroot = '/opt/gluu-server-' + appconf.gluu_version # slapd.conf file will be downloaded from server. Make ssh connection # and download it c = RemoteClient(server.hostname, ip=server.ip) try: c.startup() except ClientNotSetupException as e: flash(str(e), "danger") return redirect(url_for('index.home')) slapd_conf_file = os.path.join(chroot, 'opt/symas/etc/openldap/slapd.conf') if request.method == 'POST': config = request.form.get('conf') r = c.put_file(slapd_conf_file, config) if not r[0]: flash("Cant' saved to server: {0}".format(r[1]), "danger") else: flash('File {0} was saved on {1}'.format(slapd_conf_file, server.hostname)) return redirect(url_for('index.home')) # After editing, slapd.conf file will be uploaded to server via ssh r = c.get_file(slapd_conf_file) if not r[0]: flash("Cant't get file {0}: {1}".format(slapd_conf_file, r[1]), "success") return redirect(url_for('index.home')) config = r[1].read() return render_template('conf_editor.html', config=config, hostname=server.hostname)
def edit_slapd_conf(server_id): server = Server.query.get(server_id) appconf = AppConfiguration.query.first() if not server: print "Yoook" flash("No such server.", "warning") return redirect(url_for('index.home')) if not server.gluu_server: chroot = '/' else: chroot = '/opt/gluu-server-' + appconf.gluu_version c = RemoteClient(server.hostname, ip=server.ip) try: c.startup() except ClientNotSetupException as e: flash(str(e), "danger") return redirect(url_for('index.home')) slapd_conf_file = os.path.join(chroot, 'opt/symas/etc/openldap/slapd.conf') if request.method == 'POST': config = request.form.get('conf') r = c.put_file(slapd_conf_file, config) if not r[0]: flash("Cant' saved to server: {0}".format(r[1]), "danger") else: flash('File {0} was saved on {1}'.format(slapd_conf_file, server.hostname)) return redirect(url_for('index.home')) r = c.get_file(slapd_conf_file) if not r[0]: flash("Cant't get file {0}: {1}".format(slapd_conf_file, r[1]), "success") return redirect(url_for('index.home')) config = r[1].read() return render_template('conf_editor.html', config=config, hostname=server.hostname)
def install_cache_components(self, method): """Celery task that installs the redis, stunnel and twemproxy applications in the required servers. Redis and stunnel are installed in all the servers in the cluster. Twemproxy is installed in the load-balancer/proxy server :param self: the celery task :param method: either STANDALONE, SHARDED :return: the number of servers where both stunnel and redis were installed successfully """ tid = self.request.id installed = 0 servers = Server.query.all() for server in servers: wlogger.log(tid, "Installing Redis in server {0}".format( server.hostname), "info", server_id=server.id) ri = RedisInstaller(server, tid) redis_installed = ri.install() if redis_installed: server.redis = True wlogger.log(tid, "Redis install successful", "success", server_id=server.id) else: server.redis = False wlogger.log(tid, "Redis install failed", "fail", server_id=server.id) wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id) si = StunnelInstaller(server, tid) stunnel_installed = si.install() if stunnel_installed: server.stunnel = True wlogger.log(tid, "Stunnel install successful", "success", server_id=server.id) else: server.stunnel = False wlogger.log(tid, "Stunnel install failed", "fail", server_id=server.id) # Save the redis and stunnel install situation to the db db.session.commit() if redis_installed and stunnel_installed: installed += 1 if method != 'STANDALONE': # No need to install twemproxy for "SHARDED" configuration return True # Install twemproxy in the Nginx load balancing proxy server app_conf = AppConfiguration.query.first() host = app_conf.nginx_host rc = RemoteClient(host) try: rc.startup() except Exception as e: wlogger.log(tid, "Could not connect to {0}".format(e), "error") return False server_os = get_os_type(rc) mock_server = Server() mock_server.hostname = host wlogger.log(tid, "Installing Stunnel in proxy server") si = StunnelInstaller(mock_server, tid) stunnel_installed = si.install() if stunnel_installed: wlogger.log(tid, "Stunnel install successful", "success") else: wlogger.log(tid, "Stunnel install failed", "fail") wlogger.log(tid, "Cluster manager will now try to build Twemproxy") # 1. Setup the development tools for installation if server_os in ["Ubuntu 16", "Ubuntu 14"]: run_and_log(rc, "apt-get update", tid) run_and_log(rc, "apt-get install -y build-essential autoconf libtool", tid) elif server_os in ["CentOS 6", "CentOS 7", "RHEL 7"]: run_and_log(rc, "yum install -y wget", tid) run_and_log(rc, "yum groupinstall -y 'Development tools'", tid) if server_os == "CentOS 6": run_and_log(rc, "wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz", tid) run_and_log(rc, "tar xvfvz autoconf-2.69.tar.gz", tid) run_and_log(rc, "cd autoconf-2.69 && ./configure", tid) run_and_log(rc, "cd autoconf-2.69 && make", tid) run_and_log(rc, "cd autoconf-2.69 && make install", tid) # 2. Get the source, build & install the nutcracker binaries run_and_log(rc, "wget https://github.com/twitter/twemproxy/archive/v0.4.1.tar.gz", tid) run_and_log(rc, "tar -xf v0.4.1.tar.gz", tid) run_and_log(rc, "cd twemproxy-0.4.1", tid) run_and_log(rc, "cd twemproxy-0.4.1 && autoreconf -fvi", tid) run_and_log(rc, "cd twemproxy-0.4.1 && ./configure --prefix=/usr", tid) run_and_log(rc, "cd twemproxy-0.4.1 && make", tid) run_and_log(rc, "cd twemproxy-0.4.1 && make install", tid) # 3. Post installation - setup user and logging run_and_log(rc, "useradd nutcracker", tid) run_and_log(rc, "mkdir /var/log/nutcracker", tid) run_and_log(rc, "touch /var/log/nutcracker/nutcracker.log", tid) run_and_log(rc, "chown -R nutcracker:nutcracker /var/log/nutcracker", tid) logrotate_conf = ["/var/log/nutcracker/nutcracker*.log {", "\tweekly", "\tmissingok", "\trotate 12", "\tcompress", "\tnotifempty", "}"] rc.put_file("/etc/logrotate.d/nutcracker", "\n".join(logrotate_conf)) # 4. Add init/service scripts to run nutcracker as a service if server_os in ["Ubuntu 16", "CentOS 7", "RHEL 7"]: local = os.path.join(app.root_path, "templates", "twemproxy", "twemproxy.service") remote = "/lib/systemd/system/nutcracker.service" rc.upload(local, remote) run_and_log(rc, "systemctl enable nutcracker", tid, None) elif server_os == "Ubuntu 14": local = os.path.join(app.root_path, "templates", "twemproxy", "nutcracker.init") remote = "/etc/init.d/nutcracker" rc.upload(local, remote) run_and_log(rc, 'chmod +x /etc/init.d/nutcracker', tid) run_and_log(rc, "update-rc.d nutcracker defaults", tid) elif server_os == "CentOS 6": local = os.path.join(app.root_path, "templates", "twemproxy", "nutcracker.centos.init") remote = "/etc/rc.d/init.d/nutcracker" rc.upload(local, remote) run_and_log(rc, "chmod +x /etc/init.d/nutcracker", tid) run_and_log(rc, "chkconfig --add nutcracker", tid) run_and_log(rc, "chkconfig nutcracker on", tid) # 5. Create the default configuration file referenced in the init scripts run_and_log(rc, "mkdir -p /etc/nutcracker", tid) run_and_log(rc, "touch /etc/nutcracker/nutcracker.yml", tid) rc.close() return installed
def install_cache_cluster(self, servers_id_list, cache_servers_id_list): tid = self.request.id servers = [ Server.query.get(id) for id in servers_id_list ] cache_servers = [ CacheServer.query.get(id) for id in cache_servers_id_list ] app_conf = AppConfiguration.query.first() primary_cache = None primary_cache_server = CacheServer.query.first() for server in cache_servers: rc = __get_remote_client(server, tid) if not rc: wlogger.log(tid, "SSH connection to server failed", "error", server_id=server.id) return False ri = RedisInstaller(server, tid, rc) if app_conf.offline: if not ri.check_installed(): wlogger.log( tid, 'Redis Server was not installed. Please install Redis ' ' Server and retry.', 'error', server_id=server.id ) return False redis_installed = ri.install() wlogger.log( tid, 'Setting Redis password', 'info', server_id=server.id ) ri.set_redis_password() if redis_installed: wlogger.log(tid, "Redis install successful", "success", server_id=server.id) else: wlogger.log(tid, "Redis install failed", "fail", server_id=server.id) return False ri.run_sysctl('enable') ri.run_sysctl('restart') si = StunnelInstaller(server, tid, rc) if app_conf.offline: if not si.check_installed(): wlogger.log( tid, 'Stunnel was not installed. Please install stunnel ' 'and retry.', 'error', server_id=server.id ) return False if si.check_installed(): server.stunnel = True else: wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id) stunnel_installed = si.install() if stunnel_installed: server.stunnel = True wlogger.log(tid, "Stunnel install successful", "success", server_id=server.id) else: server.stunnel = False wlogger.log(tid, "Stunnel install failed", "fail", server_id=server.id) return False if si.os_type == 'rpm': si.upload_service_file() if si.os_type == 'deb': wlogger.log(tid, "Enabling stunnel", "debug", server_id=server.id) si.run_command("sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/stunnel4") if not primary_cache: primary_cache = server.ip if not rc.exists('/etc/stunnel/redis-server.crt'): wlogger.log(tid, "Creating SSL certificate for stunnel", "info", server_id=server.id) si.run_command( 'openssl req -x509 -nodes -days 3650 -newkey rsa:2048 ' '-batch -keyout /etc/stunnel/redis-server.key ' '-out /etc/stunnel/redis-server.crt' ) si.run_command('chmod 600 /etc/stunnel/redis-server.key') wlogger.log(tid, "Retreiving server certificate", "info", server_id=server.id) stunnel_redis_conf = ( 'pid = /run/stunnel-redis.pid\n' '[redis-server]\n' 'cert = /etc/stunnel/redis-server.crt\n' 'key = /etc/stunnel/redis-server.key\n' 'accept = {0}:{1}\n' 'connect = 127.0.0.1:6379\n' ).format(server.ip, primary_cache_server.stunnel_port) wlogger.log(tid, "Writing redis stunnel configurations", "info", server_id=server.id) rc.put_file('/etc/stunnel/stunnel.conf', stunnel_redis_conf) si.run_sysctl('enable') si.run_sysctl('restart') server.installed = True db.session.commit() rc.close() wlogger.log(tid, "2", "set_step") # retreive stunnel certificate rc = RemoteClient(primary_cache_server.hostname, ip=primary_cache_server.ip) rc.startup() stunnel_cert = rc.get_file('/etc/stunnel/redis-server.crt') if not stunnel_cert[0]: print "Can't retreive server certificate from primary cache server" return False stunnel_cert = stunnel_cert[1].read() for server in servers: rc = __get_remote_client(server, tid) if not rc: wlogger.log(tid, "SSH connection to server failed", "error", server_id=server.id) return False si = StunnelInstaller(server, tid, rc) wlogger.log(tid, "Installing Stunnel", "debug", server_id=server.id) if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'): wlogger.log(tid, "Stunnel was allready installed", "info", server_id=server.id) server.stunnel = True else: wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id) stunnel_installed = si.install() if stunnel_installed: server.stunnel = True wlogger.log(tid, "Stunnel install successful", "success", server_id=server.id) else: server.stunnel = False wlogger.log(tid, "Stunnel install failed", "fail", server_id=server.id) return False if si.os_type == 'rpm': si.upload_service_file() if si.os_type == 'deb': wlogger.log(tid, "Enabling stunnel", "debug", server_id=server.id) si.run_command("sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/stunnel4") stunnel_redis_conf = ( 'pid = /run/stunnel-redis.pid\n' '[redis-client]\n' 'client = yes\n' 'accept = 127.0.0.1:{1}\n' 'connect = {0}:{1}\n' 'CAfile = /etc/stunnel/redis-server.crt\n' 'verify = 4\n' ).format(primary_cache, primary_cache_server.stunnel_port) wlogger.log(tid, "Writing redis stunnel configurations", "info", server_id=server.id) rc.put_file('/etc/stunnel/stunnel.conf', stunnel_redis_conf) wlogger.log(tid, "Uploading server certificate", "info", server_id=server.id) rc.put_file('/etc/stunnel/redis-server.crt', stunnel_cert) si.run_sysctl('enable') si.run_sysctl('restart') if server.primary_server: server_string = 'localhost:{0}'.format(primary_cache_server.stunnel_port) __update_LDAP_cache_method(tid, server, server_string, redis_password=primary_cache_server.redis_password) wlogger.log(tid, "Restarting Gluu Server", "info", server_id=server.id) if server.os in ('RHEL 7', 'CentOS 7', 'Ubuntu 18'): si.run_command('/sbin/gluu-serverd-{} restart'.format(app_conf.gluu_version)) else: si.run_command('systemctl restart gluu-server-{}'.format(app_conf.gluu_version)) wlogger.log(tid, "3", "set_step")
def setup_sharded(tid): servers = Server.query.filter(Server.redis.is_(True)).filter( Server.stunnel.is_(True)).all() appconf = AppConfiguration.query.first() # Store the redis server info in the LDAP ports_count = len(servers) - 1 for server in servers: wlogger.log(tid, "Updating oxCacheConfiguration ...", "debug", server_id=server.id) server_string = ",".join( ['localhost:6379'] + ["localhost:700{0}".format(i) for i in xrange(ports_count)]) try: dbm = DBManager( server.hostname, 1636, server.ldap_password, ssl=True, ip=server.ip, ) except Exception as e: wlogger.log(tid, "Couldn't connect to LDAP. Error: {0}".format(e), "error", server_id=server.id) wlogger.log(tid, "Make sure your LDAP server is listening to " "connections from outside", "debug", server_id=server.id) continue entry = dbm.get_appliance_attributes('oxCacheConfiguration') cache_conf = json.loads(entry.oxCacheConfiguration.value) cache_conf['cacheProviderType'] = 'REDIS' cache_conf['redisConfiguration']['redisProviderType'] = 'SHARDED' cache_conf['redisConfiguration']['servers'] = server_string result = dbm.set_applicance_attribute('oxCacheConfiguration', [json.dumps(cache_conf)]) if not result: wlogger.log(tid, "oxCacheConfigutaion update failed", "fail", server_id=server.id) continue # generate stunnel configuration and upload it to the server wlogger.log(tid, "Setting up stunnel", "info", server_id=server.id) chdir = '/' if server.gluu_server: chdir = "/opt/gluu-server-{0}".format(appconf.gluu_version) rc = RemoteClient(server.hostname, ip=server.ip) try: rc.startup() except: wlogger.log(tid, "Could not connect to the server over SSH. " "Stunnel setup failed.", "error", server_id=server.id) continue wlogger.log(tid, "Enable stunnel start on system boot", "debug", server_id=server.id) # replace the /etc/default/stunnel4 to enable start on system startup local = os.path.join(app.root_path, 'templates', 'stunnel', 'stunnel4.default') remote = os.path.join(chdir, 'etc/default/stunnel4') rc.upload(local, remote) # setup the certificate file wlogger.log(tid, "Generating certificate for stunnel ...", "debug", server_id=server.id) prop_buffer = StringIO() propsfile = os.path.join(chdir, "install", "community-edition-setup", "setup.properties.last") rc.sftpclient.getfo(propsfile, prop_buffer) prop_buffer.seek(0) props = dict() prop_in = lambda line: line.split("=")[1].strip() for line in prop_buffer: if re.match('^countryCode', line): props['country'] = prop_in(line) if re.match('^state', line): props['state'] = prop_in(line) if re.match('^city', line): props['city'] = prop_in(line) if re.match('^orgName', line): props['org'] = prop_in(line) if re.match('^hostname', line): props['cn'] = prop_in(line) if re.match('^admin_email', line): props['email'] = prop_in(line) subject = "'/C={country}/ST={state}/L={city}/O={org}/CN={cn}" \ "/emailAddress={email}'".format(**props) cert_path = os.path.join(chdir, "etc", "stunnel", "server.crt") key_path = os.path.join(chdir, "etc", "stunnel", "server.key") pem_path = os.path.join(chdir, "etc", "stunnel", "cert.pem") cmd = [ "/usr/bin/openssl", "req", "-subj", subject, "-new", "-newkey", "rsa:2048", "-sha256", "-days", "365", "-nodes", "-x509", "-keyout", key_path, "-out", cert_path ] cin, cout, cerr = rc.run(" ".join(cmd)) rc.run("cat {cert} {key} > {pem}".format(cert=cert_path, key=key_path, pem=pem_path)) # verify certificate cin, cout, cerr = rc.run("/usr/bin/openssl verify " + pem_path) if props['cn'] in cout and props['org'] in cout: wlogger.log(tid, "Certificate generated successfully", "success", server_id=server.id) else: wlogger.log(tid, "Certificate generation failed. Add a SSL " "certificate at /etc/stunnel/cert.pem", "error", server_id=server.id) # Generate stunnel config wlogger.log(tid, "Setup stunnel listening and forwarding", "debug", server_id=server.id) sconf = [ "cert = /etc/stunnel/cert.pem", "pid = /var/run/stunnel.pid", "[redis-server]", "client = no", "accept = {0}:7777".format(server.ip), "connect = 127.0.0.1:6379" ] listen_port = 7000 for s in servers: if s.id != server.id: sconf.append("[client{0}]".format(s.id)) sconf.append("client = yes") sconf.append("accept = 127.0.0.1:{0}".format(listen_port)) sconf.append("connect = {0}:7777".format(s.ip)) listen_port += 1 rc.put_file(os.path.join(chdir, "etc/stunnel/redis-gluu.conf"), "\n".join(sconf)) return True
def install_monitoring(self): """Celery task that installs monitoring components to remote server. :param self: the celery task :return: wether monitoring were installed successfully """ tid = self.request.id installed = 0 servers = Server.query.all() app_config = AppConfiguration.query.first() for server in servers: # 1. Make SSH Connection to the remote server wlogger.log(tid, "Making SSH connection to the server {0}".format( server.hostname), "info", server_id=server.id) c = RemoteClient(server.hostname, ip=server.ip) try: c.startup() except Exception as e: wlogger.log(tid, "Cannot establish SSH connection {0}".format(e), "warning", server_id=server.id) wlogger.log(tid, "Ending server setup process.", "error", server_id=server.id) return False # 2. create monitoring directory result = c.run('mkdir -p /var/monitoring/scripts') ctext = "\n".join(result) if ctext.strip(): wlogger.log(tid, ctext, "debug", server_id=server.id) wlogger.log(tid, "Directory /var/monitoring/scripts directory " "was created", "success", server_id=server.id) # 3. Upload scripts scripts = ('cron_data_sqtile.py', 'get_data.py', 'sqlite_monitoring_tables.py') for scr in scripts: local_file = os.path.join(app.root_path, 'monitoring_scripts', scr) remote_file = '/var/monitoring/scripts/' + scr result = c.upload(local_file, remote_file) if result.startswith("Upload successful"): wlogger.log(tid, "File {} was uploaded".format(scr), "success", server_id=server.id) else: wlogger.log(tid, "File {} could not " "be uploaded: {}".format(scr, result), "error", server_id=server.id) return False # 4. Upload gluu version, no need to determine gluu version each time result = c.put_file('/var/monitoring/scripts/gluu_version.txt', app_config.gluu_version) # 5. Upload crontab entry to collect data in every 5 minutes crontab_entry = ('*/5 * * * * root python ' '/var/monitoring/scripts/cron_data_sqtile.py\n') result = c.put_file('/etc/cron.d/monitoring', crontab_entry) if not result[0]: wlogger.log(tid, "An errorr occurred while uploading crontab entry" ": {}".format(result[1]), "error", server_id=server.id) else: wlogger.log(tid, "Crontab entry was uploaded", "success", server_id=server.id) # 6. Installing packages. # 6a. First determine commands for each OS type if ('CentOS' in server.os) or ('RHEL' in server.os): package_cmd = [ 'yum install -y epel-release', 'yum repolist', 'yum install -y gcc', 'yum install -y python-devel', 'yum install -y python-pip', 'service crond restart' ] else: package_cmd = [ 'DEBIAN_FRONTEND=noninteractive apt-get install -y gcc', 'DEBIAN_FRONTEND=noninteractive apt-get install -y python-dev', 'DEBIAN_FRONTEND=noninteractive apt-get install -y python-pip', 'service cron restart', ] # 6b. These commands are common for all OS types package_cmd += [ 'pip install ldap3', 'pip install psutil', 'pip install pyDes', 'python /var/monitoring/scripts/' 'sqlite_monitoring_tables.py' ] # 6c. Executing commands wlogger.log(tid, "Installing Packages and Running Commands", "info", server_id=server.id) for cmd in package_cmd: result = c.run(cmd) wlogger.log(tid, "\n".join(result), "debug", server_id=server.id) err = False if result[2].strip(): print "Writing error", cmd if not ("pip install --upgrade pip" in result[2] or 'Redirecting to /bin/systemctl' in result[2]): wlogger.log(tid, "An error occurrued while executing " "{}: {}".format(cmd, result[2]), "error", server_id=server.id) err = True if not err: wlogger.log(tid, "Command was run successfully: {}".format(cmd), "success", server_id=server.id) server.monitoring = True db.session.commit() return True