def get_status():
status = {'redis': {}, 'stunnel': {}}
servers = Server.query.all()
check_cmd = 'python -c "import socket;s=socket.socket(socket.AF_INET,socket.SOCK_STREAM);print s.connect_ex((\'{0}\', {1}))"'
cache_servers = get_cache_servers()
stunnel_port = cache_servers[0].stunnel_port if cache_servers else None
for server in servers + cache_servers:
key = server.ip.replace('.', '_')
c = RemoteClient(host=server.hostname, ip=server.ip)
try:
c.startup()
except:
status['stunnel'][key] = False
status['redis'][key] = False
else:
status['stunnel'][key] = False
if server in cache_servers:
r = c.run(check_cmd.format('localhost', 6379))
stat = r[1].strip()
if stat == '0':
status['redis'][key] = True
else:
status['redis'][key] = False
if stunnel_port:
r = c.run(check_cmd.format(server.ip, stunnel_port))
stat = r[1].strip()
if stat == '0':
status['stunnel'][key] = True
else:
if stunnel_port:
r = c.run(check_cmd.format('localhost', '6379'))
stat = r[1].strip()
if stat == '0':
status['stunnel'][key] = True
c.close()
return jsonify(status)
class BaseInstaller(object):
"""Base class for component installers.
Args:
server (class:`clustermgr.models.Server`): the server object denoting
the server where server should be installed
tid (string): the task id of the celery task to add logs
"""
def __init__(self, server, tid):
self.server = server
self.tid = tid
self.rc = RemoteClient(server.hostname, ip=server.ip)
def install(self):
"""install() detects the os of the server and calls the appropriate
function to install redis on that server.
Returns:
boolean status of the install process
"""
try:
self.rc.startup()
except Exception as e:
wlogger.log(self.tid, "Could not connect to {0}".format(e),
"error", server_id=self.server.id)
return False
cin, cout, cerr = self.rc.run("ls /etc/*release")
files = cout.split()
cin, cout, cerr = self.rc.run("cat " + files[0])
status = False
if "Ubuntu" in cout:
status = self.install_in_ubuntu()
elif "CentOS" in cout:
status = self.install_in_centos()
else:
wlogger.log(self.tid, "Server OS is not supported. {0}".format(
cout), "error", server_id=self.server.id)
self.rc.close()
return status
def install_in_ubuntu(self):
"""This method should be overridden by the sub classes. Run the
commands needed to install your component.
Returns:
boolean status of success of the install
"""
pass
def install_in_centos(self):
"""This method should be overridden by the sub classes. Run the
commands needed to install your component.
Returns:
boolean status of success of the install
"""
pass
def run_command(self, cmd):
wlogger.log(self.tid, cmd, "debug", server_id=self.server.id)
return self.rc.run(cmd)
def install_cache_components(self, method):
"""Celery task that installs the redis, stunnel and twemproxy applications
in the required servers.
Redis and stunnel are installed in all the servers in the cluster.
Twemproxy is installed in the load-balancer/proxy server
:param self: the celery task
:param method: either STANDALONE, SHARDED
:return: the number of servers where both stunnel and redis were installed
successfully
"""
tid = self.request.id
installed = 0
servers = Server.query.all()
for server in servers:
wlogger.log(tid, "Installing Redis in server {0}".format(
server.hostname), "info", server_id=server.id)
ri = RedisInstaller(server, tid)
redis_installed = ri.install()
if redis_installed:
server.redis = True
wlogger.log(tid, "Redis install successful", "success",
server_id=server.id)
else:
server.redis = False
wlogger.log(tid, "Redis install failed", "fail",
server_id=server.id)
wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)
si = StunnelInstaller(server, tid)
stunnel_installed = si.install()
if stunnel_installed:
server.stunnel = True
wlogger.log(tid, "Stunnel install successful", "success",
server_id=server.id)
else:
server.stunnel = False
wlogger.log(tid, "Stunnel install failed", "fail",
server_id=server.id)
# Save the redis and stunnel install situation to the db
db.session.commit()
if redis_installed and stunnel_installed:
installed += 1
if method != 'STANDALONE':
# No need to install twemproxy for "SHARDED" configuration
return True
# Install twemproxy in the Nginx load balancing proxy server
app_conf = AppConfiguration.query.first()
host = app_conf.nginx_host
rc = RemoteClient(host)
try:
rc.startup()
except Exception as e:
wlogger.log(tid, "Could not connect to {0}".format(e), "error")
return False
server_os = get_os_type(rc)
mock_server = Server()
mock_server.hostname = host
wlogger.log(tid, "Installing Stunnel in proxy server")
si = StunnelInstaller(mock_server, tid)
stunnel_installed = si.install()
if stunnel_installed:
wlogger.log(tid, "Stunnel install successful", "success")
else:
wlogger.log(tid, "Stunnel install failed", "fail")
wlogger.log(tid, "Cluster manager will now try to build Twemproxy")
# 1. Setup the development tools for installation
if server_os in ["Ubuntu 16", "Ubuntu 14"]:
run_and_log(rc, "apt-get update", tid)
run_and_log(rc, "apt-get install -y build-essential autoconf libtool",
tid)
elif server_os in ["CentOS 6", "CentOS 7", "RHEL 7"]:
run_and_log(rc, "yum install -y wget", tid)
run_and_log(rc, "yum groupinstall -y 'Development tools'", tid)
if server_os == "CentOS 6":
run_and_log(rc, "wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz",
tid)
run_and_log(rc, "tar xvfvz autoconf-2.69.tar.gz", tid)
run_and_log(rc, "cd autoconf-2.69 && ./configure", tid)
run_and_log(rc, "cd autoconf-2.69 && make", tid)
run_and_log(rc, "cd autoconf-2.69 && make install", tid)
# 2. Get the source, build & install the nutcracker binaries
run_and_log(rc, "wget https://github.com/twitter/twemproxy/archive/v0.4.1.tar.gz",
tid)
run_and_log(rc, "tar -xf v0.4.1.tar.gz", tid)
run_and_log(rc, "cd twemproxy-0.4.1", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && autoreconf -fvi", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && ./configure --prefix=/usr", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && make", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && make install", tid)
# 3. Post installation - setup user and logging
run_and_log(rc, "useradd nutcracker", tid)
run_and_log(rc, "mkdir /var/log/nutcracker", tid)
run_and_log(rc, "touch /var/log/nutcracker/nutcracker.log", tid)
run_and_log(rc, "chown -R nutcracker:nutcracker /var/log/nutcracker",
tid)
logrotate_conf = ["/var/log/nutcracker/nutcracker*.log {", "\tweekly",
"\tmissingok", "\trotate 12", "\tcompress",
"\tnotifempty", "}"]
rc.put_file("/etc/logrotate.d/nutcracker", "\n".join(logrotate_conf))
# 4. Add init/service scripts to run nutcracker as a service
if server_os in ["Ubuntu 16", "CentOS 7", "RHEL 7"]:
local = os.path.join(app.root_path, "templates", "twemproxy",
"twemproxy.service")
remote = "/lib/systemd/system/nutcracker.service"
rc.upload(local, remote)
run_and_log(rc, "systemctl enable nutcracker", tid, None)
elif server_os == "Ubuntu 14":
local = os.path.join(app.root_path, "templates", "twemproxy",
"nutcracker.init")
remote = "/etc/init.d/nutcracker"
rc.upload(local, remote)
run_and_log(rc, 'chmod +x /etc/init.d/nutcracker', tid)
run_and_log(rc, "update-rc.d nutcracker defaults", tid)
elif server_os == "CentOS 6":
local = os.path.join(app.root_path, "templates", "twemproxy",
"nutcracker.centos.init")
remote = "/etc/rc.d/init.d/nutcracker"
rc.upload(local, remote)
run_and_log(rc, "chmod +x /etc/init.d/nutcracker", tid)
run_and_log(rc, "chkconfig --add nutcracker", tid)
run_and_log(rc, "chkconfig nutcracker on", tid)
# 5. Create the default configuration file referenced in the init scripts
run_and_log(rc, "mkdir -p /etc/nutcracker", tid)
run_and_log(rc, "touch /etc/nutcracker/nutcracker.yml", tid)
rc.close()
return installed
def _rotate_keys(kr, javalibs_dir, jks_path):
pub_keys = []
openid_jks_pass = random_chars()
if kr.type == "oxeleven":
token = decrypt_text(kr.oxeleven_token, kr.oxeleven_token_key,
kr.oxeleven_token_iv)
try:
# delete old keys first
print "deleting old keys"
for key_id in OxelevenKeyID.query:
status_code, out = delete_key(kr.oxeleven_url, key_id.kid,
token)
if status_code == 200 and out["deleted"]:
db.session.delete(key_id)
db.session.commit()
elif status_code == 401:
print "insufficient access to call oxEleven API"
# obtain new keys
print "obtaining new keys"
for algo in ["RS256", "RS384", "RS512", "ES256", "ES384", "ES512"]:
status_code, out = generate_key(kr.oxeleven_url,
algo,
token=token)
if status_code == 200:
key_id = OxelevenKeyID()
key_id.kid = out["kid"]
db.session.add(key_id)
db.session.commit()
pub_keys.append(out)
elif status_code == 401:
print "insufficient access to call oxEleven API"
else:
print "unable to obtain the keys from oxEleven; " \
"status code={}".format(status_code)
except requests.exceptions.ConnectionError:
print "unable to establish connection to oxEleven; skipping task"
else:
out, err, retcode = generate_jks(
openid_jks_pass,
javalibs_dir,
jks_path,
)
if retcode == 0:
json_out = json.loads(out)
pub_keys = json_out["keys"]
else:
print err
# update LDAP entry
if pub_keys and modify_oxauth_config(kr, pub_keys, openid_jks_pass):
print "pub keys has been updated"
kr.rotated_at = datetime.utcnow()
db.session.add(kr)
db.session.commit()
if kr.type == "jks":
from clustermgr.core.remote import RemoteClient
for server in Server.query:
c = RemoteClient(server.hostname)
try:
c.startup()
except Exception:
print "Couldn't connect to server %s. Can't copy JKS" % server.hostname
continue
c.upload(jks_path, server.jks_path)
c.close()
def install_cache_components(self, method, server_id_list):
"""Celery task that installs the redis, stunnel and twemproxy applications
in the required servers.
Redis and stunnel are installed in all the servers in the cluster.
Twemproxy is installed in the load-balancer/proxy server
:param self: the celery task
:param method: either STANDALONE, SHARDED
:return: the number of servers where both stunnel and redis were installed
successfully
"""
tid = self.request.id
installed = 0
servers = []
for server_id in server_id_list:
server = Server.query.get(server_id)
ri = RedisInstaller(server, tid)
ri.rc.startup()
if ri.rc.exists('/usr/bin/redis-server') or ri.rc.exists(
'/bin/redis-server'):
server.redis = True
redis_installed = 1
wlogger.log(tid,
"Redis was already installed on server {0}".format(
server.hostname),
"info",
server_id=server.id)
else:
wlogger.log(tid,
"Installing Redis in server {0}".format(
server.hostname),
"info",
server_id=server.id)
redis_installed = ri.install()
if redis_installed:
server.redis = True
wlogger.log(tid,
"Redis install successful",
"success",
server_id=server.id)
else:
server.redis = False
wlogger.log(tid,
"Redis install failed",
"fail",
server_id=server.id)
si = StunnelInstaller(server, tid)
si.rc.startup()
if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
wlogger.log(tid,
"Stunnel was allready installed",
"info",
server_id=server.id)
server.stunnel = True
stunnel_installed = 1
else:
wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)
stunnel_installed = si.install()
if stunnel_installed:
server.stunnel = True
wlogger.log(tid,
"Stunnel install successful",
"success",
server_id=server.id)
else:
server.stunnel = False
wlogger.log(tid,
"Stunnel install failed",
"fail",
server_id=server.id)
# Save the redis and stunnel install situation to the db
if redis_installed and stunnel_installed:
installed += 1
db.session.commit()
if method != 'STANDALONE':
# No need to install twemproxy for "SHARDED" configuration
return True
# Install twemproxy in the Nginx load balancing proxy server
app_conf = AppConfiguration.query.first()
mock_server = Server()
if app_conf.external_load_balancer:
mock_server.hostname = app_conf.cache_host
mock_server.ip = app_conf.cache_ip
else:
mock_server.hostname = app_conf.nginx_host
mock_server.ip = app_conf.nginx_ip
rc = RemoteClient(mock_server.hostname)
try:
rc.startup()
except Exception as e:
wlogger.log(tid, "Could not connect to {0}".format(e), "error")
return False
server_os = get_os_type(rc)
si = StunnelInstaller(mock_server, tid)
si.rc.startup()
stunnel_installed = 0
if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
wlogger.log(tid, "Stunnel was already installed on cache server")
stunnel_installed = 1
else:
wlogger.log(tid, "Installing Stunnel in cache server")
stunnel_installed = si.install()
if stunnel_installed:
wlogger.log(tid, "Stunnel install successful", "success")
else:
wlogger.log(tid, "Stunnel install failed", "fail")
print rc.exists('/usr/sbin/nutcracker')
if not rc.exists('/usr/sbin/nutcracker'):
wlogger.log(tid, "Installing Twemproxy")
# 1. Setup the development tools for installation
if server_os == "Ubuntu 14":
run_and_log(rc, "apt-get update", tid)
run_and_log(
rc,
'wget http://ftp.debian.org/debian/pool/main/n/nutcracker/nutcracker_0.4.0+dfsg-1_amd64.deb -O /tmp/nutcracker_0.4.0+dfsg-1_amd64.deb',
tid)
run_and_log(rc, "dpkg -i /tmp/nutcracker_0.4.0+dfsg-1_amd64.deb",
tid)
elif server_os == "Ubuntu 16":
run_and_log(rc, "apt-get update", tid)
run_and_log(
rc,
"DEBIAN_FRONTEND=noninteractive apt-get install -y nutcracker",
tid)
elif server_os in ["CentOS 7", "RHEL 7"]:
run_and_log(
rc,
'yum install -y https://raw.githubusercontent.com/mbaser/gluu/master/nutcracker-0.4.1-1.gluu.centos7.x86_64.rpm',
tid)
run_and_log(rc, 'chkconfig nutcracker on', tid)
elif server_os in ['CentOS 6', 'RHEL 6']:
run_and_log(
rc,
'yum install -y https://raw.githubusercontent.com/mbaser/gluu/master/nutcracker-0.4.1-1.gluu.centos6.x86_64.rpm',
tid)
run_and_log(rc, 'chkconfig nutcracker on', tid)
# 5. Create the default configuration file referenced in the init scripts
#run_and_log(rc, "mkdir -p /etc/nutcracker", tid)
run_and_log(rc, "touch /etc/nutcracker/nutcracker.yml", tid)
else:
wlogger.log(tid, "Twemproxy was already installed on cache server")
rc.close()
return installed