def test_db_with_data_loads_dashboard(self):
s1 = Server()
s1.hostname = 'test.example.com'
s1.gluu_server = True
s1.ip = '0.0.0.0'
with self.app.app_context():
db.session.add(s1)
db.session.commit()
rv = self.client.get('/')
self.assertIn('Dashboard', rv.data)
self.assertIn('test.example.com', rv.data)
def index():
"""Route for URL /server/. GET returns ServerForm to add a server,
POST accepts the ServerForm, validates and creates a new Server object
"""
appconfig = AppConfiguration.query.first()
if not appconfig:
flash(
"Kindly set default values for the application before adding"
" servers.", "info")
return redirect(url_for('index.app_configuration', next="/server/"))
form = ServerForm()
header = "New Server"
primary_server = Server.query.filter(
Server.primary_server.is_(True)).first()
if primary_server:
del form.ldap_password
del form.ldap_password_confirm
else:
header = "New Server - Primary Server"
if form.validate_on_submit():
server = Server()
server.hostname = form.hostname.data.strip()
server.ip = form.ip.data.strip()
server.mmr = False
if primary_server:
server.ldap_password = primary_server.ldap_password
else:
server.ldap_password = form.ldap_password.data.strip()
server.primary_server = True
db.session.add(server)
db.session.commit()
# start the background job to get system details
collect_server_details.delay(server.id)
return redirect(url_for('index.home'))
return render_template('new_server.html', form=form, header=header)
def index():
"""Route for URL /server/. GET returns ServerForm to add a server,
POST accepts the ServerForm, validates and creates a new Server object
"""
appconfig = AppConfiguration.query.first()
if not appconfig:
flash(
"Kindly set default values for the application before adding"
" servers.", "info")
return redirect(url_for('index.app_configuration', next="/server/"))
form = ServerForm()
pr_server = get_primary_server_id()
if pr_server:
form.primary_server.render_kw = {'disabled': 'disabled'}
if form.validate_on_submit():
server = Server()
server.gluu_server = form.gluu_server.data
server.hostname = form.hostname.data
server.ip = form.ip.data
server.ldap_password = form.ldap_password.data
server.mmr = False
server.primary_server = form.primary_server.data
db.session.add(server)
db.session.commit()
# start the background job to get system details
collect_server_details.delay(server.id)
return redirect(url_for('index.home'))
flash(
'Cluster Manager will connect to this server via SSH to perform its'
' tasks. Ensure the server running Cluster Manager has'
'"Password-less" SSH access via shared keys to the server.', 'info')
return render_template('new_server.html', form=form, header="New Server")
def index():
"""Route for URL /server/. GET returns ServerForm to add a server,
POST accepts the ServerForm, validates and creates a new Server object
"""
appconfig = AppConfiguration.query.first()
if not appconfig:
flash(
"Kindly set default values for the application before adding"
" servers.", "info")
return redirect(url_for('index.app_configuration', next="/server/"))
form = ServerForm()
header = "New Server"
primary_server = Server.query.filter(
Server.primary_server.is_(True)).first()
if primary_server:
del form.ldap_password
del form.ldap_password_confirm
else:
header = "New Server - Primary Server"
if form.validate_on_submit():
server = Server()
server.hostname = form.hostname.data.strip()
server.ip = form.ip.data.strip()
server.mmr = False
ask_passphrase = False
server_exist = Server.query.filter_by(
hostname=form.hostname.data.strip()).first()
if server_exist:
flash(
"Server with hostname {} is already in cluster".format(
server_exist.hostname), "warning")
return redirect(url_for('index.home'))
c = RemoteClient(server.hostname, server.ip)
try:
c.startup()
except ClientNotSetupException as e:
if str(e) == 'Pubkey is encrypted.':
ask_passphrase = True
flash(
"Pubkey seems to password protected. "
"After setting your passphrase re-submit this form.",
'warning')
elif str(e) == 'Could not deserialize key data.':
ask_passphrase = True
flash(
"Password your provided for pubkey did not work. "
"After setting your passphrase re-submit this form.",
'warning')
else:
flash(
"SSH connection to {} failed. Please check if your pub key is "
"added to /root/.ssh/authorized_keys on this server. Reason: {}"
.format(server.hostname, e), 'error')
#except:
# flash("SSH connection to {} failed. Please check if your pub key is "
# "asdded to /root/.ssh/authorized_keys on this server".format(
# server.hostname))
print "ask_passphrase", ask_passphrase
return render_template('new_server.html',
form=form,
header=header,
server_id=None,
ask_passphrase=ask_passphrase,
next=url_for('server.index'))
if primary_server:
server.ldap_password = primary_server.ldap_password
else:
server.ldap_password = form.ldap_password.data.strip()
server.primary_server = True
if not server.hostname == appconfig.nginx_host:
db.session.add(server)
db.session.commit()
# start the background job to get system details
collect_server_details.delay(server.id)
return redirect(url_for('index.home'))
else:
flash("Load balancer can't be used as gluu server", 'danger')
return render_template('new_server.html',
form=form,
header=header,
server_id=None)
def setup_proxied(tid):
"""Configures the servers to use the Twemproxy installed in proxy server
for Redis caching securely via stunnel.
:param tid: task id for log identification
:return: None
"""
servers = Server.query.filter(Server.redis.is_(True)).filter(
Server.stunnel.is_(True)).all()
appconf = AppConfiguration.query.first()
chdir = "/opt/gluu-server-" + appconf.gluu_version
stunnel_base_conf = [
"cert = /etc/stunnel/cert.pem",
"pid = /var/run/stunnel.pid",
"output = /var/log/stunnel4/stunnel.log"
]
proxy_stunnel_conf = stunnel_base_conf
twemproxy_servers = []
proxy_ip = socket.gethostbyname(appconf.nginx_host)
primary = Server.query.filter(Server.primary_server.is_(True)).first()
if not primary:
wlogger.log(tid, "Primary Server is not setup yet. Cannot setup "
"clustered caching.", "error")
# Setup Stunnel and Redis in each server
for server in servers:
__update_LDAP_cache_method(tid, server, 'localhost:7000', 'STANDALONE')
stunnel_conf = [
"[redis-server]",
"client = no",
"accept = {0}:7777".format(server.ip),
"connect = 127.0.0.1:6379",
"[twemproxy]",
"client = yes",
"accept = 127.0.0.1:7000",
"connect = {0}:8888".format(proxy_ip)
]
stunnel_conf = stunnel_base_conf + stunnel_conf
status = __configure_stunnel(tid, server, stunnel_conf, chdir)
if not status:
continue
# if the setup was successful add the server to the list of stunnel
# clients in the proxy server configuration
client_conf = [
"[client{0}]".format(server.id),
"client = yes",
"accept = 127.0.0.1:{0}".format(7000+server.id),
"connect = {0}:7777".format(server.ip)
]
proxy_stunnel_conf.extend(client_conf)
twemproxy_servers.append(" - 127.0.0.1:{0}:1".format(7000+server.id))
wlogger.log(tid, "Configuring the proxy server ...")
# Setup Stunnel in the proxy server
mock_server = Server()
mock_server.hostname = appconf.nginx_host
mock_server.ip = proxy_ip
rc = __get_remote_client(mock_server, tid)
if not rc:
wlogger.log(tid, "Couldn't connect to proxy server. Twemproxy setup "
"failed.", "error")
return
mock_server.os = get_os_type(rc)
# Download the setup.properties file from the primary server
local = os.path.join(app.instance_path, "setup.properties")
remote = os.path.join("/opt/gluu-server-"+appconf.gluu_version,
"install", "community-edition-setup",
"setup.properties.last")
prc = __get_remote_client(primary, tid)
prc.download(remote, local)
prc.close()
rc.upload(local, "/tmp/setup.properties")
twem_server_conf = [
"[twemproxy]",
"client = no",
"accept = {0}:8888".format(proxy_ip),
"connect = 127.0.0.1:2222"
]
proxy_stunnel_conf.extend(twem_server_conf)
status = __configure_stunnel(tid, mock_server, proxy_stunnel_conf, None,
"/tmp/setup.properties")
if not status:
return False
# Setup Twemproxy
wlogger.log(tid, "Writing Twemproxy configuration")
twemproxy_conf = [
"alpha:",
" listen: 127.0.0.1:2222",
" hash: fnv1a_64",
" distribution: ketama",
" auto_eject_hosts: true",
" redis: true",
" server_failure_limit: 2",
" timeout: 400",
" preconnect: true",
" servers:"
]
twemproxy_conf.extend(twemproxy_servers)
remote = "/etc/nutcracker/nutcracker.yml"
rc.put_file(remote, "\n".join(twemproxy_conf))
wlogger.log(tid, "Configuration complete", "success")
def step1():
pserver = Server.query.filter_by(primary_server=True).first()
if pserver and request.args.get('pass_set') != 'true':
flash("Oops this service is not for you.", 'warning')
return redirect(url_for('index.home'))
wform = WizardStep1()
if request.method == 'POST':
if wform.validate_on_submit():
replication_pw = uuid.uuid4().hex
app_conf = AppConfiguration()
app_conf.nginx_host = wform.new_hostname.data.strip()
app_conf.replication_pw = replication_pw
app_conf.nginx_ip = wform.nginx_ip.data.strip()
app_conf.modify_hosts = True
db.session.add(app_conf)
server = Server()
server.ip = wform.ip.data.strip()
server.hostname = wform.current_hostname.data.strip()
server.primary_server = True
db.session.add(app_conf)
db.session.add(server)
db.session.commit()
if request.method == 'POST' or request.args.get('pass_set') == 'true':
servers = Server.query.all()
ask_passphrase = False
c = RemoteClient(servers[0].ip, servers[0].hostname)
try:
c.startup()
except ClientNotSetupException as e:
if str(e) == 'Pubkey is encrypted.':
ask_passphrase = True
flash(
"Pubkey seems to password protected. "
"Please set passphrase.", 'warning')
elif str(e) == 'Could not deserialize key data.':
ask_passphrase = True
flash(
"Password your provided for pubkey did not work. "
"Please set valid passphrase.", 'warning')
else:
flash(
"SSH connection to {} failed. Please check if your pub key is "
"added to /root/.ssh/authorized_keys on this server. Reason: {}"
.format(servers[0].hostname, e), 'error')
return render_template(
'index_passphrase.html',
e=e,
ask_passphrase=ask_passphrase,
next=url_for('wizard.step1', pass_set='true'),
warning_text="Error accessing Stand Allone Server")
task = wizard_step1.delay()
print "TASK STARTED", task.id
servers = Server.query.all()
return render_template('wizard/wizard_logger.html',
step=1,
task_id=task.id,
servers=servers)
return render_template('wizard/step1.html', wform=wform)
def setup_proxied(tid, server_id_list):
"""Configures the servers to use the Twemproxy installed in proxy server
for Redis caching securely via stunnel.
:param tid: task id for log identification
:return: None
"""
servers = []
for server_id in server_id_list:
qserver = Server.query.filter(Server.redis.is_(True)).filter(
Server.stunnel.is_(True)).filter(Server.id.is_(server_id)).first()
if qserver:
servers.append(qserver)
appconf = AppConfiguration.query.first()
chdir = "/opt/gluu-server-" + appconf.gluu_version
if appconf.external_load_balancer:
cache_ip = appconf.cache_ip
else:
cache_ip = appconf.nginx_ip
primary = Server.query.filter(Server.primary_server.is_(True)).first()
if not primary:
wlogger.log(
tid, "Primary Server is not setup yet. Cannot setup "
"clustered caching.", "error")
# Setup Stunnel and Redis in each server
for server in servers:
#Since replication is active, we only need to update on primary server
if server.primary_server:
__update_LDAP_cache_method(tid, server, 'localhost:7000',
'STANDALONE')
stunnel_conf = [
"cert = /etc/stunnel/cert.pem", "pid = /var/run/stunnel.pid",
"output = /var/log/stunnel4/stunnel.log", "[redis-server]",
"client = no", "accept = {0}:7777".format(server.ip),
"connect = 127.0.0.1:6379", "[twemproxy]", "client = yes",
"accept = 127.0.0.1:7000", "connect = {0}:8888".format(cache_ip)
]
status = __configure_stunnel(tid, server, stunnel_conf, chdir)
if not status:
continue
wlogger.log(tid, "Configuring the cahce server ...")
# Setup Stunnel in the proxy server
mock_server = Server()
if appconf.external_load_balancer:
mock_server.hostname = appconf.cache_host
mock_server.ip = appconf.cache_ip
else:
mock_server.hostname = appconf.nginx_host
mock_server.ip = appconf.nginx_ip
rc = __get_remote_client(mock_server, tid)
if not rc:
wlogger.log(
tid, "Couldn't connect to proxy server. Twemproxy setup "
"failed.", "error")
return
mock_server.os = get_os_type(rc)
if rc.exists('/usr/bin/redis-server') or rc.exists('/bin/redis-server'):
wlogger.log(
tid, "Redis was already installed on server {0}".format(
mock_server.hostname), "info")
else:
wlogger.log(
tid, "Installing Redis in server {0}".format(mock_server.hostname),
"info")
ri = RedisInstaller(mock_server, tid)
redis_installed = ri.install()
if redis_installed:
mock_server.redis = True
wlogger.log(tid, "Redis install successful", "success")
else:
mock_server.redis = False
wlogger.log(tid, "Redis install failed", "fail")
# Download the setup.properties file from the primary server
local = os.path.join(app.instance_path, "setup.properties")
remote = os.path.join("/opt/gluu-server-" + appconf.gluu_version,
"install", "community-edition-setup",
"setup.properties.last")
prc = __get_remote_client(primary, tid)
prc.download(remote, local)
prc.close()
rc.upload(local, "/tmp/setup.properties")
proxy_stunnel_conf = make_proxy_stunnel_conf()
status = __configure_stunnel(tid, mock_server, proxy_stunnel_conf, None,
"/tmp/setup.properties")
if not status:
return False
# Setup Twemproxy
wlogger.log(tid, "Writing Twemproxy configuration")
twemproxy_conf = make_twem_proxy_conf()
remote = "/etc/nutcracker/nutcracker.yml"
rc.put_file(remote, twemproxy_conf)
run_command(tid, rc, 'service nutcracker restart')
wlogger.log(tid, "Configuration complete", "success")
def install_cache_components(self, method, server_id_list):
"""Celery task that installs the redis, stunnel and twemproxy applications
in the required servers.
Redis and stunnel are installed in all the servers in the cluster.
Twemproxy is installed in the load-balancer/proxy server
:param self: the celery task
:param method: either STANDALONE, SHARDED
:return: the number of servers where both stunnel and redis were installed
successfully
"""
tid = self.request.id
installed = 0
servers = []
for server_id in server_id_list:
server = Server.query.get(server_id)
ri = RedisInstaller(server, tid)
ri.rc.startup()
if ri.rc.exists('/usr/bin/redis-server') or ri.rc.exists(
'/bin/redis-server'):
server.redis = True
redis_installed = 1
wlogger.log(tid,
"Redis was already installed on server {0}".format(
server.hostname),
"info",
server_id=server.id)
else:
wlogger.log(tid,
"Installing Redis in server {0}".format(
server.hostname),
"info",
server_id=server.id)
redis_installed = ri.install()
if redis_installed:
server.redis = True
wlogger.log(tid,
"Redis install successful",
"success",
server_id=server.id)
else:
server.redis = False
wlogger.log(tid,
"Redis install failed",
"fail",
server_id=server.id)
si = StunnelInstaller(server, tid)
si.rc.startup()
if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
wlogger.log(tid,
"Stunnel was allready installed",
"info",
server_id=server.id)
server.stunnel = True
stunnel_installed = 1
else:
wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)
stunnel_installed = si.install()
if stunnel_installed:
server.stunnel = True
wlogger.log(tid,
"Stunnel install successful",
"success",
server_id=server.id)
else:
server.stunnel = False
wlogger.log(tid,
"Stunnel install failed",
"fail",
server_id=server.id)
# Save the redis and stunnel install situation to the db
if redis_installed and stunnel_installed:
installed += 1
db.session.commit()
if method != 'STANDALONE':
# No need to install twemproxy for "SHARDED" configuration
return True
# Install twemproxy in the Nginx load balancing proxy server
app_conf = AppConfiguration.query.first()
mock_server = Server()
if app_conf.external_load_balancer:
mock_server.hostname = app_conf.cache_host
mock_server.ip = app_conf.cache_ip
else:
mock_server.hostname = app_conf.nginx_host
mock_server.ip = app_conf.nginx_ip
rc = RemoteClient(mock_server.hostname)
try:
rc.startup()
except Exception as e:
wlogger.log(tid, "Could not connect to {0}".format(e), "error")
return False
server_os = get_os_type(rc)
si = StunnelInstaller(mock_server, tid)
si.rc.startup()
stunnel_installed = 0
if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
wlogger.log(tid, "Stunnel was already installed on cache server")
stunnel_installed = 1
else:
wlogger.log(tid, "Installing Stunnel in cache server")
stunnel_installed = si.install()
if stunnel_installed:
wlogger.log(tid, "Stunnel install successful", "success")
else:
wlogger.log(tid, "Stunnel install failed", "fail")
print rc.exists('/usr/sbin/nutcracker')
if not rc.exists('/usr/sbin/nutcracker'):
wlogger.log(tid, "Installing Twemproxy")
# 1. Setup the development tools for installation
if server_os == "Ubuntu 14":
run_and_log(rc, "apt-get update", tid)
run_and_log(
rc,
'wget http://ftp.debian.org/debian/pool/main/n/nutcracker/nutcracker_0.4.0+dfsg-1_amd64.deb -O /tmp/nutcracker_0.4.0+dfsg-1_amd64.deb',
tid)
run_and_log(rc, "dpkg -i /tmp/nutcracker_0.4.0+dfsg-1_amd64.deb",
tid)
elif server_os == "Ubuntu 16":
run_and_log(rc, "apt-get update", tid)
run_and_log(
rc,
"DEBIAN_FRONTEND=noninteractive apt-get install -y nutcracker",
tid)
elif server_os in ["CentOS 7", "RHEL 7"]:
run_and_log(
rc,
'yum install -y https://raw.githubusercontent.com/mbaser/gluu/master/nutcracker-0.4.1-1.gluu.centos7.x86_64.rpm',
tid)
run_and_log(rc, 'chkconfig nutcracker on', tid)
elif server_os in ['CentOS 6', 'RHEL 6']:
run_and_log(
rc,
'yum install -y https://raw.githubusercontent.com/mbaser/gluu/master/nutcracker-0.4.1-1.gluu.centos6.x86_64.rpm',
tid)
run_and_log(rc, 'chkconfig nutcracker on', tid)
# 5. Create the default configuration file referenced in the init scripts
#run_and_log(rc, "mkdir -p /etc/nutcracker", tid)
run_and_log(rc, "touch /etc/nutcracker/nutcracker.yml", tid)
else:
wlogger.log(tid, "Twemproxy was already installed on cache server")
rc.close()
return installed