def setup_redis_cluster(tid):
servers = Server.query.filter(Server.redis.is_(True)).filter(
Server.stunnel.is_(True)).all()
appconf = AppConfiguration.query.first()
master_conf = [
"port 7000", "cluster-enabled yes",
"cluster-config-file nodes_7000.conf", "cluster-node-timeout 5000",
"appendonly yes"
]
slave_conf = [
"port 7001", "cluster-enabled yes",
"cluster-config-file nodes_7001.conf", "cluster-node-timeout 5000",
"appendonly yes"
]
for server in servers:
rc = RemoteClient(server.hostname, ip=server.ip)
try:
rc.startup()
wlogger.log(tid,
"Connecting to server ...",
"success",
server_id=server.id)
except Exception as e:
wlogger.log(tid,
"Could not connect to the server over SSH. Error:"
"{0}\nRedis configuration failed.".format(e),
"error",
server_id=server.id)
continue
chdir = '/'
if server.gluu_server:
chdir = "/opt/gluu-server-{0}".format(appconf.gluu_version)
# upload the conf files
wlogger.log(tid,
"Uploading redis conf files...",
"debug",
server_id=server.id)
rc.put_file(os.path.join(chdir, "etc/redis/redis_7000.conf"),
"\n".join(master_conf))
rc.put_file(os.path.join(chdir, "etc/redis/redis_7001.conf"),
"\n".join(slave_conf))
# upload the modified init.d file
rc.upload(
os.path.join(app.root_path, "templates", "redis", "redis-server"),
os.path.join(chdir, "etc/init.d/redis-server"))
wlogger.log(tid,
"Configuration upload complete.",
"success",
server_id=server.id)
return True
def edit_slapd_conf(server_id):
"""This view provides editing of slapd.conf file before depoloyments."""
server = Server.query.get(server_id)
appconf = AppConfiguration.query.first()
# If there is no server with server_id return to home
if not server:
flash("No such server.", "warning")
return redirect(url_for('index.home'))
if not server.gluu_server:
chroot = '/'
else:
chroot = '/opt/gluu-server-' + appconf.gluu_version
# slapd.conf file will be downloaded from server. Make ssh connection
# and download it
c = RemoteClient(server.hostname, ip=server.ip)
try:
c.startup()
except ClientNotSetupException as e:
flash(str(e), "danger")
return redirect(url_for('index.home'))
slapd_conf_file = os.path.join(chroot, 'opt/symas/etc/openldap/slapd.conf')
if request.method == 'POST':
config = request.form.get('conf')
r = c.put_file(slapd_conf_file, config)
if not r[0]:
flash("Cant' saved to server: {0}".format(r[1]), "danger")
else:
flash('File {0} was saved on {1}'.format(slapd_conf_file,
server.hostname))
return redirect(url_for('index.home'))
# After editing, slapd.conf file will be uploaded to server via ssh
r = c.get_file(slapd_conf_file)
if not r[0]:
flash("Cant't get file {0}: {1}".format(slapd_conf_file, r[1]),
"success")
return redirect(url_for('index.home'))
config = r[1].read()
return render_template('conf_editor.html',
config=config,
hostname=server.hostname)
def edit_slapd_conf(server_id):
server = Server.query.get(server_id)
appconf = AppConfiguration.query.first()
if not server:
print "Yoook"
flash("No such server.", "warning")
return redirect(url_for('index.home'))
if not server.gluu_server:
chroot = '/'
else:
chroot = '/opt/gluu-server-' + appconf.gluu_version
c = RemoteClient(server.hostname, ip=server.ip)
try:
c.startup()
except ClientNotSetupException as e:
flash(str(e), "danger")
return redirect(url_for('index.home'))
slapd_conf_file = os.path.join(chroot, 'opt/symas/etc/openldap/slapd.conf')
if request.method == 'POST':
config = request.form.get('conf')
r = c.put_file(slapd_conf_file, config)
if not r[0]:
flash("Cant' saved to server: {0}".format(r[1]), "danger")
else:
flash('File {0} was saved on {1}'.format(slapd_conf_file,
server.hostname))
return redirect(url_for('index.home'))
r = c.get_file(slapd_conf_file)
if not r[0]:
flash("Cant't get file {0}: {1}".format(slapd_conf_file, r[1]),
"success")
return redirect(url_for('index.home'))
config = r[1].read()
return render_template('conf_editor.html',
config=config,
hostname=server.hostname)
def install_cache_components(self, method):
"""Celery task that installs the redis, stunnel and twemproxy applications
in the required servers.
Redis and stunnel are installed in all the servers in the cluster.
Twemproxy is installed in the load-balancer/proxy server
:param self: the celery task
:param method: either STANDALONE, SHARDED
:return: the number of servers where both stunnel and redis were installed
successfully
"""
tid = self.request.id
installed = 0
servers = Server.query.all()
for server in servers:
wlogger.log(tid, "Installing Redis in server {0}".format(
server.hostname), "info", server_id=server.id)
ri = RedisInstaller(server, tid)
redis_installed = ri.install()
if redis_installed:
server.redis = True
wlogger.log(tid, "Redis install successful", "success",
server_id=server.id)
else:
server.redis = False
wlogger.log(tid, "Redis install failed", "fail",
server_id=server.id)
wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)
si = StunnelInstaller(server, tid)
stunnel_installed = si.install()
if stunnel_installed:
server.stunnel = True
wlogger.log(tid, "Stunnel install successful", "success",
server_id=server.id)
else:
server.stunnel = False
wlogger.log(tid, "Stunnel install failed", "fail",
server_id=server.id)
# Save the redis and stunnel install situation to the db
db.session.commit()
if redis_installed and stunnel_installed:
installed += 1
if method != 'STANDALONE':
# No need to install twemproxy for "SHARDED" configuration
return True
# Install twemproxy in the Nginx load balancing proxy server
app_conf = AppConfiguration.query.first()
host = app_conf.nginx_host
rc = RemoteClient(host)
try:
rc.startup()
except Exception as e:
wlogger.log(tid, "Could not connect to {0}".format(e), "error")
return False
server_os = get_os_type(rc)
mock_server = Server()
mock_server.hostname = host
wlogger.log(tid, "Installing Stunnel in proxy server")
si = StunnelInstaller(mock_server, tid)
stunnel_installed = si.install()
if stunnel_installed:
wlogger.log(tid, "Stunnel install successful", "success")
else:
wlogger.log(tid, "Stunnel install failed", "fail")
wlogger.log(tid, "Cluster manager will now try to build Twemproxy")
# 1. Setup the development tools for installation
if server_os in ["Ubuntu 16", "Ubuntu 14"]:
run_and_log(rc, "apt-get update", tid)
run_and_log(rc, "apt-get install -y build-essential autoconf libtool",
tid)
elif server_os in ["CentOS 6", "CentOS 7", "RHEL 7"]:
run_and_log(rc, "yum install -y wget", tid)
run_and_log(rc, "yum groupinstall -y 'Development tools'", tid)
if server_os == "CentOS 6":
run_and_log(rc, "wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz",
tid)
run_and_log(rc, "tar xvfvz autoconf-2.69.tar.gz", tid)
run_and_log(rc, "cd autoconf-2.69 && ./configure", tid)
run_and_log(rc, "cd autoconf-2.69 && make", tid)
run_and_log(rc, "cd autoconf-2.69 && make install", tid)
# 2. Get the source, build & install the nutcracker binaries
run_and_log(rc, "wget https://github.com/twitter/twemproxy/archive/v0.4.1.tar.gz",
tid)
run_and_log(rc, "tar -xf v0.4.1.tar.gz", tid)
run_and_log(rc, "cd twemproxy-0.4.1", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && autoreconf -fvi", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && ./configure --prefix=/usr", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && make", tid)
run_and_log(rc, "cd twemproxy-0.4.1 && make install", tid)
# 3. Post installation - setup user and logging
run_and_log(rc, "useradd nutcracker", tid)
run_and_log(rc, "mkdir /var/log/nutcracker", tid)
run_and_log(rc, "touch /var/log/nutcracker/nutcracker.log", tid)
run_and_log(rc, "chown -R nutcracker:nutcracker /var/log/nutcracker",
tid)
logrotate_conf = ["/var/log/nutcracker/nutcracker*.log {", "\tweekly",
"\tmissingok", "\trotate 12", "\tcompress",
"\tnotifempty", "}"]
rc.put_file("/etc/logrotate.d/nutcracker", "\n".join(logrotate_conf))
# 4. Add init/service scripts to run nutcracker as a service
if server_os in ["Ubuntu 16", "CentOS 7", "RHEL 7"]:
local = os.path.join(app.root_path, "templates", "twemproxy",
"twemproxy.service")
remote = "/lib/systemd/system/nutcracker.service"
rc.upload(local, remote)
run_and_log(rc, "systemctl enable nutcracker", tid, None)
elif server_os == "Ubuntu 14":
local = os.path.join(app.root_path, "templates", "twemproxy",
"nutcracker.init")
remote = "/etc/init.d/nutcracker"
rc.upload(local, remote)
run_and_log(rc, 'chmod +x /etc/init.d/nutcracker', tid)
run_and_log(rc, "update-rc.d nutcracker defaults", tid)
elif server_os == "CentOS 6":
local = os.path.join(app.root_path, "templates", "twemproxy",
"nutcracker.centos.init")
remote = "/etc/rc.d/init.d/nutcracker"
rc.upload(local, remote)
run_and_log(rc, "chmod +x /etc/init.d/nutcracker", tid)
run_and_log(rc, "chkconfig --add nutcracker", tid)
run_and_log(rc, "chkconfig nutcracker on", tid)
# 5. Create the default configuration file referenced in the init scripts
run_and_log(rc, "mkdir -p /etc/nutcracker", tid)
run_and_log(rc, "touch /etc/nutcracker/nutcracker.yml", tid)
rc.close()
return installed
def install_cache_cluster(self, servers_id_list, cache_servers_id_list):
tid = self.request.id
servers = [ Server.query.get(id) for id in servers_id_list ]
cache_servers = [ CacheServer.query.get(id) for id in cache_servers_id_list ]
app_conf = AppConfiguration.query.first()
primary_cache = None
primary_cache_server = CacheServer.query.first()
for server in cache_servers:
rc = __get_remote_client(server, tid)
if not rc:
wlogger.log(tid, "SSH connection to server failed", "error", server_id=server.id)
return False
ri = RedisInstaller(server, tid, rc)
if app_conf.offline:
if not ri.check_installed():
wlogger.log(
tid,
'Redis Server was not installed. Please install Redis '
' Server and retry.',
'error',
server_id=server.id
)
return False
redis_installed = ri.install()
wlogger.log(
tid,
'Setting Redis password',
'info',
server_id=server.id
)
ri.set_redis_password()
if redis_installed:
wlogger.log(tid, "Redis install successful", "success",
server_id=server.id)
else:
wlogger.log(tid, "Redis install failed", "fail",
server_id=server.id)
return False
ri.run_sysctl('enable')
ri.run_sysctl('restart')
si = StunnelInstaller(server, tid, rc)
if app_conf.offline:
if not si.check_installed():
wlogger.log(
tid,
'Stunnel was not installed. Please install stunnel '
'and retry.',
'error',
server_id=server.id
)
return False
if si.check_installed():
server.stunnel = True
else:
wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)
stunnel_installed = si.install()
if stunnel_installed:
server.stunnel = True
wlogger.log(tid, "Stunnel install successful", "success",
server_id=server.id)
else:
server.stunnel = False
wlogger.log(tid, "Stunnel install failed", "fail",
server_id=server.id)
return False
if si.os_type == 'rpm':
si.upload_service_file()
if si.os_type == 'deb':
wlogger.log(tid, "Enabling stunnel", "debug", server_id=server.id)
si.run_command("sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/stunnel4")
if not primary_cache:
primary_cache = server.ip
if not rc.exists('/etc/stunnel/redis-server.crt'):
wlogger.log(tid, "Creating SSL certificate for stunnel", "info",
server_id=server.id)
si.run_command(
'openssl req -x509 -nodes -days 3650 -newkey rsa:2048 '
'-batch -keyout /etc/stunnel/redis-server.key '
'-out /etc/stunnel/redis-server.crt'
)
si.run_command('chmod 600 /etc/stunnel/redis-server.key')
wlogger.log(tid, "Retreiving server certificate", "info",
server_id=server.id)
stunnel_redis_conf = (
'pid = /run/stunnel-redis.pid\n'
'[redis-server]\n'
'cert = /etc/stunnel/redis-server.crt\n'
'key = /etc/stunnel/redis-server.key\n'
'accept = {0}:{1}\n'
'connect = 127.0.0.1:6379\n'
).format(server.ip, primary_cache_server.stunnel_port)
wlogger.log(tid, "Writing redis stunnel configurations", "info",
server_id=server.id)
rc.put_file('/etc/stunnel/stunnel.conf', stunnel_redis_conf)
si.run_sysctl('enable')
si.run_sysctl('restart')
server.installed = True
db.session.commit()
rc.close()
wlogger.log(tid, "2", "set_step")
# retreive stunnel certificate
rc = RemoteClient(primary_cache_server.hostname, ip=primary_cache_server.ip)
rc.startup()
stunnel_cert = rc.get_file('/etc/stunnel/redis-server.crt')
if not stunnel_cert[0]:
print "Can't retreive server certificate from primary cache server"
return False
stunnel_cert = stunnel_cert[1].read()
for server in servers:
rc = __get_remote_client(server, tid)
if not rc:
wlogger.log(tid, "SSH connection to server failed", "error", server_id=server.id)
return False
si = StunnelInstaller(server, tid, rc)
wlogger.log(tid, "Installing Stunnel", "debug", server_id=server.id)
if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
wlogger.log(tid, "Stunnel was allready installed", "info",
server_id=server.id)
server.stunnel = True
else:
wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)
stunnel_installed = si.install()
if stunnel_installed:
server.stunnel = True
wlogger.log(tid, "Stunnel install successful", "success",
server_id=server.id)
else:
server.stunnel = False
wlogger.log(tid, "Stunnel install failed", "fail",
server_id=server.id)
return False
if si.os_type == 'rpm':
si.upload_service_file()
if si.os_type == 'deb':
wlogger.log(tid, "Enabling stunnel", "debug", server_id=server.id)
si.run_command("sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/stunnel4")
stunnel_redis_conf = (
'pid = /run/stunnel-redis.pid\n'
'[redis-client]\n'
'client = yes\n'
'accept = 127.0.0.1:{1}\n'
'connect = {0}:{1}\n'
'CAfile = /etc/stunnel/redis-server.crt\n'
'verify = 4\n'
).format(primary_cache, primary_cache_server.stunnel_port)
wlogger.log(tid, "Writing redis stunnel configurations", "info",
server_id=server.id)
rc.put_file('/etc/stunnel/stunnel.conf', stunnel_redis_conf)
wlogger.log(tid, "Uploading server certificate", "info",
server_id=server.id)
rc.put_file('/etc/stunnel/redis-server.crt', stunnel_cert)
si.run_sysctl('enable')
si.run_sysctl('restart')
if server.primary_server:
server_string = 'localhost:{0}'.format(primary_cache_server.stunnel_port)
__update_LDAP_cache_method(tid, server, server_string, redis_password=primary_cache_server.redis_password)
wlogger.log(tid, "Restarting Gluu Server", "info",
server_id=server.id)
if server.os in ('RHEL 7', 'CentOS 7', 'Ubuntu 18'):
si.run_command('/sbin/gluu-serverd-{} restart'.format(app_conf.gluu_version))
else:
si.run_command('systemctl restart gluu-server-{}'.format(app_conf.gluu_version))
wlogger.log(tid, "3", "set_step")
def setup_sharded(tid):
servers = Server.query.filter(Server.redis.is_(True)).filter(
Server.stunnel.is_(True)).all()
appconf = AppConfiguration.query.first()
# Store the redis server info in the LDAP
ports_count = len(servers) - 1
for server in servers:
wlogger.log(tid,
"Updating oxCacheConfiguration ...",
"debug",
server_id=server.id)
server_string = ",".join(
['localhost:6379'] +
["localhost:700{0}".format(i) for i in xrange(ports_count)])
try:
dbm = DBManager(
server.hostname,
1636,
server.ldap_password,
ssl=True,
ip=server.ip,
)
except Exception as e:
wlogger.log(tid,
"Couldn't connect to LDAP. Error: {0}".format(e),
"error",
server_id=server.id)
wlogger.log(tid, "Make sure your LDAP server is listening to "
"connections from outside",
"debug",
server_id=server.id)
continue
entry = dbm.get_appliance_attributes('oxCacheConfiguration')
cache_conf = json.loads(entry.oxCacheConfiguration.value)
cache_conf['cacheProviderType'] = 'REDIS'
cache_conf['redisConfiguration']['redisProviderType'] = 'SHARDED'
cache_conf['redisConfiguration']['servers'] = server_string
result = dbm.set_applicance_attribute('oxCacheConfiguration',
[json.dumps(cache_conf)])
if not result:
wlogger.log(tid,
"oxCacheConfigutaion update failed",
"fail",
server_id=server.id)
continue
# generate stunnel configuration and upload it to the server
wlogger.log(tid, "Setting up stunnel", "info", server_id=server.id)
chdir = '/'
if server.gluu_server:
chdir = "/opt/gluu-server-{0}".format(appconf.gluu_version)
rc = RemoteClient(server.hostname, ip=server.ip)
try:
rc.startup()
except:
wlogger.log(tid, "Could not connect to the server over SSH. "
"Stunnel setup failed.",
"error",
server_id=server.id)
continue
wlogger.log(tid,
"Enable stunnel start on system boot",
"debug",
server_id=server.id)
# replace the /etc/default/stunnel4 to enable start on system startup
local = os.path.join(app.root_path, 'templates', 'stunnel',
'stunnel4.default')
remote = os.path.join(chdir, 'etc/default/stunnel4')
rc.upload(local, remote)
# setup the certificate file
wlogger.log(tid,
"Generating certificate for stunnel ...",
"debug",
server_id=server.id)
prop_buffer = StringIO()
propsfile = os.path.join(chdir, "install", "community-edition-setup",
"setup.properties.last")
rc.sftpclient.getfo(propsfile, prop_buffer)
prop_buffer.seek(0)
props = dict()
prop_in = lambda line: line.split("=")[1].strip()
for line in prop_buffer:
if re.match('^countryCode', line):
props['country'] = prop_in(line)
if re.match('^state', line):
props['state'] = prop_in(line)
if re.match('^city', line):
props['city'] = prop_in(line)
if re.match('^orgName', line):
props['org'] = prop_in(line)
if re.match('^hostname', line):
props['cn'] = prop_in(line)
if re.match('^admin_email', line):
props['email'] = prop_in(line)
subject = "'/C={country}/ST={state}/L={city}/O={org}/CN={cn}" \
"/emailAddress={email}'".format(**props)
cert_path = os.path.join(chdir, "etc", "stunnel", "server.crt")
key_path = os.path.join(chdir, "etc", "stunnel", "server.key")
pem_path = os.path.join(chdir, "etc", "stunnel", "cert.pem")
cmd = [
"/usr/bin/openssl", "req", "-subj", subject, "-new", "-newkey",
"rsa:2048", "-sha256", "-days", "365", "-nodes", "-x509",
"-keyout", key_path, "-out", cert_path
]
cin, cout, cerr = rc.run(" ".join(cmd))
rc.run("cat {cert} {key} > {pem}".format(cert=cert_path,
key=key_path,
pem=pem_path))
# verify certificate
cin, cout, cerr = rc.run("/usr/bin/openssl verify " + pem_path)
if props['cn'] in cout and props['org'] in cout:
wlogger.log(tid,
"Certificate generated successfully",
"success",
server_id=server.id)
else:
wlogger.log(tid, "Certificate generation failed. Add a SSL "
"certificate at /etc/stunnel/cert.pem",
"error",
server_id=server.id)
# Generate stunnel config
wlogger.log(tid,
"Setup stunnel listening and forwarding",
"debug",
server_id=server.id)
sconf = [
"cert = /etc/stunnel/cert.pem", "pid = /var/run/stunnel.pid",
"[redis-server]", "client = no",
"accept = {0}:7777".format(server.ip), "connect = 127.0.0.1:6379"
]
listen_port = 7000
for s in servers:
if s.id != server.id:
sconf.append("[client{0}]".format(s.id))
sconf.append("client = yes")
sconf.append("accept = 127.0.0.1:{0}".format(listen_port))
sconf.append("connect = {0}:7777".format(s.ip))
listen_port += 1
rc.put_file(os.path.join(chdir, "etc/stunnel/redis-gluu.conf"),
"\n".join(sconf))
return True
def install_monitoring(self):
"""Celery task that installs monitoring components to remote server.
:param self: the celery task
:return: wether monitoring were installed successfully
"""
tid = self.request.id
installed = 0
servers = Server.query.all()
app_config = AppConfiguration.query.first()
for server in servers:
# 1. Make SSH Connection to the remote server
wlogger.log(tid,
"Making SSH connection to the server {0}".format(
server.hostname),
"info",
server_id=server.id)
c = RemoteClient(server.hostname, ip=server.ip)
try:
c.startup()
except Exception as e:
wlogger.log(tid,
"Cannot establish SSH connection {0}".format(e),
"warning",
server_id=server.id)
wlogger.log(tid,
"Ending server setup process.",
"error",
server_id=server.id)
return False
# 2. create monitoring directory
result = c.run('mkdir -p /var/monitoring/scripts')
ctext = "\n".join(result)
if ctext.strip():
wlogger.log(tid, ctext, "debug", server_id=server.id)
wlogger.log(tid, "Directory /var/monitoring/scripts directory "
"was created",
"success",
server_id=server.id)
# 3. Upload scripts
scripts = ('cron_data_sqtile.py', 'get_data.py',
'sqlite_monitoring_tables.py')
for scr in scripts:
local_file = os.path.join(app.root_path, 'monitoring_scripts', scr)
remote_file = '/var/monitoring/scripts/' + scr
result = c.upload(local_file, remote_file)
if result.startswith("Upload successful"):
wlogger.log(tid,
"File {} was uploaded".format(scr),
"success",
server_id=server.id)
else:
wlogger.log(tid,
"File {} could not "
"be uploaded: {}".format(scr, result),
"error",
server_id=server.id)
return False
# 4. Upload gluu version, no need to determine gluu version each time
result = c.put_file('/var/monitoring/scripts/gluu_version.txt',
app_config.gluu_version)
# 5. Upload crontab entry to collect data in every 5 minutes
crontab_entry = ('*/5 * * * * root python '
'/var/monitoring/scripts/cron_data_sqtile.py\n')
result = c.put_file('/etc/cron.d/monitoring', crontab_entry)
if not result[0]:
wlogger.log(tid,
"An errorr occurred while uploading crontab entry"
": {}".format(result[1]),
"error",
server_id=server.id)
else:
wlogger.log(tid,
"Crontab entry was uploaded",
"success",
server_id=server.id)
# 6. Installing packages.
# 6a. First determine commands for each OS type
if ('CentOS' in server.os) or ('RHEL' in server.os):
package_cmd = [
'yum install -y epel-release', 'yum repolist',
'yum install -y gcc', 'yum install -y python-devel',
'yum install -y python-pip', 'service crond restart'
]
else:
package_cmd = [
'DEBIAN_FRONTEND=noninteractive apt-get install -y gcc',
'DEBIAN_FRONTEND=noninteractive apt-get install -y python-dev',
'DEBIAN_FRONTEND=noninteractive apt-get install -y python-pip',
'service cron restart',
]
# 6b. These commands are common for all OS types
package_cmd += [
'pip install ldap3', 'pip install psutil', 'pip install pyDes',
'python /var/monitoring/scripts/'
'sqlite_monitoring_tables.py'
]
# 6c. Executing commands
wlogger.log(tid,
"Installing Packages and Running Commands",
"info",
server_id=server.id)
for cmd in package_cmd:
result = c.run(cmd)
wlogger.log(tid, "\n".join(result), "debug", server_id=server.id)
err = False
if result[2].strip():
print "Writing error", cmd
if not ("pip install --upgrade pip" in result[2]
or 'Redirecting to /bin/systemctl' in result[2]):
wlogger.log(tid,
"An error occurrued while executing "
"{}: {}".format(cmd, result[2]),
"error",
server_id=server.id)
err = True
if not err:
wlogger.log(tid,
"Command was run successfully: {}".format(cmd),
"success",
server_id=server.id)
server.monitoring = True
db.session.commit()
return True