예제 #1
0
class RemoteClientTestCase(unittest.TestCase):
    def setUp(self):
        with patch("clustermgr.core.remote.SSHClient") as mock_client:
            self.sshclient = mock_client.return_value
            # self.sshclient.open_sftp.return_value = MagicMock(name="sftp")
            self.rc = RemoteClient('server')
            self.rc.startup()

    @patch('clustermgr.core.remote.SSHClient')
    def test_starup_falls_back_to_ip(self, mock_client):
        instance = mock_client.return_value
        instance.connect = MagicMock(side_effect=[SSHException, None])
        RemoteClient('server', ip='0.0.0.0').startup()
        instance.connect.assert_called_with('0.0.0.0',
                                            port=22,
                                            username='******')

    def test_download_calls_sftpclient_get(self):
        rv = self.rc.download('remote', 'local')
        self.rc.sftpclient.get.assert_called_with('remote', 'local')
        assert "successful" in rv

    def test_download_returns_errors_when_get_throws_errors(self):
        self.rc.sftpclient.get.side_effect = OSError
        rv = self.rc.download('remote', 'local_1')
        self.assertIn('Error: Local', rv)

        self.rc.sftpclient.get.side_effect = IOError
        rv = self.rc.download('remote', 'local_2')
        self.assertIn('Error: Remote', rv)

    def test_upload_calls_sftpclient_put(self):
        rv = self.rc.upload('local', 'remote')
        self.rc.sftpclient.put.assert_called_with('local', 'remote')
        assert "successful" in rv

    def test_upload_returns_errors_when_put_throws_errors(self):
        self.rc.sftpclient.put.side_effect = IOError
        rv = self.rc.upload('local', 'remote')
        self.assertIn('Error: Remote', rv)

        self.rc.sftpclient.put.side_effect = OSError
        rv = self.rc.upload('local_2', 'remote_2')
        self.assertIn('Error: Local', rv)

    def test_upload_and_download_throws_error_if_sftpclient_is_none(self):
        self.rc.sftpclient = None
        with self.assertRaises(ClientNotSetupException):
            self.rc.upload('s', 't')
        with self.assertRaises(ClientNotSetupException):
            self.rc.download('s', 't')

    def test_exists_and_run_throws_error_if_client_is_none(self):
        self.rc.client = None
        with self.assertRaises(ClientNotSetupException):
            self.rc.exists('s')
        with self.assertRaises(ClientNotSetupException):
            self.rc.run('s')
예제 #2
0
def setup_server(self, server_id, conffile):
    """This Task sets up a standalone server with only OpenLDAP installed as
    per the request.

    As the task proceeds the various status are logged to the WebLogger under
    the uniqueID of the task. This lets the web interface to poll for the
    near-realtime updates.

    Args:
        server_id (int): the primary key of the LDAPServer object
        conffile (string): complete path of the slapd.conf generated via webui
    """
    server = LDAPServer.query.get(server_id)
    tid = self.request.id

    wlogger.log(tid, "Connecting to the server %s" % server.hostname)
    c = RemoteClient(server.hostname)
    try:
        c.startup()
    except Exception as e:
        wlogger.log(tid, "Cannot establish SSH connection {0}".format(e),
                    "error")

    wlogger.log(tid, "Retrying with the IP address")
    c = RemoteClient(server.ip)
    try:
        c.startup()
    except Exception as e:
        wlogger.log(tid, "Cannot establish SSH connection {0}".format(e),
                    "error")
        wlogger.log(tid, "Ending server setup process.", "error")
        return False

    wlogger.log(tid, 'Starting premilinary checks')
    # 1. Check OpenLDAP is installed
    if c.exists('/opt/symas/bin/slaptest'):
        wlogger.log(tid, 'Checking if OpenLDAP is installed', 'success')
    else:
        wlogger.log(tid, 'Cheking if OpenLDAP is installed', 'fail')
        wlogger.log(
            tid, 'Kindly install OpenLDAP on the server and refresh'
            ' this page to try setup again.')
        return

    # 2. symas-openldap.conf file exists
    if c.exists('/opt/symas/etc/openldap/symas-openldap.conf'):
        wlogger.log(tid, 'Checking symas-openldap.conf exists', 'success')
    else:
        wlogger.log(tid, 'Checking if symas-openldap.conf exists', 'fail')
        wlogger.log(
            tid, 'Configure OpenLDAP with /opt/gluu/etc/openldap'
            '/symas-openldap.conf', 'warning')
        return

    # 3. Certificates
    if server.tls_cacert:
        if c.exists(server.tls_cacert):
            wlogger.log(tid, 'Checking TLS CA Certificate', 'success')
        else:
            wlogger.log(tid, 'Checking TLS CA Certificate', 'fail')
    if server.tls_servercert:
        if c.exists(server.tls_servercert):
            wlogger.log(tid, 'Checking TLS Server Certificate', 'success')
        else:
            wlogger.log(tid, 'Checking TLS Server Certificate', 'fail')
    if server.tls_serverkey:
        if c.exists(server.tls_serverkey):
            wlogger.log(tid, 'Checking TLS Server Key', 'success')
        else:
            wlogger.log(tid, 'Checking TLS Server Key', 'fail')

    # 4. Data directories
    wlogger.log(tid, "Checking for data and schema folders for LDAP")
    conf = open(conffile, 'r')
    for line in conf:
        if re.match('^directory', line):
            folder = line.split()[1]
            if not c.exists(folder):
                run_command(tid, c, 'mkdir -p ' + folder)
            else:
                wlogger.log(tid, folder, 'success')

    # 5. Copy Gluu Schema files
    wlogger.log(tid, "Copying Schema files to server")
    if not c.exists('/opt/gluu/schema/openldap'):
        run_command(tid, c, 'mkdir -p /opt/gluu/schema/openldap')
    gluu_schemas = os.listdir(os.path.join(app.static_folder, 'schema'))
    for schema in gluu_schemas:
        upload_file(tid, c, os.path.join(app.static_folder, 'schema', schema),
                    "/opt/gluu/schema/openldap/" + schema)
    # 6. Copy User's custom schema files
    schemas = os.listdir(app.config['SCHEMA_DIR'])
    for schema in schemas:
        upload_file(tid, c, os.path.join(app.config['SCHEMA_DIR'], schema),
                    "/opt/gluu/schema/openldap/" + schema)

    # 7. Setup slapd.conf
    wlogger.log(tid, "Copying slapd.conf file to remote server")
    upload_file(tid, c, conffile, '/opt/symas/etc/openldap/slapd.conf')

    wlogger.log(tid, "Restarting LDAP server to validate slapd.conf")
    # IMPORTANT:
    # Restart allows the server to create missing mdb files for accesslog so
    # slapd.conf -> slapd.d conversion runs without error
    run_command(tid, c, 'service solserver restart')

    # 8. Generate OLC slapd.d
    wlogger.log(tid, "Migrating from slapd.conf to slapd.d OnlineConfig (OLC)")
    run_command(tid, c, 'service solserver stop')
    run_command(tid, c, 'rm -rf /opt/symas/etc/openldap/slapd.d')
    run_command(tid, c, 'mkdir -p /opt/symas/etc/openldap/slapd.d')
    run_command(
        tid, c, '/opt/symas/bin/slaptest -f /opt/symas/etc/openldap/slapd.conf'
        ' -F /opt/symas/etc/openldap/slapd.d')

    # 9. Restart the solserver with the new configuration
    wlogger.log(
        tid, "Starting LDAP server with OLC configuraion. Any future"
        "changes to slapd.conf will have NO effect on the LDAP server")
    log = run_command(tid, c, 'service solserver start')
    if 'failed' in log:
        wlogger.log(tid, "OpenLDAP server failed to start.", "error")
        wlogger.log(tid, "Debugging slapd...", "info")
        run_command(tid, "service solserver start -d 1")

    # Everything is done. Set the flag to based on the messages
    msgs = wlogger.get_messages(tid)
    setup_success = True
    for msg in msgs:
        setup_success = setup_success and msg['level'] != 'error'
    server.setup = setup_success
    db.session.commit()
예제 #3
0
def configure_gluu_server(self, server_id, conffile):
    server = LDAPServer.query.get(server_id)
    tid = self.request.id
    chdir = '/opt/gluu-server-' + server.gluu_version

    wlogger.log(tid, "Connecting to the server %s" % server.hostname)
    c = RemoteClient(server.hostname)
    try:
        c.startup()
    except Exception as e:
        wlogger.log(tid, "Cannot establish SSH connection {0}".format(e),
                    "error")

    wlogger.log(tid, "Retrying with the IP address")
    c = RemoteClient(server.ip)
    try:
        c.startup()
    except Exception as e:
        wlogger.log(tid, "Cannot establish SSH connection {0}".format(e),
                    "error")
        wlogger.log(tid, "Ending server setup process.", "error")
        return False

    # Since it is a Gluu Server, a number of checks can be avoided
    # 1. Check if OpenLDAP is installed
    # 2. Check if symas-openldap.conf files exists
    # 3. Check for certificates - They will be at /etc/certs

    # 4. Existance of data directories - this is necassr check as we will be
    #    enabling accesslog DIT, maybe others by admin in the conf editor
    wlogger.log(tid, "Checking existing data and schema folders for LDAP")
    conf = open(conffile, 'r')
    for line in conf:
        if re.match('^directory', line):
            folder = line.split()[1]
            if not c.exists(os.path.join(chdir, folder)):
                run_command(tid, c, 'mkdir -p ' + folder, chdir)
            else:
                wlogger.log(tid, folder, 'success')

    # 5. Gluu Schema file will be present - no checks required

    # 6. Copy User's custom schema files if any
    schemas = os.listdir(app.config['SCHEMA_DIR'])
    if len(schemas):
        wlogger.log(tid, "Copying custom schema files to the server")
        for schema in schemas:
            local = os.path.join(app.config['SCHEMA_DIR'], schema)
            remote = chdir + "/opt/gluu/schema/openldap/" + schema
            upload_file(tid, c, local, remote)

    # 7. Copy the slapd.conf
    wlogger.log(tid, "Copying slapd.conf file to the server")
    upload_file(tid, c, conffile,
                chdir + "/opt/symas/etc/openaldap/slapd.conf")

    wlogger.log(tid, "Restarting LDAP server to validate slapd.conf")
    # IMPORTANT:
    # Restart allows the server to create the mdb files for accesslog so
    # slaptest doesn't throw errors during OLC generation
    run_command(tid, c, 'service solserver restart', chdir)

    # 8. Download openldap.crt to be used in other servers for ldaps
    wlogger.log(tid, "Downloading SSL Certificate to be used in other servers")
    remote = chdir + '/etc/certs/openldap.crt'
    local = os.path.join(app.config["CERTS_DIR"],
                         "{0}.crt".format(server.hostname))
    download_file(tid, c, remote, local)

    # 9. Generate OLC slapd.d
    wlogger.log(tid, "Convert slapd.conf to slapd.d OLC")
    run_command(tid, c, 'service solserver stop', chdir)
    run_command(tid, c, "rm -rf /opt/symas/etc/openldap/slapd.d", chdir)
    run_command(tid, c, "mkdir /opt/symas/etc/openldap/slapd.d", chdir)
    run_command(
        tid, c, "/opt/symas/bin/slaptest -f /opt/symas/etc/openldap/"
        "slapd.conf -F /opt/symas/etc/openldap/slapd.d", chdir)

    # 10. Reset ownerships
    run_command(tid, c, "chown -R ldap:ldap /opt/gluu/data", chdir)
    run_command(tid, c, "chown -R ldap:ldap /opt/gluu/schema/openldap", chdir)
    run_command(tid, c, "chown -R ldap:ldap /opt/symas/etc/openldap/slapd.d",
                chdir)

    # 11. Restart the solserver with the new OLC configuration
    wlogger.log(tid, "Restarting LDAP server with OLC configuration")
    log = run_command(tid, c, "service solserver start", chdir)
    if 'failed' in log:
        wlogger.log(
            tid, "There seems to be some issue in starting the server."
            "Running LDAP server in debug mode for troubleshooting")
        run_command(tid, c, "service solserver start -d 1", chdir)

    # Everything is done. Set the flag to based on the messages
    msgs = wlogger.get_messages(tid)
    setup_success = True
    for msg in msgs:
        setup_success = setup_success and msg['level'] != 'error'
    server.setup = setup_success
    db.session.commit()
예제 #4
0
파일: cache.py 프로젝트: pks-os/cluster-mgr
def install_cache_components(self, method, server_id_list):
    """Celery task that installs the redis, stunnel and twemproxy applications
    in the required servers.

    Redis and stunnel are installed in all the servers in the cluster.
    Twemproxy is installed in the load-balancer/proxy server

    :param self: the celery task
    :param method: either STANDALONE, SHARDED

    :return: the number of servers where both stunnel and redis were installed
        successfully
    """

    tid = self.request.id
    installed = 0

    servers = []

    for server_id in server_id_list:

        server = Server.query.get(server_id)

        ri = RedisInstaller(server, tid)
        ri.rc.startup()
        if ri.rc.exists('/usr/bin/redis-server') or ri.rc.exists(
                '/bin/redis-server'):
            server.redis = True
            redis_installed = 1
            wlogger.log(tid,
                        "Redis was already installed on server {0}".format(
                            server.hostname),
                        "info",
                        server_id=server.id)
        else:
            wlogger.log(tid,
                        "Installing Redis in server {0}".format(
                            server.hostname),
                        "info",
                        server_id=server.id)
            redis_installed = ri.install()
            if redis_installed:
                server.redis = True
                wlogger.log(tid,
                            "Redis install successful",
                            "success",
                            server_id=server.id)
            else:
                server.redis = False
                wlogger.log(tid,
                            "Redis install failed",
                            "fail",
                            server_id=server.id)

        si = StunnelInstaller(server, tid)
        si.rc.startup()
        if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
            wlogger.log(tid,
                        "Stunnel was allready installed",
                        "info",
                        server_id=server.id)
            server.stunnel = True
            stunnel_installed = 1
        else:
            wlogger.log(tid, "Installing Stunnel", "info", server_id=server.id)

            stunnel_installed = si.install()
            if stunnel_installed:
                server.stunnel = True
                wlogger.log(tid,
                            "Stunnel install successful",
                            "success",
                            server_id=server.id)
            else:
                server.stunnel = False
                wlogger.log(tid,
                            "Stunnel install failed",
                            "fail",
                            server_id=server.id)
            # Save the redis and stunnel install situation to the db

        if redis_installed and stunnel_installed:
            installed += 1

        db.session.commit()
    if method != 'STANDALONE':
        # No need to install twemproxy for "SHARDED" configuration
        return True

    # Install twemproxy in the Nginx load balancing proxy server
    app_conf = AppConfiguration.query.first()

    mock_server = Server()

    if app_conf.external_load_balancer:
        mock_server.hostname = app_conf.cache_host
        mock_server.ip = app_conf.cache_ip
    else:
        mock_server.hostname = app_conf.nginx_host
        mock_server.ip = app_conf.nginx_ip

    rc = RemoteClient(mock_server.hostname)

    try:
        rc.startup()
    except Exception as e:
        wlogger.log(tid, "Could not connect to {0}".format(e), "error")
        return False

    server_os = get_os_type(rc)

    si = StunnelInstaller(mock_server, tid)
    si.rc.startup()
    stunnel_installed = 0
    if si.rc.exists('/usr/bin/stunnel') or si.rc.exists('/bin/stunnel'):
        wlogger.log(tid, "Stunnel was already installed on cache server")
        stunnel_installed = 1
    else:
        wlogger.log(tid, "Installing Stunnel in cache server")
        stunnel_installed = si.install()
        if stunnel_installed:
            wlogger.log(tid, "Stunnel install successful", "success")
        else:
            wlogger.log(tid, "Stunnel install failed", "fail")

    print rc.exists('/usr/sbin/nutcracker')

    if not rc.exists('/usr/sbin/nutcracker'):

        wlogger.log(tid, "Installing Twemproxy")
        # 1. Setup the development tools for installation
        if server_os == "Ubuntu 14":
            run_and_log(rc, "apt-get update", tid)
            run_and_log(
                rc,
                'wget http://ftp.debian.org/debian/pool/main/n/nutcracker/nutcracker_0.4.0+dfsg-1_amd64.deb -O /tmp/nutcracker_0.4.0+dfsg-1_amd64.deb',
                tid)
            run_and_log(rc, "dpkg -i /tmp/nutcracker_0.4.0+dfsg-1_amd64.deb",
                        tid)
        elif server_os == "Ubuntu 16":
            run_and_log(rc, "apt-get update", tid)
            run_and_log(
                rc,
                "DEBIAN_FRONTEND=noninteractive apt-get install -y nutcracker",
                tid)
        elif server_os in ["CentOS 7", "RHEL 7"]:
            run_and_log(
                rc,
                'yum install -y https://raw.githubusercontent.com/mbaser/gluu/master/nutcracker-0.4.1-1.gluu.centos7.x86_64.rpm',
                tid)
            run_and_log(rc, 'chkconfig nutcracker on', tid)
        elif server_os in ['CentOS 6', 'RHEL 6']:
            run_and_log(
                rc,
                'yum install -y https://raw.githubusercontent.com/mbaser/gluu/master/nutcracker-0.4.1-1.gluu.centos6.x86_64.rpm',
                tid)
            run_and_log(rc, 'chkconfig nutcracker on', tid)

        # 5. Create the default configuration file referenced in the init scripts
        #run_and_log(rc, "mkdir -p /etc/nutcracker", tid)
        run_and_log(rc, "touch /etc/nutcracker/nutcracker.yml", tid)
    else:
        wlogger.log(tid, "Twemproxy was already installed on cache server")
    rc.close()
    return installed