예제 #1
0
    def recover_nginx_configuration(self):
        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)

        nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        backup_file_location = "{}/config/osiris-instances/{}.conf.backup".format(self.config.nginx.root, self.instance.name)
        backup_content = nginx_remote.get_file(backup_file_location)
        nginx_remote.put_file(nginx_file_location, backup_content)
        return success_log("Succesfully recovered backup from".format(backup_file_location))
예제 #2
0
    def create_max_nginx_entry(self):
        nginx_params = {
            'instance_name': self.instance.name,
            'server': self.config.server,
            'server_dns': self.config.server_dns,
            'bigmax_port': BIGMAX_BASE_PORT,
            'max_port': int(self.instance.index) + MAX_BASE_PORT,
            'nginx_root_folder': self.config.nginx.root,
            'max_root_folder': self.buildout.folder
        }
        nginxentry = MAX_NGINX_ENTRY.format(**nginx_params)

        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)

        nginx_file_location = "{}/config/max-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        nginx_remote.put_file(nginx_file_location, nginxentry)
        return success_log("Succesfully created {}".format(nginx_file_location))
예제 #3
0
    def create_oauth_nginx_entry(self):
        global_allowed_ips = self.config.oauth.allowed_ips
        instance_allowed_ips = self.get_instance_allowed_ips()
        allowed_ips = instance_allowed_ips + global_allowed_ips
        nginx_params = {
            'instance_name': self.instance.name,
            'server': self.config.oauth.server,
            'server_dns': self.config.oauth.server_dns,
            'osiris_port': int(self.instance.index) + OSIRIS_BASE_PORT,
            'buildout_folder': self.config.nginx.root,
            'allowed_ips': '\n      '.join(['allow {};'.format(ip) for ip in allowed_ips])
        }
        nginxentry = OSIRIS_NGINX_ENTRY.format(**nginx_params)

        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
        nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        nginx_remote.put_file(nginx_file_location, nginxentry)
        return success_log("Succesfully created {}".format(nginx_file_location))
예제 #4
0
 def __init__(self, user, server, filename, target_lines=None, filters=[]):
     self.logfile = filename
     threading.Thread.__init__(self)
     self.remote = RemoteConnection(user, server)
     self.count = 0
     self.target_lines = target_lines
     self.show_progress_bar = self.target_lines is not None
     self.process = None
     self.finished = False
     self.filters = filters
예제 #5
0
    def __init__(self, config, *args, **kwargs):
        self.config = config

        self._instances = {}
        self.process_prefix = 'osiris_'
        self.remote = RemoteConnection(self.config.ssh_user, self.config.server)
        self.buildout = RemoteBuildoutHelper(self.remote, self.config.python_interpreter, self)

        if not self.remote.file_exists(self.config.instances_root):
            self.remote.mkdir(self.config.instances_root)
예제 #6
0
class LogEcho(threading.Thread):
    def __init__(self, user, server, filename, target_lines=None, filters=[]):
        self.logfile = filename
        threading.Thread.__init__(self)
        self.remote = RemoteConnection(user, server)
        self.count = 0
        self.target_lines = target_lines
        self.show_progress_bar = self.target_lines is not None
        self.process = None
        self.finished = False
        self.filters = filters

    def stop(self):
        print
        try:
            self.process.kill()
        except:
            print padded_log('An error occurred when stopping logger')

    def run(self):
        def tail_log(line, stdin, process):
            # At first line reception, store echo logger process
            # and instantiate progress bar if needed
            if self.process is None:
                self.process = process
                if self.show_progress_bar:
                    self.pacman = Pacman(text='    Progress')

            # Determine if we want to count/print the line based on filters setting
            matched_filter = re.search(r'({})'.format('|'.join(self.filters)), line)
            this_line_is_good = matched_filter or self.filters == []

            # If line is good and we're not reached the 100%
            if this_line_is_good and not self.finished:
                # Update progress bar if in progressbar mode
                if self.show_progress_bar:
                    self.count += 1
                    percent = (100 * self.count) / self.target_lines
                    percent = percent if percent <= 100 else 100
                    self.finished = percent == 100
                    self.pacman.progress(percent)
                # or print line
                else:
                    print '    ' + line.rstrip()

        try:
            code, stdout = self.remote.execute(
                'tail -F -n0 {}'.format(self.logfile),
                _out=tail_log
            )
        except:
            pass
예제 #7
0
class MaxServer(SupervisorHelpers, NginxHelpers, CommonSteps, PyramidServer):

    def __init__(self, config, *args, **kwargs):
        self.config = config

        self.process_prefix = 'max_'
        self._client = None
        self._instances = {}
        self.instance = None
        self.remote = RemoteConnection(self.config.ssh_user, self.config.server)
        self.buildout = RemoteBuildoutHelper(self.remote, self.config.python_interpreter, self)

        if not self.remote.file_exists(self.config.instances_root):
            self.remote.mkdir(self.config.instances_root)

    def get_client(self, instance_name, username='', password=''):
        instance_info = self.get_instance(instance_name)
        client = MaxClient(instance_info['server']['dns'])
        if username and password:
            client.login(username=username, password=password)
        return client

    def get_running_version(self, instance_name):
        instance_info = self.get_instance(instance_name)
        return requests.get('{}/info'.format(instance_info['server']['dns'])).json().get('version', '???')

    def get_expected_version(self, instance_name):
        versions = self.remote.get_file('{}/versions.cfg'.format(self.buildout.folder))
        return re.search(r'\smax\s=\s(.*?)\s', versions, re.MULTILINE).groups()[0]

    def get_instance(self, instance_name):
        if instance_name not in self._instances:
            max_ini = self.buildout.config_files.get(instance_name, {}).get('max.ini', '')
            if not max_ini:
                return {}

            maxconfig = parse_ini_from(max_ini)
            port_index = int(maxconfig['server:main']['port']) - MAX_BASE_PORT

            instance = OrderedDict()
            instance['name'] = instance_name
            instance['port_index'] = port_index
            instance['mongo_database'] = maxconfig['app:main']['mongodb.db_name']
            instance['server'] = {
                'direct': 'http://{}:{}'.format(self.config.server, maxconfig['server:main']['port']),
                'dns': maxconfig['app:main']['max.server']
            }
            instance['oauth'] = maxconfig['app:main']['max.oauth_server']
            self._instances[instance_name] = instance
        return self._instances[instance_name]

    def test_activity(self, instance_name, ldap_branch, restricted_user_password):

        progress_log('Testing UTalk activity notifications')

        # Get a maxclient for this instance
        padded_log("Getting instance information")
        instance_info = self.get_instance(instance_name)
        restricted_password = restricted_user_password
        client = self.get_client(instance_name, username='******', password=restricted_password)

        padded_log("Setting up test clients")
        test_users = [
            ('ulearn.testuser1', 'UTestuser1'),
            ('ulearn.testuser2', 'UTestuser2')
        ]

        utalk_clients = []
        max_clients = []

        # Syncronization primitives
        wait_for_others = AsyncResult()
        counter = ReadyCounter(wait_for_others)

        for user, password in test_users:
            max_clients.append(self.get_client(
                instance_name,
                username=user,
                password=password)
            )

            # Create websocket clients
            utalk_clients.append(getUtalkClient(
                instance_info['server']['dns'],
                instance_name,
                user,
                password,
                quiet=False
            ))
            counter.add()

        # Create users
        padded_log("Creating users and conversations")
        client.people['ulearn.testuser1'].post()
        client.people['ulearn.testuser2'].post()

        # Admin creates context with notifications enabled and subscribes users to it
        context = client.contexts.post(url='http://testcontext', displayName='Test Context', notifications=True)
        client.people['ulearn.testuser1'].subscriptions.post(object_url='http://testcontext')
        client.people['ulearn.testuser2'].subscriptions.post(object_url='http://testcontext')

        def post_activity():
            max_clients[0].people['ulearn.testuser1'].activities.post(
                object_content='Hola',
                contexts=[{"url": "http://testcontext", "objectType": "context"}]
            )

        # Prepare test messages for clients
        # First argument are messages to send (conversation, message)
        # Second argument are messages to expect (conversation, sender, message)
        # Third argument is a syncronization event to wait for all clients to be listening
        # Fourth argument is a method to trigger when client ready

        arguments1 = [
            [

            ],
            [
                ('test', 'test')
            ],
            counter,
            post_activity
        ]

        arguments2 = [
            [

            ],
            [
                ('test', 'test')
            ],
            counter,
            None
        ]

        padded_log("Starting websockets and waiting for messages . . .")

        greenlets = [
            gevent.spawn(utalk_clients[0].test, *arguments1),
            gevent.spawn(utalk_clients[1].test, *arguments2)
        ]

        gevent.joinall(greenlets, timeout=20, raise_error=True)

        success = None not in [g.value for g in greenlets]

        if success:
            padded_success('Websocket test passed')
        else:
            padded_error('Websocket test failed, Timed out')

    # Steps used in commands. Some of them defined in gummanager.libs.mixins

    def configure_instance(self):

        customizations = {
            'mongodb-config': {
                'replica_set': self.config.mongodb.replica_set,
                'cluster_hosts': self.config.mongodb.cluster
            },
            'max-config': {
                'name': self.instance.name,
            },
            'ports': {
                'port_index': '{:0>2}'.format(self.instance.index),
            },
            'rabbitmq-config': {
                'username': self.config.rabbitmq.username,
                'password': self.config.rabbitmq.password
            },
            'hosts': {
                'max': self.config.server_dns,
                'oauth': self.config.oauth.server_dns,
                'rabbitmq': self.config.rabbitmq.server

            }
        }

        self.buildout.configure_file('customizeme.cfg', customizations),
        return success_log('Succesfully configured {}/customizeme.cfg'.format(self.buildout.folder))

    def set_mongodb_indexes(self):
        new_instance_folder = '{}/{}'.format(
            self.config.instances_root,
            self.instance.name
        )
        code, stdout = self.remote.execute('cd {0} && ./bin/max.mongoindexes'.format(new_instance_folder))
        added = 'Creating' in stdout or 'already exists' in stdout

        if added:
            return success_log("Succesfully added indexes")
        else:
            return error_log("Error on adding indexes")

    def configure_max_security_settings(self):
        new_instance_folder = '{}/{}'.format(
            self.config.instances_root,
            self.instance.name
        )
        self.buildout.folder = new_instance_folder

        self.remote.execute('cd {} && ./bin/max.security reset'.format(new_instance_folder))
        code, stdout = self.remote.execute('cd {} && ./bin/max.security add {}'.format(new_instance_folder, self.config.authorized_user))
        added = 'restart max process' in stdout
        # Force read the new configuration files

        if added:
            return success_log("Succesfully configured security settings")
        else:
            return error_log("Error configuring security settings")

    def create_max_nginx_entry(self):
        nginx_params = {
            'instance_name': self.instance.name,
            'server': self.config.server,
            'server_dns': self.config.server_dns,
            'bigmax_port': BIGMAX_BASE_PORT,
            'max_port': int(self.instance.index) + MAX_BASE_PORT,
            'nginx_root_folder': self.config.nginx.root,
            'max_root_folder': self.buildout.folder
        }
        nginxentry = MAX_NGINX_ENTRY.format(**nginx_params)

        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)

        nginx_file_location = "{}/config/max-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        nginx_remote.put_file(nginx_file_location, nginxentry)
        return success_log("Succesfully created {}".format(nginx_file_location))

    def commit_local_changes(self):
        self.buildout.commit_to_local_branch(
            self.config.local_git_branch,
            files=[
                'customizeme.cfg',
                'mongoauth.cfg'
            ])
        return success_log("Succesfully commited local changes")

    def add_instance_to_bigmax(self):
        instances_file = ''
        if self.remote.file_exists(self.config.bigmax_instances_list):
            instances_file = self.remote.get_file(self.config.bigmax_instances_list, do_raise=True)
        if '[{}]'.format(self.instance.name) not in instances_file:
            linebreak = '\n' if instances_file else ''
            instances_file += linebreak + BIGMAX_INSTANCE_ENTRY.format(**{
                "server_dns": self.config.server_dns,
                "instance_name": self.instance.name,
            })
            self.remote.put_file(self.config.bigmax_instances_list, instances_file, do_raise=True)
        else:
            return success_log("Instance {} already in bigmax instance list".format(self.instance.name))

        return success_log("Succesfully added {} to bigmax instance list".format(self.instance.name))

    def update_buildout(self):
        result = self.buildout.upgrade(self.config.maxserver_buildout_branch, self.config.local_git_branch)
        return success(result, "Succesfully commited local changes")

    def reload_instance(self):
        self.restart(self.instance.name)
        sleep(1)
        status = self.get_status(self.instance.name)
        if status['status'] == 'running':
            return success_log("Succesfully restarted max {}".format(self.instance.name))
        else:
            return error_log('Max instance {} is not running'.format(self.instance.name))

    def check_version(self):
        running_version = self.get_running_version(self.instance.name)
        expected_version = self.get_expected_version(self.instance.name)

        if running_version == expected_version:
            return success_log("Max {} is running on {}".format(self.instance.name, running_version))
        else:
            return success_log("Max {} is running on {}, but {} was expected".format(self.instance.name, running_version, expected_version))

    # Commands

    @command
    def new_instance(self, instance_name, port_index, oauth_instance=None, logecho=None, rabbitmq_url=None):

        self.buildout.cfgfile = self.config.max.cfg_file
        self.buildout.logecho = logecho
        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        self.set_instance(
            name=instance_name,
            index=port_index,
            oauth=oauth_instance if oauth_instance is not None else instance_name,
        )

        yield step_log('Cloning buildout')
        yield self.clone_buildout()

        yield step_log('Bootstraping buildout')
        yield self.bootstrap_buildout()

        yield step_log('Configuring customizeme.cfg')
        yield self.configure_instance()

        yield step_log('Configuring mongoauth.cfg')
        yield self.configure_mongoauth()

        yield step_log('Executing buildout')
        yield self.execute_buildout()

        yield step_log('Adding indexes to mongodb')
        yield self.set_mongodb_indexes()

        yield step_log('Configuring default permissions settings')
        yield self.configure_max_security_settings()

        yield step_log('Creating nginx entry for max')
        yield self.create_max_nginx_entry()

        yield step_log('Commiting to local branch')
        yield self.commit_local_changes()

        yield step_log('Changing permissions')
        yield self.set_filesystem_permissions()

        yield step_log('Adding instance to supervisor config')
        yield self.configure_supervisor()

    @command
    def upgrade(self, instance_name, logecho=None):
        self.buildout.cfgfile = self.config.max.cfg_file
        self.buildout.logecho = logecho
        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        self.set_instance(
            name=instance_name,
        )

        yield step_log('Updating buildout')
        yield self.update_buildout()

        yield step_log('Executing buildout')
        yield self.execute_buildout(update=True)

        yield step_log('Changing permissions')
        yield self.set_filesystem_permissions()

        # yield step_log('Reloading max')
        yield self.reload_instance()

        yield step_log('Checking running version')
        yield self.check_version()
예제 #8
0
 def __init__(self, config, *args, **kwargs):
     self.config = config
     self.remote = RemoteConnection(self.config.maxbunny.ssh_user, self.config.maxbunny.server)
예제 #9
0
class UTalkServer(TokenHelper, object):

    def __init__(self, config, *args, **kwargs):
        self.config = config
        self.remote = RemoteConnection(self.config.maxbunny.ssh_user, self.config.maxbunny.server)

    def getUtalkClient(self, maxserver, username, password, quiet=False):
        client = UTalkTestClient(
            maxserver=maxserver,
            username=username,
            password=password,
            quiet=quiet
        )
        return client

    def add_entry(self, **configuration):
        instances_file = ''
        if self.remote.file_exists(self.config.maxbunny.instances_list):
            instances_file = self.remote.get_file(self.config.maxbunny.instances_list, do_raise=True)
        if '[{name}]'.format(**configuration) not in instances_file:
            linebreak = '\n' if instances_file else ''
            instances_file += linebreak + MAXBUNNY_INSTANCE_ENTRY.format(**configuration)
            self.remote.put_file(self.config.maxbunny.instances_list, instances_file, do_raise=True)
        else:
            return success_log("Instance {name} already in maxbunny instance list".format(**configuration))

        return success_log("Succesfully added {name} to maxbunny instance list".format(**configuration))

    def add_instance(self, **configuration):
        token = self.get_token(
            configuration['oauthserver']['server']['dns'],
            configuration['restricted_user'],
            configuration['restricted_user_password']
        )

        try:
            yield step_log('Adding entry')
            yield self.add_entry(
                language=configuration['language'],
                name=configuration['name'],
                hashtag=configuration['hashtag'],
                server=configuration['maxserver']['server']['dns'],
                restricted_user=configuration['restricted_user'],
                restricted_user_token=token
            )

        except StepError as error:
            yield error_log(error.message)

    def test(self, domain, restricted_password):
        progress_log('Testing UTalk websocket communication')

        # Get a maxclient for this instance
        padded_log("Getting instance information")

        domain_info = self.getDomainInfo(domain)
        client = self.config.max.get_client(domain, username='******', password=restricted_password)

        padded_log("Setting up test clients")
        test_users = [
            ('ulearn.testuser1', 'UTestuser1'),
            ('ulearn.testuser2', 'UTestuser2')
        ]

        utalk_clients = []
        max_clients = []

        # Syncronization primitives
        wait_for_others = AsyncResult()
        counter = ReadyCounter(wait_for_others)

        for user, password in test_users:
            max_clients.append(self.config.max.get_client(
                domain,
                username=user,
                password=password)
            )

            # Create websocket clients
            utalk_clients.append(self.getUtalkClient(
                domain_info['max']['server']['dns'],
                user,
                password,
                quiet=True
            ))
            counter.add()

        # Create users
        padded_log("Creating users and conversations")
        client.people['ulearn.testuser1'].post()
        client.people['ulearn.testuser2'].post()

        # user 1 creates conversation with user 2
        conversation = max_clients[0].conversations.post(
            contexts=[{"objectType": "conversation", "participants": [test_users[0][0], test_users[1][0]]}],
            object_content='Initial message'
        )
        conversation_id = conversation['contexts'][0]['id']

        # Prepare test messages for clients
        # First argument are messages to send (conversation, message)
        # Second argument are messages to expect (conversation, sender, message)
        # Third argument is a syncronization event to wait for all clients to be listening

        arguments1 = [
            [
                (conversation_id, 'First message from 1'),
                (conversation_id, 'Second message from 1')
            ],
            [
                (conversation_id, test_users[1][0], 'First message from 2'),
                (conversation_id, test_users[1][0], 'Second message from 2')
            ],
            counter,
        ]

        arguments2 = [
            [
                (conversation_id, 'First message from 2'),
                (conversation_id, 'Second message from 2')
            ],
            [
                (conversation_id, test_users[0][0], 'First message from 1'),
                (conversation_id, test_users[0][0], 'Second message from 1')
            ],
            counter
        ]

        padded_log("Starting websockets and waiting for messages . . .")

        utalk_clients[0].setup(*arguments1)
        utalk_clients[1].setup(*arguments2)

        greenlets = [
            gevent.spawn(utalk_clients[0].connect),
            gevent.spawn(utalk_clients[1].connect)
        ]

        gevent.joinall(greenlets, timeout=60, raise_error=True)
        success = None not in [g.value for g in greenlets]

        utalk_clients[0].teardown()
        utalk_clients[1].teardown()

        # Cleanup
        max_clients[0].conversations[conversation_id].delete()

        #client.people['ulearn.testuser1'].delete()
        #client.people['ulearn.testuser2'].delete()

        if success:
            padded_success('Websocket test passed')
        else:
            padded_error('Websocket test failed, Timed out')
예제 #10
0
 def __init__(self, *args, **kwargs):
     super(ULearnServer, self).__init__(*args, **kwargs)
     self.prefes = RemoteConnection(self.config.prefe_ssh_user, self.config.prefe_server)
예제 #11
0
class ULearnServer(GenwebServer):
    _remote_config_files = {}

    def __init__(self, *args, **kwargs):
        super(ULearnServer, self).__init__(*args, **kwargs)
        self.prefes = RemoteConnection(self.config.prefe_ssh_user, self.config.prefe_server)

    def reload_nginx_configuration(self):
        progress_log("Reloading nginx configuration")
        padded_log("Testing configuration")
        code, stdout = self.prefes.execute("/etc/init.d/nginx configtest")
        if code == 0 and "done" in stdout:
            padded_success("Configuration test passed")
        else:
            padded_error("Configuration test failed")
            return None

        code, stdout = self.prefes.execute("/etc/init.d/nginx reload")
        if code == 0 and "done" in stdout:
            padded_success("Nginx reloaded succesfully")
        else:
            padded_error("Error reloading nginx")
            return None

    def setup_nginx(self, site, max_url):

        nginx_params = {"instance_name": site.plonesite, "max_server": max_url, "mountpoint_id": site.mountpoint}
        nginxentry = ULEARN_NGINX_ENTRY.format(**nginx_params)

        success = self.prefes.put_file(
            "{}/config/ulearn-instances/{}.conf".format(self.config.prefe_nginx_root, site.plonesite), nginxentry
        )

        if success:
            return success_log(
                "Succesfully created {}/config/ulearn-instances/{}.conf".format(
                    self.config.prefe_nginx_root, site.plonesite
                )
            )
        else:
            return error_log("Error when generating nginx config file for ulean")

    def get_instance(self, environment, mountpoint, plonesite):
        site = UlearnSite(environment, mountpoint, plonesite, "", "")
        settings = site.get_settings()
        return settings

    def add_user(self, instance, user):
        """
        """
        pass

    @staticmethod
    def check_users(users):
        """
            Check user list consistency, raises on validation errors
        """

        # Look for repeated users
        usernames = [a["username"] for a in users]
        duplicates = [k for k, v in Counter(usernames).items() if v > 1]
        if duplicates:
            raise Exception("Found duplicated users: {}".format(", ".join(duplicates)))

        # Look for repeated emails
        emails = [a["email"] for a in users]
        duplicates = [k for k, v in Counter(emails).items() if v > 1]
        if duplicates:
            raise Exception("Found duplicated emails: {}".format(", ".join(duplicates)))

        # Look for users withuot password
        users_without_password = [a["username"] for a in users if a["password"].strip() == ""]
        if users_without_password:
            raise Exception("Found users without password: {}".format(", ".join(users_without_password)))

    # COMMANDS

    def batch_add_users(self, instance, usersfile):
        site = UlearnSite(self.get_environment(instance["environment"]), instance["mountpoint"], instance["plonesite"])
        try:
            users = read_users_file(usersfile, required_fields=["username", "fullname", "email", "password"])
        except Exception as exc:
            error_message = "Error parsing users file {}: {{}}".format(usersfile)
            yield raising_error_log(error_message.format(exc.message))

        try:
            self.check_users(users)
        except Exception as exc:
            yield raising_error_log(exc.message)

        yield step_log("Creating {} users ".format(len(users)))
        for count, user in enumerate(users, start=1):
            if not user:
                yield error_log("Error parsing user at line #{}".format(count))
                continue
            succeeded = site.add_user(**user)
            if not succeeded.get("error", False):
                yield success_log(succeeded["message"])
            else:
                yield error_log(succeeded["message"])

    def batch_subscribe_users(self, instance, subscriptionsfile):
        site = UlearnSite(self.get_environment(instance["environment"]), instance["mountpoint"], instance["plonesite"])
        try:
            communities = read_subscriptions_file(subscriptionsfile, required_fields=["owners", "readers", "editors"])
        except Exception as exc:
            error_message = "Error parsing subscriptionsfile file {}: {{}}".format(subscriptionsfile)
            yield raising_error_log(error_message.format(exc.message))

        for community in communities:
            yield step_log("Subscribing users to {}".format(community["url"]))

            succeeded = site.subscribe_users(**community)
            if not succeeded.get("error", False):
                yield success_log(succeeded["message"])
            else:
                yield error_log(succeeded["message"])

    def new_instance(
        self,
        instance_name,
        environment,
        mountpoint,
        title,
        language,
        max_name,
        max_direct_url,
        oauth_name,
        ldap_branch,
        ldap_password,
        logecho,
    ):

        environment = self.get_environment(environment)
        site = UlearnSite(environment, mountpoint, instance_name, title, language, logecho)

        yield step_log("Creating Plone site")
        yield site.create(packages=["ulearn.core:default"])

        yield step_log("Setting up homepage")
        yield site.setup_homepage()

        yield step_log("Setting up ldap")
        yield site.setup_ldap(ldap_branch, self.config.ldap)

        yield step_log("Setting up max")
        yield site.setup_max(max_name, oauth_name, ldap_branch)

        yield step_log("Rebuilding catalog")
        yield site.rebuild_catalog()

        yield step_log("Setting up nginx entry @ {}".format(self.config.prefe_server))
        yield self.setup_nginx(site, max_direct_url)
예제 #12
0
class OauthServer(SupervisorHelpers, NginxHelpers, CommonSteps, TokenHelper, PyramidServer):

    def __init__(self, config, *args, **kwargs):
        self.config = config

        self._instances = {}
        self.process_prefix = 'osiris_'
        self.remote = RemoteConnection(self.config.ssh_user, self.config.server)
        self.buildout = RemoteBuildoutHelper(self.remote, self.config.python_interpreter, self)

        if not self.remote.file_exists(self.config.instances_root):
            self.remote.mkdir(self.config.instances_root)

    def update_buildout(self):
        result = self.buildout.upgrade(self.config.maxserver_buildout_branch, self.config.local_git_branch)
        return success(result, "Succesfully commited local changes")

    def reload_instance(self):
        self.restart(self.instance.name)
        sleep(1)
        status = self.get_status(self.instance.name)
        if status['status'] == 'running':
            return success_log("Succesfully restarted oauth {}".format(self.instance.name))
        else:
            return error_log('Oauth instance {} is not running'.format(self.instance.name))

    def get_instance(self, instance_name):
        if instance_name not in self._instances:
            osiris_ini = self.buildout.config_files[instance_name].get('osiris.ini', '')
            if not osiris_ini:
                return {}
            osiris = parse_ini_from(osiris_ini)

            ldap_ini = self.buildout.config_files[instance_name].get('ldap.ini', '')
            if not ldap_ini:
                return {}
            ldap = parse_ini_from(ldap_ini)

            port_index = int(osiris['server:main']['port']) - OSIRIS_BASE_PORT

            instance = OrderedDict()
            instance['name'] = instance_name
            instance['port_index'] = port_index
            instance['mongo_database'] = osiris['app:main']['osiris.store.db']
            instance['server'] = {
                'direct': 'http://{}:{}'.format(self.config.server, osiris['server:main']['port']),
                'dns': 'https://{}/{}'.format(self.config.server_dns, instance_name)
            }

            instance['ldap'] = {
                'server': ldap['ldap']['server'],
                'basedn': ldap['ldap']['userbasedn'],
                'branch': re.match(r"ou=(.*?),", ldap['ldap']['userbasedn']).groups()[0]
            }
            self._instances[instance_name] = instance
        return self._instances[instance_name]

    def test(self, instance_name, username, password):
        instance = self.get_instance(instance_name)
        try:
            yield step_log('Testing oauth server @ {}'.format(instance['server']['dns']))

            yield message_log('Checking server health')

            try:
                status = requests.get(instance['server']['dns'], verify=True).status_code
            except requests.exceptions.SSLError:
                yield error_log('SSL certificate verification failed')
                yield message_log('Continuing test without certificate check')

            try:
                status = requests.get(instance['server']['dns'], verify=False).status_code
            except requests.ConnectionError:
                yield raising_error_log('Connection error, check nginx is running, and dns resolves as expected.')
            except:
                yield raising_error_log('Unknown error trying to access oauth server. Check params and try again')
            else:
                if status == 500:
                    yield raising_error_log('Error on oauth server, Possible causes:\n  - ldap configuration error (bad server url?)\n  - Mongodb configuration error (bad replicaset name or hosts list?)\nCheck osiris log for more information.')
                elif status == 502:
                    yield raising_error_log('Server not respoding at {}. Check that:\n  - osiris process is running\n  - nginx upstream definition is pointing to the right host:port.'.format(instance['server']['dns']))
                elif status == 504:
                    yield raising_error_log('Gateway timeout. Probably oauth server is giving timeout trying to contact ldap server')
                elif status == 404:
                    yield raising_error_log('There\'s no oauth server at {}. Chech there\'s an nginx entry for this server.'.format(instance['server']['dns']))
                elif status != 200:
                    yield raising_error_log('Server {} responded with {} code. Check osiris logs.'.format(instance['server']['dns'], status))

            yield message_log('Retrieving token for "{}"'.format(username))
            token = self.get_token(instance['server']['dns'], username, password)
            succeeded_retrieve_token = token is not None

            if not succeeded_retrieve_token:
                yield raising_error_log('Error retreiving token. Check username/password and try again')

            yield message_log('Checking retreived token')
            succeeded_check_token = self.check_token(instance['server']['dns'], username, token)

            if not succeeded_check_token:
                yield raising_error_log('Error retreiving token')

            if succeeded_check_token and succeeded_retrieve_token:
                yield success_log('Oauth server check passed')
            else:
                yield raising_error_log('Oauth server check failed')

        except StepError as error:
            yield error_log(error.message)

    # Steps used in commands. Some of them defined in gummanager.libs.mixins

    def configure_instance(self):
        customizations = {
            'mongodb-config': {
                'replica_set': self.config.mongodb.replica_set,
                'cluster_hosts': self.config.mongodb.cluster
            },
            'osiris-config': {
                'name': self.instance.name,
            },
            'ports': {
                'port_index': '{:0>2}'.format(self.instance.index),
            },

        }

        self.buildout.configure_file('customizeme.cfg', customizations),
        return success_log('Succesfully configured {}/customizeme.cfg'.format(self.buildout.folder))

    def configure_ldap(self):
        """
            Configure the right settings for ldap based on if :
            branches option enabled or disabled
        """

        if self.config.ldap.branches.enabled:
            effective_admin_dn = 'cn={admin_cn},ou={branch},{base_dn}'.format(branch=self.instance.ldap, **self.config.ldap.branches)
            effective_admin_password = self.config.ldap.branches.admin_password
            effective_users_base_dn = 'ou={},{}'.format(self.instance.ldap, self.config.ldap.branches.base_dn)
            effective_groups_base_dn = 'ou=groups,ou={},{}'.format(self.instance.ldap, self.config.ldap.branches.base_dn)
        else:
            effective_admin_dn = self.config.ldap.admin_dn
            effective_admin_password = self.config.ldap.admin_password
            effective_users_base_dn = self.config.ldap.users_base_dn
            effective_groups_base_dn = self.config.ldap.group_base_dn

        ldapini = configure_ini(
            string=LDAP_INI,
            params={
                'ldap': {
                    'server': self.config.ldap.server,
                    'password': effective_admin_password,
                    'userbind': effective_admin_dn,
                    'userbasedn': effective_users_base_dn,
                    'groupbasedn': effective_groups_base_dn
                }
            }
        )
        ldap_ini_location = "{}/config/ldap.ini".format(self.buildout.folder)
        self.remote.put_file(ldap_ini_location, ldapini)
        return success_log('Succesfully configured {}'.format(ldap_ini_location))

    def create_oauth_nginx_entry(self):
        global_allowed_ips = self.config.oauth.allowed_ips
        instance_allowed_ips = self.get_instance_allowed_ips()
        allowed_ips = instance_allowed_ips + global_allowed_ips
        nginx_params = {
            'instance_name': self.instance.name,
            'server': self.config.oauth.server,
            'server_dns': self.config.oauth.server_dns,
            'osiris_port': int(self.instance.index) + OSIRIS_BASE_PORT,
            'buildout_folder': self.config.nginx.root,
            'allowed_ips': '\n      '.join(['allow {};'.format(ip) for ip in allowed_ips])
        }
        nginxentry = OSIRIS_NGINX_ENTRY.format(**nginx_params)

        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
        nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        nginx_remote.put_file(nginx_file_location, nginxentry)
        return success_log("Succesfully created {}".format(nginx_file_location))

    def backup_nginx_configuration(self):
        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)

        nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        backup_file_location = "{}/config/osiris-instances/{}.conf.backup".format(self.config.nginx.root, self.instance.name)
        backup_content = nginx_remote.get_file(nginx_file_location)
        nginx_remote.put_file(backup_file_location, backup_content)
        return success_log("Succesfully backed up to {}".format(backup_file_location))

    def recover_nginx_configuration(self):
        nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)

        nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
        backup_file_location = "{}/config/osiris-instances/{}.conf.backup".format(self.config.nginx.root, self.instance.name)
        backup_content = nginx_remote.get_file(backup_file_location)
        nginx_remote.put_file(nginx_file_location, backup_content)
        return success_log("Succesfully recovered backup from".format(backup_file_location))

    def commit_local_changes(self, message=None):
        self.buildout.commit_to_local_branch(
            self.config.local_git_branch,
            files=[
                'customizeme.cfg',
                'mongoauth.cfg'
            ],
            message=message)
        return success_log("Succesfully commited local changes")

    def get_instance_allowed_ips(self):
        """
            Get the ips that are currently configured on [osiris-config] allowed_ips.

            If there's only one ip configured, the value will be a string, so we adapt it.
        """
        configured_ips = self.buildout.read_configuration_file('customizeme.cfg')['osiris-config'].get('allowed_ips', [])
        configured_ips = [a.strip() for a in configured_ips.split(' ') if a.strip()] if isinstance(configured_ips, str) else configured_ips
        return configured_ips

    def add_new_bypass_allowed_ip(self, ip):
        configured_ips = self.get_instance_allowed_ips()
        allowed_ips = list(set(configured_ips + [ip]))

        if not set(allowed_ips).symmetric_difference(set(configured_ips)):
            return message_log('No changes to allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))

        customizations = {
            'osiris-config': {
                'allowed_ips': allowed_ips,
            },
        }

        self.buildout.configure_file('customizeme.cfg', customizations)
        return success_log('Succesfully updated allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))

    def remove_bypass_allowed_ip(self, ip):
        configured_ips = self.get_instance_allowed_ips()
        allowed_ips = list(set(configured_ips) - set([ip]))

        if not set(allowed_ips).symmetric_difference(set(configured_ips)):
            return message_log('No changes to allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))

        customizations = {
            'osiris-config': {
                'allowed_ips': allowed_ips,
            }
        }

        self.buildout.configure_file('customizeme.cfg', customizations)
        return success_log('Succesfully updated allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))

    # COMMANDS

    @command
    def new_instance(self, instance_name, port_index, ldap_branch=None, logecho=None):

        self.buildout.cfgfile = self.config.oauth.cfg_file
        self.buildout.logecho = logecho
        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        self.set_instance(
            name=instance_name,
            index=port_index,
            ldap=ldap_branch if ldap_branch is not None else instance_name
        )

        yield step_log('Cloning buildout')
        yield self.clone_buildout()

        yield step_log('Bootstraping buildout')
        yield self.bootstrap_buildout()

        yield step_log('Configuring customizeme.cfg')
        yield self.configure_instance()

        yield step_log('Configuring mongoauth.cfg')
        yield self.configure_mongoauth()

        yield step_log('Configuring ldap.ini')
        yield self.configure_ldap()

        yield step_log('Creating nginx entry for oauth')
        yield self.create_oauth_nginx_entry()

        yield step_log('Executing buildout')
        yield self.execute_buildout()

        yield step_log('Commiting to local branch')
        yield self.commit_local_changes()

        yield step_log('Changing permissions')
        yield self.set_filesystem_permissions()

        yield step_log('Adding instance to supervisor config')
        yield self.configure_supervisor()

    @command
    def upgrade(self, instance_name, logecho=None):
        self.buildout.cfgfile = self.config.oauth.cfg_file
        self.buildout.logecho = logecho
        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        self.set_instance(
            name=instance_name,
        )

        yield step_log('Updating buildout')
        yield self.update_buildout()

        yield step_log('Executing buildout')
        yield self.execute_buildout(update=True)

        yield step_log('Changing permissions')
        yield self.set_filesystem_permissions()

        # yield step_log('Reloading oauth')
        yield self.reload_instance()

    @command
    def reconfigure_nginx(self, instance_name):
        self.buildout.cfgfile = self.config.oauth.cfg_file
        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        instance = self.get_instance(instance_name)
        self.set_instance(
            name=instance_name,
            index=instance['port_index']

        )

        yield step_log('Backing up current configuration')
        yield self.backup_nginx_configuration()

        yield step_log('Creating nginx entry for oauth')
        yield self.create_oauth_nginx_entry()

        yield step_log('Testing new nginx configuration')
        status = self.test_nginx()
        if status[0] == 0:
            self.recover_nginx_configuration()
        yield status

        yield step_log('Reloading nginx')
        yield self.reload_nginx()

    @command
    def add_allowed_ip(self, instance_name, new_ip):
        instance = self.get_instance(instance_name)
        self.set_instance(
            name=instance_name,
            index=instance['port_index']
        )

        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        yield step_log('Backing up current configuration')
        yield self.backup_nginx_configuration()

        yield step_log('Adding new allowed ip')
        yield self.add_new_bypass_allowed_ip(new_ip)

        yield step_log('Creating nginx entry for oauth')
        yield self.create_oauth_nginx_entry()

        yield step_log('Testing new nginx configuration')
        status = self.test_nginx()
        if status[0] == 0:
            yield self.recover_nginx_configuration()
        yield status

        yield step_log('Commiting to local branch')
        yield self.commit_local_changes(message='Added allowed ip')

        yield step_log('Changing permissions')
        yield self.set_filesystem_permissions()

        yield step_log('Reloading nginx')
        yield self.reload_nginx()

    @command
    def remove_allowed_ip(self, instance_name, existing_ip):
        instance = self.get_instance(instance_name)
        self.set_instance(
            name=instance_name,
            index=instance['port_index']
        )

        self.buildout.folder = '{}/{}'.format(
            self.config.instances_root,
            instance_name
        )

        yield step_log('Backing up current configuration')
        yield self.backup_nginx_configuration()

        yield step_log('Adding new allowed ip')
        yield self.remove_bypass_allowed_ip(existing_ip)

        yield step_log('Creating nginx entry for oauth')
        yield self.create_oauth_nginx_entry()

        yield step_log('Testing new nginx configuration')
        status = self.test_nginx()
        if status[0] == 0:
            yield self.recover_nginx_configuration()
        yield status

        yield step_log('Commiting to local branch')
        yield self.commit_local_changes(message='Removed allowed ip')

        yield step_log('Changing permissions')
        yield self.set_filesystem_permissions()

        yield step_log('Reloading nginx')
        yield self.reload_nginx()