Exemple #1
0
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        POOLS['clusterexecpool'].spawn(
            clusterexec.clusterexec, name, 'restart', self.store)
        key = '/commissaire/cluster/{0}/restart'.format(name)
        cluster_restart_default = {
            'status': 'in_process',
            'restarted': [],
            'in_process': [],
            'started_at': datetime.datetime.utcnow().isoformat(),
            'finished_at': None
        }
        cluster_restart = ClusterRestart(**cluster_restart_default)
        self.store.set(key, cluster_restart.to_json())
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        try:
            store_manager = cherrypy.engine.publish('get-store-manager')[0]
            cluster_restart = store_manager.get(ClusterRestart.new(name=name))
            self.logger.debug('Found a ClusterRestart for {0}'.format(name))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return
        except:
            # This means one doesn't already exist
            pass

        # TODO: Move to a poll?
        store_manager = cherrypy.engine.publish('get-store-manager')[0]
        args = (store_manager.clone(), name, 'restart')
        p = Process(target=clusterexec, args=args)
        p.start()

        self.logger.debug('Started restart in clusterexecpool for {0}'.format(
            name))

        cluster_restart = ClusterRestart.new(
            name=name,
            status='in_process',
            started_at=datetime.datetime.utcnow().isoformat()
        )
        store_manager = cherrypy.engine.publish('get-store-manager')[0]
        store_manager.save(cluster_restart)
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        try:
            store_manager = cherrypy.engine.publish('get-store-manager')[0]
            cluster_restart = store_manager.get(ClusterRestart.new(name=name))
            self.logger.debug('Found a ClusterRestart for {0}'.format(name))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return
        except:
            # This means one doesn't already exist
            pass

        # TODO: Move to a poll?
        store_manager = cherrypy.engine.publish('get-store-manager')[0]
        args = (store_manager.clone(), name, 'restart')
        p = Process(target=clusterexec, args=args)
        p.start()

        self.logger.debug('Started restart in clusterexecpool for {0}'.format(
            name))

        cluster_restart = ClusterRestart.new(
            name=name,
            status='in_process',
            started_at=datetime.datetime.utcnow().isoformat()
        )
        store_manager = cherrypy.engine.publish('get-store-manager')[0]
        store_manager.save(cluster_restart)
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
Exemple #4
0
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        try:
            cluster_restart = ClusterRestart.retrieve(name)
            self.logger.debug('Found a ClusterRestart for {0}'.format(name))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return
        except:
            # This means one doesn't already exist
            pass

        # TODO: Move to a poll?
        p = Process(target=clusterexec, args=(name, 'restart'))
        p.start()

        self.logger.debug('Started restart in clusterexecpool for {0}'.format(
            name))
        cluster_restart_default = {
            'status': 'in_process',
            'restarted': [],
            'in_process': [],
            'started_at': datetime.datetime.utcnow().isoformat(),
            'finished_at': None
        }
        cluster_restart = ClusterRestart(**cluster_restart_default)
        cluster_restart.save(name)
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
Exemple #5
0
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(self.store, name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        key = '/commissaire/cluster/{0}/restart'.format(name)
        try:
            etcd_resp = self.store.get(key)
            self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
            cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return
        except etcd.EtcdKeyNotFound:
            pass

        POOLS['clusterexecpool'].spawn(
            clusterexec.clusterexec, name, 'restart', self.store)
        self.logger.debug('Started restart in clusterexecpool for {0}'.format(
            name))
        cluster_restart_default = {
            'status': 'in_process',
            'restarted': [],
            'in_process': [],
            'started_at': datetime.datetime.utcnow().isoformat(),
            'finished_at': None
        }
        cluster_restart = ClusterRestart(**cluster_restart_default)
        self.store.set(key, cluster_restart.to_json())
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
Exemple #6
0
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        key = '/commissaire/cluster/{0}/restart'.format(name)
        etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
        if not error:
            self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
            cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return

        # TODO: Move to a poll?
        p = Process(target=clusterexec, args=(name, 'restart'))
        p.start()

        self.logger.debug('Started restart in clusterexecpool for {0}'.format(
            name))
        cluster_restart_default = {
            'status': 'in_process',
            'restarted': [],
            'in_process': [],
            'started_at': datetime.datetime.utcnow().isoformat(),
            'finished_at': None
        }
        cluster_restart = ClusterRestart(**cluster_restart_default)
        cherrypy.engine.publish('store-save', key, cluster_restart.to_json())
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
Exemple #7
0
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        key = '/commissaire/cluster/{0}/restart'.format(name)
        etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
        if not error:
            self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
            cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return

        # TODO: Move to a poll?
        p = Process(target=clusterexec, args=(name, 'restart'))
        p.start()

        self.logger.debug(
            'Started restart in clusterexecpool for {0}'.format(name))
        cluster_restart_default = {
            'status': 'in_process',
            'restarted': [],
            'in_process': [],
            'started_at': datetime.datetime.utcnow().isoformat(),
            'finished_at': None
        }
        cluster_restart = ClusterRestart(**cluster_restart_default)
        cherrypy.engine.publish('store-save', key, cluster_restart.to_json())
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
Exemple #8
0
    def on_put(self, req, resp, name):
        """
        Handles PUT (or "initiate") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        # Make sure the cluster name is valid.
        if not util.etcd_cluster_exists(self.store, name):
            self.logger.info(
                'Restart PUT requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        # If the operation is already in progress, return the current
        # status with response code 200 OK.
        key = '/commissaire/cluster/{0}/restart'.format(name)
        try:
            etcd_resp = self.store.get(key)
            self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
            cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
            if not cluster_restart.finished_at:
                self.logger.debug(
                    'Cluster {0} restart already in progress'.format(name))
                resp.status = falcon.HTTP_200
                req.context['model'] = cluster_restart
                return
        except etcd.EtcdKeyNotFound:
            pass

        POOLS['clusterexecpool'].spawn(clusterexec.clusterexec, name,
                                       'restart', self.store)
        self.logger.debug(
            'Started restart in clusterexecpool for {0}'.format(name))
        cluster_restart_default = {
            'status': 'in_process',
            'restarted': [],
            'in_process': [],
            'started_at': datetime.datetime.utcnow().isoformat(),
            'finished_at': None
        }
        cluster_restart = ClusterRestart(**cluster_restart_default)
        self.store.set(key, cluster_restart.to_json())
        resp.status = falcon.HTTP_201
        req.context['model'] = cluster_restart
    def on_get(self, req, resp, name):
        """
        Handles GET (or "status") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart GET requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        try:
            store_manager = cherrypy.engine.publish('get-store-manager')[0]
            cluster_restart = store_manager.get(ClusterRestart.new(name=name))
        except:
            # Return "204 No Content" if we have no status,
            # meaning no restart is in progress.  The client
            # can't be expected to know that, so it's not a
            # client error (4xx).
            self.logger.debug((
                'Restart GET requested for {0} but no restart '
                'has ever been executed.').format(name))
            resp.status = falcon.HTTP_204
            return
        resp.status = falcon.HTTP_200
        req.context['model'] = cluster_restart
Exemple #10
0
    def on_get(self, req, resp, name):
        """
        Handles GET (or "status") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        key = '/commissaire/cluster/{0}/restart'.format(name)
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart GET requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return
        status, error = cherrypy.engine.publish('store-get', key)[0]
        self.logger.debug('Etcd Response: {0}'.format(status))
        if error:
            # Return "204 No Content" if we have no status,
            # meaning no restart is in progress.  The client
            # can't be expected to know that, so it's not a
            # client error (4xx).
            self.logger.debug(('Restart GET requested for {0} but no restart '
                               'has ever been executed.').format(name))
            resp.status = falcon.HTTP_204
            return
        resp.status = falcon.HTTP_200
        req.context['model'] = ClusterRestart(**json.loads(status.value))
    def on_get(self, req, resp, name):
        """
        Handles GET (or "status") requests for a Cluster restart.

        :param req: Request instance that will be passed through.
        :type req: falcon.Request
        :param resp: Response instance that will be passed through.
        :type resp: falcon.Response
        :param name: The name of the Cluster being restarted.
        :type name: str
        """
        if not util.etcd_cluster_exists(name):
            self.logger.info(
                'Restart GET requested for nonexistent cluster {0}'.format(
                    name))
            resp.status = falcon.HTTP_404
            return

        try:
            store_manager = cherrypy.engine.publish('get-store-manager')[0]
            cluster_restart = store_manager.get(ClusterRestart.new(name=name))
        except:
            # Return "204 No Content" if we have no status,
            # meaning no restart is in progress.  The client
            # can't be expected to know that, so it's not a
            # client error (4xx).
            self.logger.debug((
                'Restart GET requested for {0} but no restart '
                'has ever been executed.').format(name))
            resp.status = falcon.HTTP_204
            return
        resp.status = falcon.HTTP_200
        req.context['model'] = cluster_restart
#: Cluster model with HOST for most tests
CLUSTER_WITH_HOST = Cluster.new(
    name='cluster',
    status='ok',
    hostset=[HOST],
)
#: Cluster model with flattened HOST for tests
CLUSTER_WITH_FLAT_HOST = Cluster.new(
    name='cluster',
    status='ok',
    hostset=[HOST.address],
)
#: ClusterRestart model for most tests
CLUSTER_RESTART = ClusterRestart.new(name='cluster',
                                     status='ok',
                                     restarted=[],
                                     in_process=[],
                                     started_at='',
                                     finished_at='')
#: ClusterUpgrade model for most tests
CLUSTER_UPGRADE = ClusterUpgrade.new(name='cluster',
                                     status='ok',
                                     upgraded=[],
                                     in_process=[],
                                     started_at='',
                                     finished_at='')
#: ClusterDeploy model for most tests
CLUSTER_DEPLOY = ClusterDeploy.new(name='cluster',
                                   status='ok',
                                   version='1.0',
                                   deployed=[],
                                   in_process=[],
Exemple #13
0
def clusterexec(store_manager, cluster_name, command, kwargs={}):
    """
    Remote executes a shell commands across a cluster.

    :param store_manager: Proxy object for remtote stores
    :type store_manager: commissaire.store.StoreHandlerManager
    :param cluster_name: Name of the cluster to act on
    :type cluster_name: str
    :param command: Top-level command to execute
    :type command: str
    :param kwargs: Keyword arguments for the command
    :type kwargs: dict
    """
    logger = logging.getLogger('clusterexec')

    # TODO: This is a hack and should really be done elsewhere
    command_args = ()
    if command == 'upgrade':
        finished_hosts_key = 'upgraded'
        model_instance = ClusterUpgrade.new(
            name=cluster_name,
            status='in_process',
            started_at=datetime.datetime.utcnow().isoformat(),
            upgraded=[],
            in_process=[],
        )
    elif command == 'restart':
        finished_hosts_key = 'restarted'
        model_instance = ClusterRestart.new(
            name=cluster_name,
            status='in_process',
            started_at=datetime.datetime.utcnow().isoformat(),
            restarted=[],
            in_process=[],
        )
    elif command == 'deploy':
        finished_hosts_key = 'deployed'
        version = kwargs.get('version', '')
        command_args = (version,)
        model_instance = ClusterDeploy.new(
            name=cluster_name,
            status='in_process',
            started_at=datetime.datetime.utcnow().isoformat(),
            version=version,
            deployed=[],
            in_process=[],
        )

    end_status = 'finished'

    try:
        # Set the initial status in the store
        logger.info('Setting initial status.')
        logger.debug('Status={0}'.format(model_instance.to_json()))
        store_manager.save(model_instance)
    except Exception as error:
        logger.error(
            'Unable to save initial state for "{0}" clusterexec due to '
            '{1}: {2}'.format(cluster_name, type(error), error))
        return

    # Collect all host addresses in the cluster
    try:
        cluster = store_manager.get(Cluster.new(
            name=cluster_name, status='', hostset=[]))
    except Exception as error:
        logger.warn(
            'Unable to continue for cluster "{0}" due to '
            '{1}: {2}. Returning...'.format(cluster_name, type(error), error))
        return

    if cluster.hostset:
        logger.debug(
            '{0} hosts in cluster "{1}"'.format(
                len(cluster.hostset), cluster_name))
    else:
        logger.warn('No hosts in cluster "{0}"'.format(cluster_name))

    # TODO: Find better way to do this
    try:
        hosts = store_manager.list(Hosts(hosts=[]))
    except Exception as error:
        logger.warn(
            'No hosts in the cluster. Error: {0}. Exiting clusterexec'.format(
                error))
        return

    for host in hosts.hosts:
        if host.address not in cluster.hostset:
            logger.debug(
                'Skipping {0} as it is not in this cluster.'.format(
                    host.address))
            continue  # Move on to the next one
        oscmd = get_oscmd(host.os)

        # command_list is only used for logging
        command_list = getattr(oscmd, command)(*command_args)
        logger.info('Executing {0} on {1}...'.format(
            command_list, host.address))

        model_instance.in_process.append(host.address)
        try:
            store_manager.save(model_instance)
        except Exception as error:
            logger.error(
                'Unable to save in_process state for "{0}" clusterexec due to '
                '{1}: {2}'.format(cluster_name, type(error), error))
            return

        key = TemporarySSHKey(host, logger)
        key.create()

        try:
            transport = ansibleapi.Transport(host.remote_user)
            exe = getattr(transport, command)
            result, facts = exe(
                host.address, key.path, oscmd, kwargs)
        # XXX: ansibleapi explicitly raises Exception()
        except Exception as ex:
            # If there was a failure set the end_status and break out
            end_status = 'failed'
            logger.error('Clusterexec {0} for {1} failed: {2}: {3}'.format(
                command, host.address, type(ex), ex))
            break
        finally:
            try:
                key.remove()
                logger.debug('Removed temporary key file {0}'.format(key.path))
            except:
                logger.warn(
                    'Unable to remove the temporary key file: {0}'.format(
                        key.path))

        # Set the finished hosts
        new_finished_hosts = getattr(
            model_instance, finished_hosts_key) + [host.address]
        setattr(
            model_instance,
            finished_hosts_key,
            new_finished_hosts)
        try:
            idx = model_instance.in_process.index(host.address)
            model_instance.in_process.pop(idx)
        except ValueError:
            logger.warn('Host {0} was not in_process for {1} {2}'.format(
                host['address'], command, cluster_name))
        try:
            store_manager.save(model_instance)
            logger.info('Finished executing {0} for {1} in {2}'.format(
                command, host.address, cluster_name))
        except Exception as error:
            logger.error(
                'Unable to save cluster state for "{0}" clusterexec due to '
                '{1}: {2}'.format(cluster_name, type(error), error))
            return

    # Final set of command result
    model_instance.finished_at = datetime.datetime.utcnow().isoformat()
    model_instance.status = end_status

    logger.info('Cluster {0} final {1} status: {2}'.format(
        cluster_name, command, model_instance.to_json()))

    try:
        store_manager.save(model_instance)
    except Exception as error:
        logger.error(
            'Unable to save final state for "{0}" clusterexec due to '
            '{1}: {2}'.format(cluster_name, type(error), error))

    logger.info('Clusterexec stopping')
CLUSTER_WITH_HOST = Cluster.new(
    name='cluster',
    status='ok',
    hostset=[HOST],
)
#: Cluster model with flattened HOST for tests
CLUSTER_WITH_FLAT_HOST = Cluster.new(
    name='cluster',
    status='ok',
    hostset=[HOST.address],
)
#: ClusterRestart model for most tests
CLUSTER_RESTART = ClusterRestart.new(
    name='cluster',
    status='ok',
    restarted=[],
    in_process=[],
    started_at='',
    finished_at= ''
)
#: ClusterUpgrade model for most tests
CLUSTER_UPGRADE = ClusterUpgrade.new(
    name='cluster',
    status='ok',
    upgraded=[],
    in_process=[],
    started_at='',
    finished_at= ''
)
#: ClusterDeploy model for most tests
CLUSTER_DEPLOY = ClusterDeploy.new(
    name='cluster',