Exemple #1
0
    def _load(plugin):
        """
        Import a module by file name.
        :param plugin: A plugin to load.
        :type plugin: Plugin
        :return: The loaded plugin.
        :rtype: Plugin
        """
        Remote.clear()
        Actions.clear()
        Plugin.add(plugin)
        try:
            path = plugin.descriptor.main.plugin
            if path:
                Plugin.add(plugin, path)
                plugin.impl = __import__(path, {}, {}, [path.split('.')[-1]])
            else:
                path = PluginLoader._find(plugin.name)
                plugin.impl = imp.load_source(plugin.name, path)

            log.info('plugin:%s loaded using: %s', plugin.name, path)

            for fn in Remote.find(plugin.impl.__name__):
                fn.gofer.plugin = plugin

            plugin.dispatcher += Remote.collated()
            plugin.actions = Actions.collated()
            plugin.delegate = Delegate()
            plugin.load()
            return plugin
        except Exception:
            log.exception('plugin:%s, import failed', plugin.name)
            Plugin.delete(plugin)
Exemple #2
0
 def test_init(self):
     Delegate.load.append(1)
     Delegate.unload.append(2)
     d = Delegate()
     self.assertEqual(Delegate.load, [])
     self.assertEqual(Delegate.unload, [])
     self.assertEqual(d.load, [1])
     self.assertEqual(d.unload, [2])
Exemple #3
0
 def __init__(self, descriptor, path):
     """
     :param descriptor: The plugin descriptor.
     :type descriptor: PluginDescriptor
     :param path: The plugin descriptor path.
     :type path: str
     """
     self.__mutex = RLock()
     self.path = path
     self.descriptor = descriptor
     self.pool = ThreadPool(int(descriptor.main.threads or 1))
     self.impl = None
     self.actions = []
     self.dispatcher = Dispatcher()
     self.whiteboard = Whiteboard()
     self.scheduler = Scheduler(self)
     self.delegate = Delegate()
     self.authenticator = None
     self.consumer = None
Exemple #4
0
 def __init__(self, descriptor, path):
     """
     :param descriptor: The plugin descriptor.
     :type descriptor: PluginDescriptor
     :param path: The plugin descriptor path.
     :type path: str
     """
     self.__mutex = RLock()
     self.path = path
     self.descriptor = descriptor
     self.pool = ThreadPool(int(descriptor.main.threads or 1))
     self.impl = None
     self.actions = []
     self.dispatcher = Dispatcher()
     self.whiteboard = Whiteboard()
     self.scheduler = Scheduler(self)
     self.delegate = Delegate()
     self.authenticator = None
     self.consumer = None
Exemple #5
0
class Plugin(object):
    """
    Represents a plugin.
    :ivar descriptor: descriptor.
    :type descriptor: PluginDescriptor
    :param path: The descriptor path.
    :type path: str
    :ivar pool: The main thread pool.
    :type pool: ThreadPool
    :ivar impl: The plugin implementation.
    :ivar impl: module
    :ivar actions: List of: gofer.action.Action.
    :type actions: list
    :ivar dispatcher: The RMI dispatcher.
    :type dispatcher: Dispatcher
    :ivar whiteboard: The plugin whiteboard.
    :type whiteboard: Whiteboard
    :ivar authenticator: The plugin message authenticator.
    :type authenticator: gofer.messaging.auth.Authenticator
    :ivar consumer: An AMQP request consumer.
    :type consumer: gofer.rmi.consumer.RequestConsumer.
    """

    container = Container()
    
    @staticmethod
    def add(plugin, *names):
        """
        Add the plugin.
        :param plugin: The plugin to add.
        :type plugin: Plugin
        :return: The added plugin
        :rtype: Plugin
        """
        Plugin.container.add(plugin, *names)
    
    @staticmethod
    def delete(plugin):
        """
        Delete the plugin.
        :param plugin: The plugin to delete.
        :type plugin: Plugin
        """
        Plugin.container.delete(plugin)
        if not plugin.impl:
            # not loaded
            return
        mod = plugin.impl.__name__
        del sys.modules[mod]
    
    @staticmethod
    def find(name):
        """
        Find a plugin by name or path.
        :param name: A plugin name
        :type name: str
        :return: The plugin when found.
        :rtype: Plugin 
        """
        return Plugin.container.find(name)

    @staticmethod
    def all():
        """
        Get a unique list of loaded plugins.
        :return: A list of plugins
        :rtype: list
        """
        return Plugin.container.all()

    def __init__(self, descriptor, path):
        """
        :param descriptor: The plugin descriptor.
        :type descriptor: PluginDescriptor
        :param path: The plugin descriptor path.
        :type path: str
        """
        self.__mutex = RLock()
        self.path = path
        self.descriptor = descriptor
        self.pool = ThreadPool(int(descriptor.main.threads or 1))
        self.impl = None
        self.actions = []
        self.dispatcher = Dispatcher()
        self.whiteboard = Whiteboard()
        self.scheduler = Scheduler(self)
        self.delegate = Delegate()
        self.authenticator = None
        self.consumer = None

    @property
    def name(self):
        return self.cfg.main.name

    @property
    def cfg(self):
        return self.descriptor

    @property
    def uuid(self):
        return self.cfg.messaging.uuid

    @property
    def url(self):
        return self.cfg.messaging.url

    @property
    def enabled(self):
        return get_bool(self.cfg.main.enabled)

    @property
    def connector(self):
        return Connector(self.url)

    @property
    def node(self):
        model = BrokerModel(self)
        return model.node

    @property
    def forward(self):
        _list = self.cfg.main.forward
        _list = [p.strip() for p in _list.split(',')]
        return set(_list)

    @property
    def accept(self):
        _list = self.cfg.main.accept
        _list = [p.strip() for p in _list.split(',')]
        return set(_list)

    @property
    def is_started(self):
        return self.scheduler.isAlive()

    @synchronized
    def start(self):
        """
        Start the plugin.
        - attach
        - start scheduler
        """
        if self.is_started:
            # already started
            return
        self.attach()
        self.scheduler.start()

    @synchronized
    def shutdown(self, teardown=True):
        """
        Shutdown the plugin.
        - detach
        - shutdown the thread pool.
        - shutdown the scheduler.
        :param teardown: Teardown the broker model.
        :type teardown: bool
        :return: List of pending requests.
        :rtype: list
        """
        if not self.is_started:
            # not started
            return []
        self.detach(teardown)
        pending = self.pool.shutdown()
        self.scheduler.shutdown()
        self.scheduler.join()
        return pending

    @synchronized
    def refresh(self):
        """
        Refresh the AMQP configurations using the plugin configuration.
        """
        connector = Connector(self.url)
        messaging = self.cfg.messaging
        connector.heartbeat = get_integer(messaging.heartbeat)
        connector.ssl.ca_certificate = messaging.cacert
        connector.ssl.client_key = messaging.clientkey
        connector.ssl.client_certificate = messaging.clientcert
        connector.ssl.host_validation = messaging.host_validation
        connector.add()

    @attach
    @synchronized
    def attach(self):
        """
        Attach (connect) to AMQP connector using the specified uuid.
        """
        self.detach(False)
        self.refresh()
        model = BrokerModel(self)
        model.setup()
        node = Node(model.queue)
        consumer = RequestConsumer(node, self)
        consumer.authenticator = self.authenticator
        consumer.start()
        self.consumer = consumer
        log.info('plugin:%s, attached => %s', self.name, self.node)

    @synchronized
    def detach(self, teardown=True):
        """
        Detach (disconnect) from AMQP connector.
        :param teardown: Teardown the broker model.
        :type teardown: bool
        """
        if not self.consumer:
            # not attached
            return
        self.consumer.shutdown()
        self.consumer.join()
        self.consumer = None
        log.info('plugin:%s, detached [%s]', self.name, self.node)
        if teardown:
            model = BrokerModel(self)
            model.teardown()

    def provides(self, name):
        """
        Get whether the plugin provides the name.
        :param name: A class name.
        :type name: str
        :return: True if provides.
        :raise: bool
        """
        return self.dispatcher.provides(name)

    def dispatch(self, request):
        """
        Dispatch (invoke) the specified RMI request.
        :param request: An RMI request
        :type request: gofer.Document
        :return: The RMI returned.
        """
        dispatcher = self.dispatcher
        call = Document(request.request)
        if not self.provides(call.classname):
            for plugin in Plugin.all():
                if plugin == self:
                    continue
                if not plugin.provides(call.classname):
                    # not provided
                    continue
                valid = set()
                valid.add('*')
                valid.add(plugin.name)
                if not valid.intersection(self.forward):
                    # (forwarding) not approved
                    continue
                valid = set()
                valid.add('*')
                valid.add(self.name)
                if not valid.intersection(plugin.accept):
                    # (accept) not approved
                    continue
                dispatcher = plugin.dispatcher
                break
        return dispatcher.dispatch(request)

    @synchronized
    def load(self):
        """
        Load the plugin.
        """
        self.delegate.loaded()
        path = self.cfg.messaging.authenticator
        if not path:
            # not configured
            return
        path = path.split('.')
        mod = '.'.join(path[:-1])
        mod = __import__(mod, {}, {}, [path[-1]])
        self.authenticator = mod.Authenticator()

    @synchronized
    def unload(self):
        """
        Unload the plugin.
        - Detach.
        - Delete the plugin.
        - Abort scheduled requests.
        - Plugin shutdown.
        - Purge pending requests.
        """
        Plugin.delete(self)
        self.shutdown()
        self.delegate.unloaded()
        self.scheduler.pending.delete()
        log.info('plugin:%s, unloaded', self.name)

    @synchronized
    def reload(self):
        """
        Reload the plugin.
        - Detach.
        - Delete the plugin.
        - Abort scheduled requests.
        - Plugin shutdown.
        - Reload plugin.
        - Reschedule pending work to reloaded plugin.
        """
        Plugin.delete(self)
        scheduled = self.shutdown(False)
        self.delegate.unloaded()
        plugin = PluginLoader.load(self.path)
        if plugin:
            for call in scheduled:
                if isinstance(call.fn, Task):
                    task = call.fn
                    task.transaction.plugin = plugin
                plugin.pool.schedule(call)
            plugin.start()
        log.info('plugin:%s, reloaded', self.name)
        return plugin
Exemple #6
0
 def test_unloaded_failed(self):
     d = Delegate()
     d.unload = [Mock(side_effect=ValueError), Mock()]
     d.unloaded()
     for fn in d.unload:
         fn.asssert_called_once_with()
Exemple #7
0
 def test_unloaded(self):
     d = Delegate()
     d.unload = [Mock(), Mock()]
     d.unloaded()
     for fn in d.unload:
         fn.asssert_called_once_with()
Exemple #8
0
 def test_unloaded_failed(self):
     d = Delegate()
     d.unload = [Mock(side_effect=ValueError), Mock()]
     d.unloaded()
     for fn in d.unload:
         fn.asssert_called_once_with()
Exemple #9
0
 def test_unloaded(self):
     d = Delegate()
     d.unload = [Mock(), Mock()]
     d.unloaded()
     for fn in d.unload:
         fn.asssert_called_once_with()
Exemple #10
0
class Plugin(object):
    """
    Represents a plugin.
    :ivar descriptor: descriptor.
    :type descriptor: PluginDescriptor
    :param path: The descriptor path.
    :type path: str
    :ivar pool: The main thread pool.
    :type pool: ThreadPool
    :ivar impl: The plugin implementation.
    :ivar impl: module
    :ivar actions: List of: gofer.action.Action.
    :type actions: list
    :ivar dispatcher: The RMI dispatcher.
    :type dispatcher: Dispatcher
    :ivar whiteboard: The plugin whiteboard.
    :type whiteboard: Whiteboard
    :ivar authenticator: The plugin message authenticator.
    :type authenticator: gofer.messaging.auth.Authenticator
    :ivar consumer: An AMQP request consumer.
    :type consumer: gofer.rmi.consumer.RequestConsumer.
    """

    container = Container()

    @staticmethod
    def add(plugin, *names):
        """
        Add the plugin.
        :param plugin: The plugin to add.
        :type plugin: Plugin
        :return: The added plugin
        :rtype: Plugin
        """
        Plugin.container.add(plugin, *names)

    @staticmethod
    def delete(plugin):
        """
        Delete the plugin.
        :param plugin: The plugin to delete.
        :type plugin: Plugin
        """
        Plugin.container.delete(plugin)
        if not plugin.impl:
            # not loaded
            return
        mod = plugin.impl.__name__
        del sys.modules[mod]

    @staticmethod
    def find(name):
        """
        Find a plugin by name or path.
        :param name: A plugin name
        :type name: str
        :return: The plugin when found.
        :rtype: Plugin 
        """
        return Plugin.container.find(name)

    @staticmethod
    def all():
        """
        Get a unique list of loaded plugins.
        :return: A list of plugins
        :rtype: list
        """
        return Plugin.container.all()

    def __init__(self, descriptor, path):
        """
        :param descriptor: The plugin descriptor.
        :type descriptor: PluginDescriptor
        :param path: The plugin descriptor path.
        :type path: str
        """
        self.__mutex = RLock()
        self.path = path
        self.descriptor = descriptor
        self.pool = ThreadPool(int(descriptor.main.threads or 1))
        self.impl = None
        self.actions = []
        self.dispatcher = Dispatcher()
        self.whiteboard = Whiteboard()
        self.scheduler = Scheduler(self)
        self.delegate = Delegate()
        self.authenticator = None
        self.consumer = None

    @property
    def name(self):
        return self.cfg.main.name

    @property
    def cfg(self):
        return self.descriptor

    @property
    def uuid(self):
        return self.cfg.messaging.uuid

    @property
    def url(self):
        return self.cfg.messaging.url

    @property
    def enabled(self):
        return get_bool(self.cfg.main.enabled)

    @property
    def connector(self):
        return Connector(self.url)

    @property
    def node(self):
        model = BrokerModel(self)
        return model.node

    @property
    def forward(self):
        _list = self.cfg.main.forward
        _list = [p.strip() for p in _list.split(',')]
        return set(_list)

    @property
    def accept(self):
        _list = self.cfg.main.accept
        _list = [p.strip() for p in _list.split(',')]
        return set(_list)

    @property
    def is_started(self):
        return self.scheduler.isAlive()

    @property
    def latency(self):
        return float(self.cfg.main.latency)

    @synchronized
    def start(self):
        """
        Start the plugin.
        - attach
        - start scheduler
        """
        if self.is_started:
            # already started
            return
        self.attach()
        self.scheduler.start()

    @synchronized
    def shutdown(self, teardown=True):
        """
        Shutdown the plugin.
        - detach
        - shutdown the thread pool.
        - shutdown the scheduler.
        :param teardown: Teardown the broker model.
        :type teardown: bool
        :return: List of pending requests.
        :rtype: list
        """
        if not self.is_started:
            # not started
            return []
        self.detach(teardown)
        pending = self.pool.shutdown()
        self.scheduler.shutdown()
        self.scheduler.join()
        return pending

    @synchronized
    def refresh(self):
        """
        Refresh the AMQP configurations using the plugin configuration.
        """
        connector = Connector(self.url)
        messaging = self.cfg.messaging
        connector.heartbeat = get_integer(messaging.heartbeat)
        connector.ssl.ca_certificate = messaging.cacert
        connector.ssl.client_key = messaging.clientkey
        connector.ssl.client_certificate = messaging.clientcert
        connector.ssl.host_validation = messaging.host_validation
        connector.add()

    @attach
    @synchronized
    def attach(self):
        """
        Attach (connect) to AMQP connector using the specified uuid.
        """
        self.detach(False)
        self.refresh()
        model = BrokerModel(self)
        model.setup()
        node = Node(model.queue)
        consumer = RequestConsumer(node, self)
        consumer.authenticator = self.authenticator
        consumer.start()
        self.consumer = consumer
        log.info('plugin:%s, attached => %s', self.name, self.node)

    @synchronized
    def detach(self, teardown=True):
        """
        Detach (disconnect) from AMQP connector.
        :param teardown: Teardown the broker model.
        :type teardown: bool
        """
        if not self.consumer:
            # not attached
            return
        self.consumer.shutdown()
        self.consumer.join()
        self.consumer = None
        log.info('plugin:%s, detached [%s]', self.name, self.node)
        if teardown:
            model = BrokerModel(self)
            model.teardown()

    def provides(self, name):
        """
        Get whether the plugin provides the name.
        :param name: A class name.
        :type name: str
        :return: True if provides.
        :raise: bool
        """
        return self.dispatcher.provides(name)

    def dispatch(self, request):
        """
        Dispatch (invoke) the specified RMI request.
        :param request: An RMI request
        :type request: gofer.Document
        :return: The RMI returned.
        """
        dispatcher = self.dispatcher
        call = Document(request.request)
        if not self.provides(call.classname):
            for plugin in Plugin.all():
                if plugin == self:
                    continue
                if not plugin.provides(call.classname):
                    # not provided
                    continue
                valid = set()
                valid.add('*')
                valid.add(plugin.name)
                if not valid.intersection(self.forward):
                    # (forwarding) not approved
                    continue
                valid = set()
                valid.add('*')
                valid.add(self.name)
                if not valid.intersection(plugin.accept):
                    # (accept) not approved
                    continue
                dispatcher = plugin.dispatcher
                break
        return dispatcher.dispatch(request)

    @synchronized
    def load(self):
        """
        Load the plugin.
        """
        self.delegate.loaded()
        path = self.cfg.messaging.authenticator
        if not path:
            # not configured
            return
        path = path.split('.')
        mod = '.'.join(path[:-1])
        mod = __import__(mod, {}, {}, [path[-1]])
        self.authenticator = mod.Authenticator()

    @synchronized
    def unload(self):
        """
        Unload the plugin.
        - Detach.
        - Delete the plugin.
        - Abort scheduled requests.
        - Plugin shutdown.
        - Purge pending requests.
        """
        Plugin.delete(self)
        self.shutdown()
        self.delegate.unloaded()
        self.scheduler.pending.delete()
        log.info('plugin:%s, unloaded', self.name)

    @synchronized
    def reload(self):
        """
        Reload the plugin.
        - Detach.
        - Delete the plugin.
        - Abort scheduled requests.
        - Plugin shutdown.
        - Reload plugin.
        - Reschedule pending work to reloaded plugin.
        """
        Plugin.delete(self)
        scheduled = self.shutdown(False)
        self.delegate.unloaded()
        plugin = PluginLoader.load(self.path)
        if plugin:
            for call in scheduled:
                if isinstance(call.fn, Task):
                    task = call.fn
                    task.transaction.plugin = plugin
                plugin.pool.schedule(call)
            plugin.start()
        log.info('plugin:%s, reloaded', self.name)
        return plugin