def test(calls=100): results = [] pool = ThreadPool(9) pool.start() for n in range(calls): pool.run(fn1, n, results) while len(results) < calls: sleep(1) print('shutdown pool...') pool.shutdown()
class Builtin(object): """ The builtin pseudo-plugin. """ @staticmethod def _dispatcher(): collator = collation.Collator() classes, _ = collator({f[1]: {} for f in inspection.methods(Admin)}) dispatcher = Dispatcher() dispatcher += classes return dispatcher def __init__(self, plugin): """ :param plugin: A real plugin. :type plugin: gofer.agent.plugin.Plugin """ Admin.container = plugin.container self.pool = ThreadPool(3) self.dispatcher = self._dispatcher() self.plugin = plugin self.latency = 0 @property def url(self): return self.plugin.url @property def authenticator(self): return self.plugin.authenticator def provides(self, name): """ Get whether the plugin provides the name. :param name: A class name. :type name: str :return: True if provides. :raise: bool """ return self.dispatcher.provides(name) def dispatch(self, request): """ Dispatch (invoke) the specified RMI request. :param request: An RMI request :type request: gofer.Document :return: The RMI returned. """ return self.dispatcher.dispatch(request) def start(self): """ Start the plugin. """ self.pool.start() def shutdown(self): """ Shutdown the plugin. """ self.pool.shutdown()
class Plugin(object): """ Represents a plugin. :ivar descriptor: descriptor. :type descriptor: PluginDescriptor :param path: The descriptor path. :type path: str :ivar pool: The main thread pool. :type pool: ThreadPool :ivar impl: The plugin implementation. :ivar impl: module :ivar actions: List of: gofer.action.Action. :type actions: list :ivar dispatcher: The RMI dispatcher. :type dispatcher: Dispatcher :ivar whiteboard: The plugin whiteboard. :type whiteboard: Whiteboard :ivar authenticator: The plugin message authenticator. :type authenticator: gofer.messaging.auth.Authenticator :ivar consumer: An AMQP request consumer. :type consumer: gofer.rmi.consumer.RequestConsumer. """ container = Container() @staticmethod def add(plugin, *names): """ Add the plugin. :param plugin: The plugin to add. :type plugin: Plugin :return: The added plugin :rtype: Plugin """ Plugin.container.add(plugin, *names) @staticmethod def delete(plugin): """ Delete the plugin. :param plugin: The plugin to delete. :type plugin: Plugin """ Plugin.container.delete(plugin) if not plugin.impl: # not loaded return mod = plugin.impl.__name__ del sys.modules[mod] @staticmethod def find(name): """ Find a plugin by name or path. :param name: A plugin name :type name: str :return: The plugin when found. :rtype: Plugin """ return Plugin.container.find(name) @staticmethod def all(): """ Get a unique list of loaded plugins. :return: A list of plugins :rtype: list """ return Plugin.container.all() def __init__(self, descriptor, path): """ :param descriptor: The plugin descriptor. :type descriptor: PluginDescriptor :param path: The plugin descriptor path. :type path: str """ self.__mutex = RLock() self.path = path self.descriptor = descriptor self.pool = ThreadPool(int(descriptor.main.threads or 1)) self.impl = None self.actions = [] self.dispatcher = Dispatcher() self.whiteboard = Whiteboard() self.scheduler = Scheduler(self) self.delegate = Delegate() self.authenticator = None self.consumer = None @property def name(self): return self.cfg.main.name @property def cfg(self): return self.descriptor @property def url(self): return self.cfg.messaging.url @property def enabled(self): return get_bool(self.cfg.main.enabled) @property def connector(self): return Connector(self.url) @property def node(self): model = BrokerModel(self) return model.node @property def forward(self): _list = self.cfg.main.forward _list = [p.strip() for p in _list.split(',')] return set(_list) @property def accept(self): _list = self.cfg.main.accept _list = [p.strip() for p in _list.split(',')] return set(_list) @property def is_started(self): return self.scheduler.isAlive() @property def latency(self): return float(self.cfg.main.latency) @synchronized def start(self): """ Start the plugin. - attach - start scheduler """ if self.is_started: # already started return self.attach() self.scheduler.start() self.pool.start() @synchronized def shutdown(self, teardown=True, hard=False): """ Shutdown the plugin. - detach - limit messaging repairing - shutdown the thread pool. - shutdown the scheduler. :param teardown: Teardown the broker model. :type teardown: bool :param hard: Abort threads in the pool. When not aborted, work in progress will attempt to be completed before shutdown. :type hard: bool :return: List of pending requests. :rtype: list """ if not self.is_started: # not started return [] self.detach(teardown) pending = self.pool.shutdown(hard=hard) self.scheduler.shutdown() self.scheduler.join() return pending @synchronized def refresh(self): """ Refresh the AMQP configurations using the plugin configuration. """ connector = Connector(self.url) messaging = self.cfg.messaging connector.heartbeat = get_integer(messaging.heartbeat) connector.ssl.ca_certificate = messaging.cacert connector.ssl.client_key = messaging.clientkey connector.ssl.client_certificate = messaging.clientcert connector.ssl.host_validation = messaging.host_validation connector.add() @attach @synchronized def attach(self): """ Attach (connect) to AMQP connector using the specified uuid. """ self.detach(False) self.refresh() model = BrokerModel(self) model.setup() node = Node(model.queue) consumer = RequestConsumer(node, self) consumer.authenticator = self.authenticator consumer.start() self.consumer = consumer log.info('plugin:%s, attached => %s', self.name, self.node) @synchronized def detach(self, teardown=True): """ Detach (disconnect) from AMQP connector. :param teardown: Teardown the broker model. :type teardown: bool """ if not self.consumer: # not attached return self.consumer.shutdown() self.consumer.join() self.consumer = None log.info('plugin:%s, detached [%s]', self.name, self.node) if teardown: model = BrokerModel(self) model.teardown() def provides(self, name): """ Get whether the plugin provides the name. :param name: A class name. :type name: str :return: True if provides. :raise: bool """ return self.dispatcher.provides(name) def dispatch(self, request): """ Dispatch (invoke) the specified RMI request. :param request: An RMI request :type request: gofer.Document :return: The RMI returned. """ dispatcher = self.dispatcher call = Document(request.request) if not self.provides(call.classname): for plugin in Plugin.all(): if plugin == self: continue if not plugin.provides(call.classname): # not provided continue valid = set() valid.add('*') valid.add(plugin.name) if not valid.intersection(self.forward): # (forwarding) not approved continue valid = set() valid.add('*') valid.add(self.name) if not valid.intersection(plugin.accept): # (accept) not approved continue dispatcher = plugin.dispatcher break return dispatcher.dispatch(request) @synchronized def load(self): """ Load the plugin. """ self.delegate.loaded() path = self.cfg.messaging.authenticator if not path: # not configured return path = path.split('.') mod = '.'.join(path[:-1]) mod = __import__(mod, {}, {}, [path[-1]]) self.authenticator = mod.Authenticator() @synchronized def unload(self): """ Unload the plugin. - Detach. - Delete the plugin. - Abort scheduled requests. - Plugin shutdown. - Purge pending requests. """ Plugin.delete(self) self.shutdown(hard=True) self.delegate.unloaded() self.scheduler.pending.delete() log.info('plugin:%s, unloaded', self.name) @synchronized def reload(self): """ Reload the plugin. - Detach. - Delete the plugin. - Abort scheduled requests. - Plugin shutdown. - Reload plugin. - Reschedule pending work to reloaded plugin. """ Plugin.delete(self) scheduled = self.shutdown(teardown=False) self.delegate.unloaded() plugin = PluginLoader.load(self.path) if plugin: for call in scheduled: if isinstance(call.fn, Task): task = call.fn task.transaction.plugin = plugin plugin.pool.queue.put(call) plugin.start() log.info('plugin:%s, reloaded', self.name) return plugin