def test(calls=100):
results = []
pool = ThreadPool(9)
for n in range(calls):
pool.run(fn1, n, results)
while len(results) < calls:
sleep(1)
print 'shutdown pool...'
pool.shutdown()
def __init__(self, plugin):
"""
:param plugin: A real plugin.
:type plugin: gofer.agent.plugin.Plugin
"""
Admin.container = plugin.container
self.pool = ThreadPool(3)
self.dispatcher = self._dispatcher()
self.plugin = plugin
self.latency = 0
class Builtin(object):
"""
The builtin pseudo-plugin.
"""
def __init__(self, plugin):
"""
:param plugin: A real plugin.
:type plugin: gofer.agent.plugin.Plugin
"""
self.pool = ThreadPool(3)
self.dispatcher = Dispatcher()
self.dispatcher += [Admin(plugin.container)]
self.plugin = plugin
self.latency = 0
@property
def url(self):
return self.plugin.url
@property
def authenticator(self):
return self.plugin.authenticator
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
return self.dispatcher.dispatch(request)
def shutdown(self):
"""
Shutdown the plugin.
"""
self.pool.shutdown()
def test(calls=100):
results = []
pool = ThreadPool(9)
pool.start()
for n in range(calls):
pool.run(fn1, n, results)
while len(results) < calls:
sleep(1)
print('shutdown pool...')
pool.shutdown()
def __init__(self, plugin):
"""
:param plugin: A real plugin.
:type plugin: gofer.agent.plugin.Plugin
"""
self.pool = ThreadPool(3)
self.dispatcher = Dispatcher()
self.dispatcher += [Admin(plugin.container)]
self.plugin = plugin
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
class Plugin(object):
"""
Represents a plugin.
:ivar descriptor: descriptor.
:type descriptor: PluginDescriptor
:param path: The descriptor path.
:type path: str
:ivar pool: The main thread pool.
:type pool: ThreadPool
:ivar impl: The plugin implementation.
:ivar impl: module
:ivar actions: List of: gofer.action.Action.
:type actions: list
:ivar dispatcher: The RMI dispatcher.
:type dispatcher: Dispatcher
:ivar whiteboard: The plugin whiteboard.
:type whiteboard: Whiteboard
:ivar authenticator: The plugin message authenticator.
:type authenticator: gofer.messaging.auth.Authenticator
:ivar consumer: An AMQP request consumer.
:type consumer: gofer.rmi.consumer.RequestConsumer.
"""
container = Container()
@staticmethod
def add(plugin, *names):
"""
Add the plugin.
:param plugin: The plugin to add.
:type plugin: Plugin
:return: The added plugin
:rtype: Plugin
"""
Plugin.container.add(plugin, *names)
@staticmethod
def delete(plugin):
"""
Delete the plugin.
:param plugin: The plugin to delete.
:type plugin: Plugin
"""
Plugin.container.delete(plugin)
if not plugin.impl:
# not loaded
return
mod = plugin.impl.__name__
del sys.modules[mod]
@staticmethod
def find(name):
"""
Find a plugin by name or path.
:param name: A plugin name
:type name: str
:return: The plugin when found.
:rtype: Plugin
"""
return Plugin.container.find(name)
@staticmethod
def all():
"""
Get a unique list of loaded plugins.
:return: A list of plugins
:rtype: list
"""
return Plugin.container.all()
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
@property
def name(self):
return self.cfg.main.name
@property
def cfg(self):
return self.descriptor
@property
def uuid(self):
return self.cfg.messaging.uuid
@property
def url(self):
return self.cfg.messaging.url
@property
def enabled(self):
return get_bool(self.cfg.main.enabled)
@property
def connector(self):
return Connector(self.url)
@property
def node(self):
model = BrokerModel(self)
return model.node
@property
def forward(self):
_list = self.cfg.main.forward
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def accept(self):
_list = self.cfg.main.accept
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def is_started(self):
return self.scheduler.isAlive()
@synchronized
def start(self):
"""
Start the plugin.
- attach
- start scheduler
"""
if self.is_started:
# already started
return
self.attach()
self.scheduler.start()
@synchronized
def shutdown(self, teardown=True):
"""
Shutdown the plugin.
- detach
- shutdown the thread pool.
- shutdown the scheduler.
:param teardown: Teardown the broker model.
:type teardown: bool
:return: List of pending requests.
:rtype: list
"""
if not self.is_started:
# not started
return []
self.detach(teardown)
pending = self.pool.shutdown()
self.scheduler.shutdown()
self.scheduler.join()
return pending
@synchronized
def refresh(self):
"""
Refresh the AMQP configurations using the plugin configuration.
"""
connector = Connector(self.url)
messaging = self.cfg.messaging
connector.heartbeat = get_integer(messaging.heartbeat)
connector.ssl.ca_certificate = messaging.cacert
connector.ssl.client_key = messaging.clientkey
connector.ssl.client_certificate = messaging.clientcert
connector.ssl.host_validation = messaging.host_validation
connector.add()
@attach
@synchronized
def attach(self):
"""
Attach (connect) to AMQP connector using the specified uuid.
"""
self.detach(False)
self.refresh()
model = BrokerModel(self)
model.setup()
node = Node(model.queue)
consumer = RequestConsumer(node, self)
consumer.authenticator = self.authenticator
consumer.start()
self.consumer = consumer
log.info('plugin:%s, attached => %s', self.name, self.node)
@synchronized
def detach(self, teardown=True):
"""
Detach (disconnect) from AMQP connector.
:param teardown: Teardown the broker model.
:type teardown: bool
"""
if not self.consumer:
# not attached
return
self.consumer.shutdown()
self.consumer.join()
self.consumer = None
log.info('plugin:%s, detached [%s]', self.name, self.node)
if teardown:
model = BrokerModel(self)
model.teardown()
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
dispatcher = self.dispatcher
call = Document(request.request)
if not self.provides(call.classname):
for plugin in Plugin.all():
if plugin == self:
continue
if not plugin.provides(call.classname):
# not provided
continue
valid = set()
valid.add('*')
valid.add(plugin.name)
if not valid.intersection(self.forward):
# (forwarding) not approved
continue
valid = set()
valid.add('*')
valid.add(self.name)
if not valid.intersection(plugin.accept):
# (accept) not approved
continue
dispatcher = plugin.dispatcher
break
return dispatcher.dispatch(request)
@synchronized
def load(self):
"""
Load the plugin.
"""
self.delegate.loaded()
path = self.cfg.messaging.authenticator
if not path:
# not configured
return
path = path.split('.')
mod = '.'.join(path[:-1])
mod = __import__(mod, {}, {}, [path[-1]])
self.authenticator = mod.Authenticator()
@synchronized
def unload(self):
"""
Unload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Purge pending requests.
"""
Plugin.delete(self)
self.shutdown()
self.delegate.unloaded()
self.scheduler.pending.delete()
log.info('plugin:%s, unloaded', self.name)
@synchronized
def reload(self):
"""
Reload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Reload plugin.
- Reschedule pending work to reloaded plugin.
"""
Plugin.delete(self)
scheduled = self.shutdown(False)
self.delegate.unloaded()
plugin = PluginLoader.load(self.path)
if plugin:
for call in scheduled:
if isinstance(call.fn, Task):
task = call.fn
task.transaction.plugin = plugin
plugin.pool.schedule(call)
plugin.start()
log.info('plugin:%s, reloaded', self.name)
return plugin
class Builtin(object):
"""
The builtin pseudo-plugin.
"""
@staticmethod
def _dispatcher():
collator = collation.Collator()
classes, _ = collator({f[1]: {} for f in inspection.methods(Admin)})
dispatcher = Dispatcher()
dispatcher += classes
return dispatcher
def __init__(self, plugin):
"""
:param plugin: A real plugin.
:type plugin: gofer.agent.plugin.Plugin
"""
Admin.container = plugin.container
self.pool = ThreadPool(3)
self.dispatcher = self._dispatcher()
self.plugin = plugin
self.latency = 0
@property
def url(self):
return self.plugin.url
@property
def authenticator(self):
return self.plugin.authenticator
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
return self.dispatcher.dispatch(request)
def start(self):
"""
Start the plugin.
"""
self.pool.start()
def shutdown(self):
"""
Shutdown the plugin.
"""
self.pool.shutdown()
class Plugin(object):
"""
Represents a plugin.
:ivar descriptor: descriptor.
:type descriptor: PluginDescriptor
:param path: The descriptor path.
:type path: str
:ivar pool: The main thread pool.
:type pool: ThreadPool
:ivar impl: The plugin implementation.
:ivar impl: module
:ivar actions: List of: gofer.action.Action.
:type actions: list
:ivar dispatcher: The RMI dispatcher.
:type dispatcher: Dispatcher
:ivar whiteboard: The plugin whiteboard.
:type whiteboard: Whiteboard
:ivar authenticator: The plugin message authenticator.
:type authenticator: gofer.messaging.auth.Authenticator
:ivar consumer: An AMQP request consumer.
:type consumer: gofer.rmi.consumer.RequestConsumer.
"""
container = Container()
@staticmethod
def add(plugin, *names):
"""
Add the plugin.
:param plugin: The plugin to add.
:type plugin: Plugin
:return: The added plugin
:rtype: Plugin
"""
Plugin.container.add(plugin, *names)
@staticmethod
def delete(plugin):
"""
Delete the plugin.
:param plugin: The plugin to delete.
:type plugin: Plugin
"""
Plugin.container.delete(plugin)
if not plugin.impl:
# not loaded
return
mod = plugin.impl.__name__
del sys.modules[mod]
@staticmethod
def find(name):
"""
Find a plugin by name or path.
:param name: A plugin name
:type name: str
:return: The plugin when found.
:rtype: Plugin
"""
return Plugin.container.find(name)
@staticmethod
def all():
"""
Get a unique list of loaded plugins.
:return: A list of plugins
:rtype: list
"""
return Plugin.container.all()
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
@property
def name(self):
return self.cfg.main.name
@property
def cfg(self):
return self.descriptor
@property
def uuid(self):
return self.cfg.messaging.uuid
@property
def url(self):
return self.cfg.messaging.url
@property
def enabled(self):
return get_bool(self.cfg.main.enabled)
@property
def connector(self):
return Connector(self.url)
@property
def node(self):
model = BrokerModel(self)
return model.node
@property
def forward(self):
_list = self.cfg.main.forward
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def accept(self):
_list = self.cfg.main.accept
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def is_started(self):
return self.scheduler.isAlive()
@property
def latency(self):
return float(self.cfg.main.latency)
@synchronized
def start(self):
"""
Start the plugin.
- attach
- start scheduler
"""
if self.is_started:
# already started
return
self.attach()
self.scheduler.start()
@synchronized
def shutdown(self, teardown=True):
"""
Shutdown the plugin.
- detach
- shutdown the thread pool.
- shutdown the scheduler.
:param teardown: Teardown the broker model.
:type teardown: bool
:return: List of pending requests.
:rtype: list
"""
if not self.is_started:
# not started
return []
self.detach(teardown)
pending = self.pool.shutdown()
self.scheduler.shutdown()
self.scheduler.join()
return pending
@synchronized
def refresh(self):
"""
Refresh the AMQP configurations using the plugin configuration.
"""
connector = Connector(self.url)
messaging = self.cfg.messaging
connector.heartbeat = get_integer(messaging.heartbeat)
connector.ssl.ca_certificate = messaging.cacert
connector.ssl.client_key = messaging.clientkey
connector.ssl.client_certificate = messaging.clientcert
connector.ssl.host_validation = messaging.host_validation
connector.add()
@attach
@synchronized
def attach(self):
"""
Attach (connect) to AMQP connector using the specified uuid.
"""
self.detach(False)
self.refresh()
model = BrokerModel(self)
model.setup()
node = Node(model.queue)
consumer = RequestConsumer(node, self)
consumer.authenticator = self.authenticator
consumer.start()
self.consumer = consumer
log.info('plugin:%s, attached => %s', self.name, self.node)
@synchronized
def detach(self, teardown=True):
"""
Detach (disconnect) from AMQP connector.
:param teardown: Teardown the broker model.
:type teardown: bool
"""
if not self.consumer:
# not attached
return
self.consumer.shutdown()
self.consumer.join()
self.consumer = None
log.info('plugin:%s, detached [%s]', self.name, self.node)
if teardown:
model = BrokerModel(self)
model.teardown()
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
dispatcher = self.dispatcher
call = Document(request.request)
if not self.provides(call.classname):
for plugin in Plugin.all():
if plugin == self:
continue
if not plugin.provides(call.classname):
# not provided
continue
valid = set()
valid.add('*')
valid.add(plugin.name)
if not valid.intersection(self.forward):
# (forwarding) not approved
continue
valid = set()
valid.add('*')
valid.add(self.name)
if not valid.intersection(plugin.accept):
# (accept) not approved
continue
dispatcher = plugin.dispatcher
break
return dispatcher.dispatch(request)
@synchronized
def load(self):
"""
Load the plugin.
"""
self.delegate.loaded()
path = self.cfg.messaging.authenticator
if not path:
# not configured
return
path = path.split('.')
mod = '.'.join(path[:-1])
mod = __import__(mod, {}, {}, [path[-1]])
self.authenticator = mod.Authenticator()
@synchronized
def unload(self):
"""
Unload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Purge pending requests.
"""
Plugin.delete(self)
self.shutdown()
self.delegate.unloaded()
self.scheduler.pending.delete()
log.info('plugin:%s, unloaded', self.name)
@synchronized
def reload(self):
"""
Reload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Reload plugin.
- Reschedule pending work to reloaded plugin.
"""
Plugin.delete(self)
scheduled = self.shutdown(False)
self.delegate.unloaded()
plugin = PluginLoader.load(self.path)
if plugin:
for call in scheduled:
if isinstance(call.fn, Task):
task = call.fn
task.transaction.plugin = plugin
plugin.pool.schedule(call)
plugin.start()
log.info('plugin:%s, reloaded', self.name)
return plugin
