class Builtin(object):
"""
The builtin pseudo-plugin.
"""
def __init__(self, plugin):
"""
:param plugin: A real plugin.
:type plugin: gofer.agent.plugin.Plugin
"""
self.pool = ThreadPool(3)
self.dispatcher = Dispatcher()
self.dispatcher += [Admin(plugin.container)]
self.plugin = plugin
self.latency = 0
@property
def url(self):
return self.plugin.url
@property
def authenticator(self):
return self.plugin.authenticator
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
return self.dispatcher.dispatch(request)
def shutdown(self):
"""
Shutdown the plugin.
"""
self.pool.shutdown()
class Plugin(object):
"""
Represents a plugin.
:ivar descriptor: descriptor.
:type descriptor: PluginDescriptor
:param path: The descriptor path.
:type path: str
:ivar pool: The main thread pool.
:type pool: ThreadPool
:ivar impl: The plugin implementation.
:ivar impl: module
:ivar actions: List of: gofer.action.Action.
:type actions: list
:ivar dispatcher: The RMI dispatcher.
:type dispatcher: Dispatcher
:ivar whiteboard: The plugin whiteboard.
:type whiteboard: Whiteboard
:ivar authenticator: The plugin message authenticator.
:type authenticator: gofer.messaging.auth.Authenticator
:ivar consumer: An AMQP request consumer.
:type consumer: gofer.rmi.consumer.RequestConsumer.
"""
container = Container()
@staticmethod
def add(plugin, *names):
"""
Add the plugin.
:param plugin: The plugin to add.
:type plugin: Plugin
:return: The added plugin
:rtype: Plugin
"""
Plugin.container.add(plugin, *names)
@staticmethod
def delete(plugin):
"""
Delete the plugin.
:param plugin: The plugin to delete.
:type plugin: Plugin
"""
Plugin.container.delete(plugin)
if not plugin.impl:
# not loaded
return
mod = plugin.impl.__name__
del sys.modules[mod]
@staticmethod
def find(name):
"""
Find a plugin by name or path.
:param name: A plugin name
:type name: str
:return: The plugin when found.
:rtype: Plugin
"""
return Plugin.container.find(name)
@staticmethod
def all():
"""
Get a unique list of loaded plugins.
:return: A list of plugins
:rtype: list
"""
return Plugin.container.all()
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
@property
def name(self):
return self.cfg.main.name
@property
def cfg(self):
return self.descriptor
@property
def uuid(self):
return self.cfg.messaging.uuid
@property
def url(self):
return self.cfg.messaging.url
@property
def enabled(self):
return get_bool(self.cfg.main.enabled)
@property
def connector(self):
return Connector(self.url)
@property
def node(self):
model = BrokerModel(self)
return model.node
@property
def forward(self):
_list = self.cfg.main.forward
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def accept(self):
_list = self.cfg.main.accept
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def is_started(self):
return self.scheduler.isAlive()
@synchronized
def start(self):
"""
Start the plugin.
- attach
- start scheduler
"""
if self.is_started:
# already started
return
self.attach()
self.scheduler.start()
@synchronized
def shutdown(self, teardown=True):
"""
Shutdown the plugin.
- detach
- shutdown the thread pool.
- shutdown the scheduler.
:param teardown: Teardown the broker model.
:type teardown: bool
:return: List of pending requests.
:rtype: list
"""
if not self.is_started:
# not started
return []
self.detach(teardown)
pending = self.pool.shutdown()
self.scheduler.shutdown()
self.scheduler.join()
return pending
@synchronized
def refresh(self):
"""
Refresh the AMQP configurations using the plugin configuration.
"""
connector = Connector(self.url)
messaging = self.cfg.messaging
connector.heartbeat = get_integer(messaging.heartbeat)
connector.ssl.ca_certificate = messaging.cacert
connector.ssl.client_key = messaging.clientkey
connector.ssl.client_certificate = messaging.clientcert
connector.ssl.host_validation = messaging.host_validation
connector.add()
@attach
@synchronized
def attach(self):
"""
Attach (connect) to AMQP connector using the specified uuid.
"""
self.detach(False)
self.refresh()
model = BrokerModel(self)
model.setup()
node = Node(model.queue)
consumer = RequestConsumer(node, self)
consumer.authenticator = self.authenticator
consumer.start()
self.consumer = consumer
log.info('plugin:%s, attached => %s', self.name, self.node)
@synchronized
def detach(self, teardown=True):
"""
Detach (disconnect) from AMQP connector.
:param teardown: Teardown the broker model.
:type teardown: bool
"""
if not self.consumer:
# not attached
return
self.consumer.shutdown()
self.consumer.join()
self.consumer = None
log.info('plugin:%s, detached [%s]', self.name, self.node)
if teardown:
model = BrokerModel(self)
model.teardown()
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
dispatcher = self.dispatcher
call = Document(request.request)
if not self.provides(call.classname):
for plugin in Plugin.all():
if plugin == self:
continue
if not plugin.provides(call.classname):
# not provided
continue
valid = set()
valid.add('*')
valid.add(plugin.name)
if not valid.intersection(self.forward):
# (forwarding) not approved
continue
valid = set()
valid.add('*')
valid.add(self.name)
if not valid.intersection(plugin.accept):
# (accept) not approved
continue
dispatcher = plugin.dispatcher
break
return dispatcher.dispatch(request)
@synchronized
def load(self):
"""
Load the plugin.
"""
self.delegate.loaded()
path = self.cfg.messaging.authenticator
if not path:
# not configured
return
path = path.split('.')
mod = '.'.join(path[:-1])
mod = __import__(mod, {}, {}, [path[-1]])
self.authenticator = mod.Authenticator()
@synchronized
def unload(self):
"""
Unload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Purge pending requests.
"""
Plugin.delete(self)
self.shutdown()
self.delegate.unloaded()
self.scheduler.pending.delete()
log.info('plugin:%s, unloaded', self.name)
@synchronized
def reload(self):
"""
Reload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Reload plugin.
- Reschedule pending work to reloaded plugin.
"""
Plugin.delete(self)
scheduled = self.shutdown(False)
self.delegate.unloaded()
plugin = PluginLoader.load(self.path)
if plugin:
for call in scheduled:
if isinstance(call.fn, Task):
task = call.fn
task.transaction.plugin = plugin
plugin.pool.schedule(call)
plugin.start()
log.info('plugin:%s, reloaded', self.name)
return plugin
class Plugin(object):
"""
Represents a plugin.
@ivar name: The plugin name.
@type name: str
@ivar synonyms: The plugin synonyms.
@type synonyms: list
@ivar descriptor: The plugin descriptor.
@type descriptor: str
@cvar plugins: The dict of loaded plugins.
@type plugins: dict
"""
plugins = {}
@classmethod
def add(cls, plugin):
"""
Add the plugin.
@param plugin: The plugin to add.
@type plugin: L{Plugin}
@return: The added plugin
@rtype: L{Plugin}
"""
cls.plugins[plugin.name] = plugin
for syn in plugin.synonyms:
if syn == plugin.name:
continue
cls.plugins[syn] = plugin
return plugin
@classmethod
def delete(cls, plugin):
"""
Delete the plugin.
@param plugin: The plugin to delete.
@type plugin: L{Plugin}
"""
for k,v in cls.plugins.items():
if v == plugin:
del cls.plugins[k]
return plugin
@classmethod
def find(cls, name):
"""
Find a plugin by name or synonym.
@param name: A plugin name or synonym.
@type name: str
@return: The plugin when found.
@rtype: L{Plugin}
"""
return cls.plugins.get(name)
@classmethod
def all(cls):
"""
Get a unique list of loaded plugins.
@return: A list of plugins
@rtype: list
"""
unique = []
for p in cls.plugins.values():
if p in unique:
continue
unique.append(p)
return unique
def __init__(self, name, descriptor, synonyms=[]):
"""
@param name: The plugin name.
@type name: str
@param descriptor: The plugin descriptor.
@type descriptor: str
@param synonyms: The plugin synonyms.
@type synonyms: list
"""
self.name = name
self.descriptor = descriptor
self.synonyms = []
for syn in synonyms:
if syn == name:
continue
self.synonyms.append(syn)
self.__mutex = RLock()
self.__pool = None
self.impl = None
self.actions = []
self.dispatcher = Dispatcher([])
self.whiteboard = Whiteboard()
self.consumer = None
def names(self):
"""
Get I{all} the names by which the plugin can be found.
@return: A list of name and synonyms.
@rtype: list
"""
names = [self.name]
names += self.synonyms
return names
def enabled(self):
"""
Get whether the plugin is enabled.
@return: True if enabled.
@rtype: bool
"""
cfg = self.cfg()
try:
return int(nvl(cfg.main.enabled, 0))
except:
return 0
def getuuid(self):
"""
Get the plugin's messaging UUID.
@return: The plugin's messaging UUID.
@rtype: str
"""
self.__lock()
try:
cfg = self.cfg()
return nvl(cfg.messaging.uuid)
finally:
self.__unlock()
def geturl(self):
"""
Get the broker URL
@return: The broker URL
@rtype: str
"""
main = Config()
cfg = self.cfg()
return nvl(cfg.messaging.url,
nvl(main.messaging.url))
def getbroker(self):
"""
Get the amqp broker for this plugin. Each plugin can
connect to a different broker.
@return: The broker if configured.
@rtype: L{Broker}
"""
cfg = self.cfg()
main = Config()
broker = Broker(self.geturl())
broker.cacert = \
nvl(cfg.messaging.cacert,
nvl(main.messaging.cacert))
broker.clientcert = \
nvl(cfg.messaging.clientcert,
nvl(main.messaging.clientcert))
log.info('broker (qpid) configured: %s', broker)
return broker
def getpool(self):
"""
Get the plugin's thread pool.
@return: ThreadPool.
"""
if self.__pool is None:
n = self.nthreads()
self.__pool = ThreadPool(1, n, duplex=False)
return self.__pool
def setuuid(self, uuid, save=False):
"""
Set the plugin's UUID.
@param uuid: The new UUID.
@type uuid: str
@param save: Save to plugin descriptor.
@type save: bool
"""
self.__lock()
try:
cfg = self.cfg()
if uuid:
cfg.messaging.uuid = uuid
else:
delattr(cfg.messaging, 'uuid')
if save:
cfg.write()
finally:
self.__unlock()
def seturl(self, url, save=False):
"""
Set the plugin's URL.
@param url: The new URL.
@type url: str
@param save: Save to plugin descriptor.
@type save: bool
"""
self.__lock()
try:
cfg = self.cfg()
if url:
cfg.messaging.url = url
else:
delattr(cfg.messaging, 'url')
if save:
cfg.write()
finally:
self.__unlock()
def nthreads(self):
"""
Get the number of theads in the plugin's pool.
@return: number of theads.
@rtype: int
"""
main = Config()
cfg = self.cfg()
value = \
nvl(cfg.messaging.threads,
nvl(main.messaging.threads, 1))
value = int(value)
assert(value >= 1)
return value
def attach(self, uuid=None):
"""
Attach (connect) to AMQP broker using the specified uuid.
@param uuid: The (optional) messaging UUID.
@type uuid: str
"""
cfg = self.cfg()
if not uuid:
uuid = self.getuuid()
broker = self.getbroker()
url = broker.url
queue = Queue(uuid)
consumer = RequestConsumer(queue, url=url)
consumer.start()
self.consumer = consumer
def detach(self):
"""
Detach (disconnect) from AMQP broker (if connected).
"""
if self.consumer:
self.consumer.close()
self.consumer = None
return True
else:
return False
def cfg(self):
"""
Get the plugin descriptor.
@return: The plugin descriptor
@rtype: L{Config}
"""
return self.descriptor
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
@param request: An RMI request
@type request: L{Envelope}
@return: The RMI returned.
"""
return self.dispatcher.dispatch(request)
def provides(self, name):
"""
Get whether a plugin provides the specified class.
@param name: A class (or module) name.
@type name: str
@return: True if provides.
@rtype: bool
"""
return self.dispatcher.provides(name)
def export(self, name):
"""
Export an object defined in the plugin (module).
The name must reference a class or function object.
@param name: A name (class|function)
@type name: str
@return: The named item.
@rtype: (class|function)
@raise NameError: when not found
"""
try:
obj = getattr(self.impl, name)
valid = inspect.isclass(obj) or inspect.isfunction(obj)
if valid:
return obj
raise TypeError, '(%s) must be class|function' % name
except AttributeError:
raise NameError(name)
def __lock(self):
self.__mutex.acquire()
def __unlock(self):
self.__mutex.release()
class Plugin(object):
"""
Represents a plugin.
:ivar descriptor: descriptor.
:type descriptor: PluginDescriptor
:param path: The descriptor path.
:type path: str
:ivar pool: The main thread pool.
:type pool: ThreadPool
:ivar impl: The plugin implementation.
:ivar impl: module
:ivar actions: List of: gofer.action.Action.
:type actions: list
:ivar dispatcher: The RMI dispatcher.
:type dispatcher: Dispatcher
:ivar whiteboard: The plugin whiteboard.
:type whiteboard: Whiteboard
:ivar authenticator: The plugin message authenticator.
:type authenticator: gofer.messaging.auth.Authenticator
:ivar consumer: An AMQP request consumer.
:type consumer: gofer.rmi.consumer.RequestConsumer.
"""
container = Container()
@staticmethod
def add(plugin, *names):
"""
Add the plugin.
:param plugin: The plugin to add.
:type plugin: Plugin
:return: The added plugin
:rtype: Plugin
"""
Plugin.container.add(plugin, *names)
@staticmethod
def delete(plugin):
"""
Delete the plugin.
:param plugin: The plugin to delete.
:type plugin: Plugin
"""
Plugin.container.delete(plugin)
if not plugin.impl:
# not loaded
return
mod = plugin.impl.__name__
del sys.modules[mod]
@staticmethod
def find(name):
"""
Find a plugin by name or path.
:param name: A plugin name
:type name: str
:return: The plugin when found.
:rtype: Plugin
"""
return Plugin.container.find(name)
@staticmethod
def all():
"""
Get a unique list of loaded plugins.
:return: A list of plugins
:rtype: list
"""
return Plugin.container.all()
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
@property
def name(self):
return self.cfg.main.name
@property
def cfg(self):
return self.descriptor
@property
def uuid(self):
return self.cfg.messaging.uuid
@property
def url(self):
return self.cfg.messaging.url
@property
def enabled(self):
return get_bool(self.cfg.main.enabled)
@property
def connector(self):
return Connector(self.url)
@property
def node(self):
model = BrokerModel(self)
return model.node
@property
def forward(self):
_list = self.cfg.main.forward
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def accept(self):
_list = self.cfg.main.accept
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def is_started(self):
return self.scheduler.isAlive()
@property
def latency(self):
return float(self.cfg.main.latency)
@synchronized
def start(self):
"""
Start the plugin.
- attach
- start scheduler
"""
if self.is_started:
# already started
return
self.attach()
self.scheduler.start()
@synchronized
def shutdown(self, teardown=True):
"""
Shutdown the plugin.
- detach
- shutdown the thread pool.
- shutdown the scheduler.
:param teardown: Teardown the broker model.
:type teardown: bool
:return: List of pending requests.
:rtype: list
"""
if not self.is_started:
# not started
return []
self.detach(teardown)
pending = self.pool.shutdown()
self.scheduler.shutdown()
self.scheduler.join()
return pending
@synchronized
def refresh(self):
"""
Refresh the AMQP configurations using the plugin configuration.
"""
connector = Connector(self.url)
messaging = self.cfg.messaging
connector.heartbeat = get_integer(messaging.heartbeat)
connector.ssl.ca_certificate = messaging.cacert
connector.ssl.client_key = messaging.clientkey
connector.ssl.client_certificate = messaging.clientcert
connector.ssl.host_validation = messaging.host_validation
connector.add()
@attach
@synchronized
def attach(self):
"""
Attach (connect) to AMQP connector using the specified uuid.
"""
self.detach(False)
self.refresh()
model = BrokerModel(self)
model.setup()
node = Node(model.queue)
consumer = RequestConsumer(node, self)
consumer.authenticator = self.authenticator
consumer.start()
self.consumer = consumer
log.info('plugin:%s, attached => %s', self.name, self.node)
@synchronized
def detach(self, teardown=True):
"""
Detach (disconnect) from AMQP connector.
:param teardown: Teardown the broker model.
:type teardown: bool
"""
if not self.consumer:
# not attached
return
self.consumer.shutdown()
self.consumer.join()
self.consumer = None
log.info('plugin:%s, detached [%s]', self.name, self.node)
if teardown:
model = BrokerModel(self)
model.teardown()
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
dispatcher = self.dispatcher
call = Document(request.request)
if not self.provides(call.classname):
for plugin in Plugin.all():
if plugin == self:
continue
if not plugin.provides(call.classname):
# not provided
continue
valid = set()
valid.add('*')
valid.add(plugin.name)
if not valid.intersection(self.forward):
# (forwarding) not approved
continue
valid = set()
valid.add('*')
valid.add(self.name)
if not valid.intersection(plugin.accept):
# (accept) not approved
continue
dispatcher = plugin.dispatcher
break
return dispatcher.dispatch(request)
@synchronized
def load(self):
"""
Load the plugin.
"""
self.delegate.loaded()
path = self.cfg.messaging.authenticator
if not path:
# not configured
return
path = path.split('.')
mod = '.'.join(path[:-1])
mod = __import__(mod, {}, {}, [path[-1]])
self.authenticator = mod.Authenticator()
@synchronized
def unload(self):
"""
Unload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Purge pending requests.
"""
Plugin.delete(self)
self.shutdown()
self.delegate.unloaded()
self.scheduler.pending.delete()
log.info('plugin:%s, unloaded', self.name)
@synchronized
def reload(self):
"""
Reload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Reload plugin.
- Reschedule pending work to reloaded plugin.
"""
Plugin.delete(self)
scheduled = self.shutdown(False)
self.delegate.unloaded()
plugin = PluginLoader.load(self.path)
if plugin:
for call in scheduled:
if isinstance(call.fn, Task):
task = call.fn
task.transaction.plugin = plugin
plugin.pool.schedule(call)
plugin.start()
log.info('plugin:%s, reloaded', self.name)
return plugin