def __startScheduler(self, plugins):
"""
Start the RMI scheduler.
@param plugins: A list of loaded plugins.
@type plugins: list
@return: The started scheduler thread.
@rtype: L{Scheduler}
"""
scheduler = Scheduler(plugins)
scheduler.start()
return scheduler
def test_run_raised(self, aborted, select_plugin, pending):
plugin = Mock()
sn = 1234
pending.return_value.get.return_value = Document(sn=sn)
select_plugin.side_effect = ValueError
aborted.side_effect = [False, True]
# test
scheduler = Scheduler(plugin)
scheduler.run()
# validation
pending.return_value.commit.assert_called_once_with(sn)
def test_run(self, builtin, pending, task, select_plugin, tx, aborted):
_builtin = Mock(name='builtin')
plugin = Mock(name='plugin')
task_list = [
Mock(name='task-1'),
Mock(name='task-2'),
]
tx_list = [
Mock(name='tx-1'),
Mock(name='tx-2'),
]
request_list = [
Document(sn=1),
Document(sn=2),
]
task.side_effect = task_list
tx.side_effect = tx_list
aborted.side_effect = [False, False, True]
pending.return_value.get.side_effect = request_list
builtin.return_value = _builtin
builtin.return_value.provides.side_effect = [True, False]
select_plugin.side_effect = [_builtin, plugin]
# test
scheduler = Scheduler(plugin)
scheduler.run()
# validation
_builtin.pool.run.assert_called_once_with(task_list[0])
plugin.pool.run.assert_called_once_with(task_list[1])
self.assertEqual(
select_plugin.call_args_list,
[
((request_list[0],), {}),
((request_list[1],), {}),
])
self.assertEqual(
tx.call_args_list,
[
((_builtin, pending.return_value, request_list[0]), {}),
((plugin, pending.return_value, request_list[1]), {})
])
self.assertEqual(
task.call_args_list,
[
((tx_list[0],), {}),
((tx_list[1],), {}),
])
def test_run(self, builtin, pending, task, select_plugin, tx, aborted):
builtin.return_value = Mock(name='builtin')
plugin = Mock(name='plugin')
task_list = [
Mock(name='task-1'),
Mock(name='task-2'),
]
tx_list = [
Mock(name='tx-1'),
Mock(name='tx-2'),
]
request_list = [
Document(sn=1),
Document(sn=2),
]
task.side_effect = task_list
tx.side_effect = tx_list
aborted.side_effect = [False, False, True]
pending.return_value.get.side_effect = request_list
builtin.return_value.provides.side_effect = [True, False]
select_plugin.side_effect = [builtin.return_value, plugin]
# test
scheduler = Scheduler(plugin)
scheduler.run()
# validation
builtin.return_value.pool.run.assert_called_once_with(task_list[0])
plugin.pool.run.assert_called_once_with(task_list[1])
self.assertEqual(
select_plugin.call_args_list,
[
((request_list[0],), {}),
((request_list[1],), {}),
])
self.assertEqual(
tx.call_args_list,
[
((builtin.return_value, pending.return_value, request_list[0]), {}),
((plugin, pending.return_value, request_list[1]), {})
])
self.assertEqual(
task.call_args_list,
[
((tx_list[0],), {}),
((tx_list[1],), {}),
])
def test_init(self, builtin, pending, set_daemon):
plugin = Mock()
scheduler = Scheduler(plugin)
pending.assert_called_once_with(plugin.name)
builtin.assert_called_once_with(plugin)
set_daemon.assert_called_with(True)
self.assertEqual(scheduler.plugin, plugin)
self.assertEqual(scheduler.pending, pending.return_value)
self.assertEqual(scheduler.builtin, builtin.return_value)
def test_select_plugin(self, builtin):
plugin = Mock()
request = Document(request={'classname': 'A'})
scheduler = Scheduler(plugin)
# find builtin
builtin.return_value.provides.return_value = True
selected = scheduler.select_plugin(request)
self.assertEqual(selected, builtin.return_value)
# find plugin
builtin.return_value.provides.return_value = False
selected = scheduler.select_plugin(request)
self.assertEqual(selected, plugin)
self.assertEqual(
builtin.return_value.provides.call_args_list,
[
(('A',), {}),
(('A',), {})
])
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
class Plugin(object):
"""
Represents a plugin.
:ivar descriptor: descriptor.
:type descriptor: PluginDescriptor
:param path: The descriptor path.
:type path: str
:ivar pool: The main thread pool.
:type pool: ThreadPool
:ivar impl: The plugin implementation.
:ivar impl: module
:ivar actions: List of: gofer.action.Action.
:type actions: list
:ivar dispatcher: The RMI dispatcher.
:type dispatcher: Dispatcher
:ivar whiteboard: The plugin whiteboard.
:type whiteboard: Whiteboard
:ivar authenticator: The plugin message authenticator.
:type authenticator: gofer.messaging.auth.Authenticator
:ivar consumer: An AMQP request consumer.
:type consumer: gofer.rmi.consumer.RequestConsumer.
"""
container = Container()
@staticmethod
def add(plugin, *names):
"""
Add the plugin.
:param plugin: The plugin to add.
:type plugin: Plugin
:return: The added plugin
:rtype: Plugin
"""
Plugin.container.add(plugin, *names)
@staticmethod
def delete(plugin):
"""
Delete the plugin.
:param plugin: The plugin to delete.
:type plugin: Plugin
"""
Plugin.container.delete(plugin)
if not plugin.impl:
# not loaded
return
mod = plugin.impl.__name__
del sys.modules[mod]
@staticmethod
def find(name):
"""
Find a plugin by name or path.
:param name: A plugin name
:type name: str
:return: The plugin when found.
:rtype: Plugin
"""
return Plugin.container.find(name)
@staticmethod
def all():
"""
Get a unique list of loaded plugins.
:return: A list of plugins
:rtype: list
"""
return Plugin.container.all()
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
@property
def name(self):
return self.cfg.main.name
@property
def cfg(self):
return self.descriptor
@property
def uuid(self):
return self.cfg.messaging.uuid
@property
def url(self):
return self.cfg.messaging.url
@property
def enabled(self):
return get_bool(self.cfg.main.enabled)
@property
def connector(self):
return Connector(self.url)
@property
def node(self):
model = BrokerModel(self)
return model.node
@property
def forward(self):
_list = self.cfg.main.forward
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def accept(self):
_list = self.cfg.main.accept
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def is_started(self):
return self.scheduler.isAlive()
@synchronized
def start(self):
"""
Start the plugin.
- attach
- start scheduler
"""
if self.is_started:
# already started
return
self.attach()
self.scheduler.start()
@synchronized
def shutdown(self, teardown=True):
"""
Shutdown the plugin.
- detach
- shutdown the thread pool.
- shutdown the scheduler.
:param teardown: Teardown the broker model.
:type teardown: bool
:return: List of pending requests.
:rtype: list
"""
if not self.is_started:
# not started
return []
self.detach(teardown)
pending = self.pool.shutdown()
self.scheduler.shutdown()
self.scheduler.join()
return pending
@synchronized
def refresh(self):
"""
Refresh the AMQP configurations using the plugin configuration.
"""
connector = Connector(self.url)
messaging = self.cfg.messaging
connector.heartbeat = get_integer(messaging.heartbeat)
connector.ssl.ca_certificate = messaging.cacert
connector.ssl.client_key = messaging.clientkey
connector.ssl.client_certificate = messaging.clientcert
connector.ssl.host_validation = messaging.host_validation
connector.add()
@attach
@synchronized
def attach(self):
"""
Attach (connect) to AMQP connector using the specified uuid.
"""
self.detach(False)
self.refresh()
model = BrokerModel(self)
model.setup()
node = Node(model.queue)
consumer = RequestConsumer(node, self)
consumer.authenticator = self.authenticator
consumer.start()
self.consumer = consumer
log.info('plugin:%s, attached => %s', self.name, self.node)
@synchronized
def detach(self, teardown=True):
"""
Detach (disconnect) from AMQP connector.
:param teardown: Teardown the broker model.
:type teardown: bool
"""
if not self.consumer:
# not attached
return
self.consumer.shutdown()
self.consumer.join()
self.consumer = None
log.info('plugin:%s, detached [%s]', self.name, self.node)
if teardown:
model = BrokerModel(self)
model.teardown()
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
dispatcher = self.dispatcher
call = Document(request.request)
if not self.provides(call.classname):
for plugin in Plugin.all():
if plugin == self:
continue
if not plugin.provides(call.classname):
# not provided
continue
valid = set()
valid.add('*')
valid.add(plugin.name)
if not valid.intersection(self.forward):
# (forwarding) not approved
continue
valid = set()
valid.add('*')
valid.add(self.name)
if not valid.intersection(plugin.accept):
# (accept) not approved
continue
dispatcher = plugin.dispatcher
break
return dispatcher.dispatch(request)
@synchronized
def load(self):
"""
Load the plugin.
"""
self.delegate.loaded()
path = self.cfg.messaging.authenticator
if not path:
# not configured
return
path = path.split('.')
mod = '.'.join(path[:-1])
mod = __import__(mod, {}, {}, [path[-1]])
self.authenticator = mod.Authenticator()
@synchronized
def unload(self):
"""
Unload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Purge pending requests.
"""
Plugin.delete(self)
self.shutdown()
self.delegate.unloaded()
self.scheduler.pending.delete()
log.info('plugin:%s, unloaded', self.name)
@synchronized
def reload(self):
"""
Reload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Reload plugin.
- Reschedule pending work to reloaded plugin.
"""
Plugin.delete(self)
scheduled = self.shutdown(False)
self.delegate.unloaded()
plugin = PluginLoader.load(self.path)
if plugin:
for call in scheduled:
if isinstance(call.fn, Task):
task = call.fn
task.transaction.plugin = plugin
plugin.pool.schedule(call)
plugin.start()
log.info('plugin:%s, reloaded', self.name)
return plugin
def test_shutdown(self, builtin, abort):
plugin = Mock()
scheduler = Scheduler(plugin)
scheduler.shutdown()
builtin.return_value.shutdown.assert_called_once_with()
abort.assert_called_once_with()
def test_add(self, pending):
plugin = Mock()
request = Mock()
scheduler = Scheduler(plugin)
scheduler.add(request)
pending.return_value.put.assert_called_once_with(request)
class Plugin(object):
"""
Represents a plugin.
:ivar descriptor: descriptor.
:type descriptor: PluginDescriptor
:param path: The descriptor path.
:type path: str
:ivar pool: The main thread pool.
:type pool: ThreadPool
:ivar impl: The plugin implementation.
:ivar impl: module
:ivar actions: List of: gofer.action.Action.
:type actions: list
:ivar dispatcher: The RMI dispatcher.
:type dispatcher: Dispatcher
:ivar whiteboard: The plugin whiteboard.
:type whiteboard: Whiteboard
:ivar authenticator: The plugin message authenticator.
:type authenticator: gofer.messaging.auth.Authenticator
:ivar consumer: An AMQP request consumer.
:type consumer: gofer.rmi.consumer.RequestConsumer.
"""
container = Container()
@staticmethod
def add(plugin, *names):
"""
Add the plugin.
:param plugin: The plugin to add.
:type plugin: Plugin
:return: The added plugin
:rtype: Plugin
"""
Plugin.container.add(plugin, *names)
@staticmethod
def delete(plugin):
"""
Delete the plugin.
:param plugin: The plugin to delete.
:type plugin: Plugin
"""
Plugin.container.delete(plugin)
if not plugin.impl:
# not loaded
return
mod = plugin.impl.__name__
del sys.modules[mod]
@staticmethod
def find(name):
"""
Find a plugin by name or path.
:param name: A plugin name
:type name: str
:return: The plugin when found.
:rtype: Plugin
"""
return Plugin.container.find(name)
@staticmethod
def all():
"""
Get a unique list of loaded plugins.
:return: A list of plugins
:rtype: list
"""
return Plugin.container.all()
def __init__(self, descriptor, path):
"""
:param descriptor: The plugin descriptor.
:type descriptor: PluginDescriptor
:param path: The plugin descriptor path.
:type path: str
"""
self.__mutex = RLock()
self.path = path
self.descriptor = descriptor
self.pool = ThreadPool(int(descriptor.main.threads or 1))
self.impl = None
self.actions = []
self.dispatcher = Dispatcher()
self.whiteboard = Whiteboard()
self.scheduler = Scheduler(self)
self.delegate = Delegate()
self.authenticator = None
self.consumer = None
@property
def name(self):
return self.cfg.main.name
@property
def cfg(self):
return self.descriptor
@property
def uuid(self):
return self.cfg.messaging.uuid
@property
def url(self):
return self.cfg.messaging.url
@property
def enabled(self):
return get_bool(self.cfg.main.enabled)
@property
def connector(self):
return Connector(self.url)
@property
def node(self):
model = BrokerModel(self)
return model.node
@property
def forward(self):
_list = self.cfg.main.forward
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def accept(self):
_list = self.cfg.main.accept
_list = [p.strip() for p in _list.split(',')]
return set(_list)
@property
def is_started(self):
return self.scheduler.isAlive()
@property
def latency(self):
return float(self.cfg.main.latency)
@synchronized
def start(self):
"""
Start the plugin.
- attach
- start scheduler
"""
if self.is_started:
# already started
return
self.attach()
self.scheduler.start()
@synchronized
def shutdown(self, teardown=True):
"""
Shutdown the plugin.
- detach
- shutdown the thread pool.
- shutdown the scheduler.
:param teardown: Teardown the broker model.
:type teardown: bool
:return: List of pending requests.
:rtype: list
"""
if not self.is_started:
# not started
return []
self.detach(teardown)
pending = self.pool.shutdown()
self.scheduler.shutdown()
self.scheduler.join()
return pending
@synchronized
def refresh(self):
"""
Refresh the AMQP configurations using the plugin configuration.
"""
connector = Connector(self.url)
messaging = self.cfg.messaging
connector.heartbeat = get_integer(messaging.heartbeat)
connector.ssl.ca_certificate = messaging.cacert
connector.ssl.client_key = messaging.clientkey
connector.ssl.client_certificate = messaging.clientcert
connector.ssl.host_validation = messaging.host_validation
connector.add()
@attach
@synchronized
def attach(self):
"""
Attach (connect) to AMQP connector using the specified uuid.
"""
self.detach(False)
self.refresh()
model = BrokerModel(self)
model.setup()
node = Node(model.queue)
consumer = RequestConsumer(node, self)
consumer.authenticator = self.authenticator
consumer.start()
self.consumer = consumer
log.info('plugin:%s, attached => %s', self.name, self.node)
@synchronized
def detach(self, teardown=True):
"""
Detach (disconnect) from AMQP connector.
:param teardown: Teardown the broker model.
:type teardown: bool
"""
if not self.consumer:
# not attached
return
self.consumer.shutdown()
self.consumer.join()
self.consumer = None
log.info('plugin:%s, detached [%s]', self.name, self.node)
if teardown:
model = BrokerModel(self)
model.teardown()
def provides(self, name):
"""
Get whether the plugin provides the name.
:param name: A class name.
:type name: str
:return: True if provides.
:raise: bool
"""
return self.dispatcher.provides(name)
def dispatch(self, request):
"""
Dispatch (invoke) the specified RMI request.
:param request: An RMI request
:type request: gofer.Document
:return: The RMI returned.
"""
dispatcher = self.dispatcher
call = Document(request.request)
if not self.provides(call.classname):
for plugin in Plugin.all():
if plugin == self:
continue
if not plugin.provides(call.classname):
# not provided
continue
valid = set()
valid.add('*')
valid.add(plugin.name)
if not valid.intersection(self.forward):
# (forwarding) not approved
continue
valid = set()
valid.add('*')
valid.add(self.name)
if not valid.intersection(plugin.accept):
# (accept) not approved
continue
dispatcher = plugin.dispatcher
break
return dispatcher.dispatch(request)
@synchronized
def load(self):
"""
Load the plugin.
"""
self.delegate.loaded()
path = self.cfg.messaging.authenticator
if not path:
# not configured
return
path = path.split('.')
mod = '.'.join(path[:-1])
mod = __import__(mod, {}, {}, [path[-1]])
self.authenticator = mod.Authenticator()
@synchronized
def unload(self):
"""
Unload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Purge pending requests.
"""
Plugin.delete(self)
self.shutdown()
self.delegate.unloaded()
self.scheduler.pending.delete()
log.info('plugin:%s, unloaded', self.name)
@synchronized
def reload(self):
"""
Reload the plugin.
- Detach.
- Delete the plugin.
- Abort scheduled requests.
- Plugin shutdown.
- Reload plugin.
- Reschedule pending work to reloaded plugin.
"""
Plugin.delete(self)
scheduled = self.shutdown(False)
self.delegate.unloaded()
plugin = PluginLoader.load(self.path)
if plugin:
for call in scheduled:
if isinstance(call.fn, Task):
task = call.fn
task.transaction.plugin = plugin
plugin.pool.schedule(call)
plugin.start()
log.info('plugin:%s, reloaded', self.name)
return plugin