def test_panoptes_plugin_manager(self):
matching_test_plugins_found = 0
context = TestPanoptesPluginManagerContext()
plugin_manager = PanoptesPluginManager(
plugin_type='polling',
plugin_class=PanoptesPollingPlugin,
panoptes_context=context)
plugins = plugin_manager.getPluginsOfCategory('polling')
self.assertGreater(len(plugins), 0)
for plugin in plugins:
if 'Test Polling Plugin' in plugin.name:
matching_test_plugins_found += 1
self.assertEqual(matching_test_plugins_found, 3)
polling_plugin = plugin_manager.getPluginByName(
name='Test Polling Plugin', category='polling')
self.assertEqual(polling_plugin.name, 'Test Polling Plugin')
self.assertEqual(
os.path.split(polling_plugin.config_filename)[1],
'plugin_polling_test.panoptes-plugin')
polling_plugin_second_instance = plugin_manager.getPluginByName(
name='Test Polling Plugin Second Instance', category='polling')
self.assertEqual(polling_plugin_second_instance.name,
'Test Polling Plugin Second Instance')
self.assertEqual(
os.path.split(polling_plugin_second_instance.config_filename)[1],
'plugin_polling_test_second_instance.panoptes-plugin')
def test_panoptes_plugin_manager(self):
matching_test_plugins_found = 0
context = TestPanoptesPluginManagerContext()
plugin_manager = PanoptesPluginManager(
plugin_type=u'polling',
plugin_class=PanoptesPollingPlugin,
panoptes_context=context)
plugins = plugin_manager.getPluginsOfCategory(u'polling')
self.assertGreater(len(plugins), 0)
for plugin in plugins:
if u'Test Polling Plugin' in plugin.name:
matching_test_plugins_found += 1
self.assertEqual(matching_test_plugins_found, 3)
polling_plugin = plugin_manager.getPluginByName(
name=u'Test Polling Plugin', category=u'polling')
self.assertEqual(polling_plugin.name, u'Test Polling Plugin')
self.assertEqual(
os.path.split(polling_plugin.config_filename)[1],
u'plugin_polling_test.panoptes-plugin')
polling_plugin_second_instance = plugin_manager.getPluginByName(
name=u'Test Polling Plugin Second Instance', category=u'polling')
self.assertEqual(polling_plugin_second_instance.name,
u'Test Polling Plugin Second Instance')
self.assertEqual(
os.path.split(polling_plugin_second_instance.config_filename)[1],
u'plugin_polling_test_second_instance.panoptes-plugin')
# Test unloading of th plugin modules, including error handling
yapsy_modules = [
m for m in sys.modules.keys()
if m.startswith(u'yapsy_loaded_plugin')
]
self.assertEqual(len(yapsy_modules), 3)
with LogCapture(attributes=self.extract) as log_capture:
del sys.modules['yapsy_loaded_plugin_Test_Polling_Plugin_0']
plugin_manager.unload_modules()
log_capture.check_present((
'panoptes.tests.test_plugin_manager', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin_0'
), (
'panoptes.tests.test_plugin_manager', 'ERROR',
'Error trying to delete module "yapsy_loaded_plugin_Test_Polling_Plugin_0"'
), (
'panoptes.tests.test_plugin_manager', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin_2_0'
), ('panoptes.tests.test_plugin_manager', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin_Second_Instance_0'
),
order_matters=False)
yapsy_modules = [
m for m in sys.modules.keys()
if m.startswith(u'yapsy_loaded_plugin')
]
self.assertEqual(len(yapsy_modules), 0)
def execute_plugin(self):
"""
This method loads, executes and returns the results of the plugin
The steps involved are:
* Create a PanoptesPluginManager for the type of plugins we are interested in and then try and load the \
named plugin
* Check if the plugin should be executed right now based on the last execution and results time and the \
configured execution and results caching time
* setup a PluginContext and attempt to take a lock on the plugin name and it's signature \
(which is hash of the plugin's config and data)
* Try and run the plugin
* If the plugin execution returns without errors, then update the last execution time
* Pass the results obtained to the callback function
* If the callback function succeeds, then update the last successful run time
Returns:
None
"""
logger = self._logger
config = self._panoptes_context.config_dict
utc_epoch = int(time.time())
logger.info(u'Attempting to execute plugin "%s"' % self._plugin_name)
try:
plugin_manager = PanoptesPluginManager(
plugin_type=self._plugin_type,
plugin_class=self._plugin_class,
plugin_info_class=self._plugin_info_class,
plugin_data=self._plugin_data,
panoptes_context=self._panoptes_context,
kv_store_class=self._plugin_agent_kv_store_class)
plugin = plugin_manager.getPluginByName(name=self._plugin_name,
category=self._plugin_type)
except Exception as e:
logger.error(u'Error trying to load plugin "%s": %s' %
(self._plugin_name, repr(e)))
return
if not plugin:
logger.warn(u'No plugin named "%s" found in "%s"' %
(self._plugin_name,
config[self._plugin_type][u'plugins_paths']))
return
if not plugin.execute_now:
return
try:
plugin_context = self._get_context(plugin)
except:
self.exception(plugin, u'Could not setup context for plugin')
return
self.info(
plugin,
u'Attempting to get lock for plugin "%s"' % self._plugin_name)
try:
lock = plugin.lock
except:
self.exception(plugin, u'Error in acquiring lock')
return
if lock is None or not lock.locked:
return
self.info(plugin, u'Acquired lock')
self.info(
plugin,
u'Going to run plugin "{}", version "{}", which last executed at {} (UTC) ({} seconds ago) and '
u'last produced results at {} (UTC) ({} seconds ago), module mtime {} (UTC), config mtime {} ('
u'UTC)'.format(plugin.name, plugin.version, plugin.last_executed,
plugin.last_executed_age, plugin.last_results,
plugin.last_results_age, plugin.moduleMtime,
plugin.configMtime))
results = None
plugin_start_time = time.time()
try:
results = plugin.plugin_object.run(plugin_context)
except:
self.exception(plugin, u'Failed to execute plugin')
finally:
plugin_end_time = time.time()
plugin.last_executed = utc_epoch
if results is None:
self.warn(plugin, u'Plugin did not return any results')
elif not isinstance(results, self._plugin_result_class):
self.warn(
plugin,
u'Plugin returned an unexpected result type: "{}"'.format(
type(results).__name__))
else:
self.info(
plugin, u'Plugin returned a result set with {} members'.format(
len(results)))
if len(results) > 0:
# Non-empty result set - send the results to the callback function
callback_start_time = time.time()
try:
self._results_callback(self._panoptes_context, results,
plugin)
except Exception as e:
self.exception(
plugin,
u'Results callback function failed: {}'.format(e))
return
finally:
callback_end_time = time.time()
self.info(
plugin,
u'Callback function ran in {:0.2f} seconds'.format(
callback_end_time - callback_start_time))
# If the callback was successful, then set the last results time
# The logic behind this is: in case the callback fails, then the
# plugin should be re-executed again after the plugin
# execute_frequency seconds - the execution should not be
# preempted by the results caching logic, which
# depends on the last results time in the KV store
plugin.last_results = utc_epoch
self.info(
plugin, u'Ran in {:0.2f} seconds'.format(plugin_end_time -
plugin_start_time))
try:
lock.release()
except:
self.exception(plugin, u'Failed to release lock for plugin')
else:
self.info(plugin, u'Released lock')
plugin_manager.unload_modules()
gc_start_time = time.time()
gc.collect()
gc_end_time = time.time()
self.info(
plugin, u'GC took %.2f seconds. There are %d garbage objects.' %
(gc_end_time - gc_start_time, len(gc.garbage)))