def test_helpers(self):
mock_panoptes_plugin_info = create_autospec(PanoptesPluginInfo)
mock_panoptes_plugin_info.execute_frequency = 5.75
# Assert 'round' is called by testing against non-integer product
self.assertNotEqual(expires(mock_panoptes_plugin_info),
mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_EXPIRES_MULTIPLE)
self.assertEqual(expires(mock_panoptes_plugin_info),
round(mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_EXPIRES_MULTIPLE))
# Test failure when called on a non-instance of PanoptesPluginInfo
with self.assertRaises(AssertionError):
expires(PanoptesPluginInfo)
# Assert 'round' is called by testing against non-integer product
self.assertNotEqual(time_limit(mock_panoptes_plugin_info),
mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_TIME_LIMIT_MULTIPLE)
self.assertEqual(time_limit(mock_panoptes_plugin_info),
round(mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_TIME_LIMIT_MULTIPLE))
# Test failure when called on a non-instance of PanoptesPluginInfo
with self.assertRaises(AssertionError):
time_limit(PanoptesPluginInfo)
def polling_plugin_scheduler_task(celery_beat_service):
"""
This function is the workhorse of the Polling Plugin Scheduler module. It detects changes in plugins and their
configuration and updates the Celery Beat schedule accordingly.
Args:
celery_beat_service (celery.beat.Service): The Celery Beat Service instance associated with this Plugin\
Scheduler
Returns:
None
"""
start_time = time.time()
try:
resource_cache = PanoptesResourceCache(panoptes_context)
resource_cache.setup_resource_cache()
except:
logger.exception(u'Could not create resource cache, skipping cycle')
return
try:
plugin_manager = PanoptesPluginManager(
plugin_type=u'polling',
plugin_class=PanoptesPollingPlugin,
plugin_info_class=PanoptesPollingPluginInfo,
panoptes_context=panoptes_context,
kv_store_class=PanoptesPollingPluginAgentKeyValueStore)
plugins = plugin_manager.getPluginsOfCategory(category_name=u'polling')
logger.info(u'Found %d plugins' % len(plugins))
except:
logger.exception(
u'Error trying to load Polling plugins, skipping cycle')
return
new_schedule = dict()
for plugin in plugins:
logger.info(u'Found plugin "%s", version %s at %s ' %
(plugin.name, plugin.version, plugin.path))
try:
logger.info(u'Plugin "%s" has configuration: %s' %
(plugin.name, plugin.config))
logger.info(
u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)'
% (plugin.name, plugin.moduleMtime, plugin.configMtime))
if plugin.execute_frequency <= 0:
logger.info(
u'Plugin %s has an invalid execution frequency (%d), skipping plugin'
% (plugin.name, plugin.execute_frequency))
continue
if not plugin.resource_filter:
logger.info(
u'Plugin "%s" does not have any resource filter specified, skipping plugin'
% plugin.name)
continue
except PanoptesPluginConfigurationError as e:
logger.error(
u'Error reading/parsing configuration for plugin "%s", skipping plugin. Error: %s'
% (plugin.name, repr(e)))
try:
resource_set = resource_cache.get_resources(plugin.resource_filter)
except Exception as e:
logger.info(
u'Error in applying resource filter "%s" for plugin "%s", skipping plugin: %s'
% (plugin.resource_filter, plugin.name, repr(e)))
continue
if len(resource_set) == 0:
logger.info(
u'No resources found for plugin "%s" after applying resource filter "%s", skipping plugin'
% (plugin.name, plugin.resource_filter))
logger.info(u'Length of resource set {} for plugin {}'.format(
len(resource_set), plugin.name))
for resource in resource_set:
logger.debug(
u'Going to add task for plugin "%s" with execute frequency %d, args "%s", resources %s'
% (plugin.name, plugin.execute_frequency, plugin.config,
resource))
plugin.data = resource
task_name = u':'.join([
plugin.normalized_name, plugin.signature,
str(resource.resource_id)
])
new_schedule[task_name] = {
u'task': const.POLLING_PLUGIN_AGENT_MODULE_NAME,
u'schedule': timedelta(seconds=plugin.execute_frequency),
u'args': (plugin.name, resource.serialization_key),
u'last_run_at':
datetime.utcfromtimestamp(plugin.last_executed),
u'options': {
u'expires': expires(plugin),
u'time_limit': time_limit(plugin)
}
}
resource_cache.close_resource_cache()
logger.info(
'Going to unload plugin modules. Length of sys.modules before unloading modules: %d'
% len(sys.modules))
plugin_manager.unload_modules()
logger.info(
'Unloaded plugin modules. Length of sys.modules after unloading modules: %d'
% len(sys.modules))
try:
scheduler = celery_beat_service.scheduler
scheduler.update(logger, new_schedule)
end_time = time.time()
logger.info(u'Scheduled %d tasks in %.2fs' %
(len(new_schedule), end_time - start_time))
logger.info(u'RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)
except:
logger.exception(u'Error in updating schedule for Polling Plugins')
logger.info('RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)
def polling_plugin_get_schedule():
start_time = time.time()
try:
resource_cache = PanoptesResourceCache(panoptes_context)
resource_cache.setup_resource_cache()
except Exception as e:
logger.exception(u'Could not create resource cache. {}'.format(
repr(e)))
return {}
try:
plugin_manager = PanoptesPluginManager(
plugin_type=u'polling',
plugin_class=PanoptesPollingPlugin,
plugin_info_class=PanoptesPollingPluginInfo,
panoptes_context=panoptes_context,
kv_store_class=PanoptesPollingPluginAgentKeyValueStore)
plugins = plugin_manager.getPluginsOfCategory(category_name=u'polling')
logger.info(u'Found %d plugins' % len(plugins))
except Exception as e:
logger.exception(
u'Error trying to load Polling plugins, skipping cycle. {}'.format(
repr(e)))
return {}
new_schedule = dict()
for plugin in plugins:
logger.info(u'Found plugin "%s", version %s at %s ' %
(plugin.name, plugin.version, plugin.path))
try:
logger.info(u'Plugin "%s" has configuration: %s' %
(plugin.name, plugin.config))
logger.info(
u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)'
% (plugin.name, plugin.moduleMtime, plugin.configMtime))
if plugin.execute_frequency <= 0:
logger.info(
u'Plugin %s has an invalid execution frequency (%d), skipping plugin'
% (plugin.name, plugin.execute_frequency))
continue
if not plugin.resource_filter:
logger.info(
u'Plugin "%s" does not have any resource filter specified, skipping plugin'
% plugin.name)
continue
except PanoptesPluginConfigurationError as e:
logger.error(
u'Error reading/parsing configuration for plugin "%s", skipping plugin. Error: %s'
% (plugin.name, repr(e)))
try:
resource_set = resource_cache.get_resources(plugin.resource_filter)
except Exception as e:
logger.info(
u'Error in applying resource filter "%s" for plugin "%s", skipping plugin: %s'
% (plugin.resource_filter, plugin.name, repr(e)))
continue
if len(resource_set) == 0:
logger.info(
u'No resources found for plugin "%s" after applying resource filter "%s", skipping plugin'
% (plugin.name, plugin.resource_filter))
logger.info(u'Length of resource set {} for plugin {}'.format(
len(resource_set), plugin.name))
for resource in resource_set:
logger.debug(
u'Going to add task for plugin "%s" with execute frequency %d, args "%s", resources %s'
% (plugin.name, plugin.execute_frequency, plugin.config,
resource))
plugin.data = resource
task_name = u':'.join([
plugin.normalized_name, plugin.signature,
str(resource.resource_id)
])
new_schedule[task_name] = {
u'task': const.POLLING_PLUGIN_AGENT_MODULE_NAME,
u'schedule': timedelta(seconds=plugin.execute_frequency),
u'args': (plugin.name, resource.serialization_key),
u'last_run_at':
datetime.utcfromtimestamp(plugin.last_executed),
u'options': {
u'expires': expires(plugin),
u'time_limit': time_limit(plugin)
}
}
resource_cache.close_resource_cache()
logger.info(
'Going to unload plugin modules. Length of sys.modules before unloading modules: %d'
% len(sys.modules))
plugin_manager.unload_modules()
logger.info(
'Unloaded plugin modules. Length of sys.modules after unloading modules: %d'
% len(sys.modules))
logger.info('Created Schedule With {} entries in {} seconds'.format(
len(new_schedule.keys()),
time.time() - start_time))
return new_schedule
def discovery_plugin_scheduler_task(celery_beat_service):
"""
This function is the workhorse of the Discovery Plugin Scheduler module. It detects changes in plugins and their
configuration and updates the Celery Beat schedule accordingly.
Args:
celery_beat_service (celery.beat.Service): The Celery Beat Service object associated with this Plugin Scheduler
Returns:
None
"""
start_time = time.time()
try:
plugin_manager = PanoptesPluginManager(
plugin_type=u'discovery',
plugin_class=PanoptesDiscoveryPlugin,
plugin_info_class=PanoptesPluginInfo,
panoptes_context=panoptes_context,
kv_store_class=PanoptesDiscoveryPluginAgentKeyValueStore)
plugins = plugin_manager.getPluginsOfCategory(
category_name=u'discovery')
except:
logger.exception(
u'Error trying to load Discovery plugins, skipping cycle')
return
new_schedule = dict()
for plugin in plugins:
logger.info(u'Found plugin "%s", version %s at %s ' %
(plugin.name, plugin.version, plugin.path))
try:
logger.info(u'Plugin "%s" has configuration: %s' %
(plugin.name, plugin.config))
logger.info(
u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)'
% (plugin.name, plugin.moduleMtime, plugin.configMtime))
if plugin.execute_frequency <= 0:
logger.info(
u'Plugin %s has an invalid execution frequency (%d), skipping plugin'
% (plugin.name, plugin.execute_frequency))
continue
except PanoptesPluginConfigurationError as e:
logger.error(
u'Error reading/parsing configuration for plugin %s, skipping plugin. Error: %s'
% (plugin.name, repr(e)))
logger.debug(
u'Going to add task for plugin "%s" with execute frequency %d and args "%s"'
% (plugin.name, plugin.execute_frequency, plugin.config))
task_name = u':'.join([plugin.normalized_name, plugin.signature])
new_schedule[task_name] = {
u'task': const.DISCOVERY_PLUGIN_AGENT_MODULE_NAME,
u'schedule': timedelta(seconds=plugin.execute_frequency),
u'last_run_at': datetime.utcfromtimestamp(plugin.last_executed),
u'args': (plugin.name, ),
u'options': {
u'expires': expires(plugin),
u'time_limit': time_limit(plugin)
}
}
logger.info(
u'Going to unload plugin modules. Length of sys.modules before unloading modules: %d'
% len(sys.modules))
plugin_manager.unload_modules()
logger.info(
u'Unloaded plugin modules. Length of sys.modules after unloading modules: %d'
% len(sys.modules))
try:
scheduler = celery_beat_service.scheduler
scheduler.update(logger, new_schedule)
end_time = time.time()
logger.info(u'Scheduled %d tasks in %.2fs' %
(len(new_schedule), end_time - start_time))
except:
logger.exception(u'Error in updating schedule for Discovery Plugins')
logger.info(u'RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)