def test_helpers(self): mock_panoptes_plugin_info = create_autospec(PanoptesPluginInfo) mock_panoptes_plugin_info.execute_frequency = 5.75 # Assert 'round' is called by testing against non-integer product self.assertNotEqual(expires(mock_panoptes_plugin_info), mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_EXPIRES_MULTIPLE) self.assertEqual(expires(mock_panoptes_plugin_info), round(mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_EXPIRES_MULTIPLE)) # Test failure when called on a non-instance of PanoptesPluginInfo with self.assertRaises(AssertionError): expires(PanoptesPluginInfo) # Assert 'round' is called by testing against non-integer product self.assertNotEqual(time_limit(mock_panoptes_plugin_info), mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_TIME_LIMIT_MULTIPLE) self.assertEqual(time_limit(mock_panoptes_plugin_info), round(mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_TIME_LIMIT_MULTIPLE)) # Test failure when called on a non-instance of PanoptesPluginInfo with self.assertRaises(AssertionError): time_limit(PanoptesPluginInfo)
def polling_plugin_scheduler_task(celery_beat_service): """ This function is the workhorse of the Polling Plugin Scheduler module. It detects changes in plugins and their configuration and updates the Celery Beat schedule accordingly. Args: celery_beat_service (celery.beat.Service): The Celery Beat Service instance associated with this Plugin\ Scheduler Returns: None """ start_time = time.time() try: resource_cache = PanoptesResourceCache(panoptes_context) resource_cache.setup_resource_cache() except: logger.exception(u'Could not create resource cache, skipping cycle') return try: plugin_manager = PanoptesPluginManager( plugin_type=u'polling', plugin_class=PanoptesPollingPlugin, plugin_info_class=PanoptesPollingPluginInfo, panoptes_context=panoptes_context, kv_store_class=PanoptesPollingPluginAgentKeyValueStore) plugins = plugin_manager.getPluginsOfCategory(category_name=u'polling') logger.info(u'Found %d plugins' % len(plugins)) except: logger.exception( u'Error trying to load Polling plugins, skipping cycle') return new_schedule = dict() for plugin in plugins: logger.info(u'Found plugin "%s", version %s at %s ' % (plugin.name, plugin.version, plugin.path)) try: logger.info(u'Plugin "%s" has configuration: %s' % (plugin.name, plugin.config)) logger.info( u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)' % (plugin.name, plugin.moduleMtime, plugin.configMtime)) if plugin.execute_frequency <= 0: logger.info( u'Plugin %s has an invalid execution frequency (%d), skipping plugin' % (plugin.name, plugin.execute_frequency)) continue if not plugin.resource_filter: logger.info( u'Plugin "%s" does not have any resource filter specified, skipping plugin' % plugin.name) continue except PanoptesPluginConfigurationError as e: logger.error( u'Error reading/parsing configuration for plugin "%s", skipping plugin. Error: %s' % (plugin.name, repr(e))) try: resource_set = resource_cache.get_resources(plugin.resource_filter) except Exception as e: logger.info( u'Error in applying resource filter "%s" for plugin "%s", skipping plugin: %s' % (plugin.resource_filter, plugin.name, repr(e))) continue if len(resource_set) == 0: logger.info( u'No resources found for plugin "%s" after applying resource filter "%s", skipping plugin' % (plugin.name, plugin.resource_filter)) logger.info(u'Length of resource set {} for plugin {}'.format( len(resource_set), plugin.name)) for resource in resource_set: logger.debug( u'Going to add task for plugin "%s" with execute frequency %d, args "%s", resources %s' % (plugin.name, plugin.execute_frequency, plugin.config, resource)) plugin.data = resource task_name = u':'.join([ plugin.normalized_name, plugin.signature, str(resource.resource_id) ]) new_schedule[task_name] = { u'task': const.POLLING_PLUGIN_AGENT_MODULE_NAME, u'schedule': timedelta(seconds=plugin.execute_frequency), u'args': (plugin.name, resource.serialization_key), u'last_run_at': datetime.utcfromtimestamp(plugin.last_executed), u'options': { u'expires': expires(plugin), u'time_limit': time_limit(plugin) } } resource_cache.close_resource_cache() logger.info( 'Going to unload plugin modules. Length of sys.modules before unloading modules: %d' % len(sys.modules)) plugin_manager.unload_modules() logger.info( 'Unloaded plugin modules. Length of sys.modules after unloading modules: %d' % len(sys.modules)) try: scheduler = celery_beat_service.scheduler scheduler.update(logger, new_schedule) end_time = time.time() logger.info(u'Scheduled %d tasks in %.2fs' % (len(new_schedule), end_time - start_time)) logger.info(u'RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss) except: logger.exception(u'Error in updating schedule for Polling Plugins') logger.info('RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)
def polling_plugin_get_schedule(): start_time = time.time() try: resource_cache = PanoptesResourceCache(panoptes_context) resource_cache.setup_resource_cache() except Exception as e: logger.exception(u'Could not create resource cache. {}'.format( repr(e))) return {} try: plugin_manager = PanoptesPluginManager( plugin_type=u'polling', plugin_class=PanoptesPollingPlugin, plugin_info_class=PanoptesPollingPluginInfo, panoptes_context=panoptes_context, kv_store_class=PanoptesPollingPluginAgentKeyValueStore) plugins = plugin_manager.getPluginsOfCategory(category_name=u'polling') logger.info(u'Found %d plugins' % len(plugins)) except Exception as e: logger.exception( u'Error trying to load Polling plugins, skipping cycle. {}'.format( repr(e))) return {} new_schedule = dict() for plugin in plugins: logger.info(u'Found plugin "%s", version %s at %s ' % (plugin.name, plugin.version, plugin.path)) try: logger.info(u'Plugin "%s" has configuration: %s' % (plugin.name, plugin.config)) logger.info( u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)' % (plugin.name, plugin.moduleMtime, plugin.configMtime)) if plugin.execute_frequency <= 0: logger.info( u'Plugin %s has an invalid execution frequency (%d), skipping plugin' % (plugin.name, plugin.execute_frequency)) continue if not plugin.resource_filter: logger.info( u'Plugin "%s" does not have any resource filter specified, skipping plugin' % plugin.name) continue except PanoptesPluginConfigurationError as e: logger.error( u'Error reading/parsing configuration for plugin "%s", skipping plugin. Error: %s' % (plugin.name, repr(e))) try: resource_set = resource_cache.get_resources(plugin.resource_filter) except Exception as e: logger.info( u'Error in applying resource filter "%s" for plugin "%s", skipping plugin: %s' % (plugin.resource_filter, plugin.name, repr(e))) continue if len(resource_set) == 0: logger.info( u'No resources found for plugin "%s" after applying resource filter "%s", skipping plugin' % (plugin.name, plugin.resource_filter)) logger.info(u'Length of resource set {} for plugin {}'.format( len(resource_set), plugin.name)) for resource in resource_set: logger.debug( u'Going to add task for plugin "%s" with execute frequency %d, args "%s", resources %s' % (plugin.name, plugin.execute_frequency, plugin.config, resource)) plugin.data = resource task_name = u':'.join([ plugin.normalized_name, plugin.signature, str(resource.resource_id) ]) new_schedule[task_name] = { u'task': const.POLLING_PLUGIN_AGENT_MODULE_NAME, u'schedule': timedelta(seconds=plugin.execute_frequency), u'args': (plugin.name, resource.serialization_key), u'last_run_at': datetime.utcfromtimestamp(plugin.last_executed), u'options': { u'expires': expires(plugin), u'time_limit': time_limit(plugin) } } resource_cache.close_resource_cache() logger.info( 'Going to unload plugin modules. Length of sys.modules before unloading modules: %d' % len(sys.modules)) plugin_manager.unload_modules() logger.info( 'Unloaded plugin modules. Length of sys.modules after unloading modules: %d' % len(sys.modules)) logger.info('Created Schedule With {} entries in {} seconds'.format( len(new_schedule.keys()), time.time() - start_time)) return new_schedule
def discovery_plugin_scheduler_task(celery_beat_service): """ This function is the workhorse of the Discovery Plugin Scheduler module. It detects changes in plugins and their configuration and updates the Celery Beat schedule accordingly. Args: celery_beat_service (celery.beat.Service): The Celery Beat Service object associated with this Plugin Scheduler Returns: None """ start_time = time.time() try: plugin_manager = PanoptesPluginManager( plugin_type=u'discovery', plugin_class=PanoptesDiscoveryPlugin, plugin_info_class=PanoptesPluginInfo, panoptes_context=panoptes_context, kv_store_class=PanoptesDiscoveryPluginAgentKeyValueStore) plugins = plugin_manager.getPluginsOfCategory( category_name=u'discovery') except: logger.exception( u'Error trying to load Discovery plugins, skipping cycle') return new_schedule = dict() for plugin in plugins: logger.info(u'Found plugin "%s", version %s at %s ' % (plugin.name, plugin.version, plugin.path)) try: logger.info(u'Plugin "%s" has configuration: %s' % (plugin.name, plugin.config)) logger.info( u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)' % (plugin.name, plugin.moduleMtime, plugin.configMtime)) if plugin.execute_frequency <= 0: logger.info( u'Plugin %s has an invalid execution frequency (%d), skipping plugin' % (plugin.name, plugin.execute_frequency)) continue except PanoptesPluginConfigurationError as e: logger.error( u'Error reading/parsing configuration for plugin %s, skipping plugin. Error: %s' % (plugin.name, repr(e))) logger.debug( u'Going to add task for plugin "%s" with execute frequency %d and args "%s"' % (plugin.name, plugin.execute_frequency, plugin.config)) task_name = u':'.join([plugin.normalized_name, plugin.signature]) new_schedule[task_name] = { u'task': const.DISCOVERY_PLUGIN_AGENT_MODULE_NAME, u'schedule': timedelta(seconds=plugin.execute_frequency), u'last_run_at': datetime.utcfromtimestamp(plugin.last_executed), u'args': (plugin.name, ), u'options': { u'expires': expires(plugin), u'time_limit': time_limit(plugin) } } logger.info( u'Going to unload plugin modules. Length of sys.modules before unloading modules: %d' % len(sys.modules)) plugin_manager.unload_modules() logger.info( u'Unloaded plugin modules. Length of sys.modules after unloading modules: %d' % len(sys.modules)) try: scheduler = celery_beat_service.scheduler scheduler.update(logger, new_schedule) end_time = time.time() logger.info(u'Scheduled %d tasks in %.2fs' % (len(new_schedule), end_time - start_time)) except: logger.exception(u'Error in updating schedule for Discovery Plugins') logger.info(u'RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)