Пример #1
0
    def test_helpers(self):
        mock_panoptes_plugin_info = create_autospec(PanoptesPluginInfo)
        mock_panoptes_plugin_info.execute_frequency = 5.75

        #  Assert 'round' is called by testing against non-integer product
        self.assertNotEqual(expires(mock_panoptes_plugin_info),
                            mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_EXPIRES_MULTIPLE)
        self.assertEqual(expires(mock_panoptes_plugin_info),
                         round(mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_EXPIRES_MULTIPLE))

        #  Test failure when called on a non-instance of PanoptesPluginInfo
        with self.assertRaises(AssertionError):
            expires(PanoptesPluginInfo)

        #  Assert 'round' is called by testing against non-integer product
        self.assertNotEqual(time_limit(mock_panoptes_plugin_info),
                            mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_TIME_LIMIT_MULTIPLE)
        self.assertEqual(time_limit(mock_panoptes_plugin_info),
                         round(mock_panoptes_plugin_info.execute_frequency * PLUGIN_AGENT_PLUGIN_TIME_LIMIT_MULTIPLE))

        #  Test failure when called on a non-instance of PanoptesPluginInfo
        with self.assertRaises(AssertionError):
            time_limit(PanoptesPluginInfo)
Пример #2
0
def polling_plugin_scheduler_task(celery_beat_service):
    """
    This function is the workhorse of the Polling Plugin Scheduler module. It detects changes in plugins and their
    configuration and updates the Celery Beat schedule accordingly.

    Args:
        celery_beat_service (celery.beat.Service): The Celery Beat Service instance associated with this Plugin\
        Scheduler

    Returns:
        None

    """
    start_time = time.time()

    try:
        resource_cache = PanoptesResourceCache(panoptes_context)
        resource_cache.setup_resource_cache()
    except:
        logger.exception(u'Could not create resource cache, skipping cycle')
        return

    try:
        plugin_manager = PanoptesPluginManager(
            plugin_type=u'polling',
            plugin_class=PanoptesPollingPlugin,
            plugin_info_class=PanoptesPollingPluginInfo,
            panoptes_context=panoptes_context,
            kv_store_class=PanoptesPollingPluginAgentKeyValueStore)
        plugins = plugin_manager.getPluginsOfCategory(category_name=u'polling')
        logger.info(u'Found %d plugins' % len(plugins))
    except:
        logger.exception(
            u'Error trying to load Polling plugins, skipping cycle')
        return

    new_schedule = dict()

    for plugin in plugins:
        logger.info(u'Found plugin "%s", version %s at %s ' %
                    (plugin.name, plugin.version, plugin.path))

        try:
            logger.info(u'Plugin "%s" has configuration: %s' %
                        (plugin.name, plugin.config))
            logger.info(
                u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)'
                % (plugin.name, plugin.moduleMtime, plugin.configMtime))

            if plugin.execute_frequency <= 0:
                logger.info(
                    u'Plugin %s has an invalid execution frequency (%d), skipping plugin'
                    % (plugin.name, plugin.execute_frequency))
                continue

            if not plugin.resource_filter:
                logger.info(
                    u'Plugin "%s" does not have any resource filter specified, skipping plugin'
                    % plugin.name)
                continue
        except PanoptesPluginConfigurationError as e:
            logger.error(
                u'Error reading/parsing configuration for plugin "%s", skipping plugin. Error: %s'
                % (plugin.name, repr(e)))

        try:
            resource_set = resource_cache.get_resources(plugin.resource_filter)
        except Exception as e:
            logger.info(
                u'Error in applying resource filter "%s" for plugin "%s", skipping plugin: %s'
                % (plugin.resource_filter, plugin.name, repr(e)))
            continue

        if len(resource_set) == 0:
            logger.info(
                u'No resources found for plugin "%s" after applying resource filter "%s", skipping plugin'
                % (plugin.name, plugin.resource_filter))

        logger.info(u'Length of resource set {} for plugin {}'.format(
            len(resource_set), plugin.name))

        for resource in resource_set:
            logger.debug(
                u'Going to add task for plugin "%s" with execute frequency %d, args "%s", resources %s'
                % (plugin.name, plugin.execute_frequency, plugin.config,
                   resource))

            plugin.data = resource

            task_name = u':'.join([
                plugin.normalized_name, plugin.signature,
                str(resource.resource_id)
            ])

            new_schedule[task_name] = {
                u'task': const.POLLING_PLUGIN_AGENT_MODULE_NAME,
                u'schedule': timedelta(seconds=plugin.execute_frequency),
                u'args': (plugin.name, resource.serialization_key),
                u'last_run_at':
                datetime.utcfromtimestamp(plugin.last_executed),
                u'options': {
                    u'expires': expires(plugin),
                    u'time_limit': time_limit(plugin)
                }
            }

    resource_cache.close_resource_cache()

    logger.info(
        'Going to unload plugin modules. Length of sys.modules before unloading modules: %d'
        % len(sys.modules))
    plugin_manager.unload_modules()
    logger.info(
        'Unloaded plugin modules. Length of sys.modules after unloading modules: %d'
        % len(sys.modules))

    try:
        scheduler = celery_beat_service.scheduler
        scheduler.update(logger, new_schedule)

        end_time = time.time()
        logger.info(u'Scheduled %d tasks in %.2fs' %
                    (len(new_schedule), end_time - start_time))
        logger.info(u'RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)

    except:
        logger.exception(u'Error in updating schedule for Polling Plugins')

    logger.info('RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)
Пример #3
0
def polling_plugin_get_schedule():

    start_time = time.time()

    try:
        resource_cache = PanoptesResourceCache(panoptes_context)
        resource_cache.setup_resource_cache()
    except Exception as e:
        logger.exception(u'Could not create resource cache. {}'.format(
            repr(e)))
        return {}

    try:
        plugin_manager = PanoptesPluginManager(
            plugin_type=u'polling',
            plugin_class=PanoptesPollingPlugin,
            plugin_info_class=PanoptesPollingPluginInfo,
            panoptes_context=panoptes_context,
            kv_store_class=PanoptesPollingPluginAgentKeyValueStore)
        plugins = plugin_manager.getPluginsOfCategory(category_name=u'polling')
        logger.info(u'Found %d plugins' % len(plugins))
    except Exception as e:
        logger.exception(
            u'Error trying to load Polling plugins, skipping cycle. {}'.format(
                repr(e)))
        return {}

    new_schedule = dict()

    for plugin in plugins:
        logger.info(u'Found plugin "%s", version %s at %s ' %
                    (plugin.name, plugin.version, plugin.path))

        try:
            logger.info(u'Plugin "%s" has configuration: %s' %
                        (plugin.name, plugin.config))
            logger.info(
                u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)'
                % (plugin.name, plugin.moduleMtime, plugin.configMtime))

            if plugin.execute_frequency <= 0:
                logger.info(
                    u'Plugin %s has an invalid execution frequency (%d), skipping plugin'
                    % (plugin.name, plugin.execute_frequency))
                continue

            if not plugin.resource_filter:
                logger.info(
                    u'Plugin "%s" does not have any resource filter specified, skipping plugin'
                    % plugin.name)
                continue
        except PanoptesPluginConfigurationError as e:
            logger.error(
                u'Error reading/parsing configuration for plugin "%s", skipping plugin. Error: %s'
                % (plugin.name, repr(e)))

        try:
            resource_set = resource_cache.get_resources(plugin.resource_filter)
        except Exception as e:
            logger.info(
                u'Error in applying resource filter "%s" for plugin "%s", skipping plugin: %s'
                % (plugin.resource_filter, plugin.name, repr(e)))
            continue

        if len(resource_set) == 0:
            logger.info(
                u'No resources found for plugin "%s" after applying resource filter "%s", skipping plugin'
                % (plugin.name, plugin.resource_filter))

        logger.info(u'Length of resource set {} for plugin {}'.format(
            len(resource_set), plugin.name))

        for resource in resource_set:
            logger.debug(
                u'Going to add task for plugin "%s" with execute frequency %d, args "%s", resources %s'
                % (plugin.name, plugin.execute_frequency, plugin.config,
                   resource))

            plugin.data = resource

            task_name = u':'.join([
                plugin.normalized_name, plugin.signature,
                str(resource.resource_id)
            ])

            new_schedule[task_name] = {
                u'task': const.POLLING_PLUGIN_AGENT_MODULE_NAME,
                u'schedule': timedelta(seconds=plugin.execute_frequency),
                u'args': (plugin.name, resource.serialization_key),
                u'last_run_at':
                datetime.utcfromtimestamp(plugin.last_executed),
                u'options': {
                    u'expires': expires(plugin),
                    u'time_limit': time_limit(plugin)
                }
            }

    resource_cache.close_resource_cache()

    logger.info(
        'Going to unload plugin modules. Length of sys.modules before unloading modules: %d'
        % len(sys.modules))
    plugin_manager.unload_modules()
    logger.info(
        'Unloaded plugin modules. Length of sys.modules after unloading modules: %d'
        % len(sys.modules))

    logger.info('Created Schedule With {} entries in {} seconds'.format(
        len(new_schedule.keys()),
        time.time() - start_time))

    return new_schedule
def discovery_plugin_scheduler_task(celery_beat_service):
    """
    This function is the workhorse of the Discovery Plugin Scheduler module. It detects changes in plugins and their
    configuration and updates the Celery Beat schedule accordingly.

    Args:
        celery_beat_service (celery.beat.Service): The Celery Beat Service object associated with this Plugin Scheduler

    Returns:
        None
    """

    start_time = time.time()

    try:
        plugin_manager = PanoptesPluginManager(
            plugin_type=u'discovery',
            plugin_class=PanoptesDiscoveryPlugin,
            plugin_info_class=PanoptesPluginInfo,
            panoptes_context=panoptes_context,
            kv_store_class=PanoptesDiscoveryPluginAgentKeyValueStore)
        plugins = plugin_manager.getPluginsOfCategory(
            category_name=u'discovery')
    except:
        logger.exception(
            u'Error trying to load Discovery plugins, skipping cycle')
        return

    new_schedule = dict()

    for plugin in plugins:
        logger.info(u'Found plugin "%s", version %s at %s ' %
                    (plugin.name, plugin.version, plugin.path))

        try:
            logger.info(u'Plugin "%s" has configuration: %s' %
                        (plugin.name, plugin.config))
            logger.info(
                u'Plugin %s has plugin module time %s (UTC) and config mtime %s (UTC)'
                % (plugin.name, plugin.moduleMtime, plugin.configMtime))

            if plugin.execute_frequency <= 0:
                logger.info(
                    u'Plugin %s has an invalid execution frequency (%d), skipping plugin'
                    % (plugin.name, plugin.execute_frequency))
                continue
        except PanoptesPluginConfigurationError as e:
            logger.error(
                u'Error reading/parsing configuration for plugin %s, skipping plugin. Error: %s'
                % (plugin.name, repr(e)))

        logger.debug(
            u'Going to add task for plugin "%s" with execute frequency %d and args "%s"'
            % (plugin.name, plugin.execute_frequency, plugin.config))

        task_name = u':'.join([plugin.normalized_name, plugin.signature])

        new_schedule[task_name] = {
            u'task': const.DISCOVERY_PLUGIN_AGENT_MODULE_NAME,
            u'schedule': timedelta(seconds=plugin.execute_frequency),
            u'last_run_at': datetime.utcfromtimestamp(plugin.last_executed),
            u'args': (plugin.name, ),
            u'options': {
                u'expires': expires(plugin),
                u'time_limit': time_limit(plugin)
            }
        }

    logger.info(
        u'Going to unload plugin modules. Length of sys.modules before unloading modules: %d'
        % len(sys.modules))

    plugin_manager.unload_modules()
    logger.info(
        u'Unloaded plugin modules. Length of sys.modules after unloading modules: %d'
        % len(sys.modules))

    try:
        scheduler = celery_beat_service.scheduler
        scheduler.update(logger, new_schedule)

        end_time = time.time()
        logger.info(u'Scheduled %d tasks in %.2fs' %
                    (len(new_schedule), end_time - start_time))
    except:
        logger.exception(u'Error in updating schedule for Discovery Plugins')

    logger.info(u'RSS memory: %dKB' % getrusage(RUSAGE_SELF).ru_maxrss)