コード例 #1
0
def parse_crawler_config(config_path='crawler.conf'):
    global _config

    # 1. get configs
    _config = ConfigObj(infile=misc.execution_path(config_path),
                        configspec=misc.execution_path(CONFIG_SPEC_PATH))

    # 2. apply defaults
    vdt = Validator()
    _config.validate(vdt)
コード例 #2
0
def load_env_plugin(plugin_places=[misc.execution_path('plugins')],
                    environment='cloudsight'):
    pm = PluginManager(plugin_info_ext='plugin')

    # Normalize the paths to the location of this file.
    # XXX-ricarkol: there has to be a better way to do this.
    plugin_places = [misc.execution_path(x) for x in plugin_places]

    pm.setPluginPlaces(plugin_places)
    pm.setCategoriesFilter({"RuntimeEnvironment" : IRuntimeEnvironment})
    pm.collectPlugins()

    for env_plugin in pm.getAllPlugins():
       # There should be only one plugin of the given category and type;
       # but in case there are more, pick the first one.
       if env_plugin.plugin_object.get_environment_name() == environment:
           return env_plugin.plugin_object
    raise RuntimeEnvironmentPluginNotFound('Could not find a valid runtime '
                                 'environment plugin at %s' % plugin_places)
コード例 #3
0
def load_env_plugin(plugin_places=[misc.execution_path('plugins')],
                    environment='cloudsight'):
    pm = PluginManager(plugin_info_ext='plugin')

    # Normalize the paths to the location of this file.
    # XXX-ricarkol: there has to be a better way to do this.
    plugin_places = [misc.execution_path(x) for x in plugin_places]

    pm.setPluginPlaces(plugin_places)
    pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
    pm.collectPlugins()

    for env_plugin in pm.getAllPlugins():
        # There should be only one plugin of the given category and type;
        # but in case there are more, pick the first one.
        if env_plugin.plugin_object.get_environment_name() == environment:
            return env_plugin.plugin_object
    raise RuntimeEnvironmentPluginNotFound(
        'Could not find a valid "%s" runtime '
        'environment plugin at %s' % (environment, plugin_places))
コード例 #4
0
def _load_plugins(plugin_places=[misc.execution_path('plugins')],
                  category_filter={},
                  filter_func=lambda *arg: True):
    pm = PluginManager(plugin_info_ext='plugin')

    # Normalize the paths to the location of this file.
    # XXX-ricarkol: there has to be a better way to do this.
    plugin_places = [misc.execution_path(x) for x in plugin_places]

    pm.setPluginPlaces(plugin_places)
    pm.setCategoriesFilter(category_filter)
    pm.collectPlugins()

    config = config_parser.get_config()
    enabled_plugins = [p for p in config['crawlers']]

    for plugin in pm.getAllPlugins():
        if filter_func(plugin.plugin_object, plugin.name, enabled_plugins):
            plugin_args = {}
            if plugin.name in config['crawlers']:
                plugin_args = config['crawlers'][plugin.name]
            yield (plugin.plugin_object, plugin_args)
コード例 #5
0
def reload_vm_crawl_plugins(
        plugin_places=[misc.execution_path('plugins')],
        features=config_parser.get_config()['general']['features_to_crawl'],
        plugin_mode=config_parser.get_config()['general']['plugin_mode']):
    global vm_crawl_plugins

    if plugin_mode is False:  # aka override via --features CLI
        filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
            plugin_obj.get_feature() in features)
    else:
        filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
            plugin_name in enabled_plugins)

    vm_crawl_plugins = list(
        _load_plugins(plugin_places + ['plugins'],
                      category_filter={"crawler": IVMCrawler},
                      filter_func=filter_func))
コード例 #6
0
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
                      environment='cloudsight'):
    global runtime_env
    _plugins = list(
        _load_plugins(plugin_places,
                      category_filter={"env": IRuntimeEnvironment},
                      filter_func=lambda plugin, *unused: plugin.
                      get_environment_name() == environment))

    try:
        (runtime_env, unused_args) = _plugins[0]
    except (TypeError, IndexError):
        raise RuntimeEnvironmentPluginNotFound('Could not find a valid "%s" '
                                               'environment plugin at %s' %
                                               (environment, plugin_places))

    return runtime_env
コード例 #7
0
def reload_container_crawl_plugins(
        plugin_places=[misc.execution_path('plugins')],
        features=config_parser.get_config()['general']['features_to_crawl'],
        plugin_mode=config_parser.get_config()['general']['plugin_mode']):
    global container_crawl_plugins

    # using --plugin_mode  to override plugins for legacy CLI based invocation

    if plugin_mode is False:  # aka override via --features CLI
        filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
            plugin_obj.get_feature() in features)
    else:
        filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
            plugin_name in enabled_plugins)

    container_crawl_plugins = list(
        _load_plugins(plugin_places + ['plugins'],
                      category_filter={"crawler": IContainerCrawler},
                      filter_func=filter_func))
コード例 #8
0
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
                      environment='cloudsight'):
    global runtime_env
    runtime_env = load_env_plugin(plugin_places, environment)
コード例 #9
0
    # Normalize the paths to the location of this file.
    # XXX-ricarkol: there has to be a better way to do this.
    plugin_places = [misc.execution_path(x) for x in plugin_places]

    pm.setPluginPlaces(plugin_places)
    pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
    pm.collectPlugins()

    for env_plugin in pm.getAllPlugins():
        # There should be only one plugin of the given category and type;
        # but in case there are more, pick the first one.
        if env_plugin.plugin_object.get_environment_name() == environment:
            return env_plugin.plugin_object
    raise RuntimeEnvironmentPluginNotFound('Could not find a valid "%s" runtime '
                                           'environment plugin at %s' %
                                           (environment, plugin_places))

def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
                      environment='cloudsight'):
    global runtime_env
    runtime_env = load_env_plugin(plugin_places, environment)

# default runtime environment: cloudsigth and plugins in 'plugins/'
runtime_env = load_env_plugin(plugin_places=[misc.execution_path('plugins')],
                              environment='cloudsight')

def get_runtime_env_plugin():
    global runtime_env
    return runtime_env
コード例 #10
0
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
                      environment='cloudsight'):
    global runtime_env
    runtime_env = load_env_plugin(plugin_places, environment)
コード例 #11
0
    plugin_places = [misc.execution_path(x) for x in plugin_places]

    pm.setPluginPlaces(plugin_places)
    pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
    pm.collectPlugins()

    for env_plugin in pm.getAllPlugins():
        # There should be only one plugin of the given category and type;
        # but in case there are more, pick the first one.
        if env_plugin.plugin_object.get_environment_name() == environment:
            return env_plugin.plugin_object
    raise RuntimeEnvironmentPluginNotFound(
        'Could not find a valid "%s" runtime '
        'environment plugin at %s' % (environment, plugin_places))


def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
                      environment='cloudsight'):
    global runtime_env
    runtime_env = load_env_plugin(plugin_places, environment)


# default runtime environment: cloudsigth and plugins in 'plugins/'
runtime_env = load_env_plugin(plugin_places=[misc.execution_path('plugins')],
                              environment='cloudsight')


def get_runtime_env_plugin():
    global runtime_env
    return runtime_env