def parse_crawler_config(config_path='crawler.conf'):
global _config
# 1. get configs
_config = ConfigObj(infile=misc.execution_path(config_path),
configspec=misc.execution_path(CONFIG_SPEC_PATH))
# 2. apply defaults
vdt = Validator()
_config.validate(vdt)
def load_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter({"RuntimeEnvironment" : IRuntimeEnvironment})
pm.collectPlugins()
for env_plugin in pm.getAllPlugins():
# There should be only one plugin of the given category and type;
# but in case there are more, pick the first one.
if env_plugin.plugin_object.get_environment_name() == environment:
return env_plugin.plugin_object
raise RuntimeEnvironmentPluginNotFound('Could not find a valid runtime '
'environment plugin at %s' % plugin_places)
def load_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
pm.collectPlugins()
for env_plugin in pm.getAllPlugins():
# There should be only one plugin of the given category and type;
# but in case there are more, pick the first one.
if env_plugin.plugin_object.get_environment_name() == environment:
return env_plugin.plugin_object
raise RuntimeEnvironmentPluginNotFound(
'Could not find a valid "%s" runtime '
'environment plugin at %s' % (environment, plugin_places))
def _load_plugins(plugin_places=[misc.execution_path('plugins')],
category_filter={},
filter_func=lambda *arg: True):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter(category_filter)
pm.collectPlugins()
config = config_parser.get_config()
enabled_plugins = [p for p in config['crawlers']]
for plugin in pm.getAllPlugins():
if filter_func(plugin.plugin_object, plugin.name, enabled_plugins):
plugin_args = {}
if plugin.name in config['crawlers']:
plugin_args = config['crawlers'][plugin.name]
yield (plugin.plugin_object, plugin_args)
def reload_vm_crawl_plugins(
plugin_places=[misc.execution_path('plugins')],
features=config_parser.get_config()['general']['features_to_crawl'],
plugin_mode=config_parser.get_config()['general']['plugin_mode']):
global vm_crawl_plugins
if plugin_mode is False: # aka override via --features CLI
filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
plugin_obj.get_feature() in features)
else:
filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
plugin_name in enabled_plugins)
vm_crawl_plugins = list(
_load_plugins(plugin_places + ['plugins'],
category_filter={"crawler": IVMCrawler},
filter_func=filter_func))
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
global runtime_env
_plugins = list(
_load_plugins(plugin_places,
category_filter={"env": IRuntimeEnvironment},
filter_func=lambda plugin, *unused: plugin.
get_environment_name() == environment))
try:
(runtime_env, unused_args) = _plugins[0]
except (TypeError, IndexError):
raise RuntimeEnvironmentPluginNotFound('Could not find a valid "%s" '
'environment plugin at %s' %
(environment, plugin_places))
return runtime_env
def reload_container_crawl_plugins(
plugin_places=[misc.execution_path('plugins')],
features=config_parser.get_config()['general']['features_to_crawl'],
plugin_mode=config_parser.get_config()['general']['plugin_mode']):
global container_crawl_plugins
# using --plugin_mode to override plugins for legacy CLI based invocation
if plugin_mode is False: # aka override via --features CLI
filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
plugin_obj.get_feature() in features)
else:
filter_func = lambda plugin_obj, plugin_name, enabled_plugins: (
plugin_name in enabled_plugins)
container_crawl_plugins = list(
_load_plugins(plugin_places + ['plugins'],
category_filter={"crawler": IContainerCrawler},
filter_func=filter_func))
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
global runtime_env
runtime_env = load_env_plugin(plugin_places, environment)
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
pm.collectPlugins()
for env_plugin in pm.getAllPlugins():
# There should be only one plugin of the given category and type;
# but in case there are more, pick the first one.
if env_plugin.plugin_object.get_environment_name() == environment:
return env_plugin.plugin_object
raise RuntimeEnvironmentPluginNotFound('Could not find a valid "%s" runtime '
'environment plugin at %s' %
(environment, plugin_places))
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
global runtime_env
runtime_env = load_env_plugin(plugin_places, environment)
# default runtime environment: cloudsigth and plugins in 'plugins/'
runtime_env = load_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight')
def get_runtime_env_plugin():
global runtime_env
return runtime_env
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
global runtime_env
runtime_env = load_env_plugin(plugin_places, environment)
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
pm.collectPlugins()
for env_plugin in pm.getAllPlugins():
# There should be only one plugin of the given category and type;
# but in case there are more, pick the first one.
if env_plugin.plugin_object.get_environment_name() == environment:
return env_plugin.plugin_object
raise RuntimeEnvironmentPluginNotFound(
'Could not find a valid "%s" runtime '
'environment plugin at %s' % (environment, plugin_places))
def reload_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
global runtime_env
runtime_env = load_env_plugin(plugin_places, environment)
# default runtime environment: cloudsigth and plugins in 'plugins/'
runtime_env = load_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight')
def get_runtime_env_plugin():
global runtime_env
return runtime_env