def collect_data(self, **kwargs):
self.logger.info("Begin collect_data")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({
"DataCollector": data_collector_plugin
})
# Tell it the default place(s) where to find plugins
self.logger.debug("Plugin directories: %s" % (kwargs['data_collector_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['data_collector_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
self.logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.start()
else:
self.logger.error("Failed to initialize plugin: %s" % (plugin.name))
plugin_cnt += 1
#Wait for the plugings to finish up.
self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
self.logger.info("%d Plugins completed in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
def loadPlugins(root_folder, plugin_name, categories_filter):
from yapsy.PluginManager import PluginManager
plugin_dirs = []
plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins')
plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins')
plugin_dirs.append(root_folder + '/plugins')
log.debug("Looking for %s-plugins in %s" % (plugin_name, ", ".join(plugin_dirs)))
manager = PluginManager()
manager.setPluginPlaces(plugin_dirs)
manager.setCategoriesFilter(categories_filter)
manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in manager.getAllPlugins():
manager.activatePluginByName(pluginInfo.name)
result = {}
for plugin in manager.getAllPlugins():
if hasattr(plugin.plugin_object, 'aliases') and isinstance(plugin.plugin_object.aliases, list):
for alias in plugin.plugin_object.aliases:
result[alias] = plugin.plugin_object
elif hasattr(plugin.plugin_object, 'alias'):
result[plugin.plugin_object.alias] = plugin.plugin_object
else:
result[plugin.name] = plugin.plugin_object
return result
def run_output_plugins(**kwargs):
logger = logging.getLogger(__name__)
logger.info("Begin run_output_plugins")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({"OutputResults": output_plugin})
# Tell it the default place(s) where to find plugins
if logger:
logger.debug("Plugin directories: %s" %
(kwargs['output_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
if logger:
logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.emit(
prediction_date=kwargs['prediction_date'].astimezone(
timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"),
execution_date=kwargs['prediction_run_date'].strftime(
"%Y-%m-%d %H:%M:%S"),
ensemble_tests=kwargs['site_model_ensemble'])
plugin_cnt += 1
else:
logger.error("Failed to initialize plugin: %s" % (plugin.name))
logger.debug("%d output plugins run in %f seconds" %
(plugin_cnt, time.time() - plugin_start_time))
logger.info("Finished run_output_plugins")
def run_output_plugins(**kwargs):
logger = logging.getLogger(__name__)
logger.info("Begin run_output_plugins")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({
"OutputResults": output_plugin
})
# Tell it the default place(s) where to find plugins
if logger:
logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
if logger:
logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.emit(prediction_date=kwargs['prediction_date'].astimezone(timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"),
execution_date=kwargs['prediction_run_date'].strftime("%Y-%m-%d %H:%M:%S"),
ensemble_tests=kwargs['site_model_ensemble'])
plugin_cnt += 1
else:
logger.error("Failed to initialize plugin: %s" % (plugin.name))
logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
logger.info("Finished run_output_plugins")
def loadPlugins(root_folder, plugin_name, categories_filter):
from yapsy.PluginManager import PluginManager
plugin_dirs = []
plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins')
plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins')
plugin_dirs.append(root_folder + '/plugins')
log.debug("Looking for %s-plugins in %s" %
(plugin_name, ", ".join(plugin_dirs)))
manager = PluginManager()
manager.setPluginPlaces(plugin_dirs)
manager.setCategoriesFilter(categories_filter)
manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in manager.getAllPlugins():
manager.activatePluginByName(pluginInfo.name)
result = {}
for plugin in manager.getAllPlugins():
if hasattr(plugin.plugin_object, 'aliases') and isinstance(
plugin.plugin_object.aliases, list):
for alias in plugin.plugin_object.aliases:
result[alias] = plugin.plugin_object
elif hasattr(plugin.plugin_object, 'alias'):
result[plugin.plugin_object.alias] = plugin.plugin_object
else:
result[plugin.name] = plugin.plugin_object
return result
def test_plugins_found(self):
pm = PluginManager()
libpath = '%s/OpenMesher/plugins' % (get_python_lib())
pm.setPluginPlaces([
'/usr/share/openmesher/plugins',
'~/.openmesher/plugins',
'./OpenMesher/plugins',
'./plugins',
libpath,
])
pm.setPluginInfoExtension('plugin')
pm.setCategoriesFilter({
'config': IOpenMesherConfigPlugin,
'package': IOpenMesherPackagePlugin,
'deploy': IOpenMesherDeployPlugin,
})
pm.collectPlugins()
for plugin in pm.getAllPlugins():
print('Author: %s' % (plugin.author))
print('Categories: %s' % (plugin.categories))
print('Copyright: %s' % (plugin.copyright))
print('Descr: %s' % (plugin.description))
print('Error: %s' % (plugin.error))
print('Name: %s' % (plugin.name))
print('Path: %s' % (plugin.path))
print('Version: %s' % (plugin.version))
print('Website: %s' % (plugin.website))
print('')
return True
def collect_data(self, **kwargs):
self.logger.info("Begin collect_data")
try:
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter(
{"DataCollector": data_collector_plugin})
# Tell it the default place(s) where to find plugins
self.logger.debug("Plugin directories: %s" %
(kwargs['data_collector_plugin_directories']))
yapsy_logger = logging.getLogger('yapsy')
yapsy_logger.setLevel(logging.DEBUG)
#yapsy_logger.parent.level = logging.DEBUG
yapsy_logger.disabled = False
simplePluginManager.setPluginPlaces(
kwargs['data_collector_plugin_directories'])
simplePluginManager.collectPlugins()
output_queue = Queue()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
plugin_start_time = time.time()
self.logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(
details=plugin.details,
queue=output_queue,
begin_date=kwargs['begin_date']):
plugin.plugin_object.start()
self.logger.info("Waiting for %s plugin to complete." %
(plugin.name))
plugin.plugin_object.join()
self.logger.info(
"%s plugin to completed in %f seconds." %
(plugin.name, time.time() - plugin_start_time))
else:
self.logger.error("Failed to initialize plugin: %s" %
(plugin.name))
plugin_cnt += 1
#Wait for the plugings to finish up.
self.logger.info("Waiting for %d plugins to complete." %
(plugin_cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
while not output_queue.empty():
results = output_queue.get()
if results[0] == data_result_types.SAMPLING_DATA_TYPE:
self.bacteria_sample_data = results[1]
self.logger.info("%d Plugins completed in %f seconds" %
(plugin_cnt, time.time() - plugin_start_time))
except Exception as e:
self.logger.exception(e)
class Plugins():
'''
Plugins
'''
def __init__(self):
'''
Constructor
'''
super(Plugins, self).__init__()
self.write_tab_dock_plugin_dict = {}
self.write_panel_dock_plugin_dict = {}
# Build the manager
self._plugin_manager = PluginManager()
# List all sub-directories of "plugins"
plugin_path = os.path.sep.join([os.getcwd(), "plugins"])
plugin_places = [plugin_path]
for dirname, dirnames, filenames in os.walk(plugin_path):
# print path to all subdirectories first.
for subdirname in dirnames:
plugin_places.append(os.path.join(dirname, subdirname))
# Tell it the default place(s) where to find plugins
self._plugin_manager.setPluginPlaces(plugin_places)
sys.path.append(plugin_path)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._plugin_manager.setCategoriesFilter({
"CoreWriteTabDockPlugin":
CoreWriteTabDockPlugin,
"CoreWritePanelDockPlugin":
CoreWritePanelDockPlugin
})
self._plugin_manager.collectPlugins()
self.load_plugins(
["CoreWriteTabDockPlugin", "CoreWritePanelDockPlugin"])
def load_plugins(self, categories):
'''
function:: load_plugins(categories)
:param categories: list
'''
for category in categories:
for pluginInfo in self._plugin_manager.getPluginsOfCategory(
category):
setattr(self,
pluginInfo.plugin_object.core_class().__name__,
pluginInfo.plugin_object.core_class())
if category is "CoreWriteTabDockPlugin":
self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \
= pluginInfo.plugin_object.core_class()
if category is "CoreWritePanelDockPlugin":
self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \
= pluginInfo.plugin_object.core_class()
class Plugins():
'''
Plugins
'''
def __init__(self):
'''
Constructor
'''
super(Plugins, self).__init__()
self.write_tab_dock_plugin_dict = {}
self.write_panel_dock_plugin_dict = {}
# Build the manager
self._plugin_manager = PluginManager()
# List all sub-directories of "plugins"
plugin_path = os.path.sep.join([os.getcwd(), "plugins"])
plugin_places = [plugin_path]
for dirname, dirnames, filenames in os.walk(plugin_path):
# print path to all subdirectories first.
for subdirname in dirnames:
plugin_places.append(os.path.join(dirname, subdirname))
# Tell it the default place(s) where to find plugins
self._plugin_manager.setPluginPlaces(plugin_places)
sys.path.append(plugin_path)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._plugin_manager.setCategoriesFilter({
"GuiWriteTabDockPlugin": GuiWriteTabDockPlugin,
"GuiWritePanelDockPlugin": GuiWritePanelDockPlugin
})
self._plugin_manager.collectPlugins()
self.load_plugins(
["GuiWriteTabDockPlugin", "GuiWritePanelDockPlugin"])
def load_plugins(self, categories):
'''
function:: load_plugins(categories)
:param categories: list
'''
for category in categories:
for pluginInfo in self._plugin_manager.getPluginsOfCategory(category):
setattr(self, pluginInfo.plugin_object.gui_class(
).__name__, pluginInfo.plugin_object.gui_class())
if category is "GuiWriteTabDockPlugin":
self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \
= pluginInfo.plugin_object.gui_class()
if category is "GuiWritePanelDockPlugin":
self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \
= pluginInfo.plugin_object.gui_class()
class PluginHandler:
def __init__(self):
self._plugin_manager = PluginManager()
self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False}
self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []}
self._modifier_plugins = []
self._analyzer_plugins = []
self._collect_all_plugins()
def _collect_all_plugins(self):
self._plugin_manager.setPluginPlaces(
["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"]
)
# setting filter categories for pluginmanager
self._plugin_manager.setCategoriesFilter(
{
# movie metadata provider
"Modifier": IModifier,
# movie metadata provider
"Comparator": IComparator,
# sub metadata provider
"Analyzer": IAnalyzer,
}
)
self._plugin_manager.collectPlugins()
def activate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def deactivate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def _toggle_activate_plugins_by_category(self, category):
plugins = self._plugin_manager.getPluginsOfCategory(category)
is_active = self._category_active[category]
for pluginInfo in plugins:
if is_active:
self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].remove(pluginInfo)
else:
self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].append(pluginInfo)
self._category_active[category] = not is_active
def get_plugins_from_category(self, category):
plugins = []
for plugin in self._plugin_from_category[category]:
plugin.plugin_object.name = plugin.name
plugin.plugin_object.description = plugin.description
plugins.append(plugin.plugin_object)
return plugins
def is_activated(self, category):
""" True if category is activated. """
return self._category_active[category]
def initialize_plugins(self):
self.__log.info("Collecting and loading plugins")
try:
plugin_manager = PluginManager()
# TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT)
plugin_manager.setCategoriesFilter({
CARTRIDGE_AGENT_PLUGIN:
ICartridgeAgentPlugin,
ARTIFACT_MGT_PLUGIN:
IArtifactManagementPlugin
})
plugin_manager.setPluginPlaces(
[self.__config.read_property(constants.PLUGINS_DIR)])
plugin_manager.collectPlugins()
# activate cartridge agent plugins
plugins = plugin_manager.getPluginsOfCategory(
CARTRIDGE_AGENT_PLUGIN)
grouped_plugins = {}
for plugin_info in plugins:
self.__log.debug("Found plugin [%s] at [%s]" %
(plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated plugin [%s]" % plugin_info.name)
mapped_events = plugin_info.description.split(",")
for mapped_event in mapped_events:
if mapped_event.strip() != "":
if grouped_plugins.get(mapped_event) is None:
grouped_plugins[mapped_event] = []
grouped_plugins[mapped_event].append(plugin_info)
# activate artifact management plugins
artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(
ARTIFACT_MGT_PLUGIN)
for plugin_info in artifact_mgt_plugins:
self.__log.debug(
"Found artifact management plugin [%s] at [%s]" %
(plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated artifact management plugin [%s]" %
plugin_info.name)
return plugin_manager, grouped_plugins, artifact_mgt_plugins
except ParameterNotFoundException as e:
self.__log.exception(
"Could not load plugins. Plugins directory not set: %s" % e)
return None, None, None
except Exception as e:
self.__log.exception("Error while loading plugin: %s" % e)
return None, None, None
def ConfigurePluginManager(categories=None, pluginLocation=None):
if categories is None:
categories = dict(Test=ITestPlugin, DAQ=IDAQPlugin, App=IAppPlugin, Decoder=IDecoderPlugin)
if pluginLocation is None:
pluginLocation = ["plugins"]
manager = PluginManager()
# todo: plugin directory and categories should be set some other way (command line arg parsing?)
manager.setPluginPlaces(pluginLocation)
manager.setCategoriesFilter(categories)
manager.collectPlugins()
return manager
def loadPlugins():
# Find all plugin directories (xxx.plugin)
dirs = [f for f in os.listdir('.') if f.endswith('.plugin')]
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(dirs)
manager.setCategoriesFilter({"plugin" : IInfoBoxPlugin})
manager.collectPlugins()
return manager
class MUCBot(sleekxmpp.ClientXMPP):
def __init__(self, jid, password, room, nick):
sleekxmpp.ClientXMPP.__init__(self, jid, password)
self.room = room
self.nick = nick
# Load plugins
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins_enabled"])
self.plugin_manager.setCategoriesFilter({
"Message" : MessagePlugin,
"Presence": PresencePlugin
})
self.plugin_manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in self.plugin_manager.getAllPlugins():
self.plugin_manager.activatePluginByName(pluginInfo.name)
self.add_event_handler("session_start", self.start)
self.add_event_handler("message", self.message)
self.add_event_handler("groupchat_message", self.muc_message)
self.add_event_handler("muc::%s::got_online" % self.room, self.muc_online)
self.add_event_handler("muc::%s::got_offline" % self.room, self.muc_offline)
def start(self, event):
self.get_roster()
self.send_presence()
if self.room and len(self.room) > 0:
self.plugin['xep_0045'].joinMUC(self.room,
self.nick,
wait=True)
def message(self, msg):
if msg['type'] in ('chat', 'normal'):
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"):
pluginInfo.plugin_object.message_received(msg)
def muc_message(self, msg):
if msg['mucnick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"):
pluginInfo.plugin_object.message_received(msg, nick=self.nick)
def muc_online(self, presence):
if presence['muc']['nick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"):
pluginInfo.plugin_object.got_online(self, presence)
def muc_offline(self, presence):
if presence['muc']['nick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"):
pluginInfo.plugin_object.got_offline(self, presence)
def get_plugins(category_filter={}, plugin_places=['plugins']):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter(category_filter)
pm.collectPlugins()
return pm.getAllPlugins()
def output_results(self, **kwargs):
self.logger.info("Begin run_output_plugins")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter(
{"OutputResults": data_collector_plugin})
# Tell it the default place(s) where to find plugins
self.logger.debug("Plugin directories: %s" %
(kwargs['output_plugin_directories']))
simplePluginManager.setPluginPlaces(
kwargs['output_plugin_directories'])
yapsy_logger = logging.getLogger('yapsy')
yapsy_logger.setLevel(logging.DEBUG)
# yapsy_logger.parent.level = logging.DEBUG
yapsy_logger.disabled = False
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
try:
self.logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(
details=plugin.details,
prediction_date=kwargs['prediction_date'].astimezone(
timezone("US/Eastern")).strftime(
"%Y-%m-%d %H:%M:%S"),
execution_date=kwargs['prediction_run_date'].strftime(
"%Y-%m-%d %H:%M:%S"),
ensemble_tests=kwargs['site_model_ensemble']):
plugin.plugin_object.start()
plugin_cnt += 1
else:
self.logger.error("Failed to initialize plugin: %s" %
(plugin.details))
except Exception as e:
self.logger.exception(e)
#Wait for the plugings to finish up.
self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
plugin.plugin_object.finalize()
self.logger.debug("%d output plugins run in %f seconds" %
(plugin_cnt, time.time() - plugin_start_time))
self.logger.info("Finished output_results")
def ConfigurePluginManager(categories=None, pluginLocation=None):
if categories is None:
categories = dict(Test=ITestPlugin,
DAQ=IDAQPlugin,
App=IAppPlugin,
Decoder=IDecoderPlugin)
if pluginLocation is None:
pluginLocation = ["plugins"]
manager = PluginManager()
# todo: plugin directory and categories should be set some other way (command line arg parsing?)
manager.setPluginPlaces(pluginLocation)
manager.setCategoriesFilter(categories)
manager.collectPlugins()
return manager
def modelBehaviorImplementations(paths):
"""
Return an iterable of plugin-info for every locatable implementation of this interface on a given path.
PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to
the schedule upon which module resolution and imports happen.
"""
manager = PluginManager()
manager.setPluginPlaces(paths)
from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior
# hey PyDev: the previous line is weird, but is actually legal
manager.setCategoriesFilter({"ModelBehavior": foreignModelBehavior})
manager.collectPlugins()
return manager.getPluginsOfCategory("ModelBehavior")
def create_plugin_manager(category_filter, plugin_place):
""" Creates a PluginManager object from the given folder according to the given filter
:param category_filter:
:param plugin_place:
:return:
:rtype: PluginManager
"""
plugin_manager = PluginManager()
plugin_manager.setCategoriesFilter(category_filter)
plugin_manager.setPluginPlaces([plugin_place])
plugin_manager.collectPlugins()
return plugin_manager
def create_plugin_manager(category_filter, plugin_place):
""" Creates a PluginManager object from the given folder according to the given filter
:param category_filter:
:param plugin_place:
:return:
:rtype: PluginManager
"""
plugin_manager = PluginManager()
plugin_manager.setCategoriesFilter(category_filter)
plugin_manager.setPluginPlaces([plugin_place])
plugin_manager.collectPlugins()
return plugin_manager
def main():
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(["plugins\\plugins1", "plugins\\plugins2"])
manager.setCategoriesFilter({
"PluginOne": category.PluginOne,
"PluginTwo": category.PluginTwo,
})
print { "PluginOne": category.PluginOne, "PluginTwo": category.PluginTwo }
manager.collectPlugins()
print manager.getPluginsOfCategory("PluginTwo")
for plugin in manager.getPluginsOfCategory("PluginTwo"):
plugin.plugin_object.print_name()
def get_manager():
categories = {
catg.ApkChecker.category: catg.ApkChecker,
catg.Unpacker.category: catg.Unpacker,
catg.ProtectChecker.category: catg.ProtectChecker,
catg.Auditor.category: catg.Auditor,
}
manager = PluginManager()
plugin_paths = map(lambda e: os.path.join(settings.PLUGINS_DIR, e),
categories.keys())
manager.setPluginPlaces(plugin_paths)
manager.setCategoriesFilter(categories)
manager.collectPlugins()
return manager
def modelBehaviorImplementations(paths):
"""
Return an iterable of plugin-info for every locatable implementation of this interface on a given path.
PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to
the schedule upon which module resolution and imports happen.
"""
manager = PluginManager()
manager.setPluginPlaces(paths)
from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior
#hey PyDev: the previous line is weird, but is actually legal
manager.setCategoriesFilter({
"ModelBehavior" : foreignModelBehavior,
})
manager.collectPlugins()
return manager.getPluginsOfCategory("ModelBehavior")
def get_plugin_manager(config):
plugin_manager = PluginManager()
plugin_manager.setPluginPlaces(
[config.get("GENERAL", "PLUGIN_DIRECTORY")]
)
plugin_manager.setCategoriesFilter({
PluginTypeName.SENSOR: ISensorPlugin,
PluginTypeName.REPOSITORY: IRepositoryPlugin,
PluginTypeName.DISPLAY: IDisplayPlugin
})
plugin_manager.collectPlugins()
return plugin_manager
def test_json_reporting_plugin_found():
"""Test that the plugin manager finds the JSON plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__),
'plugins')])
manager.setCategoriesFilter({
"Reporting": ReportingPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "upload_risk_assessment"
assert any(plugin_info.plugin_object.get_name() == 'upload_risk_assessment' for
plugin_info in manager.getPluginsOfCategory("Reporting"))
# While we're at it, verify that a plugin is named JSON Risk Assessment Reporting Plugin
assert any(plugin_info.name == 'JSON Risk Assessment Reporting Plugin' for
plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_console_reporting_plugin_found():
"""Test that the plugin manager finds the console reporting plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')])
manager.setCategoriesFilter({
"Reporting": ReportingPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "c"
assert any(plugin_info.plugin_object.get_name() == 'print_to_console'
for plugin_info in manager.getPluginsOfCategory("Reporting"))
# While we're at it, verify that a plugin is named C Discovery Plugin
assert any(plugin_info.name == 'Print To Console Reporting Plugin'
for plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_maven_discovery_plugin_found():
"""Test that the plugin manager finds the Maven discovery plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "java"
assert any(plugin_info.plugin_object.get_name() == "maven"
for plugin_info in manager.getPluginsOfCategory("Discovery"))
# While we're at it, verify that a plugin is named Maven Discovery Plugin
assert any(plugin_info.name == "Maven Discovery Plugin"
for plugin_info in manager.getPluginsOfCategory("Discovery"))
def test_yaml_discovery_plugin_found():
"""Test that the YAML discovery plugin is detected by the plugin system."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__),
'plugins')])
manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "yaml"
assert any(plugin_info.plugin_object.get_name() == 'yaml' for
plugin_info in manager.getPluginsOfCategory("Discovery"))
# While we're at it, verify that a plugin is named YAML Discovery Plugin
assert any(plugin_info.name == 'YAML Discovery Plugin' for
plugin_info in manager.getPluginsOfCategory("Discovery"))
def test_cccc_tool_plugin_found():
"""Test that the CCCC tool plugin is detected by the plugin system."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Tool": ToolPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "cccc"
assert any(plugin_info.plugin_object.get_name() == "cccc"
for plugin_info in manager.getPluginsOfCategory("Tool"))
# While we're at it, verify that a plugin is named CCCC Tool Plugin
assert any(plugin_info.name == "CCCC Tool Plugin"
for plugin_info in manager.getPluginsOfCategory("Tool"))
def test_clang_tidy_tool_plugin_found():
"""Test that the plugin manager can find the clang-tidy plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__),
'plugins')])
manager.setCategoriesFilter({
"Tool": ToolPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "clang_tidy"
assert any(plugin_info.plugin_object.get_name() == 'clang-tidy' for
plugin_info in manager.getPluginsOfCategory("Tool"))
# While we're at it, verify that a plugin is named ClangTidy Tool Plugin
assert any(plugin_info.name == 'clang-tidy Tool Plugin' for
plugin_info in manager.getPluginsOfCategory("Tool"))
def initialize_plugins(self):
self.__log.info("Collecting and loading plugins")
try:
plugin_manager = PluginManager()
# TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT)
plugin_manager.setCategoriesFilter({
CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin,
ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin
})
plugin_manager.setPluginPlaces([self.__config.read_property(constants.PLUGINS_DIR)])
plugin_manager.collectPlugins()
# activate cartridge agent plugins
plugins = plugin_manager.getPluginsOfCategory(CARTRIDGE_AGENT_PLUGIN)
grouped_plugins = {}
for plugin_info in plugins:
self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated plugin [%s]" % plugin_info.name)
mapped_events = plugin_info.description.split(",")
for mapped_event in mapped_events:
if mapped_event.strip() != "":
if grouped_plugins.get(mapped_event) is None:
grouped_plugins[mapped_event] = []
grouped_plugins[mapped_event].append(plugin_info)
# activate artifact management plugins
artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(ARTIFACT_MGT_PLUGIN)
for plugin_info in artifact_mgt_plugins:
self.__log.debug("Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name)
return plugin_manager, grouped_plugins, artifact_mgt_plugins
except ParameterNotFoundException as e:
self.__log.exception("Could not load plugins. Plugins directory not set: %s" % e)
return None, None, None
except Exception as e:
self.__log.exception("Error while loading plugin: %s" % e)
return None, None, None
def test_write_jenkins_warnings_ng_reporting_plugin_found():
"""Test that the plugin manager finds the file writing plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Reporting": ReportingPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "c"
assert any(
plugin_info.plugin_object.get_name() == "write_jenkins_warnings_ng"
for plugin_info in manager.getPluginsOfCategory("Reporting"))
# While we're at it, verify that a plugin is named C Discovery Plugin
assert any(plugin_info.name == "Write Jenkins Warnings NG Reporting Plugin"
for plugin_info in manager.getPluginsOfCategory("Reporting"))
def init_plugin_manager():
global tom_plugin_manager
from engine import BuiltinEngine, AnsibleEngine, Engine
if not holder.plugin_manager:
tom_plugin_manager = PluginManager()
tom_plugin_manager.setPluginPlaces(config.plugin_dirs)
tom_plugin_manager.setPluginInfoExtension('plug')
# 3 types of plugins, Built-in for plugin come with Tom,
# Ansible for ansible simplerunner or playbook running on tom
# User for user-defined plugin
tom_plugin_manager.setCategoriesFilter({
'Built-in': BuiltinEngine,
'Ansible': AnsibleEngine,
'User': Engine
})
holder.plugin_manager = tom_plugin_manager
else:
tom_plugin_manager = holder.plugin_manager
def init_plugin_manager():
global tom_plugin_manager
from engine import BuiltinEngine, AnsibleEngine, Engine
if not holder.plugin_manager:
tom_plugin_manager = PluginManager()
tom_plugin_manager.setPluginPlaces(config.plugin_dirs)
tom_plugin_manager.setPluginInfoExtension('plug')
# 3 types of plugins, Built-in for plugin come with Tom,
# Ansible for ansible simplerunner or playbook running on tom
# User for user-defined plugin
tom_plugin_manager.setCategoriesFilter({
'Built-in': BuiltinEngine,
'Ansible': AnsibleEngine,
'User': Engine
})
holder.plugin_manager = tom_plugin_manager
else:
tom_plugin_manager = holder.plugin_manager
def load_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter({"RuntimeEnvironment" : IRuntimeEnvironment})
pm.collectPlugins()
for env_plugin in pm.getAllPlugins():
# There should be only one plugin of the given category and type;
# but in case there are more, pick the first one.
if env_plugin.plugin_object.get_environment_name() == environment:
return env_plugin.plugin_object
raise RuntimeEnvironmentPluginNotFound('Could not find a valid runtime '
'environment plugin at %s' % plugin_places)
def __init__(self):
self.ver = "0.0.1"
wx.Frame.__init__(self, None, -1, "xshock v" + self.ver, size = (1000, 700))
plugin_mger = PluginManager(categories_filter={"toolbar":IToolBarPlugin})
plugin_mger.setPluginPlaces(["plugins"])
plugin_mger.setCategoriesFilter({
"all":IPlugin,
"toolbar":IToolBarPlugin,
"Win":IWinPlugin,
})
plugin_mger.collectPlugins()
tbs = []
nbs = []
#tb_plugin_objs = plugin_mger.getPluginsOfCategory("all")
tb_plugin_objs = plugin_mger.getAllPlugins()
for obj in tb_plugin_objs:
if isinstance (obj.plugin_object,IToolBarPlugin):
tbs.append(obj)
elif isinstance (obj.plugin_object,IWinPlugin):
nbs.append(obj)
self.tm = ECG_Timer()
vbox = wx.BoxSizer(wx.VERTICAL)
self.tb = self.createToolBar(tbs)
vbox.Add(self.tb, 0 , wx.EXPAND)
self.splitter = wx.SplitterWindow(self, -1, style = wx.SP_LIVE_UPDATE)
self.ann_tree = LogTree(self.splitter)
self.ecg_notebook = Notebook_div(self.splitter,nbs)
self.splitter.SplitVertically(self.ann_tree, self.ecg_notebook, 200)
vbox.Add(self.splitter, 2, wx.EXPAND)
self.SetSizer(vbox)
def test_isort_tool_plugin_found():
"""Test that the plugin manager can find the Isort plugin."""
if sys.version_info.major == 3 and sys.version_info.minor < 6:
pytest.skip("isort is only available for Python 3.6+, unable to test")
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Tool": ToolPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "isort"
assert any(plugin_info.plugin_object.get_name() == "isort"
for plugin_info in manager.getPluginsOfCategory("Tool"))
# While we're at it, verify that a plugin is named Isort Tool Plugin
assert any(plugin_info.name == "Isort Tool Plugin"
for plugin_info in manager.getPluginsOfCategory("Tool"))
def testChangingCategoriesFilter(self):
"""
Test the effect of setting a new category filer.
"""
spm = PluginManager(directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),"plugins")])
# load the plugins that may be found
spm.collectPlugins()
newCategory = "Mouf"
# Pre-requisite for the test
previousCategories = spm.getCategories()
self.assertTrue(len(previousCategories) >= 1)
self.assertTrue(newCategory not in previousCategories)
# change the category and see what's happening
spm.setCategoriesFilter({newCategory: IPlugin})
spm.collectPlugins()
for categoryName in previousCategories:
self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName)
self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
def load_env_plugin(plugin_places=[misc.execution_path('plugins')],
environment='cloudsight'):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment})
pm.collectPlugins()
for env_plugin in pm.getAllPlugins():
# There should be only one plugin of the given category and type;
# but in case there are more, pick the first one.
if env_plugin.plugin_object.get_environment_name() == environment:
return env_plugin.plugin_object
raise RuntimeEnvironmentPluginNotFound(
'Could not find a valid "%s" runtime '
'environment plugin at %s' % (environment, plugin_places))
class ExtensionLoader(object):
def __init__(self, core_config):
default_dir = os.path.join(os.path.expanduser("~"), "epops")
self.plugin_path = core_config.get('plugin_dir', [default_dir])
self.simple_plugin_manager = PluginManager()
self.simple_plugin_manager.setPluginPlaces(self.plugin_path)
self.simple_plugin_manager.setPluginInfoExtension('plugin-meta')
self.simple_plugin_manager.setCategoriesFilter({"Extension": IExtension})
self.pluginManager = AutoInstallPluginManager(self.plugin_path[0], self.simple_plugin_manager)
def ext_init(self):
self.simple_plugin_manager.collectPlugins()
return self.simple_plugin_manager.getAllPlugins()
def ext_load_new(self, filename):
zip_path = os.path.join(self.plugin_path[0], filename)
self.pluginManager.installFromZIP(zip_path)
self.simple_plugin_manager.collectPlugins()
return self.simple_plugin_manager.getAllPlugins()
def testChangingCategoriesFilter(self):
"""
Test the effect of setting a new category filer.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# load the plugins that may be found
spm.collectPlugins()
newCategory = "Mouf"
# Pre-requisite for the test
previousCategories = spm.getCategories()
self.assertTrue(len(previousCategories) >= 1)
self.assertTrue(newCategory not in previousCategories)
# change the category and see what's happening
spm.setCategoriesFilter({newCategory: IPlugin})
spm.collectPlugins()
for categoryName in previousCategories:
self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName)
self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
def _load_plugins(plugin_places=[misc.execution_path('plugins')],
category_filter={},
filter_func=lambda *arg: True):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter(category_filter)
pm.collectPlugins()
config = config_parser.get_config()
enabled_plugins = [p for p in config['crawlers']]
for plugin in pm.getAllPlugins():
if filter_func(plugin.plugin_object, plugin.name, enabled_plugins):
plugin_args = {}
if plugin.name in config['crawlers']:
plugin_args = config['crawlers'][plugin.name]
yield (plugin.plugin_object, plugin_args)
def test_print_json_reporting_plugin_found():
"""Test that the plugin manager finds the print json reporting plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
)
manager.setCategoriesFilter(
{
"Reporting": ReportingPlugin,
}
)
manager.collectPlugins()
assert any(
plugin_info.plugin_object.get_name() == "print_json"
for plugin_info in manager.getPluginsOfCategory("Reporting")
)
assert any(
plugin_info.name == "Print JSON Reporting Plugin"
for plugin_info in manager.getPluginsOfCategory("Reporting")
)
def load_plugins():
""" Load plugin for environment. See lib/base.py
"""
# Create plugin manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces(["pylons_yapsy_plugin/plugins/"])
# Define the various categories corresponding to the different
# kinds of plugins you have defined
manager.setCategoriesFilter({
"Menu" : menu.Menu,
"Inline" : inline.Inline,
})
manager.locatePlugins()
# Deactivate plugin
# Список деактивированных плагинов из БД
deactivatedPlugins = [plugin.name for plugin in\
s.query(DeactivatedPlugins).all()]
# Список кандидатов на загрузку
candidates = manager.getPluginCandidates()
# Список деактивированных плагинов в формате yapsy
deactivatedList = []
for candidate in candidates:
if candidate[2].name in deactivatedPlugins:
deactivatedList.append(candidate)
# Исключаем плагины которые указанны в БД
for plugin in deactivatedList:
manager.removePluginCandidate(plugin)
# Load plugins
manager.loadPlugins()
return manager, [plugin[2] for plugin in deactivatedList]
def test_groovylint_tool_plugin_found():
"""Test that the plugin manager can find the groovylint plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
)
manager.setCategoriesFilter(
{
"Tool": ToolPlugin,
}
)
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "groovylint"
assert any(
plugin_info.plugin_object.get_name() == "groovylint"
for plugin_info in manager.getPluginsOfCategory("Tool")
)
# While we're at it, verify that a plugin is named NPM Groovy Lint Tool Plugin
assert any(
plugin_info.name == "NPM Groovy Lint Tool Plugin"
for plugin_info in manager.getPluginsOfCategory("Tool")
)
def _load_plugins(
category_filter={},
filter_func=lambda *arg: True,
features=['os', 'cpu'],
plugin_places=['plugins'],
options={}):
pm = PluginManager(plugin_info_ext='plugin')
# Normalize the paths to the location of this file.
# XXX-ricarkol: there has to be a better way to do this.
plugin_places = [misc.execution_path(x) for x in plugin_places]
pm.setPluginPlaces(plugin_places)
pm.setCategoriesFilter(category_filter)
pm.collectPlugins()
config = config_parser.get_config()
enabled_plugins = []
if 'enabled_plugins' in config['general']:
enabled_plugins = config['general']['enabled_plugins']
if 'ALL' in enabled_plugins:
enabled_plugins = [p for p in config['crawlers']]
# Reading from 'crawlers' section inside crawler.conf
# Alternatively, 'ALL' can be made to signify
# all crawlers in plugins/*
for plugin in pm.getAllPlugins():
if filter_func(
plugin.plugin_object,
plugin.name,
enabled_plugins,
features):
plugin_args = get_plugin_args(plugin, config, options)
yield (plugin.plugin_object, plugin_args)
class GiantCommands(object):
_client_category = 'Client'
_server_category = 'Server'
def __init__(self):
self.client_plugins = None
self.server_plugins = None
self._plugin_manager = PluginManager(plugin_info_ext='giant')
self._plugin_manager.setCategoriesFilter({
GiantCommands._client_category : BaseGiantClient,
GiantCommands._server_category: BaseGiantServer,
})
path = os.path.dirname(os.path.realpath(__file__))
self._plugin_manager.setPluginPlaces([path + '/plugins'])
self._plugin_manager.collectPlugins()
self.plugin_infos = self._plugin_manager.getAllPlugins()
def new_plugin(self, args):
import getpass
username = getpass.getuser()
safe_language = (args.language
.replace('#', 'Sharp')
.replace('+', 'Plus')
.replace(' ', '')
.replace('-', '_')
.replace('!', 'Exclamation')
.replace('/', 'Slash')
.replace('*', 'Star'))
repo_name = safe_language.lower()+'_'+args.framework.lower()
cookiecutter('gh:lixar/giant-plugin', extra_context={
'full_name': username,
'project_language': args.language,
'project_framework': args.framework,
'project_type': args.type,
'repo_name': repo_name,
'command': repo_name.replace('_', '-')
})
def install_plugin(self, args):
import glob
input_path = args.input_plugin
plugin_file = glob.glob(input_path + '/*.giant')
with open(plugin_file[0], 'r') as plugin:
for line in plugin.readlines():
if line.startswith('Name'):
plugin_name = line.split(' = ')[1].strip()
if line.startswith('Type'):
plugin_type = line.split(' = ')[1].strip()
path = os.path.dirname(os.path.abspath(__file__))
plugins_dir = os.path.join(path, 'plugins')
plugins_dir = os.path.join(plugins_dir, plugin_type.lower())
try:
os.makedirs(plugins_dir)
except:
pass
plugins_dir = os.path.join(plugins_dir, plugin_name)
if args.symlink:
try:
os.symlink(os.path.abspath(args.input_plugin), plugins_dir)
except:
logging.error('Plugin with this name already exists.')
else:
import shutil
shutil.copytree(os.path.abspath(args.input_plugin), plugins_dir)
# TODO Update giant-1.0.0.dist-info/RECORD to have new files added.
def uninstall_plugin(self, args):
import shutil
plugin = self._get_plugin(args.command_name)
path = plugin.path
while not path.endswith('client') and not path.endswith('server'):
path, plugin_dirname = os.path.split(path)
shutil.rmtree(os.path.join(path, plugin_dirname))
def get_client_plugins(self):
if self.client_plugins == None:
self.client_plugins = self._plugin_manager.getPluginsOfCategory(GiantCommands._client_category)
return self.client_plugins
def get_server_plugins(self):
if self.server_plugins == None:
self.server_plugins = self._plugin_manager.getPluginsOfCategory(GiantCommands._server_category)
return self.server_plugins
def get_all_plugins(self):
return self.get_client_plugins() + self.get_server_plugins()
def _get_plugin(self, command_name):
return next(plugin for plugin in self.get_all_plugins() if plugin.details.get('Details', 'Command') == command_name)
def generate_client(self, args, name):
plugin_info = next(plugin_info for plugin_info in self.client_plugins if plugin_info.details.get('Details', 'Command') == name)
self._plugin_manager.activatePluginByName(plugin_info.name)
self._generate_project(args, plugin_info.plugin_object)
def generate_server(self, args, name):
plugin_info = next(plugin_info for plugin_info in self.server_plugins if plugin_info.details.get('Details', 'Command') == name)
plugin = self._plugin_manager.activatePluginByName(plugin_info.name)
self._generate_project(args, plugin_info.plugin_object)
def _generate_project(self, args, plugin):
if args.swagger_files == None:
import requests
response = requests.get(args.swagger_url)
try:
swagger = response.json()
except:
import yaml
try:
swagger = yaml.load(response.text)
except:
logging.critical("Failed to parse server response to JSON or YAML.")
exit(1)
else:
swagger = {}
for f in args.swagger_files:
if os.path.isdir(f):
files = os.listdir(f)
else:
files = [f]
for swagger_file_path in files:
with open(swagger_file_path, 'r') as swagger_file:
try:
api = json.load(swagger_file)
except:
swagger_file.seek(0)
import yaml
api = yaml.load(swagger_file)
paths = {}
for path_name, path in api['paths'].iteritems():
if len(args.swagger_files) > 1:
paths[api['host'] + path_name] = path
else:
paths[path_name] = path
api['paths'] = paths
tag = f.split('.')[0]
self.mergedicts(swagger, api, conflicts=['paths', 'definitions'], conflict_tag=tag)
plugin.setup(swagger, args.output_dir, args.force_overwrite)
print('Generating Project...')
plugin.generate()
def mergedicts(self, a, b, path=None, conflicts=[], conflict_tag=None):
"merges b into a"
if path is None: path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
self.mergedicts(a[key], b[key], path + [str(key)], conflicts=conflicts, conflict_tag=conflict_tag)
elif a[key] == b[key]:
pass # same leaf value
else:
conflict_path = '.'.join(path + [str(key)])
if any(conflict in conflicts for conflict in conflict_path.split('.')):
if conflict_tag is not None and key + conflict_tag not in a.keys():
a[key + conflict_tag] = b[key]
continue # resolve conflict using tag.
raise Exception('Conflict at ' + '.'.join(path + [str(key)]))
else:
print('Ignoring conflict ' + conflict_path)
else:
a[key] = b[key]
return a
def main():
parser = optparse.OptionParser()
parser.add_option("-l",
"--LogConfigFile",
dest="log_config_file",
help="Logging Configuration file.")
parser.add_option(
"-p",
"--PluginDirList",
dest="plugin_directories",
help="Comma separated directory list to search for plugins.")
parser.add_option(
"-s",
"--StartDateTime",
dest="start_date_time",
help=
"A date to re-run the predictions for, if not provided, the default is the current day. Format is YYYY-MM-DD HH:MM:SS."
)
(options, args) = parser.parse_args()
if (options.plugin_directories is None):
parser.print_help()
sys.exit(-1)
if options.log_config_file:
logging.config.fileConfig(options.log_config_file)
logging.getLogger('yapsy').setLevel(logging.DEBUG)
"""
log_stop_event = Event()
log_queue = Queue()
log_config_plugin = {
'version': 1,
'disable_existing_loggers': True,
'handlers': {
'queue': {
'class': 'logutils.queue.QueueHandler',
'queue': log_queue,
},
},
'root': {
'level': 'NOTSET',
'handlers': ['queue']
}
}
lp = Process(target=listener_process, name='listener',
args=(log_queue, log_stop_event, options.log_config_file))
lp.start()
"""
#logging.config.fileConfig(logfile)
#logging.config.dictConfig(log_config_plugin)
logger = logging.getLogger("WQ-Engine")
logger.info("Log file opened.")
plugin_dirs = options.plugin_directories.split(',')
#plugin_dirs = (["plugins"])
# Build the manager
simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
if logger:
logger.debug("Plugin directories: %s" % (options.plugin_directories))
simplePluginManager.setCategoriesFilter(
{"PredictionEngine": wq_prediction_engine_plugin})
simplePluginManager.setPluginPlaces(plugin_dirs)
# Load all plugins
if logger:
logger.info("Begin loading plugins")
simplePluginManager.collectPlugins()
plugin_proc_start = time.time()
cnt = 0
try:
for plugin in simplePluginManager.getAllPlugins():
if logger:
logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(ini=plugin.details.get(
"Core", "Ini"),
name=plugin.name):
plugin.plugin_object.start()
else:
logger.error("Failed to initialize plugin: %s" % (plugin.name))
cnt += 1
#Wait for the plugings to finish up.
if logger:
logger.info("Waiting for %d plugins to complete." % (cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
if logger:
logger.info("Plugins completed in %f seconds" %
(time.time() - plugin_proc_start))
logger.debug("Shutting down logger.")
"""
log_stop_event.set()
lp.join()
"""
except Exception as e:
logger.exception(e)
if logger:
logger.info("Log file closed.")
try:
from compatible.urlresolver.curlresolver import getPlugins as getURPlugins
except:
ur_plugins = False
else:
ur_plugins = True
from yapsy.PluginManager import PluginManager
from yapsy.PluginFileLocator import PluginFileLocator, PluginFileAnalyzerMathingRegex
import re
import os
plugins = PluginManager(plugin_locator=PluginFileLocator([PluginFileAnalyzerMathingRegex("all", "^[a-zA-Z][a-zA-Z_]*\.py$")]))
plugins.setCategoriesFilter({
"Page" : Page,
})
def loadExtension():
# folder from this project
path = os.path.dirname(os.path.abspath(__file__))
pathes = []
pathes.append(os.path.join(path, 'pages'))
# folder from config
# for con in ('page_extension_dir'):
# c_path = config.get(con, "")
# if len(c_path) > 1:
# pathes.append(c_path)
plugins.setPluginPlaces(pathes)
plugins.collectPlugins()
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.harasser = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
self.watchdog = None
self.plugin_trans_sem = Semaphore()
def setOptions(self, args):
self.options = vars(args)
self.args = args
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
elif is_py2 and type(inval) is unicode:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0], keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[','')
val = val.replace(']','')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i=0
for val in newvals:
st, newvals[i] = self.__convert_value(opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
unsupported_options = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
unsupported_options.append(kvkey)
if unsupported_options:
sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options)))
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
sys.exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py"));
except ImportError:
print("ERROR: unable to load LoadClasses that must contain the class loader object")
sys.exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__();
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Load plugins from each of the specified plugin dirs
for dirPath in plugindirs:
if not Path(dirPath).exists():
print("Attempted to load plugins from non-existent path:", dirPath)
continue
try:
self.loader.load(dirPath)
except Exception as e:
print("Exception caught while loading plugins:")
print(e)
sys.exit(1)
# Build plugin managers,
# class yapsy.PluginManager.PluginManager(categories_filter=None,
# directories_list=None, plugin_info_ext=None, plugin_locator=None)
# Build the stages plugin manager
self.stages = PluginManager(None, plugindirs, None, None)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager(None, plugindirs, None, None)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager(None, plugindirs, None, None)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
elif "Watchdog" == pluginInfo.plugin_object.print_name():
self.watchdog = pluginInfo.plugin_object
if self.execmd is not None and self.modcmd is not None and self.watchdog is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
# Configure harasser plugin
for pluginInfo in self.tools.getPluginsOfCategory("Harasser"):
if "Harasser" == pluginInfo.plugin_object.print_name():
self.harasser = pluginInfo.plugin_object
break
if self.harasser is None:
print("Harasser plugin was not found")
print("This is required for all TestRun plugins")
print("cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
sys.exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
sys.exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
sys.exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name",tool)
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
sys.exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
sys.exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
sys.exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " + pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object
self.logger.open(self)
return
def fill_log_interpolation(self, basestr, sublog):
if isinstance(sublog, str):
self.config.set("LOG", basestr, sublog.replace("$","$$"))
elif isinstance(sublog, dict):
for k,v in sublog.items():
self.fill_log_interpolation("%s.%s" % (basestr, k), v)
elif isinstance(sublog, list):
if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog):
self.fill_log_interpolation(basestr, {k:v for k,v in sublog})
else:
for i,v in enumerate(sublog):
self.fill_log_interpolation("%s.%d" % (basestr, i), v)
else:
self.fill_log_interpolation(basestr, str(sublog))
def expandWildCards(self, sections):
expsec = []
cpsections = list(sections)
for sec in cpsections:
if '*' in sec:
modsec = sec.split('*')
startswith = modsec[0]
endswith = modsec[-1]
findsec = modsec[1:-1]
allsections = self.config.sections()
for s in allsections:
if not s.startswith(startswith):
continue
if not s.endswith(endswith):
continue
found = True
s_tmp = s
for f in findsec:
if not f in s_tmp:
found = False
break
s_tmp = f.join(s_tmp.split(f)[1:])
if not found:
continue
expsec.append(s)
sections.remove(sec)
return sections + expsec
def fill_env_hidden_section(self):
"""fill ENV section with environment variables
"""
try:
self.config.add_section('ENV')
except configparser.DuplicateSectionError:
pass
for k,v in os.environ.items():
self.config.set('ENV', k, v.replace("$","$$"))
def fill_log_hidden_section(self):
"""Add LOG section filled with log results of stages
"""
try:
self.config.add_section('LOG')
except configparser.DuplicateSectionError:
pass
thefulllog = self.logger.getLog(None)
for e in thefulllog:
self.fill_log_interpolation(e['section'].replace(":","_"), e)
def check_for_nondefined_env_variables(self):
# Check for ENV input
required_env = []
all_file_contents = []
for testFile in self.log['inifiles']:
file_contents = open(testFile, "r").read()
file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")])
all_file_contents.append(file_contents)
if "${ENV:" in file_contents:
required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]])
env_not_found = set([e for e in required_env if e not in os.environ.keys()])
lines_with_env_not_found = []
for file_contents in all_file_contents:
lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \
for l in file_contents.split("\n") \
if sum(["${ENV:%s}"%e in l for e in env_not_found])])
if lines_with_env_not_found:
print("ERROR: Not all required environment variables are defined.")
print("ERROR: Still need:")
for l in lines_with_env_not_found:
print("ERROR: %s"%l)
sys.exit(1)
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.fill_env_hidden_section()
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file",testFile,"not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
# Check for ENV input
self.check_for_nondefined_env_variables()
# find all the sections that match the wild card and expand them
# this is simple wild carding, ie *text, text*, *text* and *
# should all work
if sections is not None:
sections = self.expandWildCards(sections)
#if sections is not None:
# expsec = []
# cpsections = list(sections)
# for sec in cpsections:
# if '*' in sec:
# modsec = sec.replace('*','')
# for s in self.config.sections():
# if modsec in s:
# expsec.append(s)
# sections.remove(sec)
# sections = sections + expsec
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if sections is not None and 0 != len(sections) and not skip:
print("ERROR: sections were specified for execution and not found:",sections)
sys.exit(1)
# set Defaults -command line args supercede .ini args
try:
if not self.options['scratchdir']:
self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratchdir')
except:
try:
self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratch')
except:
self.options['scratchdir'] = os.path.abspath('./mttscratch')
self.options['scratchdir'] = os.path.abspath(self.options['scratchdir'])
try:
if not self.options['executor']:
self.options['executor'] = self.config.get('MTTDefaults', 'executor')
except:
self.options['executor'] = 'sequential'
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']) :
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self, executor="sequential"):
self.logger.print_cmdline_args(self)
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
sys.exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
sys.exit(1)
if not self.tools.getPluginByName(executor, "Executor"):
print("Specified executor %s not found" % executor)
sys.exit(1)
# activate the specified plugin
self.tools.activatePluginByName(executor, "Executor")
# execute the provided test description
executor = self.tools.getPluginByName(executor, "Executor")
status = executor.plugin_object.execute(self)
if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']):
self.logger.verbose_print("Cleaning up scratchdir after successful run")
shutil.rmtree(self.options['scratchdir'])
return status
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0,len(opts),3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0,len(opts),3):
line = opts[i] + (max1-len(opts[i]))*sp
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
line = line + optout + (max2-len(optout))*sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i+2])):
# split the remaining column into individual words
words = opts[i+2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2)*sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i+2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:",category)
return None
if category == 'Default features':
plugin = pluginManager.getPluginByName("Standard Object Features", "ObjectFeatures")
else:
plugin = pluginManager.getPluginByName(category, "ObjectFeatures")
if plugin:
plugin_feature_names = {name: {}}
all_props = plugin.plugin_object.fill_properties(plugin_feature_names) # fill in display name and such
if all_props:
long_name = all_props[name]["displaytext"]
else:
long_name = name
return long_name
###############
# the manager #
###############
pluginManager = PluginManager()
pluginManager.setPluginPlaces(plugin_paths)
pluginManager.setCategoriesFilter({
"ObjectFeatures" : ObjectFeaturesPlugin,
"TrackingExportFormats": TrackingExportFormatPlugin
})
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
class PluginSystem:
def __init__(self, config=None, extra_plugin_paths=[]):
'Initialize a new PluginSystem. Needs no arguments.'
# Build the manager
self._mngr = PluginManager(plugin_info_ext='plugin')
# Tell it the default place(s) where to find plugins
plugin_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'plugins')
self._mngr.setPluginPlaces([plugin_path] + extra_plugin_paths)
# find the categories specified in moosecat/plugins/__init__.py
self._categories = _get_interfaces_from_module()
LOGGER.debug('Discovered following plugins categories: ' + ', '.join(self.list_categories()))
# tell yapsy about our extra categories
self._mngr.setCategoriesFilter(self._categories)
try:
self._mngr.collectPlugins()
except SystemError as e:
LOGGER.exception('Some plugin could not be loaded.')
# Get a list of plugin names to load
load_these = [pluginInfo.name for pluginInfo in self._mngr.getAllPlugins()]
if config is not None:
config.add_defaults({'plugins_to_load': load_these})
load_these = config.get('plugins_to_load')
# Actually load them
for name in load_these:
self._mngr.activatePluginByName(name)
def list_plugin_info(self):
'''
Get a list of PluginInfo objects.
These contain additionnal metadata found in the .plugin files.
See here for more information: http://yapsy.sourceforge.net/PluginInfo.html
Useful if you want to realize a list of Plugins.
:returns: A list of PluginInfo objects
'''
return self._mngr.getAllPlugins()
def list_plugin_info_by_category(self, name):
return self._mngr.getPluginsOfCategory(name)
def list_categories(self):
'Get a string list of categories.'
return self._categories.keys()
def by_name(self, plugin_name):
'Access a Plugin by its actual name - discouraged'
return self._mngr.getPluginByName(plugin_name)
@functools.lru_cache(maxsize=25)
def category(self, name):
'''
Get a list of all Plugins of a Category.
:name: The name of a category.
:returns: A list of instances of a certain Plugin Classes.
'''
cat = [info.plugin_object for info in self.list_plugin_info_by_category(name)]
cat.sort(key=lambda pobj: pobj.priority(), reverse=True)
return cat
def first(self, name):
'''
Shortcut for ``psys.categories(name)[0]``
Will return None if no plugins for this category.
:name: A Category Name, Same as in ``category()``
:returns: An Instance of a certain Plugin Class.
'''
cat = self.category(name)
if len(cat) > 0:
return cat[0]
import os
import codecs
from yapsy.PluginManager import PluginManager
from pendium.plugins import IRenderPlugin, ISearchPlugin
from pendium import app
import logging
logger = logging.getLogger(__name__)
# Populate plugins
manager = PluginManager()
manager.setPluginPlaces(["pendium/plugins"])
manager.setCategoriesFilter({"Search": ISearchPlugin,
"Render": IRenderPlugin})
manager.collectPlugins()
class PathExists(Exception):
pass
class PathNotFound(Exception):
pass
class CannotRender(Exception):
pass
class NoSearchPluginAvailable(Exception):
"""Helper for features that only take one channel.
:param fn: function that computes features
"""
results = []
slc = [slice(None)] * 4
for channel in range(image.shape[axes.c]):
slc[axes.c] = channel
#a dictionary for the channel
result = fn(image[slc], axes=axes, **kwargs)
results.append(result)
return self.combine_dicts_with_numpy(results)
###############
# the manager #
###############
pluginManager = PluginManager()
pluginManager.setPluginPlaces(plugin_paths)
pluginManager.setCategoriesFilter({
"ObjectFeatures": ObjectFeaturesPlugin,
})
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
if not os.path.isdir(app.config['UPLOAD_FOLDER']):
os.makedirs(app.config['UPLOAD_FOLDER'], 0o770)
app.jinja_env.filters['fixify'] = utils.printable
from flaskext.markdown import Markdown
from markdown.extensions.toc import TocExtension
Markdown(app, extensions=['extra', 'codehilite', QuotedExtension(), TocExtension(baselevel=2, marker='')], lazy_ol=False)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("quoted-forsooth")
logging.getLogger("werkzeug").setLevel(logging.WARNING)
manager = PluginManager()
manager.setPluginPlaces([os.path.join(os.path.dirname(__file__), 'plugins')])
manager.setCategoriesFilter({
'Report' : report_plugin.ReportPlugin
})
manager.collectPlugins()
with app.app_context():
for plugin in manager.getAllPlugins():
manager.activatePluginByName(plugin.name, plugin.category)
@app.route('/files/<path:path>')
def send_file(path):
return send_from_directory(app.config['UPLOAD_FOLDER'], path)
@app.after_request
def after_request(response):
if db.config['file_store']['type'] == 'xenodermus':
response.headers.add('Accept-Ranges', 'bytes')
return response
def main():
parser = optparse.OptionParser()
parser.add_option("-l", "--LogConfigFile", dest="log_config_file",
help="Logging Configuration file." )
parser.add_option("-p", "--PluginDirList", dest="plugin_directories",
help="Comma separated directory list to search for plugins.")
parser.add_option("-s", "--StartDateTime", dest="start_date_time",
help="A date to re-run the predictions for, if not provided, the default is the current day. Format is YYYY-MM-DD HH:MM:SS." )
(options, args) = parser.parse_args()
if(options.plugin_directories is None):
parser.print_help()
sys.exit(-1)
if options.log_config_file:
logging.config.fileConfig(options.log_config_file)
logging.getLogger('yapsy').setLevel(logging.DEBUG)
"""
log_stop_event = Event()
log_queue = Queue()
log_config_plugin = {
'version': 1,
'disable_existing_loggers': True,
'handlers': {
'queue': {
'class': 'logutils.queue.QueueHandler',
'queue': log_queue,
},
},
'root': {
'level': 'NOTSET',
'handlers': ['queue']
}
}
lp = Process(target=listener_process, name='listener',
args=(log_queue, log_stop_event, options.log_config_file))
lp.start()
"""
#logging.config.fileConfig(logfile)
#logging.config.dictConfig(log_config_plugin)
logger = logging.getLogger("WQ-Engine")
logger.info("Log file opened.")
plugin_dirs = options.plugin_directories.split(',')
#plugin_dirs = (["plugins"])
# Build the manager
simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
if logger:
logger.debug("Plugin directories: %s" % (options.plugin_directories))
simplePluginManager.setCategoriesFilter({
"PredictionEngine" : wq_prediction_engine_plugin
})
simplePluginManager.setPluginPlaces(plugin_dirs)
# Load all plugins
if logger:
logger.info("Begin loading plugins")
simplePluginManager.collectPlugins()
plugin_proc_start = time.time()
cnt = 0
try:
for plugin in simplePluginManager.getAllPlugins():
if logger:
logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(ini=plugin.details.get("Core", "Ini"),
name=plugin.name):
plugin.plugin_object.start()
else:
logger.error("Failed to initialize plugin: %s" % (plugin.name))
cnt += 1
#Wait for the plugings to finish up.
if logger:
logger.info("Waiting for %d plugins to complete." % (cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
if logger:
logger.info("Plugins completed in %f seconds" % (time.time() - plugin_proc_start))
logger.debug("Shutting down logger.")
"""
log_stop_event.set()
lp.join()
"""
except Exception as e:
logger.exception(e)
if logger:
logger.info("Log file closed.")
m = getattr(m.storage, storage_package)
STORAGE = getattr(m, storage_class)(CONFIG['storage config'])
STORAGE.load_data()
# plugin system
from yapsy.PluginManager import PluginManager
# Build the manager
pluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
myPath = os.path.dirname(__file__)
modulePath = os.path.join(myPath, "plugins")
pluginManager.setPluginPlaces([modulePath])
# Tell it about our categories
pluginManager.setCategoriesFilter({
'WebPlugin': telescope.plugins.abstract.ITelescopeWebPlugin,
'AnnouncePlugin': telescope.plugins.abstract.ITelescopeAnnouncePlugin
})
# Load all plugins
pluginManager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
PLUGIN_MANAGER = pluginManager
def go():
import bottle
import telescope.handler
class WeaponSystem(rpyc.Service):
'''
RPC Services: This is the code that does the actual password cracking
and returns the results to orbital control. Currently only supports
cracking using rainbow tables (RCrackPy)
'''
is_initialized = False
mutex = Lock()
is_busy = False
job_id = None
def initialize(self):
''' Initializes variables, this should only be called once '''
logging.info("Weapon system initializing ...")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins/"])
self.plugin_manager.setCategoriesFilter(FILTERS)
self.plugin_manager.collectPlugins()
self.plugins = {}
logging.info(
"Loaded %d plugin(s)" % len(self.plugin_manager.getAllPlugins())
)
self.__cpu__()
logging.info("Weapon system online, good hunting.")
@atomic
def on_connect(self):
''' Called when successfully connected '''
if not self.is_initialized:
self.initialize()
self.is_initialized = True
logging.info("Uplink to orbital control active")
def on_disconnect(self):
''' Called if the connection is lost/disconnected '''
logging.info("Disconnected from orbital command server.")
def __cpu__(self):
''' Detects the number of CPU cores on a system (including virtual cores) '''
if cpu_count is not None:
try:
self.cpu_cores = cpu_count()
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except NotImplementedError:
logging.error("Could not detect number of processors; assuming 1")
self.cpu_cores = 1
else:
try:
self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF"))
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except ValueError:
logging.error("Could not detect number of processors; assuming 1")
self.cpu_cores = 1
############################ [ EXPOSED METHODS ] ############################
@atomic
def exposed_crack(self, plugin_name, job_id, hashes, **kwargs):
''' Exposes plugins calls '''
assert plugin_name in self.plugins
self.is_busy = True
self.job_id = job_id
self.plugin_manager.activatePluginByName(plugin_name)
plugin = self.plugin_manager.getPluginByName(plugin_name)
results = plugin.execute(hashes, **kwargs)
self.plugin_manager.deactivatePluginByName(plugin_name)
self.job_id = None
self.is_busy = False
return results
def exposed_get_plugin_names(self):
''' Returns what algorithms can be cracked '''
logging.info("Method called: exposed_get_capabilities")
plugins = self.plugin_manager.getAllPlugins()
return [plugin.name for plugin in plugins]
def exposed_get_categories(self):
''' Return categories for which we have plugins '''
categories = []
for category in self.plugin_manager.getCategories():
if 0 < len(self.plugin_manager.getPluginsOfCategory(category)):
categories.append(category)
return categories
def exposed_get_category_plugins(self, category):
''' Get plugin names for a category '''
plugins = self.plugin_manager.getPluginsOfCategory(category)
return [plugin.name for plugin in plugins]
def exposed_get_plugin_details(self, category, plugin_name):
''' Get plugin based on name details '''
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
info = {'name': plugin.name}
info['author'] = plugin.details.get('Documentation', 'author')
info['website'] = plugin.details.get('Documentation', 'website')
info['version'] = plugin.details.get('Documentation', 'version')
info['description'] = plugin.details.get('Documentation', 'description')
info['copyright'] = plugin.details.get('Documentation', 'copyright')
info['precomputation'] = plugin.details.getboolean('Core', 'precomputation')
return info
def exposed_ping(self):
''' Returns a pong message '''
return "PONG"
def exposed_is_busy(self):
''' Returns True/False if the current system is busy (thread safe) '''
return self.is_busy
def exposed_current_job_id(self):
''' Returns the current job id (thread safe) '''
return self.job_id
def exposed_cpu_count(self):
''' Returns the number of detected cpu cores '''
return self.cpu_cores
class PluginHandler:
"""
Handles management of provider, postprocessor and converter plugins.
.. autosummary::
activate_plugins_by_category
deactivate_plugins_by_category
get_plugins_from_category
is_activated
Categories are Provider, Converter and Postprocessor.
"""
def __init__(self):
self._plugin_manager = PluginManager()
self._category_active = {
'Provider': False,
'Converter': False,
'Postprocessor': False
}
self._plugin_from_category = {
'Provider': [],
'Converter': [],
'Postprocessor': []
}
self._provider_plugins = []
self._converter_plugins = []
self._postprocessor_plugins = []
self._collect_all_plugins()
def _collect_all_plugins(self):
""" Collect all provider, converter and postprocessor plugins. """
self._plugin_manager.setPluginPlaces([
'hugin/harvest/provider',
'hugin/harvest/converter',
'hugin/harvest/postprocessor'
])
# setting filter categories for pluginmanager
self._plugin_manager.setCategoriesFilter({
# movie metadata provider
'Provider': IProvider,
# sub metadata provider
'Movie': IMovieProvider,
'Person': IPersonProvider,
# output converter
'Converter': IConverter,
# postprocessor filter
'Postprocessor': IPostprocessor
})
self._plugin_manager.collectPlugins()
def activate_plugins_by_category(self, category):
""" Activate plugins from given category. """
self._toggle_activate_plugins_by_category(category)
def deactivate_plugins_by_category(self, category):
""" Deactivate plugins from given category. """
self._toggle_activate_plugins_by_category(category)
def _toggle_activate_plugins_by_category(self, category):
plugins = self._plugin_manager.getPluginsOfCategory(category)
is_active = self._category_active[category]
for pluginInfo in plugins:
if is_active:
self._plugin_manager.deactivatePluginByName(
name=pluginInfo.name,
category=category
)
self._plugin_from_category[category].remove(pluginInfo)
else:
self._plugin_manager.activatePluginByName(
name=pluginInfo.name,
category=category
)
self._plugin_from_category[category].append(pluginInfo)
self._category_active[category] = not is_active
def get_plugins_from_category(self, category):
"""
Retrun plugins from the given categrory.
Gets plugins from given categrory. Plugin name is set according to name
given in yapsy plugin file.
:param categrory: The category plugins to load from.
:returns: A list with plugins.
"""
plugins = []
for plugin in self._plugin_from_category[category]:
plugin.plugin_object.name = plugin.name
plugin.plugin_object.description = plugin.description
plugins.append(plugin.plugin_object)
return plugins
def is_activated(self, category):
""" True if category is activated. """
return self._category_active[category]
"""Helper for features that only take one channel.
:param fn: function that computes features
"""
results = []
slc = [slice(None)] * 4
for channel in range(image.shape[axes.c]):
slc[axes.c] = channel
#a dictionary for the channel
result = fn(image[slc], axes=axes, **kwargs)
results.append(result)
return self.combine_dicts_with_numpy(results)
###############
# the manager #
###############
pluginManager = PluginManager()
pluginManager.setPluginPlaces(plugin_paths)
pluginManager.setCategoriesFilter({
"ObjectFeatures" : ObjectFeaturesPlugin,
})
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
