class StoqPluginManager:
"""
stoQ Plugin Manager Class
"""
def __init__(self):
# Define the plugin categories and the associated class.
# If we need to add a new plugin category, it must be done here.
self.plugin_categories = {
"worker": StoqWorkerPlugin,
"connector": StoqConnectorPlugin,
"reader": StoqReaderPlugin,
"source": StoqSourcePlugin,
"extractor": StoqExtractorPlugin,
"carver": StoqCarverPlugin,
"decoder": StoqDecoderPlugin
}
self.manager = PluginManager()
self.manager.setPluginInfoExtension("stoq")
self.manager.setPluginPlaces([self.plugin_dir])
self.manager.setCategoriesFilter(self.plugin_categories)
# Setup our plugin filter
self.manager = FilteredPluginManager(self.manager)
@property
def __plugindict__(self):
"""
Create a dict() of plugin class name and the plugin category
"""
plugins = {}
for category, category_class in self.plugin_categories.items():
class_str = re.search('(?<=<class \'stoq\.plugins\.)(.+)(?=.*\'>)',
str(category_class)).group(0)
plugins[class_str] = category
return plugins
def collect_plugins(self):
"""
Wrapper for yapsy.PluginManager.collectPlugins()
"""
self.manager.collectPlugins()
def get_categories(self):
"""
Wrapper for yapsy.PluginManager.getCategories()
"""
return self.manager.getCategories()
def get_plugins_of_category(self, category):
"""
Wrapper for yapsy.PluginManager.getPluginsOfCategory()
"""
return self.manager.getPluginsOfCategory(category)
def get_plugin_names_of_category(self, category):
"""
Lists plugin name of a specific category
:param str category: Category to discover plugins in
:returns: A list of discovered plugins
:rtype: list
"""
return [p.name for p in self.get_plugins_of_category(category)]
def get_plugin(self, name, category):
"""
Initializes a plugin within a specific category
:param str name: Name of plugin to get
:param str category: Category of the named plugin
:returns: plugin object
:rtype: object
"""
return self.manager.getPluginByName(name, category)
def deactivate_plugin(self, name, category):
"""
Deactivate a plugin within a specific category
:param str name: Name of plugin to deactivate
:param str category: Category of the named plugin
"""
self.manager.deactivatePluginByName(name, category)
def load_plugin(self, name, category):
"""
Load the desired plugin
:param str name: Plugin name to be loaded
:param str category: The category of plugin to be loaded
:returns: The loaded plugin object
:rtype: object
"""
# We are going to dynamically reimplement the isPluginOk method
# so only the needed plugins are loaded into memory. Much faster
# and efficient
self.manager.isPluginOk = lambda x: x.name == name
# Gather, filter, and load plugin
self.manager.locatePlugins()
self.manager.filterPlugins()
self.manager.loadPlugins()
# Initialize our plugin
plugin = self.get_plugin(name, category)
if not plugin:
self.stoq.log.warn("Plugin {}:{} failed to load".format(
category, name))
return None
for sect in plugin.details.sections():
# Let's skip over the sections that are required by our
# plugin manager. No sense in trying to overwrite.
if any([s in sect for s in ['Core', 'Documentation']]):
next
for opt in plugin.details.options(sect):
# define each configuration option as an object within
# plugin class.
# Note: In order to reduce logic, we attempt to load
# the option as a boolean. By default, this will raise
# an error which in turn will cause us to load it as
# a string.
try:
setattr(plugin.plugin_object, opt,
plugin.details.getboolean(sect, opt))
except ValueError:
value = plugin.details.get(sect, opt)
if opt.endswith("_list"):
# If our option ends with a list, let's turn it
# into one
# Example:
# worker_list = this, is, a, list
# Becomes:
# worker.worker_list = ['this', 'is', 'a', 'list']
value = [i.strip() for i in value.split(",")]
elif opt.endswith("_dict"):
value = self.loads(value)
elif opt.endswith("_tuple"):
value = tuple(i.strip() for i in value.split(","))
setattr(plugin.plugin_object, opt, value)
setattr(plugin.plugin_object, 'category', category)
plugin_path = "{}/{}/{}".format(self.plugin_dir, category, name)
setattr(plugin.plugin_object, 'plugin_path', plugin_path)
# Make sure we attempt to activate the plugin after we setattr
# from the plugin config file
plugin.plugin_object.activate(self)
return plugin.plugin_object
def get_all_plugin_names(self):
"""
List all plugin names
:returns: All plugin names
:rtype: list
"""
return [p.name for p in self.get_all_plugins()]
def get_all_plugins(self):
"""
Wrapper for yapsy.PluginManager.getAllPlugins()
"""
return self.manager.getAllPlugins()
def list_plugins(self):
"""
List all available plugins and their category
"""
# Make sure we update the filter, otherwise all plugins won't be
# visible.
self.manager.isPluginOk = lambda x: x.name != ""
self.collect_plugins()
print("Available Plugins:")
for category in self.get_categories():
print(" {0}s".format(category))
for plugin in self.get_plugins_of_category(category):
print(" - {0}v{1}{2}".format(plugin.name.ljust(20),
str(plugin.version).ljust(7),
plugin.description))
class TrackingPluginManager(object):
"""
Our plugin manager that handles the types of plugins known in this pipeline
"""
def __init__(self,
pluginPaths=['hytra/plugins'],
turnOffFeatures=[],
verbose=False):
"""
Create the plugin manager that looks inside the specified `pluginPaths` (recursively),
and if `verbose=True` then the [yapsy](http://yapsy.sourceforge.net/) plugin backend
will also show errors that occurred while trying to import plugins (useful for debugging).
"""
self._pluginPaths = pluginPaths
self._turnOffFeatures = turnOffFeatures
self._verbose = verbose
self._initializeYapsy()
self.chosen_data_provider = "LocalImageLoader"
self.chosen_feature_serializer = "LocalFeatureSerializer"
self.chosen_merger_resolver = 'GMMMergerResolver'
def __getstate__(self):
'''
We define __getstate__ and __setstate__ to exclude the loaded yapsy modules from being pickled.
See https://docs.python.org/3/library/pickle.html#pickle-state for more details.
'''
# Copy the object's state from self.__dict__ which contains
# all our instance attributes. Always use the dict.copy()
# method to avoid modifying the original state.
state = self.__dict__.copy()
# Remove the unpicklable entries.
del state['_yapsyPluginManager']
return state
def __setstate__(self, state):
# Restore instance attributes
self.__dict__.update(state)
# Restore the yapsy plugins by reading them from scratch
self._initializeYapsy()
def _initializeYapsy(self):
# Build the manager
self._yapsyPluginManager = PluginManager()
self._yapsyPluginManager = FilteredPluginManager(
self._yapsyPluginManager)
self._yapsyPluginManager.isPluginOk = lambda x: x.name not in self._turnOffFeatures
# Tell it the default place(s) where to find plugins
self._yapsyPluginManager.setPluginPlaces(self._pluginPaths)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._yapsyPluginManager.setCategoriesFilter({
"ObjectFeatureComputation":
ObjectFeatureComputationPlugin,
"TransitionFeatureVectorConstruction":
TransitionFeatureVectorConstructionPlugin,
"ImageProvider":
ImageProviderPlugin,
"FeatureSerializer":
FeatureSerializerPlugin,
"MergerResolver":
MergerResolverPlugin,
})
if self._verbose:
logging.getLogger('yapsy').setLevel(logging.DEBUG)
else:
logging.getLogger('yapsy').setLevel(logging.CRITICAL)
self._yapsyPluginManager.collectPlugins()
def _applyToAllPluginsOfCategory(self, func, category):
''' helper function to apply `func` to all plugins of the given `category` and hide all yapsy stuff '''
for pluginInfo in self._yapsyPluginManager.getPluginsOfCategory(
category):
p = pluginInfo.plugin_object
func(p)
def _getPluginOfCategory(self, name, category):
'''
helper function to access a certain plugin by `name` from a `category`.
**returns** the plugin or throws a `KeyError`
'''
pluginDict = dict(
(pluginInfo.name, pluginInfo.plugin_object)
for pluginInfo in self._yapsyPluginManager.getPluginsOfCategory(
category))
return pluginDict[name]
def applyObjectFeatureComputationPlugins(self, ndims, rawImage, labelImage,
frameNumber, rawFilename):
"""
computes the features of all plugins and returns a list of dictionaries, as well as a list of
feature names that should be ignored
"""
features = []
featureNamesToIgnore = []
def computeFeatures(plugin):
if ndims in plugin.worksForDimensions:
f = plugin.computeFeatures(rawImage, labelImage, frameNumber,
rawFilename)
features.append(f)
featureNamesToIgnore.extend(plugin.omittedFeatures)
self._applyToAllPluginsOfCategory(computeFeatures,
"ObjectFeatureComputation")
return features, featureNamesToIgnore
def applyTransitionFeatureVectorConstructionPlugins(
self, featureDictObjectA, featureDictObjectB, selectedFeatures):
"""
constructs a transition feature vector for training/prediction with a random forest from the
features of the two objects participating in the transition.
"""
featureVector = []
def appendFeatures(plugin):
f = plugin.constructFeatureVector(featureDictObjectA,
featureDictObjectB,
selectedFeatures)
featureVector.extend(f)
self._applyToAllPluginsOfCategory(
appendFeatures, "TransitionFeatureVectorConstruction")
return featureVector
def getTransitionFeatureNames(self, featureDictObjectA, featureDictObjectB,
selectedFeatures):
"""
returns a verbal description of each feature in the transition feature vector
"""
featureNames = []
def collectFeatureNames(plugin):
f = plugin.getFeatureNames(featureDictObjectA, featureDictObjectB,
selectedFeatures)
featureNames.extend(f)
self._applyToAllPluginsOfCategory(
collectFeatureNames, "TransitionFeatureVectorConstruction")
return featureNames
def setImageProvider(self, imageProviderName):
''' set the used image provier plugin name '''
self.chosen_data_provider = imageProviderName
def getImageProvider(self):
''' get an instance of the selected image provider plugin '''
return self._getPluginOfCategory(self.chosen_data_provider,
"ImageProvider")
def setFeatureSerializer(self, featureSerializerName):
''' set the used feature serializer plugin name '''
self.chosen_feature_serializer = featureSerializerName
def getFeatureSerializer(self):
''' get an instance of the selected feature serializer plugin '''
return self._getPluginOfCategory(self.chosen_feature_serializer,
"FeatureSerializer")
def setMergerResolver(self, mergerResolverName):
''' set the used merger resolver plugin name '''
self.chosen_merger_resolver = mergerResolverName
def getMergerResolver(self):
''' get an instance of the selected merger resolver plugin '''
return self._getPluginOfCategory(self.chosen_merger_resolver,
"MergerResolver")
class StoqPluginManager:
"""
stoQ Plugin Manager Class
"""
def __init__(self):
# Define the plugin categories and the associated class.
# If we need to add a new plugin category, it must be done here.
self.plugin_categories = {"worker": StoqWorkerPlugin,
"connector": StoqConnectorPlugin,
"reader": StoqReaderPlugin,
"source": StoqSourcePlugin,
"extractor": StoqExtractorPlugin,
"carver": StoqCarverPlugin,
"decoder": StoqDecoderPlugin
}
self.manager = PluginManager()
self.manager.setPluginInfoExtension("stoq")
self.manager.setPluginPlaces([self.plugin_dir])
self.manager.setCategoriesFilter(self.plugin_categories)
# Setup our plugin filter
self.manager = FilteredPluginManager(self.manager)
@property
def __plugindict__(self):
"""
Create a dict() of plugin class name and the plugin category
"""
plugins = {}
for category, category_class in self.plugin_categories.items():
class_str = re.search(r'(?<=<class \'stoq\.plugins\.)(.+)(?=.*\'>)',
str(category_class)).group(0)
plugins[class_str] = category
return plugins
def collect_plugins(self):
"""
Wrapper for yapsy.PluginManager.collectPlugins()
"""
self.manager.collectPlugins()
def get_categories(self):
"""
Wrapper for yapsy.PluginManager.getCategories()
"""
return self.manager.getCategories()
def get_plugins_of_category(self, category):
"""
Wrapper for yapsy.PluginManager.getPluginsOfCategory()
"""
return self.manager.getPluginsOfCategory(category)
def get_plugin_names_of_category(self, category):
"""
Lists plugin name of a specific category
:param str category: Category to discover plugins in
:returns: A list of discovered plugins
:rtype: list
"""
return [p.name for p in self.get_plugins_of_category(category)]
def get_plugin(self, name, category):
"""
Initializes a plugin within a specific category
:param str name: Name of plugin to get
:param str category: Category of the named plugin
:returns: plugin object
:rtype: object
"""
return self.manager.getPluginByName(name, category)
def deactivate_plugin(self, name, category):
"""
Deactivate a plugin within a specific category
:param str name: Name of plugin to deactivate
:param str category: Category of the named plugin
"""
self.manager.deactivatePluginByName(name, category)
def load_plugin(self, name, category):
"""
Load the desired plugin
:param str name: Plugin name to be loaded
:param str category: The category of plugin to be loaded
:returns: The loaded plugin object
:rtype: object
"""
# We are going to dynamically reimplement the isPluginOk method
# so only the needed plugins are loaded into memory. Much faster
# and efficient
self.manager.isPluginOk = lambda x: x.name == name
# Gather, filter, and load plugin
self.manager.locatePlugins()
self.manager.filterPlugins()
self.manager.loadPlugins()
# Initialize our plugin
plugin = self.get_plugin(name, category)
if not plugin:
self.log.warn("Plugin {}:{} failed to load".format(category, name))
return False
for sect in plugin.details.sections():
# Let's skip over the sections that are required by our
# plugin manager. No sense in trying to overwrite.
if any([s in sect for s in ['Core', 'Documentation']]):
next
for opt in plugin.details.options(sect):
# define each configuration option as an object within
# plugin class.
# Note: In order to reduce logic, we attempt to load
# the option as a boolean. By default, this will raise
# an error which in turn will cause us to load it as
# a string.
try:
setattr(plugin.plugin_object, opt,
plugin.details.getboolean(sect, opt))
except ValueError:
value = plugin.details.get(sect, opt)
if opt.endswith("_list"):
# If our option ends with a list, let's turn it
# into one
# Example:
# worker_list = this, is, a, list
# Becomes:
# worker.worker_list = ['this', 'is', 'a', 'list']
value = [i.strip() for i in value.split(",")]
elif opt.endswith("_dict"):
value = self.loads(value)
elif opt.endswith("_tuple"):
value = tuple(i.strip() for i in value.split(","))
setattr(plugin.plugin_object, opt, value)
setattr(plugin.plugin_object, 'category', category)
plugin_path = "{}/{}/{}".format(self.plugin_dir, category, name)
setattr(plugin.plugin_object, 'plugin_path', plugin_path)
# Make sure we attempt to activate the plugin after we setattr
# from the plugin config file
plugin.plugin_object.activate(self)
return plugin.plugin_object
def get_all_plugin_names(self):
"""
List all plugin names
:returns: All plugin names
:rtype: list
"""
return [p.name for p in self.get_all_plugins()]
def get_all_plugins(self):
"""
Wrapper for yapsy.PluginManager.getAllPlugins()
"""
return self.manager.getAllPlugins()
def list_plugins(self):
"""
List all available plugins and their category
"""
# Make sure we update the filter, otherwise all plugins won't be
# visible.
self.manager.isPluginOk = lambda x: x.name != ""
self.collect_plugins()
print("Available Plugins:")
for category in self.get_categories():
print(" {0}s".format(category))
for plugin in self.get_plugins_of_category(category):
print(" - {0}v{1}{2}".format(plugin.name.ljust(20),
str(plugin.version).ljust(7),
plugin.description))
class TrackingPluginManager(object):
"""
Our plugin manager that handles the types of plugins known in this pipeline
"""
def __init__(self, pluginPaths=['hytra/plugins'], turnOffFeatures=[], verbose=False):
"""
Create the plugin manager that looks inside the specified `pluginPaths` (recursively),
and if `verbose=True` then the [yapsy](http://yapsy.sourceforge.net/) plugin backend
will also show errors that occurred while trying to import plugins (useful for debugging).
"""
self._pluginPaths = pluginPaths
self._turnOffFeatures = turnOffFeatures
self._verbose = verbose
self._initializeYapsy()
self.chosen_data_provider = "LocalImageLoader"
self.chosen_feature_serializer = "LocalFeatureSerializer"
self.chosen_merger_resolver = 'GMMMergerResolver'
def __getstate__(self):
'''
We define __getstate__ and __setstate__ to exclude the loaded yapsy modules from being pickled.
See https://docs.python.org/3/library/pickle.html#pickle-state for more details.
'''
# Copy the object's state from self.__dict__ which contains
# all our instance attributes. Always use the dict.copy()
# method to avoid modifying the original state.
state = self.__dict__.copy()
# Remove the unpicklable entries.
del state['_yapsyPluginManager']
return state
def __setstate__(self, state):
# Restore instance attributes
self.__dict__.update(state)
# Restore the yapsy plugins by reading them from scratch
self._initializeYapsy()
def _initializeYapsy(self):
# Build the manager
self._yapsyPluginManager = PluginManager()
self._yapsyPluginManager = FilteredPluginManager(self._yapsyPluginManager)
self._yapsyPluginManager.isPluginOk = lambda x: x.name not in self._turnOffFeatures
# Tell it the default place(s) where to find plugins
self._yapsyPluginManager.setPluginPlaces(self._pluginPaths)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._yapsyPluginManager.setCategoriesFilter({
"ObjectFeatureComputation": ObjectFeatureComputationPlugin,
"TransitionFeatureVectorConstruction": TransitionFeatureVectorConstructionPlugin,
"ImageProvider": ImageProviderPlugin,
"FeatureSerializer": FeatureSerializerPlugin,
"MergerResolver": MergerResolverPlugin,
})
if self._verbose:
logging.getLogger('yapsy').setLevel(logging.DEBUG)
else:
logging.getLogger('yapsy').setLevel(logging.CRITICAL)
self._yapsyPluginManager.collectPlugins()
def _applyToAllPluginsOfCategory(self, func, category):
''' helper function to apply `func` to all plugins of the given `category` and hide all yapsy stuff '''
for pluginInfo in self._yapsyPluginManager.getPluginsOfCategory(category):
p = pluginInfo.plugin_object
func(p)
def _getPluginOfCategory(self, name, category):
'''
helper function to access a certain plugin by `name` from a `category`.
**returns** the plugin or throws a `KeyError`
'''
pluginDict = dict((pluginInfo.name, pluginInfo.plugin_object)
for pluginInfo in self._yapsyPluginManager.getPluginsOfCategory(category))
return pluginDict[name]
def applyObjectFeatureComputationPlugins(self, ndims, rawImage, labelImage, frameNumber, rawFilename):
"""
computes the features of all plugins and returns a list of dictionaries, as well as a list of
feature names that should be ignored
"""
features = []
featureNamesToIgnore = []
def computeFeatures(plugin):
if ndims in plugin.worksForDimensions:
f = plugin.computeFeatures(rawImage, labelImage, frameNumber, rawFilename)
features.append(f)
featureNamesToIgnore.extend(plugin.omittedFeatures)
self._applyToAllPluginsOfCategory(computeFeatures, "ObjectFeatureComputation")
return features, featureNamesToIgnore
def applyTransitionFeatureVectorConstructionPlugins(self, featureDictObjectA, featureDictObjectB, selectedFeatures):
"""
constructs a transition feature vector for training/prediction with a random forest from the
features of the two objects participating in the transition.
"""
featureVector = []
def appendFeatures(plugin):
f = plugin.constructFeatureVector(featureDictObjectA, featureDictObjectB, selectedFeatures)
featureVector.extend(f)
self._applyToAllPluginsOfCategory(appendFeatures, "TransitionFeatureVectorConstruction")
return featureVector
def getTransitionFeatureNames(self, featureDictObjectA, featureDictObjectB, selectedFeatures):
"""
returns a verbal description of each feature in the transition feature vector
"""
featureNames = []
def collectFeatureNames(plugin):
f = plugin.getFeatureNames(featureDictObjectA, featureDictObjectB, selectedFeatures)
featureNames.extend(f)
self._applyToAllPluginsOfCategory(collectFeatureNames, "TransitionFeatureVectorConstruction")
return featureNames
def setImageProvider(self, imageProviderName):
''' set the used image provier plugin name '''
self.chosen_data_provider = imageProviderName
def getImageProvider(self):
''' get an instance of the selected image provider plugin '''
return self._getPluginOfCategory(self.chosen_data_provider, "ImageProvider")
def setFeatureSerializer(self, featureSerializerName):
''' set the used feature serializer plugin name '''
self.chosen_feature_serializer = featureSerializerName
def getFeatureSerializer(self):
''' get an instance of the selected feature serializer plugin '''
return self._getPluginOfCategory(self.chosen_feature_serializer, "FeatureSerializer")
def setMergerResolver(self, mergerResolverName):
''' set the used merger resolver plugin name '''
self.chosen_merger_resolver = mergerResolverName
def getMergerResolver(self):
''' get an instance of the selected merger resolver plugin '''
return self._getPluginOfCategory(self.chosen_merger_resolver, "MergerResolver")
