def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.process")
edActionCluster = EDActionCluster()
edActionCluster.addAction(self.edPluginIndexingLabelit)
edActionCluster.addAction(self.edPluginControlIndicators)
self.edPluginIndexingLabelit.connectSUCCESS(self.doSuccessLabelitIndexing)
self.edPluginIndexingLabelit.connectFAILURE(self.doFailureLabelitIndexing)
self.edPluginControlIndicators.connectSUCCESS(self.doSuccessControlIndicators)
self.edPluginControlIndicators.connectFAILURE(self.doFailureControlIndicators)
edActionCluster.execute()
edActionCluster.synchronize()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlIndexingIndicatorsv10.process")
edActionCluster = EDActionCluster()
edActionCluster.addAction(self.__edPluginMOSFLMIndexing)
edActionCluster.addAction(self.__edPluginControlIndicators)
self.__edPluginMOSFLMIndexing.connectSUCCESS(self.doSuccessMOSFLMIndexing)
self.__edPluginMOSFLMIndexing.connectFAILURE(self.doFailureMOSFLMIndexing)
self.__edPluginControlIndicators.connectSUCCESS(self.doSuccessControlIndicators)
self.__edPluginControlIndicators.connectFAILURE(self.doFailureControlIndicators)
edActionCluster.execute()
edActionCluster.synchronize()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlIndexingIndicatorsv10.process")
edActionCluster = EDActionCluster()
edActionCluster.addAction(self.__edPluginMOSFLMIndexing)
edActionCluster.addAction(self.__edPluginControlIndicators)
self.__edPluginMOSFLMIndexing.connectSUCCESS(
self.doSuccessMOSFLMIndexing)
self.__edPluginMOSFLMIndexing.connectFAILURE(
self.doFailureMOSFLMIndexing)
self.__edPluginControlIndicators.connectSUCCESS(
self.doSuccessControlIndicators)
self.__edPluginControlIndicators.connectFAILURE(
self.doFailureControlIndicators)
edActionCluster.execute()
edActionCluster.synchronize()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.process")
edActionCluster = EDActionCluster()
if self.bDoLabelitIndexing:
edActionCluster.addAction(self.edPluginIndexingLabelit)
self.edPluginIndexingLabelit.connectSUCCESS(
self.doSuccessLabelitIndexing)
self.edPluginIndexingLabelit.connectFAILURE(
self.doFailureLabelitIndexing)
edActionCluster.addAction(self.edPluginControlIndicators)
self.edPluginControlIndicators.connectSUCCESS(
self.doSuccessControlIndicators)
self.edPluginControlIndicators.connectFAILURE(
self.doFailureControlIndicators)
edActionCluster.execute()
edActionCluster.synchronize()
class EDPluginControl(EDPlugin):
"""
An EDPluginControl is a plugin that is responsible for a EDPluginExec or EDPluginControl plugin execution:
It is responsible for:
- The EDPluginExec or EDPluginControl Workflow
- The data propagation between the EDPluginExec
- The translation between generic and specific data models via EDHandler classes
- The error/warning propagation
- The executive summaries propagation
- Execution of an "action cluster": a set of plugins can be added to a so called "action cluster"
with the method "addPluginToActionCluster". All the plugins in the cluster can then be executed
simultaneously with the method "executeActionCluster" and synchronized with the method
"synchronizeActionCluster". The number of threads used by the action cluster is by default the
number of processors available on the computer, but this value can be changed either by
calling the method "setClusterSize" or by using the configuration parameter "clusterSize".
"""
def __init__ (self):
"""
"""
EDPlugin.__init__(self)
self.__strPluginToBeControlledName = None
self.__dictControlledPlugins = {}
self.__edActionCluster = None
self.__iClusterSize = None
self.__listOfLoadedPlugins = []
def configure(self):
"""
Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
"""
EDPlugin.configure(self)
EDVerbose.DEBUG("EDPluginControl.configure")
strControlledPlugins = self.config.get("controlledPlugins", None)
if (strControlledPlugins != None):
pyListControlledPlugins = strControlledPlugins.split(",")
for strControlledPlugin in pyListControlledPlugins:
strControlledPluginName = self.getStringConfigurationParameterValue(strControlledPlugin)
if strControlledPluginName != None:
self.setControlledPluginName(strControlledPlugin, strControlledPluginName)
EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName))
clusterSize = self.config.get("clusterSize", None)
if (clusterSize != None):
self.__iClusterSize = int(strClusterSize)
EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)
def emptyListOfLoadedPlugin(self):
"""
Reset all plugins kept in memory
"""
self.__listOfLoadedPlugins = []
gc.collect()
def getListOfLoadedPlugin(self):
"""
"""
return self.__listOfLoadedPlugins
def removeLoadedPlugin(self, _plugin):
"""
Remove a plugin from the list of loaded plugins to free some memory
@param _plugin: plugin to remove
@type _plugin: instance of the class EDPlugin
"""
if _plugin in self.__listOfLoadedPlugins:
with self.locked():
self.__listOfLoadedPlugins.remove(_plugin)
self.DEBUG("EDPluginControl.removeLoadedPlugin: Caught, removed %s unreferenced objects. currently there are %i plugins" % (gc.get_count(), len(self.__listOfLoadedPlugins)))
gc.collect()
else:
self.DEBUG("EDPluginControl.removeLoadedPlugin: Missed. currently there are %i plugins" % len(self.__listOfLoadedPlugins))
def synchronizePlugins(self):
EDVerbose.DEBUG("EDPluginControl.synchronizePlugins")
bSynchronized = False
while not bSynchronized:
listPluginOrig = self.__listOfLoadedPlugins[:]
for edPlugin in listPluginOrig:
if edPlugin.isStarted() and (not edPlugin.isEnded()):
edPlugin.synchronize()
elif not edPlugin.isStarted():
time.sleep(0.01) #release GIL to let plugin start
continue
time.sleep(0.01)
with self.locked():
bSynchronized = (self.__listOfLoadedPlugins == listPluginOrig)
def loadPlugins(self):
"""
This method loads and returns a list of references to the plugins to be controlled.
The name of the plugin to be controlled is set set before calling this method using the
"setControlledPluginName" method.
The base name of the plugin to be controlled is used as the working
directory name of the plugin in question. The name of the plugin is used as
base name.
"""
EDVerbose.DEBUG("EDPluginControl.loadPlugins")
listKeys = self.__dictControlledPlugins.keys()
listLoadedPlugins = []
for strKey in listKeys:
strPluginName = self.__dictControlledPlugins[strKey]
edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
edPlugin.setBaseDirectory(self.getWorkingDirectory())
edPlugin.setBaseName(strPluginName)
listLoadedPlugins.append(edPlugin)
return listLoadedPlugins
def loadPlugin(self, _strPluginToBeControlledName=None, _strBaseName=None):
"""
This method loads and returns a reference to the plugin to be controlled.
The name of the plugin to be controlled can either be passed as an
argument, or bet set before calling this method using the
"setPluginToBeControlledName".
The base name of the plugin to be controlled is used as the working
directory name of the plugin in question. If no argument is supplied
the name of the plugin is used as base name. In the case of creation of
several plugins to be launched simultaneously, the base name should be
different for each plugin and hence must be provided explicitly.
"""
EDVerbose.DEBUG("EDPluginControl.loadPlugin")
if (_strPluginToBeControlledName is None):
strPluginName = self.__strPluginToBeControlledName
else:
strPluginName = _strPluginToBeControlledName
edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
if (edPlugin is None):
strErrorMessage = "EDPluginControl.loadPlugin : Cannot load plugin %s" % strPluginName
EDVerbose.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError, strErrorMessage
else:
self.__listOfLoadedPlugins.append(edPlugin)
edPlugin.setBaseDirectory(self.getWorkingDirectory())
if (_strBaseName is None):
# Check if base name exists. OBS! Not thread safe so please set explicitly
# _strBaseName for multi-threaded code
strRenamedPlugin = self.compactPluginName(strPluginName)
strNewWorkingDirectory = os.path.join(self.getWorkingDirectory(), strRenamedPlugin)
if (os.path.exists(strNewWorkingDirectory)):
edPlugin.setBaseName(edPlugin.createBaseName())
else:
edPlugin.setBaseName(strRenamedPlugin)
else:
edPlugin.setBaseName(_strBaseName)
return edPlugin
def setControlledPluginName(self, _strControlledPluginName, _strControlledPluginValue):
"""
Adds a name-value pair to the dictionary to map the general to the specific name of a plugin to be controlled
"""
self.__dictControlledPlugins[_strControlledPluginName] = _strControlledPluginValue
def getControlledPluginName(self, _strControlledPluginName):
"""
Returns the name of the plugin to be controlled.
"""
strPluginname = None
if self.__dictControlledPlugins.has_key(_strControlledPluginName):
strPluginname = self.__dictControlledPlugins[_strControlledPluginName]
return strPluginname
def addWarningMessages(self, _listWarningMessages):
"""
Adds a list of warning messages in the existing list of warning messages
"""
EDVerbose.DEBUG("EDPluginControl.addWarningMessages")
for strWarningMessage in _listWarningMessages:
self.addWarningMessage(strWarningMessage)
def addErrorMessages(self, _listErrorMessages):
"""
Adds a list of error messages in the existing list of error messages
"""
EDVerbose.DEBUG("EDPluginControl.addErrorMessages")
for strErrorMessage in _listErrorMessages:
self.addErrorMessage(strErrorMessage)
def retrieveFailureMessages(self, _edPlugin, _strMethodCaller):
"""
Propagates failure messages from a plugin including unexpected errors
Should be called in the plugin control method invoked when a plugin exec fails (doActionFailure<>)
"""
EDVerbose.DEBUG("EDPluginControl.retrieveFailureMessages")
self.retrieveWarningMessages(_edPlugin)
self.retrieveErrorMessages(_edPlugin, _strMethodCaller, True)
def retrieveSuccessMessages(self, _edPlugin, _strMethodCaller):
"""
Propagates success messages from a plugin
Error messages are retrieved because a plugin could end successfully with errors (depending on the use case)
In this case, there is no check for unexpected errors
"""
EDVerbose.DEBUG("EDPluginControl.retrieveSuccessMessages")
self.retrieveWarningMessages(_edPlugin)
self.retrieveErrorMessages(_edPlugin, _strMethodCaller, False)
def retrieveErrorMessages(self, _edPlugin, _strMethodCaller, _bFailure):
"""
Propagates error messages from a plugin
if _bFailure is true, this method has been called from retrieveFailureMessages
in this case, checks for potential unexpected errors coming from the EDPluginExec
"""
EDVerbose.DEBUG("EDPluginControl.retrieveErrorMessages")
listErrorMessages = _edPlugin.getListOfErrorMessages()
if (len(listErrorMessages) == 0) and (_bFailure is True):
strErrorMessage = "%s : Adding Unexpected error" % _strMethodCaller
EDVerbose.DEBUG(strErrorMessage)
listErrorMessages.append(strErrorMessage)
self.addErrorMessages(listErrorMessages)
def retrieveWarningMessages(self, _edPlugin):
"""
Propagates warning messages from a plugin
"""
EDVerbose.DEBUG("EDPluginControl.retrieveWarningMessages")
self.addWarningMessages(_edPlugin.getListOfWarningMessages())
def appendExecutiveSummary(self, _edPlugin, _strPrefix="", _bAddSeparator=True):
"""
Appends the executive summary from a plugin.
"""
EDVerbose.DEBUG("EDPluginControl.appendExecutiveSummary")
if (_bAddSeparator):
self.addExecutiveSummarySeparator()
if _edPlugin:
for strLine in _edPlugin.getListExecutiveSummaryLines():
if strLine == self.getExecutiveSummarySeparator() and _strPrefix != "":
strLine = strLine[ :-len(_strPrefix) ]
self.addExecutiveSummaryLine(_strPrefix + strLine)
def addErrorWarningMessagesToExecutiveSummary(self, _strErrorMessage="Error messages:", _strWarningMessage="Warning messages:"):
"""
Adds error and warning messages (if any) in the executive summary
"""
if len(self.getListOfErrorMessages()) != 0:
self.addExecutiveSummarySeparator()
self.addExecutiveSummaryLine(_strErrorMessage)
for strErrorMessage in self.getListOfErrorMessages():
self.addExecutiveSummaryLine(strErrorMessage)
self.addExecutiveSummarySeparator()
if len(self.getListOfWarningMessages()) != 0:
self.addExecutiveSummarySeparator()
self.addExecutiveSummaryLine(_strWarningMessage)
for warningMessage in self.getListOfWarningMessages():
self.addExecutiveSummaryLine(warningMessage)
self.addExecutiveSummarySeparator()
def addPluginToActionCluster(self, _edPlugin):
"""
This method adds a plugin instance to an action cluster.
"""
if self.__edActionCluster == None:
self.__edActionCluster = EDActionCluster()
self.__edActionCluster.addAction(_edPlugin)
self.__listOfLoadedPlugins.append(_edPlugin)
def executeActionCluster(self):
"""
This method executes the action cluster. The action cluster is executed
asynchronoulsy.
"""
if self.__iClusterSize != None:
self.__edActionCluster.setClusterSize(self.__iClusterSize)
self.__edActionCluster.execute()
def synchronizeActionCluster(self):
"""
This method synchronises the action cluster with the control plugin thread.
"""
self.__edActionCluster.synchronize()
def setClusterSize(self, _iClusterSize):
"""
This method sets the size of the action cluster, i.e. the number of threads
that will be executed simultaneously.
"""
self.__iClusterSize = _iClusterSize
def executePlugin(self, _edPlugin, _bSynchronous=False):
"""
This method is used to start executable plugins in pipeline asynchronously.
"""
if _bSynchronous:
self.executePluginSynchronous(_edPlugin)
else:
_edPlugin.execute()
def executePluginSynchronous(self, _edPlugin):
"""
This method is used to start executable plugins in pipeline synchronously.
"""
_edControlSlotSUCCESS = EDSlot()
_edControlSlotFAILURE = EDSlot()
map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod())
map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod())
_edPlugin.getSlotSUCCESS().emptyListMethod()
_edPlugin.getSlotFAILURE().emptyListMethod()
_edPlugin.executeSynchronous()
if (not _edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotSUCCESS.call(_edPlugin)
except Exception:
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!")
EDVerbose.writeErrorTrace()
_edPlugin.setFailure()
if (_edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotFAILURE.call(_edPlugin)
except Exception:
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!")
EDVerbose.writeErrorTrace()
class EDPluginControl(EDPlugin):
"""
An EDPluginControl is a plugin that is responsible for a EDPluginExec or EDPluginControl plugin execution:
It is responsible for:
- The EDPluginExec or EDPluginControl Workflow
- The data propagation between the EDPluginExec
- The translation between generic and specific data models via EDHandler classes
- The error/warning propagation
- The executive summaries propagation
- Execution of an "action cluster": a set of plugins can be added to a so called "action cluster"
with the method "addPluginToActionCluster". All the plugins in the cluster can then be executed
simultaneously with the method "executeActionCluster" and synchronized with the method
"synchronizeActionCluster". The number of threads used by the action cluster is by default the
number of processors available on the computer, but this value can be changed either by
calling the method "setClusterSize" or by using the configuration parameter "clusterSize".
"""
def __init__(self):
"""
"""
EDPlugin.__init__(self)
self.__strPluginToBeControlledName = None
self.__dictControlledPlugins = {}
self.__edActionCluster = None
self.__iClusterSize = None
self.__listOfLoadedPlugins = []
def configure(self):
"""
Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
"""
EDPlugin.configure(self)
EDVerbose.DEBUG("EDPluginControl.configure")
strControlledPlugins = self.config.get("controlledPlugins", None)
if (strControlledPlugins != None):
pyListControlledPlugins = strControlledPlugins.split(",")
for strControlledPlugin in pyListControlledPlugins:
strControlledPluginName = self.getStringConfigurationParameterValue(
strControlledPlugin)
if strControlledPluginName != None:
self.setControlledPluginName(strControlledPlugin,
strControlledPluginName)
EDVerbose.DEBUG(
"EDPluginControl.configure: setting controlled plugin %s to specific plugin %s"
% (strControlledPlugin, strControlledPluginName))
clusterSize = self.config.get("clusterSize", None)
if (clusterSize != None):
self.__iClusterSize = int(strClusterSize)
EDVerbose.DEBUG(
"EDPluginControl.configure: setting cluster size to %d" %
self.__iClusterSize)
def emptyListOfLoadedPlugin(self):
"""
Reset all plugins kept in memory
"""
self.__listOfLoadedPlugins = []
gc.collect()
def getListOfLoadedPlugin(self):
"""
"""
return self.__listOfLoadedPlugins
def removeLoadedPlugin(self, _plugin):
"""
Remove a plugin from the list of loaded plugins to free some memory
@param _plugin: plugin to remove
@type _plugin: instance of the class EDPlugin
"""
if _plugin in self.__listOfLoadedPlugins:
with self.locked():
self.__listOfLoadedPlugins.remove(_plugin)
self.DEBUG(
"EDPluginControl.removeLoadedPlugin: Caught, removed %s unreferenced objects. currently there are %i plugins"
% (gc.get_count(), len(self.__listOfLoadedPlugins)))
gc.collect()
else:
self.DEBUG(
"EDPluginControl.removeLoadedPlugin: Missed. currently there are %i plugins"
% len(self.__listOfLoadedPlugins))
def synchronizePlugins(self):
EDVerbose.DEBUG("EDPluginControl.synchronizePlugins")
bSynchronized = False
while not bSynchronized:
listPluginOrig = self.__listOfLoadedPlugins[:]
for edPlugin in listPluginOrig:
if edPlugin.isStarted() and (not edPlugin.isEnded()):
edPlugin.synchronize()
elif not edPlugin.isStarted():
time.sleep(0.01) #release GIL to let plugin start
continue
time.sleep(0.01)
with self.locked():
bSynchronized = (self.__listOfLoadedPlugins == listPluginOrig)
def loadPlugins(self):
"""
This method loads and returns a list of references to the plugins to be controlled.
The name of the plugin to be controlled is set set before calling this method using the
"setControlledPluginName" method.
The base name of the plugin to be controlled is used as the working
directory name of the plugin in question. The name of the plugin is used as
base name.
"""
EDVerbose.DEBUG("EDPluginControl.loadPlugins")
listKeys = self.__dictControlledPlugins.keys()
listLoadedPlugins = []
for strKey in listKeys:
strPluginName = self.__dictControlledPlugins[strKey]
edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
edPlugin.setBaseDirectory(self.getWorkingDirectory())
edPlugin.setBaseName(strPluginName)
listLoadedPlugins.append(edPlugin)
return listLoadedPlugins
def loadPlugin(self, _strPluginToBeControlledName=None, _strBaseName=None):
"""
This method loads and returns a reference to the plugin to be controlled.
The name of the plugin to be controlled can either be passed as an
argument, or bet set before calling this method using the
"setPluginToBeControlledName".
The base name of the plugin to be controlled is used as the working
directory name of the plugin in question. If no argument is supplied
the name of the plugin is used as base name. In the case of creation of
several plugins to be launched simultaneously, the base name should be
different for each plugin and hence must be provided explicitly.
"""
EDVerbose.DEBUG("EDPluginControl.loadPlugin")
if (_strPluginToBeControlledName is None):
strPluginName = self.__strPluginToBeControlledName
else:
strPluginName = _strPluginToBeControlledName
edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
if (edPlugin is None):
strErrorMessage = "EDPluginControl.loadPlugin : Cannot load plugin %s" % strPluginName
EDVerbose.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError, strErrorMessage
else:
self.__listOfLoadedPlugins.append(edPlugin)
edPlugin.setBaseDirectory(self.getWorkingDirectory())
if (_strBaseName is None):
# Check if base name exists. OBS! Not thread safe so please set explicitly
# _strBaseName for multi-threaded code
strRenamedPlugin = self.compactPluginName(strPluginName)
strNewWorkingDirectory = os.path.join(self.getWorkingDirectory(),
strRenamedPlugin)
if (os.path.exists(strNewWorkingDirectory)):
edPlugin.setBaseName(edPlugin.createBaseName())
else:
edPlugin.setBaseName(strRenamedPlugin)
else:
edPlugin.setBaseName(_strBaseName)
return edPlugin
def setControlledPluginName(self, _strControlledPluginName,
_strControlledPluginValue):
"""
Adds a name-value pair to the dictionary to map the general to the specific name of a plugin to be controlled
"""
self.__dictControlledPlugins[
_strControlledPluginName] = _strControlledPluginValue
def getControlledPluginName(self, _strControlledPluginName):
"""
Returns the name of the plugin to be controlled.
"""
strPluginname = None
if self.__dictControlledPlugins.has_key(_strControlledPluginName):
strPluginname = self.__dictControlledPlugins[
_strControlledPluginName]
return strPluginname
def addWarningMessages(self, _listWarningMessages):
"""
Adds a list of warning messages in the existing list of warning messages
"""
EDVerbose.DEBUG("EDPluginControl.addWarningMessages")
for strWarningMessage in _listWarningMessages:
self.addWarningMessage(strWarningMessage)
def addErrorMessages(self, _listErrorMessages):
"""
Adds a list of error messages in the existing list of error messages
"""
EDVerbose.DEBUG("EDPluginControl.addErrorMessages")
for strErrorMessage in _listErrorMessages:
self.addErrorMessage(strErrorMessage)
def retrieveFailureMessages(self, _edPlugin, _strMethodCaller):
"""
Propagates failure messages from a plugin including unexpected errors
Should be called in the plugin control method invoked when a plugin exec fails (doActionFailure<>)
"""
EDVerbose.DEBUG("EDPluginControl.retrieveFailureMessages")
self.retrieveWarningMessages(_edPlugin)
self.retrieveErrorMessages(_edPlugin, _strMethodCaller, True)
def retrieveSuccessMessages(self, _edPlugin, _strMethodCaller):
"""
Propagates success messages from a plugin
Error messages are retrieved because a plugin could end successfully with errors (depending on the use case)
In this case, there is no check for unexpected errors
"""
EDVerbose.DEBUG("EDPluginControl.retrieveSuccessMessages")
self.retrieveWarningMessages(_edPlugin)
self.retrieveErrorMessages(_edPlugin, _strMethodCaller, False)
def retrieveErrorMessages(self, _edPlugin, _strMethodCaller, _bFailure):
"""
Propagates error messages from a plugin
if _bFailure is true, this method has been called from retrieveFailureMessages
in this case, checks for potential unexpected errors coming from the EDPluginExec
"""
EDVerbose.DEBUG("EDPluginControl.retrieveErrorMessages")
listErrorMessages = _edPlugin.getListOfErrorMessages()
if (len(listErrorMessages) == 0) and (_bFailure is True):
strErrorMessage = "%s : Adding Unexpected error" % _strMethodCaller
EDVerbose.DEBUG(strErrorMessage)
listErrorMessages.append(strErrorMessage)
self.addErrorMessages(listErrorMessages)
def retrieveWarningMessages(self, _edPlugin):
"""
Propagates warning messages from a plugin
"""
EDVerbose.DEBUG("EDPluginControl.retrieveWarningMessages")
self.addWarningMessages(_edPlugin.getListOfWarningMessages())
def appendExecutiveSummary(self,
_edPlugin,
_strPrefix="",
_bAddSeparator=True):
"""
Appends the executive summary from a plugin.
"""
EDVerbose.DEBUG("EDPluginControl.appendExecutiveSummary")
if (_bAddSeparator):
self.addExecutiveSummarySeparator()
if _edPlugin:
for strLine in _edPlugin.getListExecutiveSummaryLines():
if strLine == self.getExecutiveSummarySeparator(
) and _strPrefix != "":
strLine = strLine[:-len(_strPrefix)]
self.addExecutiveSummaryLine(_strPrefix + strLine)
def addErrorWarningMessagesToExecutiveSummary(
self,
_strErrorMessage="Error messages:",
_strWarningMessage="Warning messages:"):
"""
Adds error and warning messages (if any) in the executive summary
"""
if len(self.getListOfErrorMessages()) != 0:
self.addExecutiveSummarySeparator()
self.addExecutiveSummaryLine(_strErrorMessage)
for strErrorMessage in self.getListOfErrorMessages():
self.addExecutiveSummaryLine(strErrorMessage)
self.addExecutiveSummarySeparator()
if len(self.getListOfWarningMessages()) != 0:
self.addExecutiveSummarySeparator()
self.addExecutiveSummaryLine(_strWarningMessage)
for warningMessage in self.getListOfWarningMessages():
self.addExecutiveSummaryLine(warningMessage)
self.addExecutiveSummarySeparator()
def addPluginToActionCluster(self, _edPlugin):
"""
This method adds a plugin instance to an action cluster.
"""
if self.__edActionCluster == None:
self.__edActionCluster = EDActionCluster()
self.__edActionCluster.addAction(_edPlugin)
self.__listOfLoadedPlugins.append(_edPlugin)
def executeActionCluster(self):
"""
This method executes the action cluster. The action cluster is executed
asynchronoulsy.
"""
if self.__iClusterSize != None:
self.__edActionCluster.setClusterSize(self.__iClusterSize)
self.__edActionCluster.execute()
def synchronizeActionCluster(self):
"""
This method synchronises the action cluster with the control plugin thread.
"""
self.__edActionCluster.synchronize()
def setClusterSize(self, _iClusterSize):
"""
This method sets the size of the action cluster, i.e. the number of threads
that will be executed simultaneously.
"""
self.__iClusterSize = _iClusterSize
def executePlugin(self, _edPlugin, _bSynchronous=False):
"""
This method is used to start executable plugins in pipeline asynchronously.
"""
if _bSynchronous:
self.executePluginSynchronous(_edPlugin)
else:
_edPlugin.execute()
def executePluginSynchronous(self, _edPlugin):
"""
This method is used to start executable plugins in pipeline synchronously.
"""
_edControlSlotSUCCESS = EDSlot()
_edControlSlotFAILURE = EDSlot()
map(_edControlSlotSUCCESS.connect,
_edPlugin.getSlotSUCCESS().getListMethod())
map(_edControlSlotFAILURE.connect,
_edPlugin.getSlotFAILURE().getListMethod())
_edPlugin.getSlotSUCCESS().emptyListMethod()
_edPlugin.getSlotFAILURE().emptyListMethod()
_edPlugin.executeSynchronous()
if (not _edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotSUCCESS.call(_edPlugin)
except Exception:
EDVerbose.DEBUG(
"EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!"
)
EDVerbose.writeErrorTrace()
_edPlugin.setFailure()
if (_edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotFAILURE.call(_edPlugin)
except Exception:
EDVerbose.DEBUG(
"EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!"
)
EDVerbose.writeErrorTrace()
class EDPluginExecMeasureOffsetv2_0(EDPluginControl):
"""
An exec plugin that takes two images and measures the offset between the two.
In facts it is not an ExecPlugin but a control plugin that:
* Converts the pair of images in colored JPEG
* Extract the SIFT descriptor of each image
* Measure the offset between the two images using the Autopano tool
* return the measured offset and the file describing the control points.
"""
def __init__(self):
"""
"""
EDPluginControl.__init__(self)
self.setXSDataInputClass(XSDataInputMeasureOffset)
self.__strControlledPluginThumbnail = "EDPluginExecThumbnailv10"
self.__strControlledPluginSift = "EDPluginExecSiftDescriptorv1_0"
self.__strControlledPluginAutopano = "EDPluginExecSiftOffsetv1_0"
self.semThumbnail = threading.Semaphore()
self.semSift = threading.Semaphore()
self.ACThumbnail = EDActionCluster()
self.ACSift = EDActionCluster()
self.xsdImages = []
self.xsdThumb = []
self.xsdKeys = []
self.xsdIdx = []
self.tCrop = [0, 0]
self.inputImages = []
self.tOffset = None
self.xsdPTO = None
def checkParameters(self):
"""
Checks the mandatory parameters.
"""
EDVerbose.DEBUG("EDPluginControlMeasureOffsetv2_0.checkParameters")
self.checkMandatoryParameters(self.getDataInput(), "Data Input is None")
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.preProcess")
sdi = self.getDataInput()
crop = sdi.getCropBorders()
if len(crop) == 2:
self.tCrop = (crop[0].getValue(), crop[1].getValue())
elif len(crop) == 1:
self.tCrop = (crop[0].getValue(), crop[0].getValue())
#
if len(sdi.getImage()) == 2:
for i in sdi.getImage():
array = openimage(i.getPath().getValue()).data
shape = array.shape
if (self.tCrop != [0, 0]) and (shape[0] > self.tCrop[0]) and (shape[1] > self.tCrop[1]):
array = array[self.tCrop[0]:-self.tCrop[0], self.tCrop[1]:-self.tCrop[1] ]
EDVerbose.DEBUG("After Crop, images have shape : (%s,%s) " % (array.shape))
self.xsdImages.append(EDUtilsArray.arrayToXSData(array))
elif len(sdi.getArray()) == 2:
if (self.tCrop == [0, 0]) :
self.xsdImages = sdi.getArray()
else:
for xsdArray in sdi.getArray():
array = EDUtilsArray.xsDataToArray(xsdArray)
shape = array.shape
if (shape[0] > self.tCrop[0]) and (shape[1] > self.tCrop[1]):
array = array[self.tCrop[0]:-self.tCrop[0], self.tCrop[1]:-self.tCrop[1] ]
EDVerbose.DEBUG("After Crop, images have shape : (%s,%s) " % (array.shape))
self.xsdImages.append(EDUtilsArray.arrayToXSData(array))
else:
strError = "EDPluginExecMeasureOffsetv2_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal()
EDVerbose.ERROR(strError)
self.setFailure()
raise RuntimeError(strError)
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.xsdImages len=%i %s" % (len(self.xsdImages), self.xsdImages))
EDAssert.equal(self.xsdImages[0].getShape() , self.xsdImages[1].getShape(), "Images have the same size")
self.xsdIdx = sdi.getIndex()
if len(self.xsdIdx) < len(self.xsdImages):
self.xsdIdx = [XSDataInteger(i) for i in range(len(self.xsdImages))]
def process(self, _edObject=None):
"""
"""
for i in range(2):
execPlugin = self.loadPlugin(self.__strControlledPluginThumbnail)
xsdin = XSDataInputExecThumbnail()
xsdin.setInputArray(self.xsdImages[i])
xsdFile = XSDataFile()
xsdFile.setPath(XSDataString(os.path.join(self.getWorkingDirectory(), "image%i.jpg" % self.xsdIdx[i].getValue())))
xsdin.setOutputPath(xsdFile)
xsdin.setLevelsColorize(XSDataBoolean(1))
xsdin.setLevelsEqualize(XSDataBoolean(1))
execPlugin.setDataInput(xsdin)
execPlugin.connectSUCCESS(self.doSuccessThumb)
execPlugin.connectFAILURE(self.doFailureThumb)
self.ACThumbnail.addAction(execPlugin)
self.ACThumbnail.execute()
while len(self.xsdThumb) < 2:
time.sleep(1)
for oneImage in self.xsdThumb:
execPlugin = self.loadPlugin(self.__strControlledPluginSift)
xsdin = XSDataInputSiftDescriptor()
xsdin.setImage(oneImage)
execPlugin.setDataInput(xsdin)
execPlugin.connectSUCCESS(self.doSuccessSift)
execPlugin.connectFAILURE(self.doFailureSift)
self.ACSift.addAction(execPlugin)
self.ACSift.execute()
#
# else:
# strError = "There are only %s images in self.xsdThumb" % len(self.xsdThumb)
# EDVerbose.ERROR(strError)
# self.setFailure()
# raise RuntimeError(strError)
################################################################################
# This should be executed only after the Sift actions cluster finishes
################################################################################
while len(self.xsdKeys) < 2:
time.sleep(1)
execPlugin = self.loadPlugin(self.__strControlledPluginAutopano)
xsdin = XSDataInputMeasureOffsetSift()
xsdin.setDescriptorFile(self.xsdKeys)
execPlugin.setDataInput(xsdin)
execPlugin.connectSUCCESS(self.doSuccessAutopano)
execPlugin.connectFAILURE(self.doFailureAutopano)
execPlugin.executeSynchronous()
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.postProcess")
# Create some output data
xsDataResult = XSDataResultMeasureOffset()
xsDataResult.setOffset(self.tOffset)
xsDataResult.setPanoFile(self.xsdPTO)
self.setDataOutput(xsDataResult)
self.xsdImages = []
def doSuccessThumb(self, _edPlugin=None):
self.semThumbnail.acquire()
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessThumb")
self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessThumb")
self.xsdThumb.append(_edPlugin.getDataOutput().getThumbnailPath())
self.semThumbnail.release()
def doFailureThumb(self, _edPlugin=None):
self.semThumbnail.acquire()
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureThumb")
self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureThumb")
self.setFailure()
strError = "Error in converting to Jpeg with this input: %s" % _edPlugin.getDataInput().marshal()
EDVerbose.ERROR(strError)
self.semThumbnail.release()
raise RuntimeError(strError)
def doSuccessSift(self, _edPlugin=None):
self.semSift.acquire()
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessSift")
self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessSift")
self.xsdKeys.append(_edPlugin.getDataOutput().getDescriptorFile())
self.semSift.release()
def doFailureSift(self, _edPlugin=None):
self.semSift.acquire()
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureSift")
self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureSift")
self.setFailure()
strError = "Error in extracting SIFT keys with this input: %s" % _edPlugin.getDataInput().marshal()
EDVerbose.ERROR(strError)
self.semSift.release()
raise RuntimeError(strError)
def doSuccessAutopano(self, _edPlugin=None):
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessAutopano")
self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessSift")
self.tOffset = _edPlugin.getDataOutput().getOffset()
self.xsdPTO = _edPlugin.getDataOutput().getPanoFile()
def doFailureAutopano(self, _edPlugin=None):
EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureAutopano")
self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureAutopano")
self.setFailure()
strError = "Error in Autopano execution of with this input: %s" % _edPlugin.getDataInput().marshal()
EDVerbose.ERROR(strError)
raise RuntimeError(strError)
