def __init__(self, _edControlPlugin, _strPluginName, _dictXSDataInput, _iNbThreads):
        """
        Initialaze data structures
        @param _edControlPlugin: Parent control plugin
        @param _strPluginName: Name of the execution plugin to run
        @param _dictXSDataInput: Dictionary of the input objects for execution plugin
        """

        EDActionCluster.__init__(self, _iNbThreads)

        self.__strPluginName = _strPluginName
        self.__dictXSDataInput = _dictXSDataInput
        self.__edControlPlugin = _edControlPlugin

        self.__xsEDPluginExecJobs = {}
 def process(self, _edObject=None):
     EDPluginControl.process(self)
     self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.process")
     edActionCluster = EDActionCluster()
     edActionCluster.addAction(self.edPluginIndexingLabelit)
     edActionCluster.addAction(self.edPluginControlIndicators)
     self.edPluginIndexingLabelit.connectSUCCESS(self.doSuccessLabelitIndexing)
     self.edPluginIndexingLabelit.connectFAILURE(self.doFailureLabelitIndexing)
     self.edPluginControlIndicators.connectSUCCESS(self.doSuccessControlIndicators)
     self.edPluginControlIndicators.connectFAILURE(self.doFailureControlIndicators)
     edActionCluster.execute()
     edActionCluster.synchronize()
 def process(self, _edObject=None):
     EDPluginControl.process(self)
     self.DEBUG("EDPluginControlIndexingIndicatorsv10.process")
     edActionCluster = EDActionCluster()
     edActionCluster.addAction(self.__edPluginMOSFLMIndexing)
     edActionCluster.addAction(self.__edPluginControlIndicators)
     self.__edPluginMOSFLMIndexing.connectSUCCESS(self.doSuccessMOSFLMIndexing)
     self.__edPluginMOSFLMIndexing.connectFAILURE(self.doFailureMOSFLMIndexing)
     self.__edPluginControlIndicators.connectSUCCESS(self.doSuccessControlIndicators)
     self.__edPluginControlIndicators.connectFAILURE(self.doFailureControlIndicators)
     edActionCluster.execute()
     edActionCluster.synchronize()
Exemple #4
0
 def addPluginToActionCluster(self, _edPlugin):
     """
     This method adds a plugin instance to an action cluster.
     """
     if self.__edActionCluster == None:
         self.__edActionCluster = EDActionCluster()
     self.__edActionCluster.addAction(_edPlugin)
     self.__listOfLoadedPlugins.append(_edPlugin)
 def testExecuteActionClusterWithFailure(self):
     """
     Test the EDActionCluster with actions ending in success
     """
     edActionCluster = EDActionCluster()
     for i in range(10):
         edAction = EDActionSuccess()
         edActionCluster.addAction(edAction)
     edActionFailure = EDActionFailure()
     edActionCluster.addAction(edActionFailure)
     edActionCluster.executeSynchronous()
     EDAssert.equal(True, edActionCluster.isFailure(), "EDActionCluster ended in failure")
 def __init__(self):
     """
     """
     EDPluginControl.__init__(self)
     self.setXSDataInputClass(XSDataInputMeasureOffset)
     self.__strControlledPluginThumbnail = "EDPluginExecThumbnailv10"
     self.__strControlledPluginSift = "EDPluginExecSiftDescriptorv1_0"
     self.__strControlledPluginAutopano = "EDPluginExecSiftOffsetv1_0"
     self.semThumbnail = threading.Semaphore()
     self.semSift = threading.Semaphore()
     self.ACThumbnail = EDActionCluster()
     self.ACSift = EDActionCluster()
     self.xsdImages = []
     self.xsdThumb = []
     self.xsdKeys = []
     self.xsdIdx = []
     self.tCrop = [0, 0]
     self.inputImages = []
     self.tOffset = None
     self.xsdPTO = None
    def preProcess(self, _edObject=None):
        EDActionCluster.preProcess(self)
        EDVerbose.DEBUG("EDParallelJobLauncher.preProcess")

        for __dictKey in self.__dictXSDataInput.keys():
            self.__xsEDPluginExecJobs[__dictKey] = self.__edControlPlugin.loadPlugin(self.__strPluginName)

            if self.__xsEDPluginExecJobs[__dictKey] is not None:

                if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not ""):
                    self.__xsEDPluginExecJobs[__dictKey].setDataInput(self.__dictXSDataInput[__dictKey])
                    self.addAction(self.__xsEDPluginExecJobs[__dictKey])
                else:
                    EDVerbose.screen(
                        "ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey].getWorkingDirectory()
                    )

            else:
                EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName)

        self.setTimeOut(self.__edControlPlugin.getTimeOut())
 def testExecuteActionClusterWithSuccess(self):
     """
     Test the EDActionCluster with actions ending in success
     """
     edActionCluster = EDActionCluster()
     for i in range(10):
         edAction = EDActionSuccess()
         edActionCluster.addAction(edAction)
     edActionCluster.executeSynchronous()
     EDAssert.equal(False, edActionCluster.isFailure(),
                    "EDActionCluster ended in success")
Exemple #9
0
 def createDataCollectionFromDataSets(self, _pyListXSDataCCP4iDataSet):
     """
     This method takes as input a list of ccp4i data sets. Each data set can contain several
     paths to image files. It runs the EDPluginControlSubWedgeAssemble plugin to read the
     experimental information from the image headers and then creates a single XSDataCollection object.
     """
     EDVerbose.DEBUG(
         "EDPluginControlCCP4iv10.createDataCollectionFromDataSets")
     xsDataCollection = None
     # We might have to run the plugin several times
     edActionCluster = EDActionCluster()
     listPluginControlSubWedgeAssemble = []
     # Prepare the action cluster
     iIndex = 1
     for xsDataCCP4iDataSet in _pyListXSDataCCP4iDataSet:
         edPluginControlSubWedgeAssemble = self.loadPlugin(
             self.__strPluginControlSubWedgeAssembleName,
             "SubWedgeAssemble-%02d" % iIndex)
         edPluginControlSubWedgeAssemble.connectSUCCESS(
             self.doSuccessActionSubWedgeAssemble)
         edPluginControlSubWedgeAssemble.connectFAILURE(
             self.doFailureActionSubWedgeAssemble)
         # Prepare the input for the sub wedge assemble plugin
         xsDataInputSubWedgeAssemble = XSDataInputSubWedgeAssemble()
         for xsDataImageFile in xsDataCCP4iDataSet.getImageFile():
             xsDataInputSubWedgeAssemble.addFile(xsDataImageFile)
         edPluginControlSubWedgeAssemble.setDataInput(
             xsDataInputSubWedgeAssemble)
         listPluginControlSubWedgeAssemble.append(
             edPluginControlSubWedgeAssemble)
         self.addPluginToActionCluster(edPluginControlSubWedgeAssemble)
         iIndex += 1
     # Run the action cluster synchronously
     self.executeActionCluster()
     self.synchronizeActionCluster()
     # Recuperate the output
     for edPluginControlSubWedgeAssemble in listPluginControlSubWedgeAssemble:
         xsDataResultSubWedgeAssemble = edPluginControlSubWedgeAssemble.getDataOutput(
         )
         for xsDataSubWedge in xsDataResultSubWedgeAssemble.getSubWedge():
             # Instantiate the xsDataCollection object if it's not already done.
             if (xsDataCollection is None):
                 xsDataCollection = XSDataCollection()
             xsDataCollection.addSubWedge(xsDataSubWedge)
     return xsDataCollection
Exemple #10
0
class EDPluginControl(EDPlugin):
    """
    An EDPluginControl is a plugin that is responsible for a EDPluginExec or EDPluginControl plugin execution:
    It is responsible for:
        - The EDPluginExec or EDPluginControl Workflow
        - The data propagation between the EDPluginExec
        - The translation between generic and specific data models via EDHandler classes
        - The error/warning propagation
        - The executive summaries propagation
        - Execution of an "action cluster": a set of plugins can be added to a so called "action cluster"
          with the method "addPluginToActionCluster". All the plugins in the cluster can then be executed
          simultaneously with the method "executeActionCluster" and synchronized with the method
          "synchronizeActionCluster". The number of threads used by the action cluster is by default the
          number of processors available on the computer, but this value can be changed either by
          calling the method "setClusterSize" or by using the configuration parameter "clusterSize".
    """
    def __init__(self):
        """
        """
        EDPlugin.__init__(self)
        self.__strPluginToBeControlledName = None
        self.__dictControlledPlugins = {}
        self.__edActionCluster = None
        self.__iClusterSize = None
        self.__listOfLoadedPlugins = []

    def configure(self):
        """
        Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
        """
        EDPlugin.configure(self)
        EDVerbose.DEBUG("EDPluginControl.configure")
        strControlledPlugins = self.config.get("controlledPlugins", None)
        if (strControlledPlugins != None):
            pyListControlledPlugins = strControlledPlugins.split(",")
            for strControlledPlugin in pyListControlledPlugins:
                strControlledPluginName = self.getStringConfigurationParameterValue(
                    strControlledPlugin)
                if strControlledPluginName != None:
                    self.setControlledPluginName(strControlledPlugin,
                                                 strControlledPluginName)
                    EDVerbose.DEBUG(
                        "EDPluginControl.configure: setting controlled plugin %s to specific plugin %s"
                        % (strControlledPlugin, strControlledPluginName))
        clusterSize = self.config.get("clusterSize", None)
        if (clusterSize != None):
            self.__iClusterSize = int(strClusterSize)
            EDVerbose.DEBUG(
                "EDPluginControl.configure: setting cluster size to %d" %
                self.__iClusterSize)

    def emptyListOfLoadedPlugin(self):
        """
        Reset all plugins kept in memory
        """
        self.__listOfLoadedPlugins = []
        gc.collect()

    def getListOfLoadedPlugin(self):
        """
        """
        return self.__listOfLoadedPlugins

    def removeLoadedPlugin(self, _plugin):
        """
        Remove a plugin from the list of loaded plugins to free some memory
        @param _plugin: plugin to remove
        @type _plugin: instance of the class EDPlugin
        """
        if _plugin in self.__listOfLoadedPlugins:
            with self.locked():
                self.__listOfLoadedPlugins.remove(_plugin)
            self.DEBUG(
                "EDPluginControl.removeLoadedPlugin: Caught, removed %s unreferenced objects. currently there are %i plugins"
                % (gc.get_count(), len(self.__listOfLoadedPlugins)))
            gc.collect()
        else:
            self.DEBUG(
                "EDPluginControl.removeLoadedPlugin: Missed. currently there are %i plugins"
                % len(self.__listOfLoadedPlugins))

    def synchronizePlugins(self):
        EDVerbose.DEBUG("EDPluginControl.synchronizePlugins")
        bSynchronized = False
        while not bSynchronized:
            listPluginOrig = self.__listOfLoadedPlugins[:]
            for edPlugin in listPluginOrig:
                if edPlugin.isStarted() and (not edPlugin.isEnded()):
                    edPlugin.synchronize()
                elif not edPlugin.isStarted():
                    time.sleep(0.01)  #release GIL to let plugin start
                    continue
            time.sleep(0.01)
            with self.locked():
                bSynchronized = (self.__listOfLoadedPlugins == listPluginOrig)

    def loadPlugins(self):
        """
        This method loads and returns a list of references to the plugins to be controlled.
        
        The name of the plugin to be controlled is set set before calling this method using the 
        "setControlledPluginName" method. 
        
        The base name of the plugin to be controlled is used as the working
        directory name of the plugin in question. The name of the plugin is used as 
        base name. 
        """
        EDVerbose.DEBUG("EDPluginControl.loadPlugins")
        listKeys = self.__dictControlledPlugins.keys()
        listLoadedPlugins = []
        for strKey in listKeys:
            strPluginName = self.__dictControlledPlugins[strKey]
            edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
            edPlugin.setBaseDirectory(self.getWorkingDirectory())
            edPlugin.setBaseName(strPluginName)
            listLoadedPlugins.append(edPlugin)
        return listLoadedPlugins

    def loadPlugin(self, _strPluginToBeControlledName=None, _strBaseName=None):
        """
        This method loads and returns a reference to the plugin to be controlled.
        
        The name of the plugin to be controlled can either be passed as an
        argument, or bet set before calling this method using the 
        "setPluginToBeControlledName". 
        
        The base name of the plugin to be controlled is used as the working
        directory name of the plugin in question. If no argument is supplied
        the name of the plugin is used as base name. In the case of creation of
        several plugins to be launched simultaneously, the base name should be
        different for each plugin and hence must be provided explicitly.
        """
        EDVerbose.DEBUG("EDPluginControl.loadPlugin")
        if (_strPluginToBeControlledName is None):
            strPluginName = self.__strPluginToBeControlledName
        else:
            strPluginName = _strPluginToBeControlledName
        edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
        if (edPlugin is None):
            strErrorMessage = "EDPluginControl.loadPlugin : Cannot load plugin %s" % strPluginName
            EDVerbose.error(strErrorMessage)
            self.addErrorMessage(strErrorMessage)
            raise RuntimeError, strErrorMessage
        else:
            self.__listOfLoadedPlugins.append(edPlugin)
        edPlugin.setBaseDirectory(self.getWorkingDirectory())
        if (_strBaseName is None):
            # Check if base name exists. OBS! Not thread safe so please set explicitly
            # _strBaseName for multi-threaded code
            strRenamedPlugin = self.compactPluginName(strPluginName)
            strNewWorkingDirectory = os.path.join(self.getWorkingDirectory(),
                                                  strRenamedPlugin)
            if (os.path.exists(strNewWorkingDirectory)):
                edPlugin.setBaseName(edPlugin.createBaseName())
            else:
                edPlugin.setBaseName(strRenamedPlugin)
        else:
            edPlugin.setBaseName(_strBaseName)
        return edPlugin

    def setControlledPluginName(self, _strControlledPluginName,
                                _strControlledPluginValue):
        """
        Adds a name-value pair to the dictionary to map the general to the specific name of a plugin to be controlled
        """
        self.__dictControlledPlugins[
            _strControlledPluginName] = _strControlledPluginValue

    def getControlledPluginName(self, _strControlledPluginName):
        """
        Returns the name of the plugin to be controlled.
        """
        strPluginname = None
        if self.__dictControlledPlugins.has_key(_strControlledPluginName):
            strPluginname = self.__dictControlledPlugins[
                _strControlledPluginName]
        return strPluginname

    def addWarningMessages(self, _listWarningMessages):
        """
        Adds a list of warning messages in the existing list of warning messages
        """
        EDVerbose.DEBUG("EDPluginControl.addWarningMessages")
        for strWarningMessage in _listWarningMessages:
            self.addWarningMessage(strWarningMessage)

    def addErrorMessages(self, _listErrorMessages):
        """
        Adds a list of error messages in the existing list of error messages
        """
        EDVerbose.DEBUG("EDPluginControl.addErrorMessages")
        for strErrorMessage in _listErrorMessages:
            self.addErrorMessage(strErrorMessage)

    def retrieveFailureMessages(self, _edPlugin, _strMethodCaller):
        """
        Propagates failure messages from a plugin including unexpected errors
        Should be called in the plugin control method invoked when a plugin exec fails (doActionFailure<>)
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveFailureMessages")
        self.retrieveWarningMessages(_edPlugin)
        self.retrieveErrorMessages(_edPlugin, _strMethodCaller, True)

    def retrieveSuccessMessages(self, _edPlugin, _strMethodCaller):
        """
        Propagates success messages from a plugin
        Error messages are retrieved because a plugin could end successfully with errors (depending on the use case)
        In this case, there is no check for unexpected errors
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveSuccessMessages")
        self.retrieveWarningMessages(_edPlugin)
        self.retrieveErrorMessages(_edPlugin, _strMethodCaller, False)

    def retrieveErrorMessages(self, _edPlugin, _strMethodCaller, _bFailure):
        """
        Propagates error messages from a plugin
        if _bFailure is true, this method has been called from retrieveFailureMessages
        in this case, checks for potential unexpected errors coming from the EDPluginExec
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveErrorMessages")
        listErrorMessages = _edPlugin.getListOfErrorMessages()
        if (len(listErrorMessages) == 0) and (_bFailure is True):
            strErrorMessage = "%s : Adding Unexpected error" % _strMethodCaller
            EDVerbose.DEBUG(strErrorMessage)
            listErrorMessages.append(strErrorMessage)
        self.addErrorMessages(listErrorMessages)

    def retrieveWarningMessages(self, _edPlugin):
        """
        Propagates warning messages from a plugin
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveWarningMessages")
        self.addWarningMessages(_edPlugin.getListOfWarningMessages())

    def appendExecutiveSummary(self,
                               _edPlugin,
                               _strPrefix="",
                               _bAddSeparator=True):
        """
        Appends the executive summary from a plugin.
        """
        EDVerbose.DEBUG("EDPluginControl.appendExecutiveSummary")
        if (_bAddSeparator):
            self.addExecutiveSummarySeparator()
        if _edPlugin:
            for strLine in _edPlugin.getListExecutiveSummaryLines():
                if strLine == self.getExecutiveSummarySeparator(
                ) and _strPrefix != "":
                    strLine = strLine[:-len(_strPrefix)]
            self.addExecutiveSummaryLine(_strPrefix + strLine)

    def addErrorWarningMessagesToExecutiveSummary(
            self,
            _strErrorMessage="Error messages:",
            _strWarningMessage="Warning messages:"):
        """
        Adds error and warning messages (if any) in the executive summary
        """
        if len(self.getListOfErrorMessages()) != 0:
            self.addExecutiveSummarySeparator()
            self.addExecutiveSummaryLine(_strErrorMessage)
            for strErrorMessage in self.getListOfErrorMessages():
                self.addExecutiveSummaryLine(strErrorMessage)
            self.addExecutiveSummarySeparator()
        if len(self.getListOfWarningMessages()) != 0:
            self.addExecutiveSummarySeparator()
            self.addExecutiveSummaryLine(_strWarningMessage)
            for warningMessage in self.getListOfWarningMessages():
                self.addExecutiveSummaryLine(warningMessage)
            self.addExecutiveSummarySeparator()

    def addPluginToActionCluster(self, _edPlugin):
        """
        This method adds a plugin instance to an action cluster.
        """
        if self.__edActionCluster == None:
            self.__edActionCluster = EDActionCluster()
        self.__edActionCluster.addAction(_edPlugin)
        self.__listOfLoadedPlugins.append(_edPlugin)

    def executeActionCluster(self):
        """
        This method executes the action cluster. The action cluster is executed
        asynchronoulsy.
        """
        if self.__iClusterSize != None:
            self.__edActionCluster.setClusterSize(self.__iClusterSize)
        self.__edActionCluster.execute()

    def synchronizeActionCluster(self):
        """
        This method synchronises the action cluster with the control plugin thread.
        """
        self.__edActionCluster.synchronize()

    def setClusterSize(self, _iClusterSize):
        """
        This method sets the size of the action cluster, i.e. the number of threads
        that will be executed simultaneously. 
        """
        self.__iClusterSize = _iClusterSize

    def executePlugin(self, _edPlugin, _bSynchronous=False):
        """
        This method is used to start executable plugins in pipeline asynchronously.
        """
        if _bSynchronous:
            self.executePluginSynchronous(_edPlugin)
        else:
            _edPlugin.execute()

    def executePluginSynchronous(self, _edPlugin):
        """
        This method is used to start executable plugins in pipeline synchronously.
        """
        _edControlSlotSUCCESS = EDSlot()
        _edControlSlotFAILURE = EDSlot()

        map(_edControlSlotSUCCESS.connect,
            _edPlugin.getSlotSUCCESS().getListMethod())
        map(_edControlSlotFAILURE.connect,
            _edPlugin.getSlotFAILURE().getListMethod())

        _edPlugin.getSlotSUCCESS().emptyListMethod()
        _edPlugin.getSlotFAILURE().emptyListMethod()

        _edPlugin.executeSynchronous()

        if (not _edPlugin.isFailure()):
            EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS")
            # Check that something doesn't go wrong in the success method!
            try:
                _edControlSlotSUCCESS.call(_edPlugin)

            except Exception:
                EDVerbose.DEBUG(
                    "EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!"
                )
                EDVerbose.writeErrorTrace()
                _edPlugin.setFailure()

        if (_edPlugin.isFailure()):
            EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE")
            # Check that something doesn't go wrong in the success method!
            try:
                _edControlSlotFAILURE.call(_edPlugin)

            except Exception:
                EDVerbose.DEBUG(
                    "EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!"
                )
                EDVerbose.writeErrorTrace()
class EDPluginControlSaxsModelingv1_0(EDPluginControl):
    """
    Basically this is a re-implementation of EDPluginControlSolutionScattering starting after Gnom and withou web page generation
    """
    classlock = Semaphore()
    configured = False
    cluster_size = 2     # duplicate from ControlPlugin
    dammif_jobs = 16     # number of dammif job to run
    unit = "NANOMETER"   # unit of the GNOM file
    symmetry = "P1"      #
    mode = "fast"        #
    # constants:  plugin names
    strPluginExecDammif = "EDPluginExecDammifv0_2"
    strPluginExecSupcomb = "EDPluginExecSupcombv0_3"
    strPluginExecDamaver = "EDPluginExecDamaverv0_3"
    strPluginExecDamfilt = "EDPluginExecDamfiltv0_3"
    strPluginExecDamstart = "EDPluginExecDamstartv0_3"
    strPluginExecDammin = "EDPluginExecDamminv0_2"
    Rg_min = 0.5  # nm
    
    
    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputSaxsModeling)
        self.edPlugin = None
        self.edPlugin = None
        self.xsGnomFile = None
        self.result = XSDataResultSaxsModeling()
        self.result.dammifModels = []
        self.graph_format = "png"
        self.dammif_plugins = []
        self.dammif = None
        self.supcomb_plugins = {}
        self.actclust_supcomb = None
        self.valid = None  # index of valid damif models
        self.mask2d = None
        self.arrayNSD = None
        self.ref = None  # reference frame number (starting ar 0)


    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginControlSaxsModelingv1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.gnomFile, "gnom output is missing")

    def configure(self):
        EDPluginControl.configure(self)
        if not self.configured:
            with self.classlock:
                if not self.configured:
                    EDPluginControl.configure(self)
                    dammif_jobs = self.config.get("dammifJobs", None)
                    if (dammif_jobs != None):
                        self.__class__.dammif_jobs = int(dammif_jobs)
                        self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting number of dammif jobs to %d" % self.dammif_jobs)
                    unit = self.config.get("unit", None)
                    if (unit != None):
                        self.__class__.unit = unit.upper()
                        self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting input units to %s" % self.unit)
                    symmetry = self.config.get("symmetry", None)
                    if (symmetry != None):
                        self.__class__.symmetry = symmetry
                        self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting symmetry to %s" % self.symmetry)
                    mode = self.config.get("mode", None)
                    if (mode != None):
                        self.__class__.mode = mode
                        self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting dammif mode to %s" % self.mode)
                    clusterSize = self.config.get("clusterSize", None)
                    if (clusterSize != None):
                        self.__class__.cluster_size = int(clusterSize)
                        self.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.cluster_size)
                    self.__class__.configured = True


    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.preProcess")
        self.xsGnomFile = self.dataInput.gnomFile
        if self.dataInput.graphFormat:
            self.graph_format = self.dataInput.graphFormat.value
        self.checkRg()
            

    def checkRg(self):
        """
        If there is nothing in the sample, Rg = 0.1 nm 
        damaver is likely to produce log files of many GB  
        """
        last_line = open(self.xsGnomFile.path.value).readlines()[-1]
#         self.WARNING("last Gnom file line is %s" % last_line)
        key = "Rg ="
        start = last_line.find(key) + len(key)
        val = last_line[start:].split()[0]
        try:
            rg = float(val)
        except ValueError:
            rg = 0.0
        if rg < self.Rg_min:
            str_err = "Radius of Giration is too small (%s<%s). Stop processing !!!!" % (rg, self.Rg_min)
            self.ERROR(str_err)
            self.setFailure()
            raise RuntimeError(str_err)

    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.process")
        xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile,
                                              unit=XSDataString(self.unit),
                                              symmetry=XSDataString(self.symmetry),
                                              mode=XSDataString(self.mode))
        for i in range(self.dammif_jobs):
            dammif = self.loadPlugin(self.strPluginExecDammif)
            dammif.connectSUCCESS(self.doSuccessExecDammif)
            dammif.connectFAILURE(self.doFailureExecDammif)
            xsd = xsDataInputDammif.copyViaDict()
            xsd.order = XSDataInteger(i + 1)
            dammif.dataInput = xsd
            self.addPluginToActionCluster(dammif)
            self.dammif_plugins.append(dammif)
        self.executeActionCluster()
        self.synchronizeActionCluster()
        for plugin in self.dammif_plugins:
            if plugin.isFailure():
                self.ERROR("dammif plugin %s-%08i failed" % (plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)
        if self.isFailure():
            return

        # retrieve results from best dammif
        self.dammif = self.bestDammif()

        self.chi2plot("chi2_R.png")
        self.result.chiRfactorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "chi2_R.png")))

        # temporary results: use best dammif
        self.result.fitFile = self.dammif.dataOutput.fitFile
        self.result.logFile = self.dammif.dataOutput.logFile
        self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile
        self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile

        # prepare an action cluster with all supcomb plugins
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for idx in range(self.dammif_jobs):
            if self.valid[idx]:
                for ser in range(idx):
                    if self.valid[ser]:
                        supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                        supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[idx].dataOutput.pdbMoleculeFile,
                                                               superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile,
                                                               name=self.dammif_plugins[ser].dataOutput.model.name)
                        self.supcomb_plugins[(idx, ser)] = supcomb
                        self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for key, plugin in self.supcomb_plugins.items():
            if plugin.isFailure():
                self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)

        if self.isFailure():
            return

        self.makeNSDarray("nsd.png")
        self.result.nsdPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png")))

        idx = self.ref
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile,
                                                       superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile,
                                                       name=self.dammif_plugins[ser].dataOutput.model.name)
                self.supcomb_plugins[(self.ref, ser)] = supcomb
                self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                plugin = self.supcomb_plugins[(self.ref, ser)]
                if plugin.isFailure():
                    self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (self.ref + 1, ser + 1, plugin.getName(), plugin.getId()))
                    self.setFailure()
                self.retrieveMessages(plugin)
        if self.isFailure():
            return

        for i in range(self.dammif_jobs):
            if i == self.ref or not self.valid[i]:
                model = self.dammif_plugins[i].dataOutput.model
            else:
                model = self.supcomb_plugins[(self.ref, i)].dataOutput.model
                model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt
#                model.chiSqrt =  self.dammif_plugins[i].dataOutput.model.chiSqrt
            self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1))
            self.result.dammifModels[i] = model


#        Now that all (valid) models are aligned we can combine them using damaver
        pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile]

        for idx in range(self.dammif_jobs):
            if self.valid[idx] and idx != self.ref:
                pdbFiles.append(self.supcomb_plugins[(self.ref, idx)].dataOutput.outputFilename)

        damaver = self.loadPlugin(self.strPluginExecDamaver)
        damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles,
                                                automatic=XSDataBoolean(False))
        damaver.connectSUCCESS(self.doSuccessExecDamaver)
        damaver.connectFAILURE(self.doFailureExecDamaver)
        damaver.executeSynchronous()

        if self.isFailure():
            return

        damfilt = self.loadPlugin(self.strPluginExecDamfilt)
        damfilt.dataInput = XSDataInputDamfilt(inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damfilt.connectSUCCESS(self.doSuccessExecDamfilt)
        damfilt.connectFAILURE(self.doFailureExecDamfilt)
        damfilt.execute()
        ########################################################################
        # TODO: This is a dead end : do it in parallel
        ########################################################################

        if self.isFailure():
            return

        damstart = self.loadPlugin(self.strPluginExecDamstart)
        damstart.dataInput = XSDataInputDamstart(inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damstart.connectSUCCESS(self.doSuccessExecDamstart)
        damstart.connectFAILURE(self.doFailureExecDamstart)
        damstart.executeSynchronous()

        if self.isFailure():
            return
        ########################################################################
        # Finally call dammin
        ########################################################################
        if self.config.get("do_dammin") in ["False", "0", False, 0]:
            return
        dammin = self.loadPlugin(self.strPluginExecDammin)
        dammin.dataInput = XSDataInputDammin(pdbInputFile=damstart.dataOutput.outputPdbFile,
                                             gnomOutputFile=self.xsGnomFile,
                                             symmetry=XSDataString(self.symmetry),
                                             mode=XSDataString(self.mode))
        dammin.connectSUCCESS(self.doSuccessExecDammin)
        dammin.connectFAILURE(self.doFailureExecDammin)
        dammin.executeSynchronous()
        # Dammin takes as lot of time ... wait here for completion

    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.postProcess")

        self.synchronizePlugins()
        # Create some output data
#        self.result.

    def finallyProcess(self, _edObject=None):
        EDPluginControl.finallyProcess(self, _edObject=_edObject)
        self.result.status = XSDataStatus(message=self.getXSDataMessage(),
                                          executiveSummary=XSDataString(os.linesep.join(self.getListExecutiveSummaryLines())))

        self.setDataOutput(self.result)
        # clean up memory
        self.dammif_plugins = []
        self.dammif = None
        self.emptyListOfLoadedPlugin()
        self.supcomb_plugins = {}
        self.actclust_supcomb = None
        gc.collect()

    def doSuccessExecDammif(self, _edPlugin=None):
        """
        Locked as dammif is called many times in parallel
        """
        with self.locked():
            self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammif")
            self.retrieveMessages(_edPlugin)
            self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammif")
            try:
                self.result.dammifModels.append(_edPlugin.dataOutput.model)
                # this has to be done only for the best model (once determined) !
#                self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile
#                self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile
#                self.result.fitFile = _edPlugin.dataOutput.fitFile
#                self.result.logFile = _edPlugin.dataOutput.logFile
            except Exception as error:
                self.ERROR("Error in doSuccessExecDammif: %s" % error)

    def doFailureExecDammif(self, _edPlugin=None):
        """
        Locked as dammif is called many times in parallel
        """

        with self.locked():
            self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammif")
            self.retrieveMessages(_edPlugin)
            self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammif")
            self.setFailure()


    def doSuccessExecDamaver(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver")
        self.retrieveMessages(_edPlugin)
        try:
            self.result.damaverModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDamaver: %s" % error)

    def doFailureExecDamaver(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamaver")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamaver")
        self.setFailure()


    def doSuccessExecDamfilt(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt")
        self.retrieveMessages(_edPlugin)
        try:
            self.result.damfiltModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDamfilt: %s" % error)



    def doFailureExecDamfilt(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt")
        self.setFailure()


    def doSuccessExecDamstart(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart")
        self.retrieveMessages(_edPlugin)
        try:
            self.result.damstartModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDamstart: %s" % error)


    def doFailureExecDamstart(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamstart")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamstart")
        self.setFailure()


    def doSuccessExecDammin(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammin")
        self.retrieveMessages(_edPlugin)
        self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin")
        try:
            self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile
            self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile
            self.result.fitFile = _edPlugin.dataOutput.fitFile
            self.result.firFile = _edPlugin.dataOutput.model.firFile
            self.result.logFile = _edPlugin.dataOutput.logFile
            self.result.damminModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDammin: %s" % error)


    def doFailureExecDammin(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammin")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin")
        #self.setFailure()


    def bestDammif(self):
        """
        Find DAMMIF run with best chi-square value
        """
        fitResultDict = dict([(plg.dataOutput.chiSqrt.value, plg)
                              for plg in self.dammif_plugins
                              if (plg.dataOutput is not None) and (plg.dataOutput.chiSqrt is not None)])
        fitResultList = fitResultDict.keys()
        fitResultList.sort()

        return fitResultDict[fitResultList[0]]

    def symlink(self, filen, link):
        """
        Create a symlink to CWD with relative path
        """
        src = os.path.abspath(filen)
        cwd = self.getWorkingDirectory()
        dest = os.path.join(cwd, link)
        os.symlink(os.path.relpath(src, cwd), dest)


    def chi2plot(self, filename=None, close=True):
        
        chi2 = numpy.array([ plg.dataOutput.chiSqrt.value for plg in self.dammif_plugins])
        chi2max = chi2.mean() + 2 * chi2.std()

        xticks = 1 + numpy.arange(self.dammif_jobs)
        fig = plt.figure(figsize=figureSize)
        ax1 = fig.add_subplot(1, 2, 1)
        ax1.bar(xticks - 0.5, chi2)
        ax1.set_ylabel(u"$\sqrt{\u03C7}$")
        ax1.set_xlabel(u"Model number")
        ax1.plot([0.5, self.dammif_jobs + 0.5], [chi2max, chi2max], "-r", label=u"$\sqrt{\u03C7}$$_{max}$ = %.3f" % chi2max)
        ax1.set_xticks(xticks)
        ax1.legend(loc=8)
        R = numpy.array([ plg.dataOutput.rfactor.value for plg in self.dammif_plugins])
        Rmax = R.mean() + 2 * R.std()
        ax2 = fig.add_subplot(1, 2, 2)
        ax2.bar(xticks - 0.5, R)
        ax2.plot([0.5, self.dammif_jobs + 0.5], [Rmax, Rmax], "-r", label=u"R$_{max}$ = %.3f" % Rmax)
        ax2.set_ylabel(u"R factor")
        ax2.set_xlabel(u"Model number")
        ax2.set_xticks(xticks)
        ax2.legend(loc=8)
#        fig.set_title("Selection of dammif models based on \u03C7$^2$")
        self.valid = (chi2 < chi2max) * (R < Rmax)
        self.mask2d = (1 - numpy.identity(self.dammif_jobs)) * numpy.outer(self.valid, self.valid)
#        print self.valid
        bbox_props = dict(fc="pink", ec="r", lw=1)
        for i in range(self.dammif_jobs):
            if not self.valid[i]:
                ax1.text(i + 0.95, chi2max / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props)
                ax2.text(i + 0.95, Rmax / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props)
        if filename:
            filename = os.path.join(self.getWorkingDirectory(), filename)
            self.log("Wrote %s" % filename)
            fig.savefig(filename)
        if close:
            fig.clf()
            plt.close(fig)
        else:
            return fig

    def makeNSDarray(self, filename=None, close=True):
        self.arrayNSD = numpy.zeros(self.mask2d.shape, numpy.float32)
        fig = plt.figure(figsize=figureSize)
        #ax1 = fig.add_subplot(1, 2, 1)
        
        ax1 = plt.subplot2grid((1,7), (0,0), colspan=4)
        ax2 = plt.subplot2grid((1,7), (0,4), colspan=3)
  
        # for now just an empty figure but a placeholder
        ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper")

        xticks = 1 + numpy.arange(self.dammif_jobs)
        lnsd = []
        for key, plugin in self.supcomb_plugins.items():
            i0, i1 = key
            nsd = plugin.dataOutput.NSD.value
            self.arrayNSD[i0, i1] = nsd
            self.arrayNSD[i1, i0] = nsd
            lnsd.append(nsd)
            #ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs)
            #ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs)
            ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size= 4, color = 'white', weight =  'demibold')
            ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=4, color = 'white' , weight =  'demibold')
        lnsd = numpy.array(lnsd)
#        print lnsd
#        print lnsd.mean() , lnsd.std(), lnsd.mean() + 2 * lnsd.std()
        nsd_max = lnsd.mean() + lnsd.std()
        data = self.arrayNSD.sum(axis=-1) / self.mask2d.sum(axis=-1)
        best_val = data[data > 0].min()
#        print data
#        print best_val
#        print numpy.where(data == best_val)
        self.ref = int(numpy.where(data == best_val)[0][-1])
#        print self.ref
        ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper")
        ax1.set_title(u"NSD correlation table")
        ax1.set_xticks(range(self.dammif_jobs))
        ax1.set_xticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)])
        ax1.set_xlim(-0.5, self.dammif_jobs - 0.5)
        ax1.set_ylim(-0.5, self.dammif_jobs - 0.5)
        ax1.set_yticks(range(self.dammif_jobs))
        ax1.set_yticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)])
        ax1.set_xlabel(u"Model number")
        ax1.set_ylabel(u"Model number")
        #ax2 = fig.add_subplot(1, 2, 2)
        ax2.bar(xticks - 0.5, data)
        ax2.plot([0.5, self.dammif_jobs + 0.5], [nsd_max, nsd_max], "-r", label=u"NSD$_{max}$ = %.2f" % nsd_max)
        ax2.set_title(u"NSD between any model \n and all others", multialignment='center')
        ax2.set_ylabel("Normalized Spatial Discrepancy")
        ax2.set_xlabel(u"Model number")
        ax2.set_xticks(xticks)
        ax2.tick_params(axis='y', labelsize=8)
        bbox_props = dict(fc="cyan", ec="b", lw=1)
       # ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=10, bbox=bbox_props)
        ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=8, bbox=bbox_props)
        ax2.legend(loc=8, prop={'size':8})
        self.valid *= (data < nsd_max)
        bbox_props = dict(fc="pink", ec="r", lw=1)
        plt.tight_layout()
        for i in range(self.dammif_jobs):
            if not self.valid[i]:
           #     ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props)
                 ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=8, bbox=bbox_props)
#        print self.valid
#        print self.ref
        if filename:
            filename = os.path.join(self.getWorkingDirectory(), filename)
            self.log("Wrote %s" % filename)
            fig.savefig(filename, dpi=1200)
        if close:
            fig.clf()
            plt.close(fig)
        else:
            return fig
class EDPluginParallelXNCFJobLauncher:
    """
    Class for starting execution plugin jobs in parallel for every input data object   
    """
    
    def __init__(self, _edControlPlugin, _strPluginName, _dictXSDataInput, _iNbThreads):
        """
        Initialaze data structures
        @param _edControlPlugin: Parent control plugin
        @param _strPluginName: Name of the execution plugin to run
        @param _dictXSDataInput: Dictionary of the input objects for execution plugin
        """
        self.__strPluginName = _strPluginName
        self.__dictXSDataInput = _dictXSDataInput
        self.__edControlPlugin = _edControlPlugin
        
        self.__xsEDPluginExecJobs = {}
        self.__xsEDActionCluster = EDActionCluster(_iNbThreads)
        
        self.__bIsFirstExecute = True
        
    def getPluginJobs(self):
        """
        Get dictionary of launched plugin jobs
        """
        return self.__xsEDPluginExecJobs 
    
    def run(self):
        """
        Initialize and run all parallel jobs
        """
        for __dictKey in self.__dictXSDataInput.keys():
            self.__xsEDPluginExecJobs[__dictKey] = self.__edControlPlugin.loadPlugin(self.__strPluginName)
            if (self.__xsEDPluginExecJobs[__dictKey] is not None):
    
                if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not "") :
                    self.__xsEDPluginExecJobs[__dictKey].setDataInput(self.__dictXSDataInput[__dictKey])
                    self.__xsEDPluginExecJobs[__dictKey].connectSUCCESS(self.__successPluginExecution)
                    self.__xsEDPluginExecJobs[__dictKey].connectFAILURE(self.__failurePluginExecution)
                    self.__xsEDActionCluster.addAction(self.__xsEDPluginExecJobs[__dictKey])
                else:
                    EDVerbose.screen("ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey].getWorkingDirectory())
            else:
                EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName)
        self.__xsEDActionCluster.setTimeOut(self.__edControlPlugin.getTimeOut())
        self.__xsEDActionCluster.executeSynchronous()
        
    def __successPluginExecution(self, _edPlugin=None):
        """
        Method called when the execution of the plugin succeeds 
        """
        EDVerbose.DEBUG("EDPluginParallelXNCFJobLauncher.__successPluginExecution")
        self.__edControlPlugin.retrieveSuccessMessages(_edPlugin, "EDPluginParallelXNCFJobLauncher.__successPluginExecution")

    def __failurePluginExecution(self, _edPlugin=None):
        """
        Method called when the execution of the plugin failed 
        """
        EDVerbose.DEBUG("EDPluginParallelXNCFJobLauncher.__failurePluginExecution")
        self.__edControlPlugin.retrieveFailureMessages(_edPlugin, "EDPluginParallelXNCFJobLauncher.__failurePluginExecutionGnom")

    def connectSUCCESS(self, _oMethod):
        self.__xsEDActionCluster.connectSUCCESS(_oMethod)
        
    def connectFAILURE(self, _oMethod):
        self.__xsEDActionCluster.connectFAILURE(_oMethod)
Exemple #13
0
class EDPluginControlSaxsModelingv1_0(EDPluginControl):
    """
    Basically this is a re-implementation of EDPluginControlSolutionScattering starting after Gnom and withou web page generation
    """
    classlock = Semaphore()
    configured = False
    cluster_size = 2  # duplicate from ControlPlugin
    dammif_jobs = 16  # number of dammif job to run
    unit = "NANOMETER"  # unit of the GNOM file
    symmetry = "P1"  #
    mode = "fast"  #
    # constants:  plugin names
    strPluginExecDammif = "EDPluginExecDammifv0_2"
    strPluginExecSupcomb = "EDPluginExecSupcombv0_3"
    strPluginExecDamaver = "EDPluginExecDamaverv0_3"
    strPluginExecDamfilt = "EDPluginExecDamfiltv0_3"
    strPluginExecDamstart = "EDPluginExecDamstartv0_3"
    strPluginExecDammin = "EDPluginExecDamminv0_2"
    Rg_min = 0.5  # nm

    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputSaxsModeling)
        self.edPlugin = None
        self.edPlugin = None
        self.xsGnomFile = None
        self.result = XSDataResultSaxsModeling()
        self.result.dammifModels = []
        self.graph_format = "png"
        self.dammif_plugins = []
        self.dammif = None
        self.supcomb_plugins = {}
        self.actclust_supcomb = None
        self.valid = None  # index of valid damif models
        self.mask2d = None
        self.arrayNSD = None
        self.ref = None  # reference frame number (starting ar 0)

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginControlSaxsModelingv1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.gnomFile,
                                      "gnom output is missing")

    def configure(self):
        EDPluginControl.configure(self)
        if not self.configured:
            with self.classlock:
                if not self.configured:
                    EDPluginControl.configure(self)
                    dammif_jobs = self.config.get("dammifJobs", None)
                    if (dammif_jobs != None):
                        self.__class__.dammif_jobs = int(dammif_jobs)
                        self.DEBUG(
                            "EDPluginControlSaxsModelingv1_0.configure: setting number of dammif jobs to %d"
                            % self.dammif_jobs)
                    unit = self.config.get("unit", None)
                    if (unit != None):
                        self.__class__.unit = unit.upper()
                        self.DEBUG(
                            "EDPluginControlSaxsModelingv1_0.configure: setting input units to %s"
                            % self.unit)
                    symmetry = self.config.get("symmetry", None)
                    if (symmetry != None):
                        self.__class__.symmetry = symmetry
                        self.DEBUG(
                            "EDPluginControlSaxsModelingv1_0.configure: setting symmetry to %s"
                            % self.symmetry)
                    mode = self.config.get("mode", None)
                    if (mode != None):
                        self.__class__.mode = mode
                        self.DEBUG(
                            "EDPluginControlSaxsModelingv1_0.configure: setting dammif mode to %s"
                            % self.mode)
                    clusterSize = self.config.get("clusterSize", None)
                    if (clusterSize != None):
                        self.__class__.cluster_size = int(clusterSize)
                        self.DEBUG(
                            "EDPluginControl.configure: setting cluster size to %d"
                            % self.cluster_size)
                    self.__class__.configured = True

    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.preProcess")
        self.xsGnomFile = self.dataInput.gnomFile
        if self.dataInput.graphFormat:
            self.graph_format = self.dataInput.graphFormat.value
        self.checkRg()

    def checkRg(self):
        """
        If there is nothing in the sample, Rg = 0.1 nm 
        damaver is likely to produce log files of many GB  
        """
        last_line = open(self.xsGnomFile.path.value).readlines()[-1]
        #         self.WARNING("last Gnom file line is %s" % last_line)
        key = "Rg ="
        start = last_line.find(key) + len(key)
        val = last_line[start:].split()[0]
        try:
            rg = float(val)
        except ValueError:
            rg = 0.0
        if rg < self.Rg_min:
            str_err = "Radius of Giration is too small (%s<%s). Stop processing !!!!" % (
                rg, self.Rg_min)
            self.ERROR(str_err)
            self.setFailure()
            raise RuntimeError(str_err)

    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.process")
        xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile,
                                              unit=XSDataString(self.unit),
                                              symmetry=XSDataString(
                                                  self.symmetry),
                                              mode=XSDataString(self.mode))
        for i in range(self.dammif_jobs):
            dammif = self.loadPlugin(self.strPluginExecDammif)
            dammif.connectSUCCESS(self.doSuccessExecDammif)
            dammif.connectFAILURE(self.doFailureExecDammif)
            xsd = xsDataInputDammif.copyViaDict()
            xsd.order = XSDataInteger(i + 1)
            dammif.dataInput = xsd
            self.addPluginToActionCluster(dammif)
            self.dammif_plugins.append(dammif)
        self.executeActionCluster()
        self.synchronizeActionCluster()
        for plugin in self.dammif_plugins:
            if plugin.isFailure():
                self.ERROR("dammif plugin %s-%08i failed" %
                           (plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)
        if self.isFailure():
            return

        # retrieve results from best dammif
        self.dammif = self.bestDammif()

        self.chi2plot("chi2_R.png")
        self.result.chiRfactorPlot = XSDataFile(
            XSDataString(os.path.join(self.getWorkingDirectory(),
                                      "chi2_R.png")))

        # temporary results: use best dammif
        self.result.fitFile = self.dammif.dataOutput.fitFile
        self.result.logFile = self.dammif.dataOutput.logFile
        self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile
        self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile

        # prepare an action cluster with all supcomb plugins
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for idx in range(self.dammif_jobs):
            if self.valid[idx]:
                for ser in range(idx):
                    if self.valid[ser]:
                        supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                        supcomb.dataInput = XSDataInputSupcomb(
                            templateFile=self.dammif_plugins[idx].dataOutput.
                            pdbMoleculeFile,
                            superimposeFile=self.dammif_plugins[ser].
                            dataOutput.pdbMoleculeFile,
                            name=self.dammif_plugins[ser].dataOutput.model.name
                        )
                        self.supcomb_plugins[(idx, ser)] = supcomb
                        self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for key, plugin in self.supcomb_plugins.items():
            if plugin.isFailure():
                self.ERROR(
                    "supcomb plugin for model pair (%i,%i) %s-%08i failed" %
                    (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)

        if self.isFailure():
            return

        self.makeNSDarray("nsd.png")
        self.result.nsdPlot = XSDataFile(
            XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png")))

        idx = self.ref
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                supcomb.dataInput = XSDataInputSupcomb(
                    templateFile=self.dammif_plugins[
                        self.ref].dataOutput.pdbMoleculeFile,
                    superimposeFile=self.dammif_plugins[ser].dataOutput.
                    pdbMoleculeFile,
                    name=self.dammif_plugins[ser].dataOutput.model.name)
                self.supcomb_plugins[(self.ref, ser)] = supcomb
                self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                plugin = self.supcomb_plugins[(self.ref, ser)]
                if plugin.isFailure():
                    self.ERROR(
                        "supcomb plugin for model pair (%i,%i) %s-%08i failed"
                        % (self.ref + 1, ser + 1, plugin.getName(),
                           plugin.getId()))
                    self.setFailure()
                self.retrieveMessages(plugin)
        if self.isFailure():
            return

        for i in range(self.dammif_jobs):
            if i == self.ref or not self.valid[i]:
                model = self.dammif_plugins[i].dataOutput.model
            else:
                model = self.supcomb_plugins[(self.ref, i)].dataOutput.model
                model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt
#                model.chiSqrt =  self.dammif_plugins[i].dataOutput.model.chiSqrt
            self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1))
            self.result.dammifModels[i] = model

#        Now that all (valid) models are aligned we can combine them using damaver
        pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile]

        for idx in range(self.dammif_jobs):
            if self.valid[idx] and idx != self.ref:
                pdbFiles.append(
                    self.supcomb_plugins[(self.ref,
                                          idx)].dataOutput.outputFilename)

        damaver = self.loadPlugin(self.strPluginExecDamaver)
        damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles,
                                               automatic=XSDataBoolean(False))
        damaver.connectSUCCESS(self.doSuccessExecDamaver)
        damaver.connectFAILURE(self.doFailureExecDamaver)
        damaver.executeSynchronous()

        if self.isFailure():
            return

        damfilt = self.loadPlugin(self.strPluginExecDamfilt)
        damfilt.dataInput = XSDataInputDamfilt(
            inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damfilt.connectSUCCESS(self.doSuccessExecDamfilt)
        damfilt.connectFAILURE(self.doFailureExecDamfilt)
        damfilt.execute()
        ########################################################################
        # TODO: This is a dead end : do it in parallel
        ########################################################################

        if self.isFailure():
            return

        damstart = self.loadPlugin(self.strPluginExecDamstart)
        damstart.dataInput = XSDataInputDamstart(
            inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damstart.connectSUCCESS(self.doSuccessExecDamstart)
        damstart.connectFAILURE(self.doFailureExecDamstart)
        damstart.executeSynchronous()

        if self.isFailure():
            return
        ########################################################################
        # Finally call dammin
        ########################################################################
        if self.config.get("do_dammin") in ["False", "0", False, 0]:
            return
        dammin = self.loadPlugin(self.strPluginExecDammin)
        dammin.dataInput = XSDataInputDammin(
            pdbInputFile=damstart.dataOutput.outputPdbFile,
            gnomOutputFile=self.xsGnomFile,
            symmetry=XSDataString(self.symmetry),
            mode=XSDataString(self.mode))
        dammin.connectSUCCESS(self.doSuccessExecDammin)
        dammin.connectFAILURE(self.doFailureExecDammin)
        dammin.executeSynchronous()
        # Dammin takes as lot of time ... wait here for completion

    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.postProcess")

        self.synchronizePlugins()
        # Create some output data
#        self.result.

    def finallyProcess(self, _edObject=None):
        EDPluginControl.finallyProcess(self, _edObject=_edObject)
        self.result.status = XSDataStatus(
            message=self.getXSDataMessage(),
            executiveSummary=XSDataString(
                os.linesep.join(self.getListExecutiveSummaryLines())))

        self.setDataOutput(self.result)
        # clean up memory
        self.dammif_plugins = []
        self.dammif = None
        self.emptyListOfLoadedPlugin()
        self.supcomb_plugins = {}
        self.actclust_supcomb = None
        gc.collect()

    def doSuccessExecDammif(self, _edPlugin=None):
        """
        Locked as dammif is called many times in parallel
        """
        with self.locked():
            self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammif")
            self.retrieveMessages(_edPlugin)
            self.retrieveSuccessMessages(
                _edPlugin,
                "EDPluginControlSaxsModelingv1_0.doFailureExecDammif")
            try:
                self.result.dammifModels.append(_edPlugin.dataOutput.model)
                # this has to be done only for the best model (once determined) !
#                self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile
#                self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile
#                self.result.fitFile = _edPlugin.dataOutput.fitFile
#                self.result.logFile = _edPlugin.dataOutput.logFile
            except Exception as error:
                self.ERROR("Error in doSuccessExecDammif: %s" % error)

    def doFailureExecDammif(self, _edPlugin=None):
        """
        Locked as dammif is called many times in parallel
        """

        with self.locked():
            self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammif")
            self.retrieveMessages(_edPlugin)
            self.retrieveFailureMessages(
                _edPlugin,
                "EDPluginControlSaxsModelingv1_0.doFailureExecDammif")
            self.setFailure()

    def doSuccessExecDamaver(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver")
        self.retrieveMessages(_edPlugin)
        try:
            self.result.damaverModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value,
                         _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDamaver: %s" % error)

    def doFailureExecDamaver(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamaver")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamaver")
        self.setFailure()

    def doSuccessExecDamfilt(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt")
        self.retrieveMessages(_edPlugin)
        try:
            self.result.damfiltModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value,
                         _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDamfilt: %s" % error)

    def doFailureExecDamfilt(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt")
        self.setFailure()

    def doSuccessExecDamstart(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart")
        self.retrieveMessages(_edPlugin)
        try:
            self.result.damstartModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value,
                         _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDamstart: %s" % error)

    def doFailureExecDamstart(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamstart")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamstart")
        self.setFailure()

    def doSuccessExecDammin(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammin")
        self.retrieveMessages(_edPlugin)
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin")
        try:
            self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile
            self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile
            self.result.fitFile = _edPlugin.dataOutput.fitFile
            self.result.firFile = _edPlugin.dataOutput.model.firFile
            self.result.logFile = _edPlugin.dataOutput.logFile
            self.result.damminModel = _edPlugin.dataOutput.model
            self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value,
                         _edPlugin.dataOutput.model.name.value + ".pdb")
        except Exception as error:
            self.ERROR("Error in doSuccessExecDammin: %s" % error)

    def doFailureExecDammin(self, _edPlugin=None):
        self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammin")
        self.retrieveMessages(_edPlugin)
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin")
        #self.setFailure()

    def bestDammif(self):
        """
        Find DAMMIF run with best chi-square value
        """
        fitResultDict = dict([(plg.dataOutput.chiSqrt.value, plg)
                              for plg in self.dammif_plugins
                              if (plg.dataOutput is not None) and (
                                  plg.dataOutput.chiSqrt is not None)])
        fitResultList = fitResultDict.keys()
        fitResultList.sort()

        return fitResultDict[fitResultList[0]]

    def symlink(self, filen, link):
        """
        Create a symlink to CWD with relative path
        """
        src = os.path.abspath(filen)
        cwd = self.getWorkingDirectory()
        dest = os.path.join(cwd, link)
        os.symlink(os.path.relpath(src, cwd), dest)

    def chi2plot(self, filename=None, close=True):

        chi2 = numpy.array(
            [plg.dataOutput.chiSqrt.value for plg in self.dammif_plugins])
        chi2max = chi2.mean() + 2 * chi2.std()

        xticks = 1 + numpy.arange(self.dammif_jobs)
        fig = plt.figure(figsize=figureSize)
        ax1 = fig.add_subplot(1, 2, 1)
        ax1.bar(xticks - 0.5, chi2)
        ax1.set_ylabel(u"$\sqrt{\u03C7}$")
        ax1.set_xlabel(u"Model number")
        ax1.plot([0.5, self.dammif_jobs + 0.5], [chi2max, chi2max],
                 "-r",
                 label=u"$\sqrt{\u03C7}$$_{max}$ = %.3f" % chi2max)
        ax1.set_xticks(xticks)
        ax1.legend(loc=8)
        R = numpy.array(
            [plg.dataOutput.rfactor.value for plg in self.dammif_plugins])
        Rmax = R.mean() + 2 * R.std()
        ax2 = fig.add_subplot(1, 2, 2)
        ax2.bar(xticks - 0.5, R)
        ax2.plot([0.5, self.dammif_jobs + 0.5], [Rmax, Rmax],
                 "-r",
                 label=u"R$_{max}$ = %.3f" % Rmax)
        ax2.set_ylabel(u"R factor")
        ax2.set_xlabel(u"Model number")
        ax2.set_xticks(xticks)
        ax2.legend(loc=8)
        #        fig.set_title("Selection of dammif models based on \u03C7$^2$")
        self.valid = (chi2 < chi2max) * (R < Rmax)
        self.mask2d = (1 - numpy.identity(self.dammif_jobs)) * numpy.outer(
            self.valid, self.valid)
        #        print self.valid
        bbox_props = dict(fc="pink", ec="r", lw=1)
        for i in range(self.dammif_jobs):
            if not self.valid[i]:
                ax1.text(i + 0.95,
                         chi2max / 2,
                         "Discarded",
                         ha="center",
                         va="center",
                         rotation=90,
                         size=10,
                         bbox=bbox_props)
                ax2.text(i + 0.95,
                         Rmax / 2,
                         "Discarded",
                         ha="center",
                         va="center",
                         rotation=90,
                         size=10,
                         bbox=bbox_props)
        if filename:
            filename = os.path.join(self.getWorkingDirectory(), filename)
            self.log("Wrote %s" % filename)
            fig.savefig(filename)
        if close:
            fig.clf()
            plt.close(fig)
        else:
            return fig

    def makeNSDarray(self, filename=None, close=True):
        self.arrayNSD = numpy.zeros(self.mask2d.shape, numpy.float32)
        fig = plt.figure(figsize=figureSize)
        #ax1 = fig.add_subplot(1, 2, 1)

        ax1 = plt.subplot2grid((1, 7), (0, 0), colspan=4)
        ax2 = plt.subplot2grid((1, 7), (0, 4), colspan=3)

        # for now just an empty figure but a placeholder
        ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper")

        xticks = 1 + numpy.arange(self.dammif_jobs)
        lnsd = []
        for key, plugin in self.supcomb_plugins.items():
            i0, i1 = key
            nsd = plugin.dataOutput.NSD.value
            self.arrayNSD[i0, i1] = nsd
            self.arrayNSD[i1, i0] = nsd
            lnsd.append(nsd)
            #ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs)
            #ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs)
            ax1.text(i0,
                     i1,
                     "%.2f" % nsd,
                     ha="center",
                     va="center",
                     size=4,
                     color='white',
                     weight='demibold')
            ax1.text(i1,
                     i0,
                     "%.2f" % nsd,
                     ha="center",
                     va="center",
                     size=4,
                     color='white',
                     weight='demibold')
        lnsd = numpy.array(lnsd)
        #        print lnsd
        #        print lnsd.mean() , lnsd.std(), lnsd.mean() + 2 * lnsd.std()
        nsd_max = lnsd.mean() + lnsd.std()
        data = self.arrayNSD.sum(axis=-1) / self.mask2d.sum(axis=-1)
        best_val = data[data > 0].min()
        #        print data
        #        print best_val
        #        print numpy.where(data == best_val)
        self.ref = int(numpy.where(data == best_val)[0][-1])
        #        print self.ref
        ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper")
        ax1.set_title(u"NSD correlation table")
        ax1.set_xticks(range(self.dammif_jobs))
        ax1.set_xticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)])
        ax1.set_xlim(-0.5, self.dammif_jobs - 0.5)
        ax1.set_ylim(-0.5, self.dammif_jobs - 0.5)
        ax1.set_yticks(range(self.dammif_jobs))
        ax1.set_yticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)])
        ax1.set_xlabel(u"Model number")
        ax1.set_ylabel(u"Model number")
        #ax2 = fig.add_subplot(1, 2, 2)
        ax2.bar(xticks - 0.5, data)
        ax2.plot([0.5, self.dammif_jobs + 0.5], [nsd_max, nsd_max],
                 "-r",
                 label=u"NSD$_{max}$ = %.2f" % nsd_max)
        ax2.set_title(u"NSD between any model \n and all others",
                      multialignment='center')
        ax2.set_ylabel("Normalized Spatial Discrepancy")
        ax2.set_xlabel(u"Model number")
        ax2.set_xticks(xticks)
        ax2.tick_params(axis='y', labelsize=8)
        bbox_props = dict(fc="cyan", ec="b", lw=1)
        # ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=10, bbox=bbox_props)
        ax2.text(self.ref + 0.95,
                 data[self.ref] / 2,
                 "Reference",
                 ha="center",
                 va="center",
                 rotation=90,
                 size=8,
                 bbox=bbox_props)
        ax2.legend(loc=8, prop={'size': 8})
        self.valid *= (data < nsd_max)
        bbox_props = dict(fc="pink", ec="r", lw=1)
        plt.tight_layout()
        for i in range(self.dammif_jobs):
            if not self.valid[i]:
                #     ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props)
                ax2.text(i + 0.95,
                         data[self.ref] / 2,
                         "Discarded",
                         ha="center",
                         va="center",
                         rotation=90,
                         size=8,
                         bbox=bbox_props)
#        print self.valid
#        print self.ref
        if filename:
            filename = os.path.join(self.getWorkingDirectory(), filename)
            self.log("Wrote %s" % filename)
            fig.savefig(filename, dpi=1200)
        if close:
            fig.clf()
            plt.close(fig)
        else:
            return fig
Exemple #14
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.process")
        xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile,
                                              unit=XSDataString(self.unit),
                                              symmetry=XSDataString(
                                                  self.symmetry),
                                              mode=XSDataString(self.mode))
        for i in range(self.dammif_jobs):
            dammif = self.loadPlugin(self.strPluginExecDammif)
            dammif.connectSUCCESS(self.doSuccessExecDammif)
            dammif.connectFAILURE(self.doFailureExecDammif)
            xsd = xsDataInputDammif.copyViaDict()
            xsd.order = XSDataInteger(i + 1)
            dammif.dataInput = xsd
            self.addPluginToActionCluster(dammif)
            self.dammif_plugins.append(dammif)
        self.executeActionCluster()
        self.synchronizeActionCluster()
        for plugin in self.dammif_plugins:
            if plugin.isFailure():
                self.ERROR("dammif plugin %s-%08i failed" %
                           (plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)
        if self.isFailure():
            return

        # retrieve results from best dammif
        self.dammif = self.bestDammif()

        self.chi2plot("chi2_R.png")
        self.result.chiRfactorPlot = XSDataFile(
            XSDataString(os.path.join(self.getWorkingDirectory(),
                                      "chi2_R.png")))

        # temporary results: use best dammif
        self.result.fitFile = self.dammif.dataOutput.fitFile
        self.result.logFile = self.dammif.dataOutput.logFile
        self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile
        self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile

        # prepare an action cluster with all supcomb plugins
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for idx in range(self.dammif_jobs):
            if self.valid[idx]:
                for ser in range(idx):
                    if self.valid[ser]:
                        supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                        supcomb.dataInput = XSDataInputSupcomb(
                            templateFile=self.dammif_plugins[idx].dataOutput.
                            pdbMoleculeFile,
                            superimposeFile=self.dammif_plugins[ser].
                            dataOutput.pdbMoleculeFile,
                            name=self.dammif_plugins[ser].dataOutput.model.name
                        )
                        self.supcomb_plugins[(idx, ser)] = supcomb
                        self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for key, plugin in self.supcomb_plugins.items():
            if plugin.isFailure():
                self.ERROR(
                    "supcomb plugin for model pair (%i,%i) %s-%08i failed" %
                    (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)

        if self.isFailure():
            return

        self.makeNSDarray("nsd.png")
        self.result.nsdPlot = XSDataFile(
            XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png")))

        idx = self.ref
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                supcomb.dataInput = XSDataInputSupcomb(
                    templateFile=self.dammif_plugins[
                        self.ref].dataOutput.pdbMoleculeFile,
                    superimposeFile=self.dammif_plugins[ser].dataOutput.
                    pdbMoleculeFile,
                    name=self.dammif_plugins[ser].dataOutput.model.name)
                self.supcomb_plugins[(self.ref, ser)] = supcomb
                self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                plugin = self.supcomb_plugins[(self.ref, ser)]
                if plugin.isFailure():
                    self.ERROR(
                        "supcomb plugin for model pair (%i,%i) %s-%08i failed"
                        % (self.ref + 1, ser + 1, plugin.getName(),
                           plugin.getId()))
                    self.setFailure()
                self.retrieveMessages(plugin)
        if self.isFailure():
            return

        for i in range(self.dammif_jobs):
            if i == self.ref or not self.valid[i]:
                model = self.dammif_plugins[i].dataOutput.model
            else:
                model = self.supcomb_plugins[(self.ref, i)].dataOutput.model
                model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt
#                model.chiSqrt =  self.dammif_plugins[i].dataOutput.model.chiSqrt
            self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1))
            self.result.dammifModels[i] = model

#        Now that all (valid) models are aligned we can combine them using damaver
        pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile]

        for idx in range(self.dammif_jobs):
            if self.valid[idx] and idx != self.ref:
                pdbFiles.append(
                    self.supcomb_plugins[(self.ref,
                                          idx)].dataOutput.outputFilename)

        damaver = self.loadPlugin(self.strPluginExecDamaver)
        damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles,
                                               automatic=XSDataBoolean(False))
        damaver.connectSUCCESS(self.doSuccessExecDamaver)
        damaver.connectFAILURE(self.doFailureExecDamaver)
        damaver.executeSynchronous()

        if self.isFailure():
            return

        damfilt = self.loadPlugin(self.strPluginExecDamfilt)
        damfilt.dataInput = XSDataInputDamfilt(
            inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damfilt.connectSUCCESS(self.doSuccessExecDamfilt)
        damfilt.connectFAILURE(self.doFailureExecDamfilt)
        damfilt.execute()
        ########################################################################
        # TODO: This is a dead end : do it in parallel
        ########################################################################

        if self.isFailure():
            return

        damstart = self.loadPlugin(self.strPluginExecDamstart)
        damstart.dataInput = XSDataInputDamstart(
            inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damstart.connectSUCCESS(self.doSuccessExecDamstart)
        damstart.connectFAILURE(self.doFailureExecDamstart)
        damstart.executeSynchronous()

        if self.isFailure():
            return
        ########################################################################
        # Finally call dammin
        ########################################################################
        if self.config.get("do_dammin") in ["False", "0", False, 0]:
            return
        dammin = self.loadPlugin(self.strPluginExecDammin)
        dammin.dataInput = XSDataInputDammin(
            pdbInputFile=damstart.dataOutput.outputPdbFile,
            gnomOutputFile=self.xsGnomFile,
            symmetry=XSDataString(self.symmetry),
            mode=XSDataString(self.mode))
        dammin.connectSUCCESS(self.doSuccessExecDammin)
        dammin.connectFAILURE(self.doFailureExecDammin)
        dammin.executeSynchronous()
class EDPluginExecMeasureOffsetv2_0(EDPluginControl):
    """
    An exec plugin that takes two images and measures the offset between the two.
    In facts it is not an ExecPlugin but a control plugin that:
    * Converts the pair of images in colored JPEG
    * Extract the SIFT descriptor of each image
    * Measure the offset between the two images using the Autopano tool
    * return the measured offset and the file describing the control points.
    """


    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputMeasureOffset)
        self.__strControlledPluginThumbnail = "EDPluginExecThumbnailv10"
        self.__strControlledPluginSift = "EDPluginExecSiftDescriptorv1_0"
        self.__strControlledPluginAutopano = "EDPluginExecSiftOffsetv1_0"
        self.semThumbnail = threading.Semaphore()
        self.semSift = threading.Semaphore()
        self.ACThumbnail = EDActionCluster()
        self.ACSift = EDActionCluster()
        self.xsdImages = []
        self.xsdThumb = []
        self.xsdKeys = []
        self.xsdIdx = []
        self.tCrop = [0, 0]
        self.inputImages = []
        self.tOffset = None
        self.xsdPTO = None

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        EDVerbose.DEBUG("EDPluginControlMeasureOffsetv2_0.checkParameters")
        self.checkMandatoryParameters(self.getDataInput(), "Data Input is None")


    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.preProcess")
        sdi = self.getDataInput()

        crop = sdi.getCropBorders()
        if len(crop) == 2:
            self.tCrop = (crop[0].getValue(), crop[1].getValue())
        elif len(crop) == 1:
            self.tCrop = (crop[0].getValue(), crop[0].getValue())

#
        if len(sdi.getImage()) == 2:
            for i in sdi.getImage():

                array = openimage(i.getPath().getValue()).data
                shape = array.shape
                if (self.tCrop != [0, 0]) and (shape[0] > self.tCrop[0]) and (shape[1] > self.tCrop[1]):
                    array = array[self.tCrop[0]:-self.tCrop[0], self.tCrop[1]:-self.tCrop[1] ]
                    EDVerbose.DEBUG("After Crop, images have shape : (%s,%s) " % (array.shape))
                self.xsdImages.append(EDUtilsArray.arrayToXSData(array))
        elif len(sdi.getArray()) == 2:
            if (self.tCrop == [0, 0]) :
                self.xsdImages = sdi.getArray()
            else:
                for xsdArray  in  sdi.getArray():
                    array = EDUtilsArray.xsDataToArray(xsdArray)
                    shape = array.shape
                    if (shape[0] > self.tCrop[0]) and (shape[1] > self.tCrop[1]):
                        array = array[self.tCrop[0]:-self.tCrop[0], self.tCrop[1]:-self.tCrop[1] ]
                        EDVerbose.DEBUG("After Crop, images have shape : (%s,%s) " % (array.shape))
                    self.xsdImages.append(EDUtilsArray.arrayToXSData(array))
        else:
            strError = "EDPluginExecMeasureOffsetv2_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal()
            EDVerbose.ERROR(strError)
            self.setFailure()
            raise RuntimeError(strError)
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.xsdImages len=%i %s" % (len(self.xsdImages), self.xsdImages))
        EDAssert.equal(self.xsdImages[0].getShape() , self.xsdImages[1].getShape(), "Images have the same size")
        self.xsdIdx = sdi.getIndex()
        if len(self.xsdIdx) < len(self.xsdImages):
            self.xsdIdx = [XSDataInteger(i) for i in range(len(self.xsdImages))]


    def process(self, _edObject=None):
        """
        """
        for  i in range(2):
            execPlugin = self.loadPlugin(self.__strControlledPluginThumbnail)
            xsdin = XSDataInputExecThumbnail()
            xsdin.setInputArray(self.xsdImages[i])
            xsdFile = XSDataFile()
            xsdFile.setPath(XSDataString(os.path.join(self.getWorkingDirectory(), "image%i.jpg" % self.xsdIdx[i].getValue())))
            xsdin.setOutputPath(xsdFile)
            xsdin.setLevelsColorize(XSDataBoolean(1))
            xsdin.setLevelsEqualize(XSDataBoolean(1))

            execPlugin.setDataInput(xsdin)
            execPlugin.connectSUCCESS(self.doSuccessThumb)
            execPlugin.connectFAILURE(self.doFailureThumb)
            self.ACThumbnail.addAction(execPlugin)
        self.ACThumbnail.execute()

        while len(self.xsdThumb) < 2:
            time.sleep(1)

        for  oneImage in self.xsdThumb:
            execPlugin = self.loadPlugin(self.__strControlledPluginSift)
            xsdin = XSDataInputSiftDescriptor()
            xsdin.setImage(oneImage)
            execPlugin.setDataInput(xsdin)
            execPlugin.connectSUCCESS(self.doSuccessSift)
            execPlugin.connectFAILURE(self.doFailureSift)
            self.ACSift.addAction(execPlugin)
        self.ACSift.execute()
#
#        else:
#            strError = "There are only %s images in self.xsdThumb" % len(self.xsdThumb)
#            EDVerbose.ERROR(strError)
#            self.setFailure()
#            raise RuntimeError(strError)

################################################################################
# This should be executed only after the Sift actions cluster finishes 
################################################################################
        while len(self.xsdKeys) < 2:
            time.sleep(1)

        execPlugin = self.loadPlugin(self.__strControlledPluginAutopano)
        xsdin = XSDataInputMeasureOffsetSift()
        xsdin.setDescriptorFile(self.xsdKeys)
        execPlugin.setDataInput(xsdin)
        execPlugin.connectSUCCESS(self.doSuccessAutopano)
        execPlugin.connectFAILURE(self.doFailureAutopano)
        execPlugin.executeSynchronous()




    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.postProcess")
        # Create some output data
        xsDataResult = XSDataResultMeasureOffset()
        xsDataResult.setOffset(self.tOffset)
        xsDataResult.setPanoFile(self.xsdPTO)
        self.setDataOutput(xsDataResult)
        self.xsdImages = []

    def doSuccessThumb(self, _edPlugin=None):
        self.semThumbnail.acquire()
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessThumb")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessThumb")
        self.xsdThumb.append(_edPlugin.getDataOutput().getThumbnailPath())
        self.semThumbnail.release()


    def doFailureThumb(self, _edPlugin=None):
        self.semThumbnail.acquire()
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureThumb")
        self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureThumb")
        self.setFailure()
        strError = "Error in converting to Jpeg with this input: %s" % _edPlugin.getDataInput().marshal()
        EDVerbose.ERROR(strError)
        self.semThumbnail.release()
        raise RuntimeError(strError)


    def doSuccessSift(self, _edPlugin=None):
        self.semSift.acquire()
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessSift")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessSift")
        self.xsdKeys.append(_edPlugin.getDataOutput().getDescriptorFile())
        self.semSift.release()


    def doFailureSift(self, _edPlugin=None):
        self.semSift.acquire()
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureSift")
        self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureSift")
        self.setFailure()
        strError = "Error in extracting SIFT keys with this input: %s" % _edPlugin.getDataInput().marshal()
        EDVerbose.ERROR(strError)
        self.semSift.release()
        raise RuntimeError(strError)


    def doSuccessAutopano(self, _edPlugin=None):
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessAutopano")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessSift")
        self.tOffset = _edPlugin.getDataOutput().getOffset()
        self.xsdPTO = _edPlugin.getDataOutput().getPanoFile()


    def doFailureAutopano(self, _edPlugin=None):
        EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureAutopano")
        self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureAutopano")
        self.setFailure()
        strError = "Error in Autopano execution of with this input: %s" % _edPlugin.getDataInput().marshal()
        EDVerbose.ERROR(strError)
        raise RuntimeError(strError)
Exemple #16
0
class EDPluginControl(EDPlugin):
    """
    An EDPluginControl is a plugin that is responsible for a EDPluginExec or EDPluginControl plugin execution:
    It is responsible for:
        - The EDPluginExec or EDPluginControl Workflow
        - The data propagation between the EDPluginExec
        - The translation between generic and specific data models via EDHandler classes
        - The error/warning propagation
        - The executive summaries propagation
        - Execution of an "action cluster": a set of plugins can be added to a so called "action cluster"
          with the method "addPluginToActionCluster". All the plugins in the cluster can then be executed
          simultaneously with the method "executeActionCluster" and synchronized with the method
          "synchronizeActionCluster". The number of threads used by the action cluster is by default the
          number of processors available on the computer, but this value can be changed either by
          calling the method "setClusterSize" or by using the configuration parameter "clusterSize".
    """

    def __init__ (self):
        """
        """
        EDPlugin.__init__(self)
        self.__strPluginToBeControlledName = None
        self.__dictControlledPlugins = {}
        self.__edActionCluster = None
        self.__iClusterSize = None
        self.__listOfLoadedPlugins = []


    def configure(self):
        """
        Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes.
        """
        EDPlugin.configure(self)
        EDVerbose.DEBUG("EDPluginControl.configure")
        strControlledPlugins = self.config.get("controlledPlugins", None)
        if (strControlledPlugins != None):
            pyListControlledPlugins = strControlledPlugins.split(",")
            for strControlledPlugin in pyListControlledPlugins:
                strControlledPluginName = self.getStringConfigurationParameterValue(strControlledPlugin)
                if strControlledPluginName != None:
                    self.setControlledPluginName(strControlledPlugin, strControlledPluginName)
                    EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName))
        clusterSize = self.config.get("clusterSize", None)
        if (clusterSize != None):
            self.__iClusterSize = int(strClusterSize)
            EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize)


    def emptyListOfLoadedPlugin(self):
        """
        Reset all plugins kept in memory
        """
        self.__listOfLoadedPlugins = []
        gc.collect()

    def getListOfLoadedPlugin(self):
        """
        """
        return self.__listOfLoadedPlugins


    def removeLoadedPlugin(self, _plugin):
        """
        Remove a plugin from the list of loaded plugins to free some memory
        @param _plugin: plugin to remove
        @type _plugin: instance of the class EDPlugin
        """
        if _plugin in self.__listOfLoadedPlugins:
            with self.locked():
                self.__listOfLoadedPlugins.remove(_plugin)
            self.DEBUG("EDPluginControl.removeLoadedPlugin: Caught, removed %s unreferenced objects. currently there are %i plugins" % (gc.get_count(), len(self.__listOfLoadedPlugins)))
            gc.collect()
        else:
            self.DEBUG("EDPluginControl.removeLoadedPlugin: Missed. currently there are %i plugins" % len(self.__listOfLoadedPlugins))


    def synchronizePlugins(self):
        EDVerbose.DEBUG("EDPluginControl.synchronizePlugins")
        bSynchronized = False
        while not bSynchronized:
            listPluginOrig = self.__listOfLoadedPlugins[:]
            for edPlugin in listPluginOrig:
                if edPlugin.isStarted() and (not edPlugin.isEnded()):
                    edPlugin.synchronize()
                elif not edPlugin.isStarted():
                    time.sleep(0.01) #release GIL to let plugin start 
                    continue
            time.sleep(0.01)
            with self.locked():
                bSynchronized = (self.__listOfLoadedPlugins == listPluginOrig)


    def loadPlugins(self):
        """
        This method loads and returns a list of references to the plugins to be controlled.
        
        The name of the plugin to be controlled is set set before calling this method using the 
        "setControlledPluginName" method. 
        
        The base name of the plugin to be controlled is used as the working
        directory name of the plugin in question. The name of the plugin is used as 
        base name. 
        """
        EDVerbose.DEBUG("EDPluginControl.loadPlugins")
        listKeys = self.__dictControlledPlugins.keys()
        listLoadedPlugins = []
        for strKey in listKeys:
            strPluginName = self.__dictControlledPlugins[strKey]
            edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
            edPlugin.setBaseDirectory(self.getWorkingDirectory())
            edPlugin.setBaseName(strPluginName)
            listLoadedPlugins.append(edPlugin)
        return listLoadedPlugins


    def loadPlugin(self, _strPluginToBeControlledName=None, _strBaseName=None):
        """
        This method loads and returns a reference to the plugin to be controlled.
        
        The name of the plugin to be controlled can either be passed as an
        argument, or bet set before calling this method using the 
        "setPluginToBeControlledName". 
        
        The base name of the plugin to be controlled is used as the working
        directory name of the plugin in question. If no argument is supplied
        the name of the plugin is used as base name. In the case of creation of
        several plugins to be launched simultaneously, the base name should be
        different for each plugin and hence must be provided explicitly.
        """
        EDVerbose.DEBUG("EDPluginControl.loadPlugin")
        if (_strPluginToBeControlledName is None):
            strPluginName = self.__strPluginToBeControlledName
        else:
            strPluginName = _strPluginToBeControlledName
        edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName)
        if (edPlugin is None):
            strErrorMessage = "EDPluginControl.loadPlugin : Cannot load plugin %s" % strPluginName
            EDVerbose.error(strErrorMessage)
            self.addErrorMessage(strErrorMessage)
            raise RuntimeError, strErrorMessage
        else:
            self.__listOfLoadedPlugins.append(edPlugin)
        edPlugin.setBaseDirectory(self.getWorkingDirectory())
        if (_strBaseName is None):
            # Check if base name exists. OBS! Not thread safe so please set explicitly
            # _strBaseName for multi-threaded code
            strRenamedPlugin = self.compactPluginName(strPluginName)
            strNewWorkingDirectory = os.path.join(self.getWorkingDirectory(), strRenamedPlugin)
            if (os.path.exists(strNewWorkingDirectory)):
                edPlugin.setBaseName(edPlugin.createBaseName())
            else:
                edPlugin.setBaseName(strRenamedPlugin)
        else:
            edPlugin.setBaseName(_strBaseName)
        return edPlugin


    def setControlledPluginName(self, _strControlledPluginName, _strControlledPluginValue):
        """
        Adds a name-value pair to the dictionary to map the general to the specific name of a plugin to be controlled
        """
        self.__dictControlledPlugins[_strControlledPluginName] = _strControlledPluginValue


    def getControlledPluginName(self, _strControlledPluginName):
        """
        Returns the name of the plugin to be controlled.
        """
        strPluginname = None
        if self.__dictControlledPlugins.has_key(_strControlledPluginName):
            strPluginname = self.__dictControlledPlugins[_strControlledPluginName]
        return strPluginname


    def addWarningMessages(self, _listWarningMessages):
        """
        Adds a list of warning messages in the existing list of warning messages
        """
        EDVerbose.DEBUG("EDPluginControl.addWarningMessages")
        for strWarningMessage in _listWarningMessages:
            self.addWarningMessage(strWarningMessage)


    def addErrorMessages(self, _listErrorMessages):
        """
        Adds a list of error messages in the existing list of error messages
        """
        EDVerbose.DEBUG("EDPluginControl.addErrorMessages")
        for strErrorMessage in _listErrorMessages:
            self.addErrorMessage(strErrorMessage)


    def retrieveFailureMessages(self, _edPlugin, _strMethodCaller):
        """
        Propagates failure messages from a plugin including unexpected errors
        Should be called in the plugin control method invoked when a plugin exec fails (doActionFailure<>)
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveFailureMessages")
        self.retrieveWarningMessages(_edPlugin)
        self.retrieveErrorMessages(_edPlugin, _strMethodCaller, True)


    def retrieveSuccessMessages(self, _edPlugin, _strMethodCaller):
        """
        Propagates success messages from a plugin
        Error messages are retrieved because a plugin could end successfully with errors (depending on the use case)
        In this case, there is no check for unexpected errors
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveSuccessMessages")
        self.retrieveWarningMessages(_edPlugin)
        self.retrieveErrorMessages(_edPlugin, _strMethodCaller, False)


    def retrieveErrorMessages(self, _edPlugin, _strMethodCaller, _bFailure):
        """
        Propagates error messages from a plugin
        if _bFailure is true, this method has been called from retrieveFailureMessages
        in this case, checks for potential unexpected errors coming from the EDPluginExec
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveErrorMessages")
        listErrorMessages = _edPlugin.getListOfErrorMessages()
        if (len(listErrorMessages) == 0) and (_bFailure is True):
            strErrorMessage = "%s : Adding Unexpected error" % _strMethodCaller
            EDVerbose.DEBUG(strErrorMessage)
            listErrorMessages.append(strErrorMessage)
        self.addErrorMessages(listErrorMessages)


    def retrieveWarningMessages(self, _edPlugin):
        """
        Propagates warning messages from a plugin
        """
        EDVerbose.DEBUG("EDPluginControl.retrieveWarningMessages")
        self.addWarningMessages(_edPlugin.getListOfWarningMessages())


    def appendExecutiveSummary(self, _edPlugin, _strPrefix="", _bAddSeparator=True):
        """
        Appends the executive summary from a plugin.
        """
        EDVerbose.DEBUG("EDPluginControl.appendExecutiveSummary")
        if (_bAddSeparator):
            self.addExecutiveSummarySeparator()
        if _edPlugin:
            for strLine in _edPlugin.getListExecutiveSummaryLines():
                if strLine == self.getExecutiveSummarySeparator() and _strPrefix != "":
                    strLine = strLine[ :-len(_strPrefix) ]
            self.addExecutiveSummaryLine(_strPrefix + strLine)


    def addErrorWarningMessagesToExecutiveSummary(self, _strErrorMessage="Error messages:", _strWarningMessage="Warning messages:"):
        """
        Adds error and warning messages (if any) in the executive summary
        """
        if len(self.getListOfErrorMessages()) != 0:
            self.addExecutiveSummarySeparator()
            self.addExecutiveSummaryLine(_strErrorMessage)
            for strErrorMessage in self.getListOfErrorMessages():
                self.addExecutiveSummaryLine(strErrorMessage)
            self.addExecutiveSummarySeparator()
        if len(self.getListOfWarningMessages()) != 0:
            self.addExecutiveSummarySeparator()
            self.addExecutiveSummaryLine(_strWarningMessage)
            for warningMessage in self.getListOfWarningMessages():
                self.addExecutiveSummaryLine(warningMessage)
            self.addExecutiveSummarySeparator()


    def addPluginToActionCluster(self, _edPlugin):
        """
        This method adds a plugin instance to an action cluster.
        """
        if self.__edActionCluster == None:
            self.__edActionCluster = EDActionCluster()
        self.__edActionCluster.addAction(_edPlugin)
        self.__listOfLoadedPlugins.append(_edPlugin)


    def executeActionCluster(self):
        """
        This method executes the action cluster. The action cluster is executed
        asynchronoulsy.
        """
        if self.__iClusterSize != None:
            self.__edActionCluster.setClusterSize(self.__iClusterSize)
        self.__edActionCluster.execute()


    def synchronizeActionCluster(self):
        """
        This method synchronises the action cluster with the control plugin thread.
        """
        self.__edActionCluster.synchronize()


    def setClusterSize(self, _iClusterSize):
        """
        This method sets the size of the action cluster, i.e. the number of threads
        that will be executed simultaneously. 
        """
        self.__iClusterSize = _iClusterSize


    def executePlugin(self, _edPlugin, _bSynchronous=False):
        """
        This method is used to start executable plugins in pipeline asynchronously.
        """
        if _bSynchronous:
            self.executePluginSynchronous(_edPlugin)
        else:
            _edPlugin.execute()


    def executePluginSynchronous(self, _edPlugin):
        """
        This method is used to start executable plugins in pipeline synchronously.
        """
        _edControlSlotSUCCESS = EDSlot()
        _edControlSlotFAILURE = EDSlot()

        map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod())
        map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod())

        _edPlugin.getSlotSUCCESS().emptyListMethod()
        _edPlugin.getSlotFAILURE().emptyListMethod()

        _edPlugin.executeSynchronous()

        if (not _edPlugin.isFailure()):
            EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS")
            # Check that something doesn't go wrong in the success method!
            try:
                _edControlSlotSUCCESS.call(_edPlugin)

            except Exception:
                EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!")
                EDVerbose.writeErrorTrace()
                _edPlugin.setFailure()

        if (_edPlugin.isFailure()):
            EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE")
            # Check that something doesn't go wrong in the success method!
            try:
                _edControlSlotFAILURE.call(_edPlugin)

            except Exception:
                EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!")
                EDVerbose.writeErrorTrace()
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlSaxsModelingv1_0.process")
        xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile,
                                              unit=XSDataString(self.unit),
                                              symmetry=XSDataString(self.symmetry),
                                              mode=XSDataString(self.mode))
        for i in range(self.dammif_jobs):
            dammif = self.loadPlugin(self.strPluginExecDammif)
            dammif.connectSUCCESS(self.doSuccessExecDammif)
            dammif.connectFAILURE(self.doFailureExecDammif)
            xsd = xsDataInputDammif.copyViaDict()
            xsd.order = XSDataInteger(i + 1)
            dammif.dataInput = xsd
            self.addPluginToActionCluster(dammif)
            self.dammif_plugins.append(dammif)
        self.executeActionCluster()
        self.synchronizeActionCluster()
        for plugin in self.dammif_plugins:
            if plugin.isFailure():
                self.ERROR("dammif plugin %s-%08i failed" % (plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)
        if self.isFailure():
            return

        # retrieve results from best dammif
        self.dammif = self.bestDammif()

        self.chi2plot("chi2_R.png")
        self.result.chiRfactorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "chi2_R.png")))

        # temporary results: use best dammif
        self.result.fitFile = self.dammif.dataOutput.fitFile
        self.result.logFile = self.dammif.dataOutput.logFile
        self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile
        self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile

        # prepare an action cluster with all supcomb plugins
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for idx in range(self.dammif_jobs):
            if self.valid[idx]:
                for ser in range(idx):
                    if self.valid[ser]:
                        supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                        supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[idx].dataOutput.pdbMoleculeFile,
                                                               superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile,
                                                               name=self.dammif_plugins[ser].dataOutput.model.name)
                        self.supcomb_plugins[(idx, ser)] = supcomb
                        self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for key, plugin in self.supcomb_plugins.items():
            if plugin.isFailure():
                self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId()))
                self.setFailure()
            self.retrieveMessages(plugin)

        if self.isFailure():
            return

        self.makeNSDarray("nsd.png")
        self.result.nsdPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png")))

        idx = self.ref
        self.actclust_supcomb = EDActionCluster(self.cluster_size)
        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                supcomb = self.loadPlugin(self.strPluginExecSupcomb)
                supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile,
                                                       superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile,
                                                       name=self.dammif_plugins[ser].dataOutput.model.name)
                self.supcomb_plugins[(self.ref, ser)] = supcomb
                self.actclust_supcomb.addAction(supcomb)
        self.actclust_supcomb.executeSynchronous()

        for ser in range(self.ref + 1, self.dammif_jobs):
            if self.valid[ser]:
                plugin = self.supcomb_plugins[(self.ref, ser)]
                if plugin.isFailure():
                    self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (self.ref + 1, ser + 1, plugin.getName(), plugin.getId()))
                    self.setFailure()
                self.retrieveMessages(plugin)
        if self.isFailure():
            return

        for i in range(self.dammif_jobs):
            if i == self.ref or not self.valid[i]:
                model = self.dammif_plugins[i].dataOutput.model
            else:
                model = self.supcomb_plugins[(self.ref, i)].dataOutput.model
                model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt
#                model.chiSqrt =  self.dammif_plugins[i].dataOutput.model.chiSqrt
            self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1))
            self.result.dammifModels[i] = model


#        Now that all (valid) models are aligned we can combine them using damaver
        pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile]

        for idx in range(self.dammif_jobs):
            if self.valid[idx] and idx != self.ref:
                pdbFiles.append(self.supcomb_plugins[(self.ref, idx)].dataOutput.outputFilename)

        damaver = self.loadPlugin(self.strPluginExecDamaver)
        damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles,
                                                automatic=XSDataBoolean(False))
        damaver.connectSUCCESS(self.doSuccessExecDamaver)
        damaver.connectFAILURE(self.doFailureExecDamaver)
        damaver.executeSynchronous()

        if self.isFailure():
            return

        damfilt = self.loadPlugin(self.strPluginExecDamfilt)
        damfilt.dataInput = XSDataInputDamfilt(inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damfilt.connectSUCCESS(self.doSuccessExecDamfilt)
        damfilt.connectFAILURE(self.doFailureExecDamfilt)
        damfilt.execute()
        ########################################################################
        # TODO: This is a dead end : do it in parallel
        ########################################################################

        if self.isFailure():
            return

        damstart = self.loadPlugin(self.strPluginExecDamstart)
        damstart.dataInput = XSDataInputDamstart(inputPdbFile=damaver.dataOutput.damaverPdbFile)
        damstart.connectSUCCESS(self.doSuccessExecDamstart)
        damstart.connectFAILURE(self.doFailureExecDamstart)
        damstart.executeSynchronous()

        if self.isFailure():
            return
        ########################################################################
        # Finally call dammin
        ########################################################################
        if self.config.get("do_dammin") in ["False", "0", False, 0]:
            return
        dammin = self.loadPlugin(self.strPluginExecDammin)
        dammin.dataInput = XSDataInputDammin(pdbInputFile=damstart.dataOutput.outputPdbFile,
                                             gnomOutputFile=self.xsGnomFile,
                                             symmetry=XSDataString(self.symmetry),
                                             mode=XSDataString(self.mode))
        dammin.connectSUCCESS(self.doSuccessExecDammin)
        dammin.connectFAILURE(self.doFailureExecDammin)
        dammin.executeSynchronous()
class EDPluginParallelXNCFJobLauncher:
    """
    Class for starting execution plugin jobs in parallel for every input data object   
    """
    def __init__(self, _edControlPlugin, _strPluginName, _dictXSDataInput,
                 _iNbThreads):
        """
        Initialaze data structures
        @param _edControlPlugin: Parent control plugin
        @param _strPluginName: Name of the execution plugin to run
        @param _dictXSDataInput: Dictionary of the input objects for execution plugin
        """
        self.__strPluginName = _strPluginName
        self.__dictXSDataInput = _dictXSDataInput
        self.__edControlPlugin = _edControlPlugin

        self.__xsEDPluginExecJobs = {}
        self.__xsEDActionCluster = EDActionCluster(_iNbThreads)

        self.__bIsFirstExecute = True

    def getPluginJobs(self):
        """
        Get dictionary of launched plugin jobs
        """
        return self.__xsEDPluginExecJobs

    def run(self):
        """
        Initialize and run all parallel jobs
        """
        for __dictKey in self.__dictXSDataInput.keys():
            self.__xsEDPluginExecJobs[
                __dictKey] = self.__edControlPlugin.loadPlugin(
                    self.__strPluginName)
            if (self.__xsEDPluginExecJobs[__dictKey] is not None):

                if (self.__dictXSDataInput[__dictKey]
                        is not None) and (self.__dictXSDataInput[__dictKey]
                                          is not ""):
                    self.__xsEDPluginExecJobs[__dictKey].setDataInput(
                        self.__dictXSDataInput[__dictKey])
                    self.__xsEDPluginExecJobs[__dictKey].connectSUCCESS(
                        self.__successPluginExecution)
                    self.__xsEDPluginExecJobs[__dictKey].connectFAILURE(
                        self.__failurePluginExecution)
                    self.__xsEDActionCluster.addAction(
                        self.__xsEDPluginExecJobs[__dictKey])
                else:
                    EDVerbose.screen("ERROR! Input data not found in " +
                                     self.__xsEDPluginExecJobs[__dictKey].
                                     getWorkingDirectory())
            else:
                EDVerbose.screen("ERROR! Plugin not found : " +
                                 self.__strPluginName)
        self.__xsEDActionCluster.setTimeOut(
            self.__edControlPlugin.getTimeOut())
        self.__xsEDActionCluster.executeSynchronous()

    def __successPluginExecution(self, _edPlugin=None):
        """
        Method called when the execution of the plugin succeeds 
        """
        EDVerbose.DEBUG(
            "EDPluginParallelXNCFJobLauncher.__successPluginExecution")
        self.__edControlPlugin.retrieveSuccessMessages(
            _edPlugin,
            "EDPluginParallelXNCFJobLauncher.__successPluginExecution")

    def __failurePluginExecution(self, _edPlugin=None):
        """
        Method called when the execution of the plugin failed 
        """
        EDVerbose.DEBUG(
            "EDPluginParallelXNCFJobLauncher.__failurePluginExecution")
        self.__edControlPlugin.retrieveFailureMessages(
            _edPlugin,
            "EDPluginParallelXNCFJobLauncher.__failurePluginExecutionGnom")

    def connectSUCCESS(self, _oMethod):
        self.__xsEDActionCluster.connectSUCCESS(_oMethod)

    def connectFAILURE(self, _oMethod):
        self.__xsEDActionCluster.connectFAILURE(_oMethod)