def __init__(self, _edControlPlugin, _strPluginName, _dictXSDataInput, _iNbThreads): """ Initialaze data structures @param _edControlPlugin: Parent control plugin @param _strPluginName: Name of the execution plugin to run @param _dictXSDataInput: Dictionary of the input objects for execution plugin """ EDActionCluster.__init__(self, _iNbThreads) self.__strPluginName = _strPluginName self.__dictXSDataInput = _dictXSDataInput self.__edControlPlugin = _edControlPlugin self.__xsEDPluginExecJobs = {}
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.process") edActionCluster = EDActionCluster() edActionCluster.addAction(self.edPluginIndexingLabelit) edActionCluster.addAction(self.edPluginControlIndicators) self.edPluginIndexingLabelit.connectSUCCESS(self.doSuccessLabelitIndexing) self.edPluginIndexingLabelit.connectFAILURE(self.doFailureLabelitIndexing) self.edPluginControlIndicators.connectSUCCESS(self.doSuccessControlIndicators) self.edPluginControlIndicators.connectFAILURE(self.doFailureControlIndicators) edActionCluster.execute() edActionCluster.synchronize()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlIndexingIndicatorsv10.process") edActionCluster = EDActionCluster() edActionCluster.addAction(self.__edPluginMOSFLMIndexing) edActionCluster.addAction(self.__edPluginControlIndicators) self.__edPluginMOSFLMIndexing.connectSUCCESS(self.doSuccessMOSFLMIndexing) self.__edPluginMOSFLMIndexing.connectFAILURE(self.doFailureMOSFLMIndexing) self.__edPluginControlIndicators.connectSUCCESS(self.doSuccessControlIndicators) self.__edPluginControlIndicators.connectFAILURE(self.doFailureControlIndicators) edActionCluster.execute() edActionCluster.synchronize()
def addPluginToActionCluster(self, _edPlugin): """ This method adds a plugin instance to an action cluster. """ if self.__edActionCluster == None: self.__edActionCluster = EDActionCluster() self.__edActionCluster.addAction(_edPlugin) self.__listOfLoadedPlugins.append(_edPlugin)
def testExecuteActionClusterWithFailure(self): """ Test the EDActionCluster with actions ending in success """ edActionCluster = EDActionCluster() for i in range(10): edAction = EDActionSuccess() edActionCluster.addAction(edAction) edActionFailure = EDActionFailure() edActionCluster.addAction(edActionFailure) edActionCluster.executeSynchronous() EDAssert.equal(True, edActionCluster.isFailure(), "EDActionCluster ended in failure")
def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputMeasureOffset) self.__strControlledPluginThumbnail = "EDPluginExecThumbnailv10" self.__strControlledPluginSift = "EDPluginExecSiftDescriptorv1_0" self.__strControlledPluginAutopano = "EDPluginExecSiftOffsetv1_0" self.semThumbnail = threading.Semaphore() self.semSift = threading.Semaphore() self.ACThumbnail = EDActionCluster() self.ACSift = EDActionCluster() self.xsdImages = [] self.xsdThumb = [] self.xsdKeys = [] self.xsdIdx = [] self.tCrop = [0, 0] self.inputImages = [] self.tOffset = None self.xsdPTO = None
def preProcess(self, _edObject=None): EDActionCluster.preProcess(self) EDVerbose.DEBUG("EDParallelJobLauncher.preProcess") for __dictKey in self.__dictXSDataInput.keys(): self.__xsEDPluginExecJobs[__dictKey] = self.__edControlPlugin.loadPlugin(self.__strPluginName) if self.__xsEDPluginExecJobs[__dictKey] is not None: if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not ""): self.__xsEDPluginExecJobs[__dictKey].setDataInput(self.__dictXSDataInput[__dictKey]) self.addAction(self.__xsEDPluginExecJobs[__dictKey]) else: EDVerbose.screen( "ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey].getWorkingDirectory() ) else: EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName) self.setTimeOut(self.__edControlPlugin.getTimeOut())
def testExecuteActionClusterWithSuccess(self): """ Test the EDActionCluster with actions ending in success """ edActionCluster = EDActionCluster() for i in range(10): edAction = EDActionSuccess() edActionCluster.addAction(edAction) edActionCluster.executeSynchronous() EDAssert.equal(False, edActionCluster.isFailure(), "EDActionCluster ended in success")
def createDataCollectionFromDataSets(self, _pyListXSDataCCP4iDataSet): """ This method takes as input a list of ccp4i data sets. Each data set can contain several paths to image files. It runs the EDPluginControlSubWedgeAssemble plugin to read the experimental information from the image headers and then creates a single XSDataCollection object. """ EDVerbose.DEBUG( "EDPluginControlCCP4iv10.createDataCollectionFromDataSets") xsDataCollection = None # We might have to run the plugin several times edActionCluster = EDActionCluster() listPluginControlSubWedgeAssemble = [] # Prepare the action cluster iIndex = 1 for xsDataCCP4iDataSet in _pyListXSDataCCP4iDataSet: edPluginControlSubWedgeAssemble = self.loadPlugin( self.__strPluginControlSubWedgeAssembleName, "SubWedgeAssemble-%02d" % iIndex) edPluginControlSubWedgeAssemble.connectSUCCESS( self.doSuccessActionSubWedgeAssemble) edPluginControlSubWedgeAssemble.connectFAILURE( self.doFailureActionSubWedgeAssemble) # Prepare the input for the sub wedge assemble plugin xsDataInputSubWedgeAssemble = XSDataInputSubWedgeAssemble() for xsDataImageFile in xsDataCCP4iDataSet.getImageFile(): xsDataInputSubWedgeAssemble.addFile(xsDataImageFile) edPluginControlSubWedgeAssemble.setDataInput( xsDataInputSubWedgeAssemble) listPluginControlSubWedgeAssemble.append( edPluginControlSubWedgeAssemble) self.addPluginToActionCluster(edPluginControlSubWedgeAssemble) iIndex += 1 # Run the action cluster synchronously self.executeActionCluster() self.synchronizeActionCluster() # Recuperate the output for edPluginControlSubWedgeAssemble in listPluginControlSubWedgeAssemble: xsDataResultSubWedgeAssemble = edPluginControlSubWedgeAssemble.getDataOutput( ) for xsDataSubWedge in xsDataResultSubWedgeAssemble.getSubWedge(): # Instantiate the xsDataCollection object if it's not already done. if (xsDataCollection is None): xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(xsDataSubWedge) return xsDataCollection
class EDPluginControl(EDPlugin): """ An EDPluginControl is a plugin that is responsible for a EDPluginExec or EDPluginControl plugin execution: It is responsible for: - The EDPluginExec or EDPluginControl Workflow - The data propagation between the EDPluginExec - The translation between generic and specific data models via EDHandler classes - The error/warning propagation - The executive summaries propagation - Execution of an "action cluster": a set of plugins can be added to a so called "action cluster" with the method "addPluginToActionCluster". All the plugins in the cluster can then be executed simultaneously with the method "executeActionCluster" and synchronized with the method "synchronizeActionCluster". The number of threads used by the action cluster is by default the number of processors available on the computer, but this value can be changed either by calling the method "setClusterSize" or by using the configuration parameter "clusterSize". """ def __init__(self): """ """ EDPlugin.__init__(self) self.__strPluginToBeControlledName = None self.__dictControlledPlugins = {} self.__edActionCluster = None self.__iClusterSize = None self.__listOfLoadedPlugins = [] def configure(self): """ Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes. """ EDPlugin.configure(self) EDVerbose.DEBUG("EDPluginControl.configure") strControlledPlugins = self.config.get("controlledPlugins", None) if (strControlledPlugins != None): pyListControlledPlugins = strControlledPlugins.split(",") for strControlledPlugin in pyListControlledPlugins: strControlledPluginName = self.getStringConfigurationParameterValue( strControlledPlugin) if strControlledPluginName != None: self.setControlledPluginName(strControlledPlugin, strControlledPluginName) EDVerbose.DEBUG( "EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName)) clusterSize = self.config.get("clusterSize", None) if (clusterSize != None): self.__iClusterSize = int(strClusterSize) EDVerbose.DEBUG( "EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize) def emptyListOfLoadedPlugin(self): """ Reset all plugins kept in memory """ self.__listOfLoadedPlugins = [] gc.collect() def getListOfLoadedPlugin(self): """ """ return self.__listOfLoadedPlugins def removeLoadedPlugin(self, _plugin): """ Remove a plugin from the list of loaded plugins to free some memory @param _plugin: plugin to remove @type _plugin: instance of the class EDPlugin """ if _plugin in self.__listOfLoadedPlugins: with self.locked(): self.__listOfLoadedPlugins.remove(_plugin) self.DEBUG( "EDPluginControl.removeLoadedPlugin: Caught, removed %s unreferenced objects. currently there are %i plugins" % (gc.get_count(), len(self.__listOfLoadedPlugins))) gc.collect() else: self.DEBUG( "EDPluginControl.removeLoadedPlugin: Missed. currently there are %i plugins" % len(self.__listOfLoadedPlugins)) def synchronizePlugins(self): EDVerbose.DEBUG("EDPluginControl.synchronizePlugins") bSynchronized = False while not bSynchronized: listPluginOrig = self.__listOfLoadedPlugins[:] for edPlugin in listPluginOrig: if edPlugin.isStarted() and (not edPlugin.isEnded()): edPlugin.synchronize() elif not edPlugin.isStarted(): time.sleep(0.01) #release GIL to let plugin start continue time.sleep(0.01) with self.locked(): bSynchronized = (self.__listOfLoadedPlugins == listPluginOrig) def loadPlugins(self): """ This method loads and returns a list of references to the plugins to be controlled. The name of the plugin to be controlled is set set before calling this method using the "setControlledPluginName" method. The base name of the plugin to be controlled is used as the working directory name of the plugin in question. The name of the plugin is used as base name. """ EDVerbose.DEBUG("EDPluginControl.loadPlugins") listKeys = self.__dictControlledPlugins.keys() listLoadedPlugins = [] for strKey in listKeys: strPluginName = self.__dictControlledPlugins[strKey] edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName) edPlugin.setBaseDirectory(self.getWorkingDirectory()) edPlugin.setBaseName(strPluginName) listLoadedPlugins.append(edPlugin) return listLoadedPlugins def loadPlugin(self, _strPluginToBeControlledName=None, _strBaseName=None): """ This method loads and returns a reference to the plugin to be controlled. The name of the plugin to be controlled can either be passed as an argument, or bet set before calling this method using the "setPluginToBeControlledName". The base name of the plugin to be controlled is used as the working directory name of the plugin in question. If no argument is supplied the name of the plugin is used as base name. In the case of creation of several plugins to be launched simultaneously, the base name should be different for each plugin and hence must be provided explicitly. """ EDVerbose.DEBUG("EDPluginControl.loadPlugin") if (_strPluginToBeControlledName is None): strPluginName = self.__strPluginToBeControlledName else: strPluginName = _strPluginToBeControlledName edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName) if (edPlugin is None): strErrorMessage = "EDPluginControl.loadPlugin : Cannot load plugin %s" % strPluginName EDVerbose.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError, strErrorMessage else: self.__listOfLoadedPlugins.append(edPlugin) edPlugin.setBaseDirectory(self.getWorkingDirectory()) if (_strBaseName is None): # Check if base name exists. OBS! Not thread safe so please set explicitly # _strBaseName for multi-threaded code strRenamedPlugin = self.compactPluginName(strPluginName) strNewWorkingDirectory = os.path.join(self.getWorkingDirectory(), strRenamedPlugin) if (os.path.exists(strNewWorkingDirectory)): edPlugin.setBaseName(edPlugin.createBaseName()) else: edPlugin.setBaseName(strRenamedPlugin) else: edPlugin.setBaseName(_strBaseName) return edPlugin def setControlledPluginName(self, _strControlledPluginName, _strControlledPluginValue): """ Adds a name-value pair to the dictionary to map the general to the specific name of a plugin to be controlled """ self.__dictControlledPlugins[ _strControlledPluginName] = _strControlledPluginValue def getControlledPluginName(self, _strControlledPluginName): """ Returns the name of the plugin to be controlled. """ strPluginname = None if self.__dictControlledPlugins.has_key(_strControlledPluginName): strPluginname = self.__dictControlledPlugins[ _strControlledPluginName] return strPluginname def addWarningMessages(self, _listWarningMessages): """ Adds a list of warning messages in the existing list of warning messages """ EDVerbose.DEBUG("EDPluginControl.addWarningMessages") for strWarningMessage in _listWarningMessages: self.addWarningMessage(strWarningMessage) def addErrorMessages(self, _listErrorMessages): """ Adds a list of error messages in the existing list of error messages """ EDVerbose.DEBUG("EDPluginControl.addErrorMessages") for strErrorMessage in _listErrorMessages: self.addErrorMessage(strErrorMessage) def retrieveFailureMessages(self, _edPlugin, _strMethodCaller): """ Propagates failure messages from a plugin including unexpected errors Should be called in the plugin control method invoked when a plugin exec fails (doActionFailure<>) """ EDVerbose.DEBUG("EDPluginControl.retrieveFailureMessages") self.retrieveWarningMessages(_edPlugin) self.retrieveErrorMessages(_edPlugin, _strMethodCaller, True) def retrieveSuccessMessages(self, _edPlugin, _strMethodCaller): """ Propagates success messages from a plugin Error messages are retrieved because a plugin could end successfully with errors (depending on the use case) In this case, there is no check for unexpected errors """ EDVerbose.DEBUG("EDPluginControl.retrieveSuccessMessages") self.retrieveWarningMessages(_edPlugin) self.retrieveErrorMessages(_edPlugin, _strMethodCaller, False) def retrieveErrorMessages(self, _edPlugin, _strMethodCaller, _bFailure): """ Propagates error messages from a plugin if _bFailure is true, this method has been called from retrieveFailureMessages in this case, checks for potential unexpected errors coming from the EDPluginExec """ EDVerbose.DEBUG("EDPluginControl.retrieveErrorMessages") listErrorMessages = _edPlugin.getListOfErrorMessages() if (len(listErrorMessages) == 0) and (_bFailure is True): strErrorMessage = "%s : Adding Unexpected error" % _strMethodCaller EDVerbose.DEBUG(strErrorMessage) listErrorMessages.append(strErrorMessage) self.addErrorMessages(listErrorMessages) def retrieveWarningMessages(self, _edPlugin): """ Propagates warning messages from a plugin """ EDVerbose.DEBUG("EDPluginControl.retrieveWarningMessages") self.addWarningMessages(_edPlugin.getListOfWarningMessages()) def appendExecutiveSummary(self, _edPlugin, _strPrefix="", _bAddSeparator=True): """ Appends the executive summary from a plugin. """ EDVerbose.DEBUG("EDPluginControl.appendExecutiveSummary") if (_bAddSeparator): self.addExecutiveSummarySeparator() if _edPlugin: for strLine in _edPlugin.getListExecutiveSummaryLines(): if strLine == self.getExecutiveSummarySeparator( ) and _strPrefix != "": strLine = strLine[:-len(_strPrefix)] self.addExecutiveSummaryLine(_strPrefix + strLine) def addErrorWarningMessagesToExecutiveSummary( self, _strErrorMessage="Error messages:", _strWarningMessage="Warning messages:"): """ Adds error and warning messages (if any) in the executive summary """ if len(self.getListOfErrorMessages()) != 0: self.addExecutiveSummarySeparator() self.addExecutiveSummaryLine(_strErrorMessage) for strErrorMessage in self.getListOfErrorMessages(): self.addExecutiveSummaryLine(strErrorMessage) self.addExecutiveSummarySeparator() if len(self.getListOfWarningMessages()) != 0: self.addExecutiveSummarySeparator() self.addExecutiveSummaryLine(_strWarningMessage) for warningMessage in self.getListOfWarningMessages(): self.addExecutiveSummaryLine(warningMessage) self.addExecutiveSummarySeparator() def addPluginToActionCluster(self, _edPlugin): """ This method adds a plugin instance to an action cluster. """ if self.__edActionCluster == None: self.__edActionCluster = EDActionCluster() self.__edActionCluster.addAction(_edPlugin) self.__listOfLoadedPlugins.append(_edPlugin) def executeActionCluster(self): """ This method executes the action cluster. The action cluster is executed asynchronoulsy. """ if self.__iClusterSize != None: self.__edActionCluster.setClusterSize(self.__iClusterSize) self.__edActionCluster.execute() def synchronizeActionCluster(self): """ This method synchronises the action cluster with the control plugin thread. """ self.__edActionCluster.synchronize() def setClusterSize(self, _iClusterSize): """ This method sets the size of the action cluster, i.e. the number of threads that will be executed simultaneously. """ self.__iClusterSize = _iClusterSize def executePlugin(self, _edPlugin, _bSynchronous=False): """ This method is used to start executable plugins in pipeline asynchronously. """ if _bSynchronous: self.executePluginSynchronous(_edPlugin) else: _edPlugin.execute() def executePluginSynchronous(self, _edPlugin): """ This method is used to start executable plugins in pipeline synchronously. """ _edControlSlotSUCCESS = EDSlot() _edControlSlotFAILURE = EDSlot() map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod()) map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod()) _edPlugin.getSlotSUCCESS().emptyListMethod() _edPlugin.getSlotFAILURE().emptyListMethod() _edPlugin.executeSynchronous() if (not _edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS") # Check that something doesn't go wrong in the success method! try: _edControlSlotSUCCESS.call(_edPlugin) except Exception: EDVerbose.DEBUG( "EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!" ) EDVerbose.writeErrorTrace() _edPlugin.setFailure() if (_edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE") # Check that something doesn't go wrong in the success method! try: _edControlSlotFAILURE.call(_edPlugin) except Exception: EDVerbose.DEBUG( "EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!" ) EDVerbose.writeErrorTrace()
class EDPluginControlSaxsModelingv1_0(EDPluginControl): """ Basically this is a re-implementation of EDPluginControlSolutionScattering starting after Gnom and withou web page generation """ classlock = Semaphore() configured = False cluster_size = 2 # duplicate from ControlPlugin dammif_jobs = 16 # number of dammif job to run unit = "NANOMETER" # unit of the GNOM file symmetry = "P1" # mode = "fast" # # constants: plugin names strPluginExecDammif = "EDPluginExecDammifv0_2" strPluginExecSupcomb = "EDPluginExecSupcombv0_3" strPluginExecDamaver = "EDPluginExecDamaverv0_3" strPluginExecDamfilt = "EDPluginExecDamfiltv0_3" strPluginExecDamstart = "EDPluginExecDamstartv0_3" strPluginExecDammin = "EDPluginExecDamminv0_2" Rg_min = 0.5 # nm def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputSaxsModeling) self.edPlugin = None self.edPlugin = None self.xsGnomFile = None self.result = XSDataResultSaxsModeling() self.result.dammifModels = [] self.graph_format = "png" self.dammif_plugins = [] self.dammif = None self.supcomb_plugins = {} self.actclust_supcomb = None self.valid = None # index of valid damif models self.mask2d = None self.arrayNSD = None self.ref = None # reference frame number (starting ar 0) def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginControlSaxsModelingv1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.gnomFile, "gnom output is missing") def configure(self): EDPluginControl.configure(self) if not self.configured: with self.classlock: if not self.configured: EDPluginControl.configure(self) dammif_jobs = self.config.get("dammifJobs", None) if (dammif_jobs != None): self.__class__.dammif_jobs = int(dammif_jobs) self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting number of dammif jobs to %d" % self.dammif_jobs) unit = self.config.get("unit", None) if (unit != None): self.__class__.unit = unit.upper() self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting input units to %s" % self.unit) symmetry = self.config.get("symmetry", None) if (symmetry != None): self.__class__.symmetry = symmetry self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting symmetry to %s" % self.symmetry) mode = self.config.get("mode", None) if (mode != None): self.__class__.mode = mode self.DEBUG("EDPluginControlSaxsModelingv1_0.configure: setting dammif mode to %s" % self.mode) clusterSize = self.config.get("clusterSize", None) if (clusterSize != None): self.__class__.cluster_size = int(clusterSize) self.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.cluster_size) self.__class__.configured = True def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.preProcess") self.xsGnomFile = self.dataInput.gnomFile if self.dataInput.graphFormat: self.graph_format = self.dataInput.graphFormat.value self.checkRg() def checkRg(self): """ If there is nothing in the sample, Rg = 0.1 nm damaver is likely to produce log files of many GB """ last_line = open(self.xsGnomFile.path.value).readlines()[-1] # self.WARNING("last Gnom file line is %s" % last_line) key = "Rg =" start = last_line.find(key) + len(key) val = last_line[start:].split()[0] try: rg = float(val) except ValueError: rg = 0.0 if rg < self.Rg_min: str_err = "Radius of Giration is too small (%s<%s). Stop processing !!!!" % (rg, self.Rg_min) self.ERROR(str_err) self.setFailure() raise RuntimeError(str_err) def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.process") xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile, unit=XSDataString(self.unit), symmetry=XSDataString(self.symmetry), mode=XSDataString(self.mode)) for i in range(self.dammif_jobs): dammif = self.loadPlugin(self.strPluginExecDammif) dammif.connectSUCCESS(self.doSuccessExecDammif) dammif.connectFAILURE(self.doFailureExecDammif) xsd = xsDataInputDammif.copyViaDict() xsd.order = XSDataInteger(i + 1) dammif.dataInput = xsd self.addPluginToActionCluster(dammif) self.dammif_plugins.append(dammif) self.executeActionCluster() self.synchronizeActionCluster() for plugin in self.dammif_plugins: if plugin.isFailure(): self.ERROR("dammif plugin %s-%08i failed" % (plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return # retrieve results from best dammif self.dammif = self.bestDammif() self.chi2plot("chi2_R.png") self.result.chiRfactorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "chi2_R.png"))) # temporary results: use best dammif self.result.fitFile = self.dammif.dataOutput.fitFile self.result.logFile = self.dammif.dataOutput.logFile self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile # prepare an action cluster with all supcomb plugins self.actclust_supcomb = EDActionCluster(self.cluster_size) for idx in range(self.dammif_jobs): if self.valid[idx]: for ser in range(idx): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[idx].dataOutput.pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name) self.supcomb_plugins[(idx, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for key, plugin in self.supcomb_plugins.items(): if plugin.isFailure(): self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return self.makeNSDarray("nsd.png") self.result.nsdPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png"))) idx = self.ref self.actclust_supcomb = EDActionCluster(self.cluster_size) for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name) self.supcomb_plugins[(self.ref, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: plugin = self.supcomb_plugins[(self.ref, ser)] if plugin.isFailure(): self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (self.ref + 1, ser + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return for i in range(self.dammif_jobs): if i == self.ref or not self.valid[i]: model = self.dammif_plugins[i].dataOutput.model else: model = self.supcomb_plugins[(self.ref, i)].dataOutput.model model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt # model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1)) self.result.dammifModels[i] = model # Now that all (valid) models are aligned we can combine them using damaver pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile] for idx in range(self.dammif_jobs): if self.valid[idx] and idx != self.ref: pdbFiles.append(self.supcomb_plugins[(self.ref, idx)].dataOutput.outputFilename) damaver = self.loadPlugin(self.strPluginExecDamaver) damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles, automatic=XSDataBoolean(False)) damaver.connectSUCCESS(self.doSuccessExecDamaver) damaver.connectFAILURE(self.doFailureExecDamaver) damaver.executeSynchronous() if self.isFailure(): return damfilt = self.loadPlugin(self.strPluginExecDamfilt) damfilt.dataInput = XSDataInputDamfilt(inputPdbFile=damaver.dataOutput.damaverPdbFile) damfilt.connectSUCCESS(self.doSuccessExecDamfilt) damfilt.connectFAILURE(self.doFailureExecDamfilt) damfilt.execute() ######################################################################## # TODO: This is a dead end : do it in parallel ######################################################################## if self.isFailure(): return damstart = self.loadPlugin(self.strPluginExecDamstart) damstart.dataInput = XSDataInputDamstart(inputPdbFile=damaver.dataOutput.damaverPdbFile) damstart.connectSUCCESS(self.doSuccessExecDamstart) damstart.connectFAILURE(self.doFailureExecDamstart) damstart.executeSynchronous() if self.isFailure(): return ######################################################################## # Finally call dammin ######################################################################## if self.config.get("do_dammin") in ["False", "0", False, 0]: return dammin = self.loadPlugin(self.strPluginExecDammin) dammin.dataInput = XSDataInputDammin(pdbInputFile=damstart.dataOutput.outputPdbFile, gnomOutputFile=self.xsGnomFile, symmetry=XSDataString(self.symmetry), mode=XSDataString(self.mode)) dammin.connectSUCCESS(self.doSuccessExecDammin) dammin.connectFAILURE(self.doFailureExecDammin) dammin.executeSynchronous() # Dammin takes as lot of time ... wait here for completion def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.postProcess") self.synchronizePlugins() # Create some output data # self.result. def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self, _edObject=_edObject) self.result.status = XSDataStatus(message=self.getXSDataMessage(), executiveSummary=XSDataString(os.linesep.join(self.getListExecutiveSummaryLines()))) self.setDataOutput(self.result) # clean up memory self.dammif_plugins = [] self.dammif = None self.emptyListOfLoadedPlugin() self.supcomb_plugins = {} self.actclust_supcomb = None gc.collect() def doSuccessExecDammif(self, _edPlugin=None): """ Locked as dammif is called many times in parallel """ with self.locked(): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammif") self.retrieveMessages(_edPlugin) self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammif") try: self.result.dammifModels.append(_edPlugin.dataOutput.model) # this has to be done only for the best model (once determined) ! # self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile # self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile # self.result.fitFile = _edPlugin.dataOutput.fitFile # self.result.logFile = _edPlugin.dataOutput.logFile except Exception as error: self.ERROR("Error in doSuccessExecDammif: %s" % error) def doFailureExecDammif(self, _edPlugin=None): """ Locked as dammif is called many times in parallel """ with self.locked(): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammif") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammif") self.setFailure() def doSuccessExecDamaver(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver") self.retrieveMessages(_edPlugin) try: self.result.damaverModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDamaver: %s" % error) def doFailureExecDamaver(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamaver") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamaver") self.setFailure() def doSuccessExecDamfilt(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt") self.retrieveMessages(_edPlugin) try: self.result.damfiltModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDamfilt: %s" % error) def doFailureExecDamfilt(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt") self.setFailure() def doSuccessExecDamstart(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart") self.retrieveMessages(_edPlugin) try: self.result.damstartModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDamstart: %s" % error) def doFailureExecDamstart(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamstart") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamstart") self.setFailure() def doSuccessExecDammin(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammin") self.retrieveMessages(_edPlugin) self.retrieveSuccessMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin") try: self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile self.result.fitFile = _edPlugin.dataOutput.fitFile self.result.firFile = _edPlugin.dataOutput.model.firFile self.result.logFile = _edPlugin.dataOutput.logFile self.result.damminModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDammin: %s" % error) def doFailureExecDammin(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammin") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages(_edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin") #self.setFailure() def bestDammif(self): """ Find DAMMIF run with best chi-square value """ fitResultDict = dict([(plg.dataOutput.chiSqrt.value, plg) for plg in self.dammif_plugins if (plg.dataOutput is not None) and (plg.dataOutput.chiSqrt is not None)]) fitResultList = fitResultDict.keys() fitResultList.sort() return fitResultDict[fitResultList[0]] def symlink(self, filen, link): """ Create a symlink to CWD with relative path """ src = os.path.abspath(filen) cwd = self.getWorkingDirectory() dest = os.path.join(cwd, link) os.symlink(os.path.relpath(src, cwd), dest) def chi2plot(self, filename=None, close=True): chi2 = numpy.array([ plg.dataOutput.chiSqrt.value for plg in self.dammif_plugins]) chi2max = chi2.mean() + 2 * chi2.std() xticks = 1 + numpy.arange(self.dammif_jobs) fig = plt.figure(figsize=figureSize) ax1 = fig.add_subplot(1, 2, 1) ax1.bar(xticks - 0.5, chi2) ax1.set_ylabel(u"$\sqrt{\u03C7}$") ax1.set_xlabel(u"Model number") ax1.plot([0.5, self.dammif_jobs + 0.5], [chi2max, chi2max], "-r", label=u"$\sqrt{\u03C7}$$_{max}$ = %.3f" % chi2max) ax1.set_xticks(xticks) ax1.legend(loc=8) R = numpy.array([ plg.dataOutput.rfactor.value for plg in self.dammif_plugins]) Rmax = R.mean() + 2 * R.std() ax2 = fig.add_subplot(1, 2, 2) ax2.bar(xticks - 0.5, R) ax2.plot([0.5, self.dammif_jobs + 0.5], [Rmax, Rmax], "-r", label=u"R$_{max}$ = %.3f" % Rmax) ax2.set_ylabel(u"R factor") ax2.set_xlabel(u"Model number") ax2.set_xticks(xticks) ax2.legend(loc=8) # fig.set_title("Selection of dammif models based on \u03C7$^2$") self.valid = (chi2 < chi2max) * (R < Rmax) self.mask2d = (1 - numpy.identity(self.dammif_jobs)) * numpy.outer(self.valid, self.valid) # print self.valid bbox_props = dict(fc="pink", ec="r", lw=1) for i in range(self.dammif_jobs): if not self.valid[i]: ax1.text(i + 0.95, chi2max / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) ax2.text(i + 0.95, Rmax / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) if filename: filename = os.path.join(self.getWorkingDirectory(), filename) self.log("Wrote %s" % filename) fig.savefig(filename) if close: fig.clf() plt.close(fig) else: return fig def makeNSDarray(self, filename=None, close=True): self.arrayNSD = numpy.zeros(self.mask2d.shape, numpy.float32) fig = plt.figure(figsize=figureSize) #ax1 = fig.add_subplot(1, 2, 1) ax1 = plt.subplot2grid((1,7), (0,0), colspan=4) ax2 = plt.subplot2grid((1,7), (0,4), colspan=3) # for now just an empty figure but a placeholder ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper") xticks = 1 + numpy.arange(self.dammif_jobs) lnsd = [] for key, plugin in self.supcomb_plugins.items(): i0, i1 = key nsd = plugin.dataOutput.NSD.value self.arrayNSD[i0, i1] = nsd self.arrayNSD[i1, i0] = nsd lnsd.append(nsd) #ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs) #ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs) ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size= 4, color = 'white', weight = 'demibold') ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=4, color = 'white' , weight = 'demibold') lnsd = numpy.array(lnsd) # print lnsd # print lnsd.mean() , lnsd.std(), lnsd.mean() + 2 * lnsd.std() nsd_max = lnsd.mean() + lnsd.std() data = self.arrayNSD.sum(axis=-1) / self.mask2d.sum(axis=-1) best_val = data[data > 0].min() # print data # print best_val # print numpy.where(data == best_val) self.ref = int(numpy.where(data == best_val)[0][-1]) # print self.ref ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper") ax1.set_title(u"NSD correlation table") ax1.set_xticks(range(self.dammif_jobs)) ax1.set_xticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)]) ax1.set_xlim(-0.5, self.dammif_jobs - 0.5) ax1.set_ylim(-0.5, self.dammif_jobs - 0.5) ax1.set_yticks(range(self.dammif_jobs)) ax1.set_yticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)]) ax1.set_xlabel(u"Model number") ax1.set_ylabel(u"Model number") #ax2 = fig.add_subplot(1, 2, 2) ax2.bar(xticks - 0.5, data) ax2.plot([0.5, self.dammif_jobs + 0.5], [nsd_max, nsd_max], "-r", label=u"NSD$_{max}$ = %.2f" % nsd_max) ax2.set_title(u"NSD between any model \n and all others", multialignment='center') ax2.set_ylabel("Normalized Spatial Discrepancy") ax2.set_xlabel(u"Model number") ax2.set_xticks(xticks) ax2.tick_params(axis='y', labelsize=8) bbox_props = dict(fc="cyan", ec="b", lw=1) # ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=8, bbox=bbox_props) ax2.legend(loc=8, prop={'size':8}) self.valid *= (data < nsd_max) bbox_props = dict(fc="pink", ec="r", lw=1) plt.tight_layout() for i in range(self.dammif_jobs): if not self.valid[i]: # ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=8, bbox=bbox_props) # print self.valid # print self.ref if filename: filename = os.path.join(self.getWorkingDirectory(), filename) self.log("Wrote %s" % filename) fig.savefig(filename, dpi=1200) if close: fig.clf() plt.close(fig) else: return fig
class EDPluginParallelXNCFJobLauncher: """ Class for starting execution plugin jobs in parallel for every input data object """ def __init__(self, _edControlPlugin, _strPluginName, _dictXSDataInput, _iNbThreads): """ Initialaze data structures @param _edControlPlugin: Parent control plugin @param _strPluginName: Name of the execution plugin to run @param _dictXSDataInput: Dictionary of the input objects for execution plugin """ self.__strPluginName = _strPluginName self.__dictXSDataInput = _dictXSDataInput self.__edControlPlugin = _edControlPlugin self.__xsEDPluginExecJobs = {} self.__xsEDActionCluster = EDActionCluster(_iNbThreads) self.__bIsFirstExecute = True def getPluginJobs(self): """ Get dictionary of launched plugin jobs """ return self.__xsEDPluginExecJobs def run(self): """ Initialize and run all parallel jobs """ for __dictKey in self.__dictXSDataInput.keys(): self.__xsEDPluginExecJobs[__dictKey] = self.__edControlPlugin.loadPlugin(self.__strPluginName) if (self.__xsEDPluginExecJobs[__dictKey] is not None): if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not "") : self.__xsEDPluginExecJobs[__dictKey].setDataInput(self.__dictXSDataInput[__dictKey]) self.__xsEDPluginExecJobs[__dictKey].connectSUCCESS(self.__successPluginExecution) self.__xsEDPluginExecJobs[__dictKey].connectFAILURE(self.__failurePluginExecution) self.__xsEDActionCluster.addAction(self.__xsEDPluginExecJobs[__dictKey]) else: EDVerbose.screen("ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey].getWorkingDirectory()) else: EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName) self.__xsEDActionCluster.setTimeOut(self.__edControlPlugin.getTimeOut()) self.__xsEDActionCluster.executeSynchronous() def __successPluginExecution(self, _edPlugin=None): """ Method called when the execution of the plugin succeeds """ EDVerbose.DEBUG("EDPluginParallelXNCFJobLauncher.__successPluginExecution") self.__edControlPlugin.retrieveSuccessMessages(_edPlugin, "EDPluginParallelXNCFJobLauncher.__successPluginExecution") def __failurePluginExecution(self, _edPlugin=None): """ Method called when the execution of the plugin failed """ EDVerbose.DEBUG("EDPluginParallelXNCFJobLauncher.__failurePluginExecution") self.__edControlPlugin.retrieveFailureMessages(_edPlugin, "EDPluginParallelXNCFJobLauncher.__failurePluginExecutionGnom") def connectSUCCESS(self, _oMethod): self.__xsEDActionCluster.connectSUCCESS(_oMethod) def connectFAILURE(self, _oMethod): self.__xsEDActionCluster.connectFAILURE(_oMethod)
class EDPluginControlSaxsModelingv1_0(EDPluginControl): """ Basically this is a re-implementation of EDPluginControlSolutionScattering starting after Gnom and withou web page generation """ classlock = Semaphore() configured = False cluster_size = 2 # duplicate from ControlPlugin dammif_jobs = 16 # number of dammif job to run unit = "NANOMETER" # unit of the GNOM file symmetry = "P1" # mode = "fast" # # constants: plugin names strPluginExecDammif = "EDPluginExecDammifv0_2" strPluginExecSupcomb = "EDPluginExecSupcombv0_3" strPluginExecDamaver = "EDPluginExecDamaverv0_3" strPluginExecDamfilt = "EDPluginExecDamfiltv0_3" strPluginExecDamstart = "EDPluginExecDamstartv0_3" strPluginExecDammin = "EDPluginExecDamminv0_2" Rg_min = 0.5 # nm def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputSaxsModeling) self.edPlugin = None self.edPlugin = None self.xsGnomFile = None self.result = XSDataResultSaxsModeling() self.result.dammifModels = [] self.graph_format = "png" self.dammif_plugins = [] self.dammif = None self.supcomb_plugins = {} self.actclust_supcomb = None self.valid = None # index of valid damif models self.mask2d = None self.arrayNSD = None self.ref = None # reference frame number (starting ar 0) def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginControlSaxsModelingv1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.gnomFile, "gnom output is missing") def configure(self): EDPluginControl.configure(self) if not self.configured: with self.classlock: if not self.configured: EDPluginControl.configure(self) dammif_jobs = self.config.get("dammifJobs", None) if (dammif_jobs != None): self.__class__.dammif_jobs = int(dammif_jobs) self.DEBUG( "EDPluginControlSaxsModelingv1_0.configure: setting number of dammif jobs to %d" % self.dammif_jobs) unit = self.config.get("unit", None) if (unit != None): self.__class__.unit = unit.upper() self.DEBUG( "EDPluginControlSaxsModelingv1_0.configure: setting input units to %s" % self.unit) symmetry = self.config.get("symmetry", None) if (symmetry != None): self.__class__.symmetry = symmetry self.DEBUG( "EDPluginControlSaxsModelingv1_0.configure: setting symmetry to %s" % self.symmetry) mode = self.config.get("mode", None) if (mode != None): self.__class__.mode = mode self.DEBUG( "EDPluginControlSaxsModelingv1_0.configure: setting dammif mode to %s" % self.mode) clusterSize = self.config.get("clusterSize", None) if (clusterSize != None): self.__class__.cluster_size = int(clusterSize) self.DEBUG( "EDPluginControl.configure: setting cluster size to %d" % self.cluster_size) self.__class__.configured = True def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.preProcess") self.xsGnomFile = self.dataInput.gnomFile if self.dataInput.graphFormat: self.graph_format = self.dataInput.graphFormat.value self.checkRg() def checkRg(self): """ If there is nothing in the sample, Rg = 0.1 nm damaver is likely to produce log files of many GB """ last_line = open(self.xsGnomFile.path.value).readlines()[-1] # self.WARNING("last Gnom file line is %s" % last_line) key = "Rg =" start = last_line.find(key) + len(key) val = last_line[start:].split()[0] try: rg = float(val) except ValueError: rg = 0.0 if rg < self.Rg_min: str_err = "Radius of Giration is too small (%s<%s). Stop processing !!!!" % ( rg, self.Rg_min) self.ERROR(str_err) self.setFailure() raise RuntimeError(str_err) def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.process") xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile, unit=XSDataString(self.unit), symmetry=XSDataString( self.symmetry), mode=XSDataString(self.mode)) for i in range(self.dammif_jobs): dammif = self.loadPlugin(self.strPluginExecDammif) dammif.connectSUCCESS(self.doSuccessExecDammif) dammif.connectFAILURE(self.doFailureExecDammif) xsd = xsDataInputDammif.copyViaDict() xsd.order = XSDataInteger(i + 1) dammif.dataInput = xsd self.addPluginToActionCluster(dammif) self.dammif_plugins.append(dammif) self.executeActionCluster() self.synchronizeActionCluster() for plugin in self.dammif_plugins: if plugin.isFailure(): self.ERROR("dammif plugin %s-%08i failed" % (plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return # retrieve results from best dammif self.dammif = self.bestDammif() self.chi2plot("chi2_R.png") self.result.chiRfactorPlot = XSDataFile( XSDataString(os.path.join(self.getWorkingDirectory(), "chi2_R.png"))) # temporary results: use best dammif self.result.fitFile = self.dammif.dataOutput.fitFile self.result.logFile = self.dammif.dataOutput.logFile self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile # prepare an action cluster with all supcomb plugins self.actclust_supcomb = EDActionCluster(self.cluster_size) for idx in range(self.dammif_jobs): if self.valid[idx]: for ser in range(idx): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb( templateFile=self.dammif_plugins[idx].dataOutput. pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser]. dataOutput.pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name ) self.supcomb_plugins[(idx, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for key, plugin in self.supcomb_plugins.items(): if plugin.isFailure(): self.ERROR( "supcomb plugin for model pair (%i,%i) %s-%08i failed" % (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return self.makeNSDarray("nsd.png") self.result.nsdPlot = XSDataFile( XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png"))) idx = self.ref self.actclust_supcomb = EDActionCluster(self.cluster_size) for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb( templateFile=self.dammif_plugins[ self.ref].dataOutput.pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser].dataOutput. pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name) self.supcomb_plugins[(self.ref, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: plugin = self.supcomb_plugins[(self.ref, ser)] if plugin.isFailure(): self.ERROR( "supcomb plugin for model pair (%i,%i) %s-%08i failed" % (self.ref + 1, ser + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return for i in range(self.dammif_jobs): if i == self.ref or not self.valid[i]: model = self.dammif_plugins[i].dataOutput.model else: model = self.supcomb_plugins[(self.ref, i)].dataOutput.model model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt # model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1)) self.result.dammifModels[i] = model # Now that all (valid) models are aligned we can combine them using damaver pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile] for idx in range(self.dammif_jobs): if self.valid[idx] and idx != self.ref: pdbFiles.append( self.supcomb_plugins[(self.ref, idx)].dataOutput.outputFilename) damaver = self.loadPlugin(self.strPluginExecDamaver) damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles, automatic=XSDataBoolean(False)) damaver.connectSUCCESS(self.doSuccessExecDamaver) damaver.connectFAILURE(self.doFailureExecDamaver) damaver.executeSynchronous() if self.isFailure(): return damfilt = self.loadPlugin(self.strPluginExecDamfilt) damfilt.dataInput = XSDataInputDamfilt( inputPdbFile=damaver.dataOutput.damaverPdbFile) damfilt.connectSUCCESS(self.doSuccessExecDamfilt) damfilt.connectFAILURE(self.doFailureExecDamfilt) damfilt.execute() ######################################################################## # TODO: This is a dead end : do it in parallel ######################################################################## if self.isFailure(): return damstart = self.loadPlugin(self.strPluginExecDamstart) damstart.dataInput = XSDataInputDamstart( inputPdbFile=damaver.dataOutput.damaverPdbFile) damstart.connectSUCCESS(self.doSuccessExecDamstart) damstart.connectFAILURE(self.doFailureExecDamstart) damstart.executeSynchronous() if self.isFailure(): return ######################################################################## # Finally call dammin ######################################################################## if self.config.get("do_dammin") in ["False", "0", False, 0]: return dammin = self.loadPlugin(self.strPluginExecDammin) dammin.dataInput = XSDataInputDammin( pdbInputFile=damstart.dataOutput.outputPdbFile, gnomOutputFile=self.xsGnomFile, symmetry=XSDataString(self.symmetry), mode=XSDataString(self.mode)) dammin.connectSUCCESS(self.doSuccessExecDammin) dammin.connectFAILURE(self.doFailureExecDammin) dammin.executeSynchronous() # Dammin takes as lot of time ... wait here for completion def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.postProcess") self.synchronizePlugins() # Create some output data # self.result. def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self, _edObject=_edObject) self.result.status = XSDataStatus( message=self.getXSDataMessage(), executiveSummary=XSDataString( os.linesep.join(self.getListExecutiveSummaryLines()))) self.setDataOutput(self.result) # clean up memory self.dammif_plugins = [] self.dammif = None self.emptyListOfLoadedPlugin() self.supcomb_plugins = {} self.actclust_supcomb = None gc.collect() def doSuccessExecDammif(self, _edPlugin=None): """ Locked as dammif is called many times in parallel """ with self.locked(): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammif") self.retrieveMessages(_edPlugin) self.retrieveSuccessMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammif") try: self.result.dammifModels.append(_edPlugin.dataOutput.model) # this has to be done only for the best model (once determined) ! # self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile # self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile # self.result.fitFile = _edPlugin.dataOutput.fitFile # self.result.logFile = _edPlugin.dataOutput.logFile except Exception as error: self.ERROR("Error in doSuccessExecDammif: %s" % error) def doFailureExecDammif(self, _edPlugin=None): """ Locked as dammif is called many times in parallel """ with self.locked(): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammif") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammif") self.setFailure() def doSuccessExecDamaver(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamaver") self.retrieveMessages(_edPlugin) try: self.result.damaverModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDamaver: %s" % error) def doFailureExecDamaver(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamaver") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamaver") self.setFailure() def doSuccessExecDamfilt(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamfilt") self.retrieveMessages(_edPlugin) try: self.result.damfiltModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDamfilt: %s" % error) def doFailureExecDamfilt(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamfilt") self.setFailure() def doSuccessExecDamstart(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doSuccessExecDamstart") self.retrieveMessages(_edPlugin) try: self.result.damstartModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDamstart: %s" % error) def doFailureExecDamstart(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDamstart") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDamstart") self.setFailure() def doSuccessExecDammin(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doSuccessExecDammin") self.retrieveMessages(_edPlugin) self.retrieveSuccessMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin") try: self.result.pdbMoleculeFile = _edPlugin.dataOutput.pdbMoleculeFile self.result.pdbSolventFile = _edPlugin.dataOutput.pdbSolventFile self.result.fitFile = _edPlugin.dataOutput.fitFile self.result.firFile = _edPlugin.dataOutput.model.firFile self.result.logFile = _edPlugin.dataOutput.logFile self.result.damminModel = _edPlugin.dataOutput.model self.symlink(_edPlugin.dataOutput.model.pdbFile.path.value, _edPlugin.dataOutput.model.name.value + ".pdb") except Exception as error: self.ERROR("Error in doSuccessExecDammin: %s" % error) def doFailureExecDammin(self, _edPlugin=None): self.DEBUG("EDPluginControlSaxsModelingv1_0.doFailureExecDammin") self.retrieveMessages(_edPlugin) self.retrieveFailureMessages( _edPlugin, "EDPluginControlSaxsModelingv1_0.doFailureExecDammin") #self.setFailure() def bestDammif(self): """ Find DAMMIF run with best chi-square value """ fitResultDict = dict([(plg.dataOutput.chiSqrt.value, plg) for plg in self.dammif_plugins if (plg.dataOutput is not None) and ( plg.dataOutput.chiSqrt is not None)]) fitResultList = fitResultDict.keys() fitResultList.sort() return fitResultDict[fitResultList[0]] def symlink(self, filen, link): """ Create a symlink to CWD with relative path """ src = os.path.abspath(filen) cwd = self.getWorkingDirectory() dest = os.path.join(cwd, link) os.symlink(os.path.relpath(src, cwd), dest) def chi2plot(self, filename=None, close=True): chi2 = numpy.array( [plg.dataOutput.chiSqrt.value for plg in self.dammif_plugins]) chi2max = chi2.mean() + 2 * chi2.std() xticks = 1 + numpy.arange(self.dammif_jobs) fig = plt.figure(figsize=figureSize) ax1 = fig.add_subplot(1, 2, 1) ax1.bar(xticks - 0.5, chi2) ax1.set_ylabel(u"$\sqrt{\u03C7}$") ax1.set_xlabel(u"Model number") ax1.plot([0.5, self.dammif_jobs + 0.5], [chi2max, chi2max], "-r", label=u"$\sqrt{\u03C7}$$_{max}$ = %.3f" % chi2max) ax1.set_xticks(xticks) ax1.legend(loc=8) R = numpy.array( [plg.dataOutput.rfactor.value for plg in self.dammif_plugins]) Rmax = R.mean() + 2 * R.std() ax2 = fig.add_subplot(1, 2, 2) ax2.bar(xticks - 0.5, R) ax2.plot([0.5, self.dammif_jobs + 0.5], [Rmax, Rmax], "-r", label=u"R$_{max}$ = %.3f" % Rmax) ax2.set_ylabel(u"R factor") ax2.set_xlabel(u"Model number") ax2.set_xticks(xticks) ax2.legend(loc=8) # fig.set_title("Selection of dammif models based on \u03C7$^2$") self.valid = (chi2 < chi2max) * (R < Rmax) self.mask2d = (1 - numpy.identity(self.dammif_jobs)) * numpy.outer( self.valid, self.valid) # print self.valid bbox_props = dict(fc="pink", ec="r", lw=1) for i in range(self.dammif_jobs): if not self.valid[i]: ax1.text(i + 0.95, chi2max / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) ax2.text(i + 0.95, Rmax / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) if filename: filename = os.path.join(self.getWorkingDirectory(), filename) self.log("Wrote %s" % filename) fig.savefig(filename) if close: fig.clf() plt.close(fig) else: return fig def makeNSDarray(self, filename=None, close=True): self.arrayNSD = numpy.zeros(self.mask2d.shape, numpy.float32) fig = plt.figure(figsize=figureSize) #ax1 = fig.add_subplot(1, 2, 1) ax1 = plt.subplot2grid((1, 7), (0, 0), colspan=4) ax2 = plt.subplot2grid((1, 7), (0, 4), colspan=3) # for now just an empty figure but a placeholder ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper") xticks = 1 + numpy.arange(self.dammif_jobs) lnsd = [] for key, plugin in self.supcomb_plugins.items(): i0, i1 = key nsd = plugin.dataOutput.NSD.value self.arrayNSD[i0, i1] = nsd self.arrayNSD[i1, i0] = nsd lnsd.append(nsd) #ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs) #ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=12 * 8 // self.dammif_jobs) ax1.text(i0, i1, "%.2f" % nsd, ha="center", va="center", size=4, color='white', weight='demibold') ax1.text(i1, i0, "%.2f" % nsd, ha="center", va="center", size=4, color='white', weight='demibold') lnsd = numpy.array(lnsd) # print lnsd # print lnsd.mean() , lnsd.std(), lnsd.mean() + 2 * lnsd.std() nsd_max = lnsd.mean() + lnsd.std() data = self.arrayNSD.sum(axis=-1) / self.mask2d.sum(axis=-1) best_val = data[data > 0].min() # print data # print best_val # print numpy.where(data == best_val) self.ref = int(numpy.where(data == best_val)[0][-1]) # print self.ref ax1.imshow(self.arrayNSD, interpolation="nearest", origin="upper") ax1.set_title(u"NSD correlation table") ax1.set_xticks(range(self.dammif_jobs)) ax1.set_xticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)]) ax1.set_xlim(-0.5, self.dammif_jobs - 0.5) ax1.set_ylim(-0.5, self.dammif_jobs - 0.5) ax1.set_yticks(range(self.dammif_jobs)) ax1.set_yticklabels([str(i) for i in range(1, 1 + self.dammif_jobs)]) ax1.set_xlabel(u"Model number") ax1.set_ylabel(u"Model number") #ax2 = fig.add_subplot(1, 2, 2) ax2.bar(xticks - 0.5, data) ax2.plot([0.5, self.dammif_jobs + 0.5], [nsd_max, nsd_max], "-r", label=u"NSD$_{max}$ = %.2f" % nsd_max) ax2.set_title(u"NSD between any model \n and all others", multialignment='center') ax2.set_ylabel("Normalized Spatial Discrepancy") ax2.set_xlabel(u"Model number") ax2.set_xticks(xticks) ax2.tick_params(axis='y', labelsize=8) bbox_props = dict(fc="cyan", ec="b", lw=1) # ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) ax2.text(self.ref + 0.95, data[self.ref] / 2, "Reference", ha="center", va="center", rotation=90, size=8, bbox=bbox_props) ax2.legend(loc=8, prop={'size': 8}) self.valid *= (data < nsd_max) bbox_props = dict(fc="pink", ec="r", lw=1) plt.tight_layout() for i in range(self.dammif_jobs): if not self.valid[i]: # ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=10, bbox=bbox_props) ax2.text(i + 0.95, data[self.ref] / 2, "Discarded", ha="center", va="center", rotation=90, size=8, bbox=bbox_props) # print self.valid # print self.ref if filename: filename = os.path.join(self.getWorkingDirectory(), filename) self.log("Wrote %s" % filename) fig.savefig(filename, dpi=1200) if close: fig.clf() plt.close(fig) else: return fig
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.process") xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile, unit=XSDataString(self.unit), symmetry=XSDataString( self.symmetry), mode=XSDataString(self.mode)) for i in range(self.dammif_jobs): dammif = self.loadPlugin(self.strPluginExecDammif) dammif.connectSUCCESS(self.doSuccessExecDammif) dammif.connectFAILURE(self.doFailureExecDammif) xsd = xsDataInputDammif.copyViaDict() xsd.order = XSDataInteger(i + 1) dammif.dataInput = xsd self.addPluginToActionCluster(dammif) self.dammif_plugins.append(dammif) self.executeActionCluster() self.synchronizeActionCluster() for plugin in self.dammif_plugins: if plugin.isFailure(): self.ERROR("dammif plugin %s-%08i failed" % (plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return # retrieve results from best dammif self.dammif = self.bestDammif() self.chi2plot("chi2_R.png") self.result.chiRfactorPlot = XSDataFile( XSDataString(os.path.join(self.getWorkingDirectory(), "chi2_R.png"))) # temporary results: use best dammif self.result.fitFile = self.dammif.dataOutput.fitFile self.result.logFile = self.dammif.dataOutput.logFile self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile # prepare an action cluster with all supcomb plugins self.actclust_supcomb = EDActionCluster(self.cluster_size) for idx in range(self.dammif_jobs): if self.valid[idx]: for ser in range(idx): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb( templateFile=self.dammif_plugins[idx].dataOutput. pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser]. dataOutput.pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name ) self.supcomb_plugins[(idx, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for key, plugin in self.supcomb_plugins.items(): if plugin.isFailure(): self.ERROR( "supcomb plugin for model pair (%i,%i) %s-%08i failed" % (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return self.makeNSDarray("nsd.png") self.result.nsdPlot = XSDataFile( XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png"))) idx = self.ref self.actclust_supcomb = EDActionCluster(self.cluster_size) for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb( templateFile=self.dammif_plugins[ self.ref].dataOutput.pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser].dataOutput. pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name) self.supcomb_plugins[(self.ref, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: plugin = self.supcomb_plugins[(self.ref, ser)] if plugin.isFailure(): self.ERROR( "supcomb plugin for model pair (%i,%i) %s-%08i failed" % (self.ref + 1, ser + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return for i in range(self.dammif_jobs): if i == self.ref or not self.valid[i]: model = self.dammif_plugins[i].dataOutput.model else: model = self.supcomb_plugins[(self.ref, i)].dataOutput.model model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt # model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1)) self.result.dammifModels[i] = model # Now that all (valid) models are aligned we can combine them using damaver pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile] for idx in range(self.dammif_jobs): if self.valid[idx] and idx != self.ref: pdbFiles.append( self.supcomb_plugins[(self.ref, idx)].dataOutput.outputFilename) damaver = self.loadPlugin(self.strPluginExecDamaver) damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles, automatic=XSDataBoolean(False)) damaver.connectSUCCESS(self.doSuccessExecDamaver) damaver.connectFAILURE(self.doFailureExecDamaver) damaver.executeSynchronous() if self.isFailure(): return damfilt = self.loadPlugin(self.strPluginExecDamfilt) damfilt.dataInput = XSDataInputDamfilt( inputPdbFile=damaver.dataOutput.damaverPdbFile) damfilt.connectSUCCESS(self.doSuccessExecDamfilt) damfilt.connectFAILURE(self.doFailureExecDamfilt) damfilt.execute() ######################################################################## # TODO: This is a dead end : do it in parallel ######################################################################## if self.isFailure(): return damstart = self.loadPlugin(self.strPluginExecDamstart) damstart.dataInput = XSDataInputDamstart( inputPdbFile=damaver.dataOutput.damaverPdbFile) damstart.connectSUCCESS(self.doSuccessExecDamstart) damstart.connectFAILURE(self.doFailureExecDamstart) damstart.executeSynchronous() if self.isFailure(): return ######################################################################## # Finally call dammin ######################################################################## if self.config.get("do_dammin") in ["False", "0", False, 0]: return dammin = self.loadPlugin(self.strPluginExecDammin) dammin.dataInput = XSDataInputDammin( pdbInputFile=damstart.dataOutput.outputPdbFile, gnomOutputFile=self.xsGnomFile, symmetry=XSDataString(self.symmetry), mode=XSDataString(self.mode)) dammin.connectSUCCESS(self.doSuccessExecDammin) dammin.connectFAILURE(self.doFailureExecDammin) dammin.executeSynchronous()
class EDPluginExecMeasureOffsetv2_0(EDPluginControl): """ An exec plugin that takes two images and measures the offset between the two. In facts it is not an ExecPlugin but a control plugin that: * Converts the pair of images in colored JPEG * Extract the SIFT descriptor of each image * Measure the offset between the two images using the Autopano tool * return the measured offset and the file describing the control points. """ def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputMeasureOffset) self.__strControlledPluginThumbnail = "EDPluginExecThumbnailv10" self.__strControlledPluginSift = "EDPluginExecSiftDescriptorv1_0" self.__strControlledPluginAutopano = "EDPluginExecSiftOffsetv1_0" self.semThumbnail = threading.Semaphore() self.semSift = threading.Semaphore() self.ACThumbnail = EDActionCluster() self.ACSift = EDActionCluster() self.xsdImages = [] self.xsdThumb = [] self.xsdKeys = [] self.xsdIdx = [] self.tCrop = [0, 0] self.inputImages = [] self.tOffset = None self.xsdPTO = None def checkParameters(self): """ Checks the mandatory parameters. """ EDVerbose.DEBUG("EDPluginControlMeasureOffsetv2_0.checkParameters") self.checkMandatoryParameters(self.getDataInput(), "Data Input is None") def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.preProcess") sdi = self.getDataInput() crop = sdi.getCropBorders() if len(crop) == 2: self.tCrop = (crop[0].getValue(), crop[1].getValue()) elif len(crop) == 1: self.tCrop = (crop[0].getValue(), crop[0].getValue()) # if len(sdi.getImage()) == 2: for i in sdi.getImage(): array = openimage(i.getPath().getValue()).data shape = array.shape if (self.tCrop != [0, 0]) and (shape[0] > self.tCrop[0]) and (shape[1] > self.tCrop[1]): array = array[self.tCrop[0]:-self.tCrop[0], self.tCrop[1]:-self.tCrop[1] ] EDVerbose.DEBUG("After Crop, images have shape : (%s,%s) " % (array.shape)) self.xsdImages.append(EDUtilsArray.arrayToXSData(array)) elif len(sdi.getArray()) == 2: if (self.tCrop == [0, 0]) : self.xsdImages = sdi.getArray() else: for xsdArray in sdi.getArray(): array = EDUtilsArray.xsDataToArray(xsdArray) shape = array.shape if (shape[0] > self.tCrop[0]) and (shape[1] > self.tCrop[1]): array = array[self.tCrop[0]:-self.tCrop[0], self.tCrop[1]:-self.tCrop[1] ] EDVerbose.DEBUG("After Crop, images have shape : (%s,%s) " % (array.shape)) self.xsdImages.append(EDUtilsArray.arrayToXSData(array)) else: strError = "EDPluginExecMeasureOffsetv2_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal() EDVerbose.ERROR(strError) self.setFailure() raise RuntimeError(strError) EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.xsdImages len=%i %s" % (len(self.xsdImages), self.xsdImages)) EDAssert.equal(self.xsdImages[0].getShape() , self.xsdImages[1].getShape(), "Images have the same size") self.xsdIdx = sdi.getIndex() if len(self.xsdIdx) < len(self.xsdImages): self.xsdIdx = [XSDataInteger(i) for i in range(len(self.xsdImages))] def process(self, _edObject=None): """ """ for i in range(2): execPlugin = self.loadPlugin(self.__strControlledPluginThumbnail) xsdin = XSDataInputExecThumbnail() xsdin.setInputArray(self.xsdImages[i]) xsdFile = XSDataFile() xsdFile.setPath(XSDataString(os.path.join(self.getWorkingDirectory(), "image%i.jpg" % self.xsdIdx[i].getValue()))) xsdin.setOutputPath(xsdFile) xsdin.setLevelsColorize(XSDataBoolean(1)) xsdin.setLevelsEqualize(XSDataBoolean(1)) execPlugin.setDataInput(xsdin) execPlugin.connectSUCCESS(self.doSuccessThumb) execPlugin.connectFAILURE(self.doFailureThumb) self.ACThumbnail.addAction(execPlugin) self.ACThumbnail.execute() while len(self.xsdThumb) < 2: time.sleep(1) for oneImage in self.xsdThumb: execPlugin = self.loadPlugin(self.__strControlledPluginSift) xsdin = XSDataInputSiftDescriptor() xsdin.setImage(oneImage) execPlugin.setDataInput(xsdin) execPlugin.connectSUCCESS(self.doSuccessSift) execPlugin.connectFAILURE(self.doFailureSift) self.ACSift.addAction(execPlugin) self.ACSift.execute() # # else: # strError = "There are only %s images in self.xsdThumb" % len(self.xsdThumb) # EDVerbose.ERROR(strError) # self.setFailure() # raise RuntimeError(strError) ################################################################################ # This should be executed only after the Sift actions cluster finishes ################################################################################ while len(self.xsdKeys) < 2: time.sleep(1) execPlugin = self.loadPlugin(self.__strControlledPluginAutopano) xsdin = XSDataInputMeasureOffsetSift() xsdin.setDescriptorFile(self.xsdKeys) execPlugin.setDataInput(xsdin) execPlugin.connectSUCCESS(self.doSuccessAutopano) execPlugin.connectFAILURE(self.doFailureAutopano) execPlugin.executeSynchronous() def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.postProcess") # Create some output data xsDataResult = XSDataResultMeasureOffset() xsDataResult.setOffset(self.tOffset) xsDataResult.setPanoFile(self.xsdPTO) self.setDataOutput(xsDataResult) self.xsdImages = [] def doSuccessThumb(self, _edPlugin=None): self.semThumbnail.acquire() EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessThumb") self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessThumb") self.xsdThumb.append(_edPlugin.getDataOutput().getThumbnailPath()) self.semThumbnail.release() def doFailureThumb(self, _edPlugin=None): self.semThumbnail.acquire() EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureThumb") self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureThumb") self.setFailure() strError = "Error in converting to Jpeg with this input: %s" % _edPlugin.getDataInput().marshal() EDVerbose.ERROR(strError) self.semThumbnail.release() raise RuntimeError(strError) def doSuccessSift(self, _edPlugin=None): self.semSift.acquire() EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessSift") self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessSift") self.xsdKeys.append(_edPlugin.getDataOutput().getDescriptorFile()) self.semSift.release() def doFailureSift(self, _edPlugin=None): self.semSift.acquire() EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureSift") self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureSift") self.setFailure() strError = "Error in extracting SIFT keys with this input: %s" % _edPlugin.getDataInput().marshal() EDVerbose.ERROR(strError) self.semSift.release() raise RuntimeError(strError) def doSuccessAutopano(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doSuccessAutopano") self.retrieveSuccessMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doSuccessSift") self.tOffset = _edPlugin.getDataOutput().getOffset() self.xsdPTO = _edPlugin.getDataOutput().getPanoFile() def doFailureAutopano(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginExecMeasureOffsetv2_0.doFailureAutopano") self.retrieveFailureMessages(_edPlugin, "EDPluginExecMeasureOffsetv2_0.doFailureAutopano") self.setFailure() strError = "Error in Autopano execution of with this input: %s" % _edPlugin.getDataInput().marshal() EDVerbose.ERROR(strError) raise RuntimeError(strError)
class EDPluginControl(EDPlugin): """ An EDPluginControl is a plugin that is responsible for a EDPluginExec or EDPluginControl plugin execution: It is responsible for: - The EDPluginExec or EDPluginControl Workflow - The data propagation between the EDPluginExec - The translation between generic and specific data models via EDHandler classes - The error/warning propagation - The executive summaries propagation - Execution of an "action cluster": a set of plugins can be added to a so called "action cluster" with the method "addPluginToActionCluster". All the plugins in the cluster can then be executed simultaneously with the method "executeActionCluster" and synchronized with the method "synchronizeActionCluster". The number of threads used by the action cluster is by default the number of processors available on the computer, but this value can be changed either by calling the method "setClusterSize" or by using the configuration parameter "clusterSize". """ def __init__ (self): """ """ EDPlugin.__init__(self) self.__strPluginToBeControlledName = None self.__dictControlledPlugins = {} self.__edActionCluster = None self.__iClusterSize = None self.__listOfLoadedPlugins = [] def configure(self): """ Gets the EDPluginControl parameters from the configuration file and stores them in class member attributes. """ EDPlugin.configure(self) EDVerbose.DEBUG("EDPluginControl.configure") strControlledPlugins = self.config.get("controlledPlugins", None) if (strControlledPlugins != None): pyListControlledPlugins = strControlledPlugins.split(",") for strControlledPlugin in pyListControlledPlugins: strControlledPluginName = self.getStringConfigurationParameterValue(strControlledPlugin) if strControlledPluginName != None: self.setControlledPluginName(strControlledPlugin, strControlledPluginName) EDVerbose.DEBUG("EDPluginControl.configure: setting controlled plugin %s to specific plugin %s" % (strControlledPlugin, strControlledPluginName)) clusterSize = self.config.get("clusterSize", None) if (clusterSize != None): self.__iClusterSize = int(strClusterSize) EDVerbose.DEBUG("EDPluginControl.configure: setting cluster size to %d" % self.__iClusterSize) def emptyListOfLoadedPlugin(self): """ Reset all plugins kept in memory """ self.__listOfLoadedPlugins = [] gc.collect() def getListOfLoadedPlugin(self): """ """ return self.__listOfLoadedPlugins def removeLoadedPlugin(self, _plugin): """ Remove a plugin from the list of loaded plugins to free some memory @param _plugin: plugin to remove @type _plugin: instance of the class EDPlugin """ if _plugin in self.__listOfLoadedPlugins: with self.locked(): self.__listOfLoadedPlugins.remove(_plugin) self.DEBUG("EDPluginControl.removeLoadedPlugin: Caught, removed %s unreferenced objects. currently there are %i plugins" % (gc.get_count(), len(self.__listOfLoadedPlugins))) gc.collect() else: self.DEBUG("EDPluginControl.removeLoadedPlugin: Missed. currently there are %i plugins" % len(self.__listOfLoadedPlugins)) def synchronizePlugins(self): EDVerbose.DEBUG("EDPluginControl.synchronizePlugins") bSynchronized = False while not bSynchronized: listPluginOrig = self.__listOfLoadedPlugins[:] for edPlugin in listPluginOrig: if edPlugin.isStarted() and (not edPlugin.isEnded()): edPlugin.synchronize() elif not edPlugin.isStarted(): time.sleep(0.01) #release GIL to let plugin start continue time.sleep(0.01) with self.locked(): bSynchronized = (self.__listOfLoadedPlugins == listPluginOrig) def loadPlugins(self): """ This method loads and returns a list of references to the plugins to be controlled. The name of the plugin to be controlled is set set before calling this method using the "setControlledPluginName" method. The base name of the plugin to be controlled is used as the working directory name of the plugin in question. The name of the plugin is used as base name. """ EDVerbose.DEBUG("EDPluginControl.loadPlugins") listKeys = self.__dictControlledPlugins.keys() listLoadedPlugins = [] for strKey in listKeys: strPluginName = self.__dictControlledPlugins[strKey] edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName) edPlugin.setBaseDirectory(self.getWorkingDirectory()) edPlugin.setBaseName(strPluginName) listLoadedPlugins.append(edPlugin) return listLoadedPlugins def loadPlugin(self, _strPluginToBeControlledName=None, _strBaseName=None): """ This method loads and returns a reference to the plugin to be controlled. The name of the plugin to be controlled can either be passed as an argument, or bet set before calling this method using the "setPluginToBeControlledName". The base name of the plugin to be controlled is used as the working directory name of the plugin in question. If no argument is supplied the name of the plugin is used as base name. In the case of creation of several plugins to be launched simultaneously, the base name should be different for each plugin and hence must be provided explicitly. """ EDVerbose.DEBUG("EDPluginControl.loadPlugin") if (_strPluginToBeControlledName is None): strPluginName = self.__strPluginToBeControlledName else: strPluginName = _strPluginToBeControlledName edPlugin = EDFactoryPluginStatic.loadPlugin(strPluginName) if (edPlugin is None): strErrorMessage = "EDPluginControl.loadPlugin : Cannot load plugin %s" % strPluginName EDVerbose.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError, strErrorMessage else: self.__listOfLoadedPlugins.append(edPlugin) edPlugin.setBaseDirectory(self.getWorkingDirectory()) if (_strBaseName is None): # Check if base name exists. OBS! Not thread safe so please set explicitly # _strBaseName for multi-threaded code strRenamedPlugin = self.compactPluginName(strPluginName) strNewWorkingDirectory = os.path.join(self.getWorkingDirectory(), strRenamedPlugin) if (os.path.exists(strNewWorkingDirectory)): edPlugin.setBaseName(edPlugin.createBaseName()) else: edPlugin.setBaseName(strRenamedPlugin) else: edPlugin.setBaseName(_strBaseName) return edPlugin def setControlledPluginName(self, _strControlledPluginName, _strControlledPluginValue): """ Adds a name-value pair to the dictionary to map the general to the specific name of a plugin to be controlled """ self.__dictControlledPlugins[_strControlledPluginName] = _strControlledPluginValue def getControlledPluginName(self, _strControlledPluginName): """ Returns the name of the plugin to be controlled. """ strPluginname = None if self.__dictControlledPlugins.has_key(_strControlledPluginName): strPluginname = self.__dictControlledPlugins[_strControlledPluginName] return strPluginname def addWarningMessages(self, _listWarningMessages): """ Adds a list of warning messages in the existing list of warning messages """ EDVerbose.DEBUG("EDPluginControl.addWarningMessages") for strWarningMessage in _listWarningMessages: self.addWarningMessage(strWarningMessage) def addErrorMessages(self, _listErrorMessages): """ Adds a list of error messages in the existing list of error messages """ EDVerbose.DEBUG("EDPluginControl.addErrorMessages") for strErrorMessage in _listErrorMessages: self.addErrorMessage(strErrorMessage) def retrieveFailureMessages(self, _edPlugin, _strMethodCaller): """ Propagates failure messages from a plugin including unexpected errors Should be called in the plugin control method invoked when a plugin exec fails (doActionFailure<>) """ EDVerbose.DEBUG("EDPluginControl.retrieveFailureMessages") self.retrieveWarningMessages(_edPlugin) self.retrieveErrorMessages(_edPlugin, _strMethodCaller, True) def retrieveSuccessMessages(self, _edPlugin, _strMethodCaller): """ Propagates success messages from a plugin Error messages are retrieved because a plugin could end successfully with errors (depending on the use case) In this case, there is no check for unexpected errors """ EDVerbose.DEBUG("EDPluginControl.retrieveSuccessMessages") self.retrieveWarningMessages(_edPlugin) self.retrieveErrorMessages(_edPlugin, _strMethodCaller, False) def retrieveErrorMessages(self, _edPlugin, _strMethodCaller, _bFailure): """ Propagates error messages from a plugin if _bFailure is true, this method has been called from retrieveFailureMessages in this case, checks for potential unexpected errors coming from the EDPluginExec """ EDVerbose.DEBUG("EDPluginControl.retrieveErrorMessages") listErrorMessages = _edPlugin.getListOfErrorMessages() if (len(listErrorMessages) == 0) and (_bFailure is True): strErrorMessage = "%s : Adding Unexpected error" % _strMethodCaller EDVerbose.DEBUG(strErrorMessage) listErrorMessages.append(strErrorMessage) self.addErrorMessages(listErrorMessages) def retrieveWarningMessages(self, _edPlugin): """ Propagates warning messages from a plugin """ EDVerbose.DEBUG("EDPluginControl.retrieveWarningMessages") self.addWarningMessages(_edPlugin.getListOfWarningMessages()) def appendExecutiveSummary(self, _edPlugin, _strPrefix="", _bAddSeparator=True): """ Appends the executive summary from a plugin. """ EDVerbose.DEBUG("EDPluginControl.appendExecutiveSummary") if (_bAddSeparator): self.addExecutiveSummarySeparator() if _edPlugin: for strLine in _edPlugin.getListExecutiveSummaryLines(): if strLine == self.getExecutiveSummarySeparator() and _strPrefix != "": strLine = strLine[ :-len(_strPrefix) ] self.addExecutiveSummaryLine(_strPrefix + strLine) def addErrorWarningMessagesToExecutiveSummary(self, _strErrorMessage="Error messages:", _strWarningMessage="Warning messages:"): """ Adds error and warning messages (if any) in the executive summary """ if len(self.getListOfErrorMessages()) != 0: self.addExecutiveSummarySeparator() self.addExecutiveSummaryLine(_strErrorMessage) for strErrorMessage in self.getListOfErrorMessages(): self.addExecutiveSummaryLine(strErrorMessage) self.addExecutiveSummarySeparator() if len(self.getListOfWarningMessages()) != 0: self.addExecutiveSummarySeparator() self.addExecutiveSummaryLine(_strWarningMessage) for warningMessage in self.getListOfWarningMessages(): self.addExecutiveSummaryLine(warningMessage) self.addExecutiveSummarySeparator() def addPluginToActionCluster(self, _edPlugin): """ This method adds a plugin instance to an action cluster. """ if self.__edActionCluster == None: self.__edActionCluster = EDActionCluster() self.__edActionCluster.addAction(_edPlugin) self.__listOfLoadedPlugins.append(_edPlugin) def executeActionCluster(self): """ This method executes the action cluster. The action cluster is executed asynchronoulsy. """ if self.__iClusterSize != None: self.__edActionCluster.setClusterSize(self.__iClusterSize) self.__edActionCluster.execute() def synchronizeActionCluster(self): """ This method synchronises the action cluster with the control plugin thread. """ self.__edActionCluster.synchronize() def setClusterSize(self, _iClusterSize): """ This method sets the size of the action cluster, i.e. the number of threads that will be executed simultaneously. """ self.__iClusterSize = _iClusterSize def executePlugin(self, _edPlugin, _bSynchronous=False): """ This method is used to start executable plugins in pipeline asynchronously. """ if _bSynchronous: self.executePluginSynchronous(_edPlugin) else: _edPlugin.execute() def executePluginSynchronous(self, _edPlugin): """ This method is used to start executable plugins in pipeline synchronously. """ _edControlSlotSUCCESS = EDSlot() _edControlSlotFAILURE = EDSlot() map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod()) map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod()) _edPlugin.getSlotSUCCESS().emptyListMethod() _edPlugin.getSlotFAILURE().emptyListMethod() _edPlugin.executeSynchronous() if (not _edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS") # Check that something doesn't go wrong in the success method! try: _edControlSlotSUCCESS.call(_edPlugin) except Exception: EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!") EDVerbose.writeErrorTrace() _edPlugin.setFailure() if (_edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE") # Check that something doesn't go wrong in the success method! try: _edControlSlotFAILURE.call(_edPlugin) except Exception: EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!") EDVerbose.writeErrorTrace()
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlSaxsModelingv1_0.process") xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile, unit=XSDataString(self.unit), symmetry=XSDataString(self.symmetry), mode=XSDataString(self.mode)) for i in range(self.dammif_jobs): dammif = self.loadPlugin(self.strPluginExecDammif) dammif.connectSUCCESS(self.doSuccessExecDammif) dammif.connectFAILURE(self.doFailureExecDammif) xsd = xsDataInputDammif.copyViaDict() xsd.order = XSDataInteger(i + 1) dammif.dataInput = xsd self.addPluginToActionCluster(dammif) self.dammif_plugins.append(dammif) self.executeActionCluster() self.synchronizeActionCluster() for plugin in self.dammif_plugins: if plugin.isFailure(): self.ERROR("dammif plugin %s-%08i failed" % (plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return # retrieve results from best dammif self.dammif = self.bestDammif() self.chi2plot("chi2_R.png") self.result.chiRfactorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "chi2_R.png"))) # temporary results: use best dammif self.result.fitFile = self.dammif.dataOutput.fitFile self.result.logFile = self.dammif.dataOutput.logFile self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile # prepare an action cluster with all supcomb plugins self.actclust_supcomb = EDActionCluster(self.cluster_size) for idx in range(self.dammif_jobs): if self.valid[idx]: for ser in range(idx): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[idx].dataOutput.pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name) self.supcomb_plugins[(idx, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for key, plugin in self.supcomb_plugins.items(): if plugin.isFailure(): self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return self.makeNSDarray("nsd.png") self.result.nsdPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png"))) idx = self.ref self.actclust_supcomb = EDActionCluster(self.cluster_size) for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: supcomb = self.loadPlugin(self.strPluginExecSupcomb) supcomb.dataInput = XSDataInputSupcomb(templateFile=self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile, superimposeFile=self.dammif_plugins[ser].dataOutput.pdbMoleculeFile, name=self.dammif_plugins[ser].dataOutput.model.name) self.supcomb_plugins[(self.ref, ser)] = supcomb self.actclust_supcomb.addAction(supcomb) self.actclust_supcomb.executeSynchronous() for ser in range(self.ref + 1, self.dammif_jobs): if self.valid[ser]: plugin = self.supcomb_plugins[(self.ref, ser)] if plugin.isFailure(): self.ERROR("supcomb plugin for model pair (%i,%i) %s-%08i failed" % (self.ref + 1, ser + 1, plugin.getName(), plugin.getId())) self.setFailure() self.retrieveMessages(plugin) if self.isFailure(): return for i in range(self.dammif_jobs): if i == self.ref or not self.valid[i]: model = self.dammif_plugins[i].dataOutput.model else: model = self.supcomb_plugins[(self.ref, i)].dataOutput.model model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt # model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1)) self.result.dammifModels[i] = model # Now that all (valid) models are aligned we can combine them using damaver pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile] for idx in range(self.dammif_jobs): if self.valid[idx] and idx != self.ref: pdbFiles.append(self.supcomb_plugins[(self.ref, idx)].dataOutput.outputFilename) damaver = self.loadPlugin(self.strPluginExecDamaver) damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles, automatic=XSDataBoolean(False)) damaver.connectSUCCESS(self.doSuccessExecDamaver) damaver.connectFAILURE(self.doFailureExecDamaver) damaver.executeSynchronous() if self.isFailure(): return damfilt = self.loadPlugin(self.strPluginExecDamfilt) damfilt.dataInput = XSDataInputDamfilt(inputPdbFile=damaver.dataOutput.damaverPdbFile) damfilt.connectSUCCESS(self.doSuccessExecDamfilt) damfilt.connectFAILURE(self.doFailureExecDamfilt) damfilt.execute() ######################################################################## # TODO: This is a dead end : do it in parallel ######################################################################## if self.isFailure(): return damstart = self.loadPlugin(self.strPluginExecDamstart) damstart.dataInput = XSDataInputDamstart(inputPdbFile=damaver.dataOutput.damaverPdbFile) damstart.connectSUCCESS(self.doSuccessExecDamstart) damstart.connectFAILURE(self.doFailureExecDamstart) damstart.executeSynchronous() if self.isFailure(): return ######################################################################## # Finally call dammin ######################################################################## if self.config.get("do_dammin") in ["False", "0", False, 0]: return dammin = self.loadPlugin(self.strPluginExecDammin) dammin.dataInput = XSDataInputDammin(pdbInputFile=damstart.dataOutput.outputPdbFile, gnomOutputFile=self.xsGnomFile, symmetry=XSDataString(self.symmetry), mode=XSDataString(self.mode)) dammin.connectSUCCESS(self.doSuccessExecDammin) dammin.connectFAILURE(self.doFailureExecDammin) dammin.executeSynchronous()
class EDPluginParallelXNCFJobLauncher: """ Class for starting execution plugin jobs in parallel for every input data object """ def __init__(self, _edControlPlugin, _strPluginName, _dictXSDataInput, _iNbThreads): """ Initialaze data structures @param _edControlPlugin: Parent control plugin @param _strPluginName: Name of the execution plugin to run @param _dictXSDataInput: Dictionary of the input objects for execution plugin """ self.__strPluginName = _strPluginName self.__dictXSDataInput = _dictXSDataInput self.__edControlPlugin = _edControlPlugin self.__xsEDPluginExecJobs = {} self.__xsEDActionCluster = EDActionCluster(_iNbThreads) self.__bIsFirstExecute = True def getPluginJobs(self): """ Get dictionary of launched plugin jobs """ return self.__xsEDPluginExecJobs def run(self): """ Initialize and run all parallel jobs """ for __dictKey in self.__dictXSDataInput.keys(): self.__xsEDPluginExecJobs[ __dictKey] = self.__edControlPlugin.loadPlugin( self.__strPluginName) if (self.__xsEDPluginExecJobs[__dictKey] is not None): if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not ""): self.__xsEDPluginExecJobs[__dictKey].setDataInput( self.__dictXSDataInput[__dictKey]) self.__xsEDPluginExecJobs[__dictKey].connectSUCCESS( self.__successPluginExecution) self.__xsEDPluginExecJobs[__dictKey].connectFAILURE( self.__failurePluginExecution) self.__xsEDActionCluster.addAction( self.__xsEDPluginExecJobs[__dictKey]) else: EDVerbose.screen("ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey]. getWorkingDirectory()) else: EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName) self.__xsEDActionCluster.setTimeOut( self.__edControlPlugin.getTimeOut()) self.__xsEDActionCluster.executeSynchronous() def __successPluginExecution(self, _edPlugin=None): """ Method called when the execution of the plugin succeeds """ EDVerbose.DEBUG( "EDPluginParallelXNCFJobLauncher.__successPluginExecution") self.__edControlPlugin.retrieveSuccessMessages( _edPlugin, "EDPluginParallelXNCFJobLauncher.__successPluginExecution") def __failurePluginExecution(self, _edPlugin=None): """ Method called when the execution of the plugin failed """ EDVerbose.DEBUG( "EDPluginParallelXNCFJobLauncher.__failurePluginExecution") self.__edControlPlugin.retrieveFailureMessages( _edPlugin, "EDPluginParallelXNCFJobLauncher.__failurePluginExecutionGnom") def connectSUCCESS(self, _oMethod): self.__xsEDActionCluster.connectSUCCESS(_oMethod) def connectFAILURE(self, _oMethod): self.__xsEDActionCluster.connectFAILURE(_oMethod)