class ProtPKPDExponentialFit(ProtPKPDFitBase): """ Fit a set of exponentials. The observed measurement is modelled as Y=sum_{i=1}^N c_i exp(-lambda_i * X).\n Confidence intervals calculated by this fitting may be pessimistic because it assumes that all model parameters are independent, which are not. Use Bootstrap estimates instead.\n Protocol created by http://www.kinestatpharma.com\n""" _label = 'fit exponentials' #--------------------------- DEFINE param functions -------------------------------------------- def _defineParams(self, form, fullForm=True): self._defineParams1(form,"t","Cp") if fullForm: form.addParam('fitType', params.EnumParam, choices=["Linear","Logarithmic","Relative"], label="Fit mode", default=1, help='Linear: sum (Cobserved-Cpredicted)^2\nLogarithmic: sum(log10(Cobserved)-log10(Cpredicted))^2\n'\ "Relative: sum ((Cobserved-Cpredicted)/Cobserved)^2") form.addParam('Nexp', params.IntParam, label="Number of exponentials", default=1, help='Number of exponentials to fit') else: self.fitType=Integer() self.fitType.set(1) self.Nexp=Integer() self.Nexp.set(1) form.addParam('bounds', params.StringParam, label="Amplitude and time constant bounds", default="", expertLevel=LEVEL_ADVANCED, help='Bounds for the c_i amplitudes and lambdas.\nExample 1: (0,10);(0,1e-2) -> c1 in (0,10), lambda1 in (0,1e-2)\n'\ 'Example 2: (0,10);(0,1e-2);(0,1);(0,1e-1) -> c1 in (0,10), lambda1 in (0,1e-2), c2 in (0,1), lambda2 in (0,1e-1)') form.addParam('confidenceInterval', params.FloatParam, label="Confidence interval=", default=95, expertLevel=LEVEL_ADVANCED, help='Confidence interval for the fitted parameters') if fullForm: form.addParam('reportX', params.StringParam, label="Evaluate at X=", default="", expertLevel=LEVEL_ADVANCED, help='Evaluate the model at these X values\nExample 1: [0,5,10,20,40,100]\nExample 2: 0:0.55:10, from 0 to 10 in steps of 0.5') else: self.reportX=String() self.reportX.set("") def getListOfFormDependencies(self): return [self.predictor.get(), self.predicted.get(), self.fitType.get(), self.bounds.get()] def createModel(self): return PKPDExponentialModel() def setupFromFormParameters(self): self.model.Nexp=self.Nexp.get() #--------------------------- INFO functions -------------------------------------------- def _warnings(self): warnings = [] experiment = self.readExperiment(self.getInputExperiment().fnPKPD,show=False) incorrectList = experiment.getNonBolusDoses() if len(incorrectList)>0: warnings.append("This protocol is meant only for intravenous bolus regimens. Check the doses for %s"%(','.join(incorrectList))) return warnings def _validate(self): errors=ProtPKPDFitBase._validate(self) if self.Nexp.get()<1: errors.append("The number of exponentials has to be larger than 0") return errors
class XmippProtProjMatch(ProtRefine3D, ProtClassify3D): """ 3D reconstruction and classification using multireference projection matching""" _label = 'projection matching' FILENAMENUMBERLENGTH = 6 def __init__(self, **args): ProtRefine3D.__init__(self, **args) ProtClassify3D.__init__(self, **args) self.numberOfCtfGroups = Integer(1) self._lastIter = Integer(0) def _initialize(self): """ This function is mean to be called after the working dir for the protocol have been set. (maybe after recovery from mapper) """ self._loadInputInfo() # Setup the dictionary with filenames templates to # be used by _getFileName createFilenameTemplates(self) # Load the values from several params generating a list # of values per iteration or references initializeLists(self) def _loadInputInfo(self): from pyworkflow.em.packages.xmipp3 import getImageLocation reference = self.input3DReferences.get() # Input can be either a single volume or a set of volumes. if isinstance(reference, Volume): # Treat the case of a single volume self.referenceFileNames = [getImageLocation(reference)] else: self.referenceFileNames = [getImageLocation(vol) for vol in reference] self.numberOfReferences = len(self.referenceFileNames) self.resolSam = reference.getSamplingRate() #--------------------------- DEFINE param functions -------------------------------------------- def _defineParams(self, form): """ Since the form definition is very very large, we have do it in a separated function. """ _defineProjectionMatchingParams(self, form) #--------------------------- INSERT steps functions -------------------------------------------- def _insertAllSteps(self): self._initialize() # Insert initial steps self._insertFunctionStep('convertInputStep') self._insertFunctionStep('executeCtfGroupsStep') # insertExecuteCtfGroupsStep(self) # insertInitAngularReferenceFileStep(self) self._insertFunctionStep('initAngularReferenceFileStep') # Steps per iteration self._insertItersSteps() # Final steps self._insertFunctionStep('createOutputStep') def _insertItersSteps(self): """ Insert several steps needed per iteration. """ for iterN in self.allIters(): dirsStep = self._insertFunctionStep('createIterDirsStep', iterN) # Insert some steps per reference volume projMatchSteps = [] for refN in self.allRefs(): # Mask the references in the iteration insertMaskReferenceStep(self, iterN, refN, prerequisites=[dirsStep]) # Create the library of projections insertAngularProjectLibraryStep(self, iterN, refN) # Projection matching steps projMatchStep = self._insertProjectionMatchingStep(iterN, refN) projMatchSteps.append(projMatchStep) # Select the reference that best fits each image self._insertFunctionStep('assignImagesToReferencesStep', iterN, prerequisites=projMatchSteps) insertAngularClassAverageStep(self, iterN, refN) # Reconstruct each reference with new averages for refN in self.allRefs(): # Create new class averages with images assigned insertReconstructionStep(self, iterN, refN) if self.doComputeResolution and self._doSplitReferenceImages[iterN]: # Reconstruct two halves of the data insertReconstructionStep(self, iterN, refN, 'Split1') insertReconstructionStep(self, iterN, refN, 'Split2') # Compute the resolution insertComputeResolutionStep(self, iterN, refN) insertFilterVolumeStep(self, iterN, refN) # Calculate both angles and shifts devitations for this iteration self._insertFunctionStep('calculateDeviationsStep', iterN) def _insertProjectionMatchingStep(self, iterN, refN): args = getProjectionMatchingArgs(self, iterN) return self._insertFunctionStep('projectionMatchingStep', iterN, refN, args) #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): """ Generated the input particles metadata expected by projection matching. And copy the generated file to be used as initial docfile for further iterations. """ from pyworkflow.em.packages.xmipp3 import writeSetOfParticles writeSetOfParticles(self.inputParticles.get(), self.selFileName, blockName=self.blockWithAllExpImages) #copyFile(self.selFileName, self._getFileName('inputParticlesDoc')) def createIterDirsStep(self, iterN): """ Create the necessary directory for a given iteration. """ iterDirs = [self._getFileName(k, iter=iterN) for k in ['iterDir', 'projMatchDirs', 'libraryDirs']] for d in iterDirs: makePath(d) return iterDirs def volumeConvertStep(self, reconstructedFilteredVolume, maskedFileName): runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName) def executeCtfGroupsStep(self, **kwargs): runExecuteCtfGroupsStep(self, **kwargs) def transformMaskStep(self, args, **kwargs): runTransformMaskStep(self, args, **kwargs) def angularProjectLibraryStep(self, iterN, refN, args, stepParams, **kwargs): runAngularProjectLibraryStep(self, iterN, refN, args, stepParams, **kwargs) def initAngularReferenceFileStep(self): runInitAngularReferenceFileStep(self) def projectionMatchingStep(self, iterN, refN, args): runProjectionMatching(self, iterN, refN, args) def assignImagesToReferencesStep(self, iterN): runAssignImagesToReferences(self, iterN) def cleanVolumeStep(self, vol1, vol2): cleanPath(vol1, vol2) def reconstructionStep(self, iterN, refN, program, method, args, suffix, mpi, threads, **kwargs): runReconstructionStep(self, iterN, refN, program, method, args, suffix, mpi, threads, **kwargs) def storeResolutionStep(self, resolIterMd, resolIterMaxMd, sampling): runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling) def calculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs): runCalculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs) def filterVolumeStep(self, iterN, refN, constantToAddToFiltration, **kwargs): runFilterVolumeStep(self, iterN, refN, constantToAddToFiltration, **kwargs) def createOutputStep(self): runCreateOutputStep(self) #--------------------------- INFO functions -------------------------------------------- def _validate(self): errors = [] if self.doCTFCorrection and not self.doAutoCTFGroup and not exists(self.setOfDefocus.get()): errors.append("Error: for non-automated ctf grouping, please provide a docfile!") if self.numberOfMpi<=1: errors.append("The number of MPI processes has to be larger than 1") xDimImg = self.inputParticles.get().getXDim() xDimVol, _, _ = self.input3DReferences.get().getDim() if xDimImg != xDimVol: errors.append("The dimensions of the volume(s) and particles must be equal!!!!") return errors def _citations(self): cites = [] return cites def _summary(self): summary = [] return summary def _methods(self): return self._summary() # summary is quite explicit and serve as methods #--------------------------- UTILS functions -------------------------------------------- def allIters(self): """ Iterate over all iterations. """ for i in range(1, self.numberOfIterations.get()+1): yield i def allRefs(self): """ Iterate over all references. """ for i in range(1, self.numberOfReferences+1): yield i def allCtfGroups(self): """ Iterate over all CTF groups. """ for i in range(1, self.numberOfCtfGroups.get() + 1): yield i def itersFloatValues(self, attributeName, firstValue=-1): """ Take the string of a given attribute and create a list of floats that will be used by the iteratioins. An special first value will be added to the list for iteration 0. """ valuesStr = self.getAttributeValue(attributeName) if valuesStr is None: raise Exception('None value for attribute: %s' % attributeName) return [firstValue] + getFloatListFromValues(valuesStr, length=self.numberOfIterations.get()) def itersBoolValues(self, attributeName, firstValue=False): """ Take the string of a given attribute and create a list of booleans that will be used by the iteratioins. An special first value will be added to the list for iteration 0. """ valuesStr = self.getAttributeValue(attributeName) if valuesStr is None: raise Exception('None value for attribute: %s' % attributeName) return [firstValue] + getBoolListFromValues(valuesStr, length=self.numberOfIterations.get()) def itersStringValues(self, attributeName, firstValue='c1'): """ Take the string of a given attribute and create a list of strings that will be used by the iteratioins. An special first value will be added to the list for iteration 0. """ valuesStr = self.getAttributeValue(attributeName) if valuesStr is None: raise Exception('None value for attribute: %s' % attributeName) return [firstValue] + getStringListFromValues(valuesStr, length=self.numberOfIterations.get()) def _getBlockFileName(self, blockName, blockNumber, filename, length=None): l = length or self.FILENAMENUMBERLENGTH return blockName + str(blockNumber).zfill(l) + '@' + filename def _getExpImagesFileName(self, filename): return self.blockWithAllExpImages + '@' + filename def _getRefBlockFileName(self, ctfBlName, ctfBlNumber, refBlName, refBlNumber, filename, length=None): l = length or self.FILENAMENUMBERLENGTH return ctfBlName + str(ctfBlNumber).zfill(l) + '_' + refBlName + str(refBlNumber).zfill(l) + '@' + filename def _getFourierMaxFrequencyOfInterest(self, iterN, refN): """ Read the corresponding resolution metadata and return the desired resolution. """ md = xmipp.MetaData(self._getFileName('resolutionXmdMax', iter=iterN, ref=refN)) return md.getValue(xmipp.MDL_RESOLUTION_FREQREAL, md.firstObject()) def calculateDeviationsStep(self, it): """ Calculate both angles and shifts devitations for all iterations """ SL = xmipp.SymList() mdIter = xmipp.MetaData() #for it in self.allIters(): mdIter.clear() SL.readSymmetryFile(self._symmetry[it]) md1 = xmipp.MetaData(self.docFileInputAngles[it]) md2 = xmipp.MetaData(self.docFileInputAngles[it-1]) #ignore disabled, md1.removeDisabled() md2.removeDisabled() #first metadata file may not have shiftx and shifty if not md2.containsLabel(xmipp.MDL_SHIFT_X): md2.addLabel(xmipp.MDL_SHIFT_X) md2.addLabel(xmipp.MDL_SHIFT_Y) md2.fillConstant(xmipp.MDL_SHIFT_X,0.) md2.fillConstant(xmipp.MDL_SHIFT_Y,0.) oldLabels=[xmipp.MDL_ANGLE_ROT, xmipp.MDL_ANGLE_TILT, xmipp.MDL_ANGLE_PSI, xmipp.MDL_SHIFT_X, xmipp.MDL_SHIFT_Y] newLabels=[xmipp.MDL_ANGLE_ROT2, xmipp.MDL_ANGLE_TILT2, xmipp.MDL_ANGLE_PSI2, xmipp.MDL_SHIFT_X2, xmipp.MDL_SHIFT_Y2] md2.renameColumn(oldLabels,newLabels) md2.addLabel(xmipp.MDL_SHIFT_X_DIFF) md2.addLabel(xmipp.MDL_SHIFT_Y_DIFF) md2.addLabel(xmipp.MDL_SHIFT_DIFF) mdIter.join1(md1, md2, xmipp.MDL_IMAGE, xmipp.INNER_JOIN) SL.computeDistance(mdIter,False,False,False) xmipp.activateMathExtensions() #operate in sqlite shiftXLabel = xmipp.label2Str(xmipp.MDL_SHIFT_X) shiftX2Label = xmipp.label2Str(xmipp.MDL_SHIFT_X2) shiftXDiff = xmipp.label2Str(xmipp.MDL_SHIFT_X_DIFF) shiftYLabel = xmipp.label2Str(xmipp.MDL_SHIFT_Y) shiftY2Label = xmipp.label2Str(xmipp.MDL_SHIFT_Y2) shiftYDiff = xmipp.label2Str(xmipp.MDL_SHIFT_Y_DIFF) shiftDiff = xmipp.label2Str(xmipp.MDL_SHIFT_DIFF) #timeStr = str(dtBegin) operateString = shiftXDiff+"="+shiftXLabel+"-"+shiftX2Label operateString += "," + shiftYDiff+"="+shiftYLabel+"-"+shiftY2Label mdIter.operate(operateString) operateString = shiftDiff+"=sqrt(" \ +shiftXDiff+"*"+shiftXDiff+"+" \ +shiftYDiff+"*"+shiftYDiff+");" mdIter.operate(operateString) iterFile = self._mdDevitationsFn(it) mdIter.write(iterFile,xmipp.MD_APPEND) self._setLastIter(it) def _mdDevitationsFn(self, it): mdFn = self._getPath('deviations.xmd') return "iter_%03d@" % it + mdFn def _setLastIter(self, iterN): self._lastIter.set(iterN) self._store(self._lastIter) def getLastIter(self): return self._lastIter.get() def _createItemMatrix(self, item, row): from pyworkflow.em.packages.xmipp3.convert import createItemMatrix import pyworkflow.em as em createItemMatrix(item, row, align=em.ALIGN_PROJ)
class QueueSystemConfig(Object): def __init__(self, **kwargs): super().__init__(**kwargs) self.name = String() # Number of cores from which the queue is mandatory # 0 means no mandatory at all # 1 will force to launch all jobs through the queue self.mandatory = Integer() self.queues = None # List for queue configurations self.submitCommand = String() # Allow to change the prefix of submission scripts # we used by default the ID.job, but in some clusters # the job script should start by a letter self.submitPrefix = String() self.checkCommand = String() self.cancelCommand = String() self.submitTemplate = String() self.jobDoneRegex = String() def hasName(self): return self.name.hasValue() def hasValue(self): return self.hasName() and len(self.queues) def getName(self): return self.name.get() def getMandatory(self): return self.mandatory.get() def getSubmitTemplate(self): return self.submitTemplate.get() def getSubmitCommand(self): return self.submitCommand.get() def getCheckCommand(self): return self.checkCommand.get() def getCancelCommand(self): return self.cancelCommand.get() def getQueues(self): return self.queues def setName(self, name): self.name.set(name) def setMandatory(self, mandatory): # This condition is to be backward compatible # when mandatory was a boolean # now it should use the number of CPU # that should force to use the queue if mandatory in ['False', 'false']: mandatory = 0 elif mandatory in ['True', 'true']: mandatory = 1 self.mandatory.set(mandatory) def setSubmitTemplate(self, submitTemplate): self.submitTemplate.set(submitTemplate) def setSubmitCommand(self, submitCommand): self.submitCommand.set(submitCommand) def setCheckCommand(self, checkCommand): self.checkCommand.set(checkCommand) def setCancelCommand(self, cancelCommand): self.cancelCommand.set(cancelCommand) def setJobDoneRegex(self, jobDoneRegex): self.jobDoneRegex.set(jobDoneRegex) def setQueues(self, queues): self.queues = queues def getQueueConfig(self, objId): if objId is not None and self.queues is not None: for queueConfig in self.queues: if objId == queueConfig.getObjId(): return queueConfig return None
class XmippProtProjMatch(ProtRefine3D, ProtClassify3D): """ 3D reconstruction and classification using multireference projection matching""" _label = 'projection matching' FILENAMENUMBERLENGTH = 6 def __init__(self, **args): ProtRefine3D.__init__(self, **args) ProtClassify3D.__init__(self, **args) self.numberOfCtfGroups = Integer(1) self._lastIter = Integer(0) def _initialize(self): """ This function is mean to be called after the working dir for the protocol have been set. (maybe after recovery from mapper) """ self._loadInputInfo() # Setup the dictionary with filenames templates to # be used by _getFileName createFilenameTemplates(self) # Load the values from several params generating a list # of values per iteration or references initializeLists(self) def _loadInputInfo(self): from pyworkflow.em.packages.xmipp3 import getImageLocation reference = self.input3DReferences.get( ) # Input can be either a single volume or a set of volumes. if isinstance(reference, Volume): # Treat the case of a single volume self.referenceFileNames = [getImageLocation(reference)] else: self.referenceFileNames = [ getImageLocation(vol) for vol in reference ] self.numberOfReferences = len(self.referenceFileNames) self.resolSam = reference.getSamplingRate() #--------------------------- DEFINE param functions -------------------------------------------- def _defineParams(self, form): """ Since the form definition is very very large, we have do it in a separated function. """ _defineProjectionMatchingParams(self, form) #--------------------------- INSERT steps functions -------------------------------------------- def _insertAllSteps(self): self._initialize() # Insert initial steps self._insertFunctionStep('convertInputStep') self._insertFunctionStep('executeCtfGroupsStep') # insertExecuteCtfGroupsStep(self) # insertInitAngularReferenceFileStep(self) self._insertFunctionStep('initAngularReferenceFileStep') # Steps per iteration self._insertItersSteps() # Final steps self._insertFunctionStep('createOutputStep') def _insertItersSteps(self): """ Insert several steps needed per iteration. """ for iterN in self.allIters(): dirsStep = self._insertFunctionStep('createIterDirsStep', iterN) # Insert some steps per reference volume projMatchSteps = [] for refN in self.allRefs(): # Mask the references in the iteration insertMaskReferenceStep(self, iterN, refN, prerequisites=[dirsStep]) # Create the library of projections insertAngularProjectLibraryStep(self, iterN, refN) # Projection matching steps projMatchStep = self._insertProjectionMatchingStep(iterN, refN) projMatchSteps.append(projMatchStep) # Select the reference that best fits each image self._insertFunctionStep('assignImagesToReferencesStep', iterN, prerequisites=projMatchSteps) insertAngularClassAverageStep(self, iterN, refN) # Reconstruct each reference with new averages for refN in self.allRefs(): # Create new class averages with images assigned insertReconstructionStep(self, iterN, refN) if self.doComputeResolution and self._doSplitReferenceImages[ iterN]: # Reconstruct two halves of the data insertReconstructionStep(self, iterN, refN, 'Split1') insertReconstructionStep(self, iterN, refN, 'Split2') # Compute the resolution insertComputeResolutionStep(self, iterN, refN) insertFilterVolumeStep(self, iterN, refN) # Calculate both angles and shifts devitations for this iteration self._insertFunctionStep('calculateDeviationsStep', iterN) def _insertProjectionMatchingStep(self, iterN, refN): args = getProjectionMatchingArgs(self, iterN) return self._insertFunctionStep('projectionMatchingStep', iterN, refN, args) #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): """ Generated the input particles metadata expected by projection matching. And copy the generated file to be used as initial docfile for further iterations. """ from pyworkflow.em.packages.xmipp3 import writeSetOfParticles writeSetOfParticles(self.inputParticles.get(), self.selFileName, blockName=self.blockWithAllExpImages) #copyFile(self.selFileName, self._getFileName('inputParticlesDoc')) def createIterDirsStep(self, iterN): """ Create the necessary directory for a given iteration. """ iterDirs = [ self._getFileName(k, iter=iterN) for k in ['iterDir', 'projMatchDirs', 'libraryDirs'] ] for d in iterDirs: makePath(d) return iterDirs def volumeConvertStep(self, reconstructedFilteredVolume, maskedFileName): runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName) def executeCtfGroupsStep(self, **kwargs): runExecuteCtfGroupsStep(self, **kwargs) def transformMaskStep(self, program, args, **kwargs): runTransformMaskStep(self, program, args, **kwargs) def angularProjectLibraryStep(self, iterN, refN, args, stepParams, **kwargs): runAngularProjectLibraryStep(self, iterN, refN, args, stepParams, **kwargs) def initAngularReferenceFileStep(self): runInitAngularReferenceFileStep(self) def projectionMatchingStep(self, iterN, refN, args): runProjectionMatching(self, iterN, refN, args) def assignImagesToReferencesStep(self, iterN): runAssignImagesToReferences(self, iterN) def cleanVolumeStep(self, vol1, vol2): cleanPath(vol1, vol2) def reconstructionStep(self, iterN, refN, program, method, args, suffix, **kwargs): runReconstructionStep(self, iterN, refN, program, method, args, suffix, **kwargs) def storeResolutionStep(self, resolIterMd, resolIterMaxMd, sampling): runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling) def calculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs): runCalculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs) def filterVolumeStep(self, iterN, refN, constantToAddToFiltration, **kwargs): runFilterVolumeStep(self, iterN, refN, constantToAddToFiltration, **kwargs) def createOutputStep(self): runCreateOutputStep(self) #--------------------------- INFO functions -------------------------------------------- def _validate(self): errors = [] if self.doCTFCorrection: if not self.doAutoCTFGroup and not exists(self.setOfDefocus.get()): errors.append("Error: for non-automated ctf grouping, " "please provide a docfile!") if not self.inputParticles.get().hasCTF(): errors.append("Error: for doing CTF correction the input " "particles should have CTF information.") if self.numberOfMpi <= 1: errors.append( "The number of MPI processes has to be larger than 1") self._validateDim(self.inputParticles.get(), self.input3DReferences.get(), errors, 'Input particles', 'Reference volume') return errors def _citations(self): cites = [] return cites def _summary(self): summary = [] return summary def _methods(self): return self._summary( ) # summary is quite explicit and serve as methods #--------------------------- UTILS functions -------------------------------------------- def allIters(self): """ Iterate over all iterations. """ for i in range(1, self.numberOfIterations.get() + 1): yield i def allRefs(self): """ Iterate over all references. """ for i in range(1, self.numberOfReferences + 1): yield i def allCtfGroups(self): """ Iterate over all CTF groups. """ for i in range(1, self.numberOfCtfGroups.get() + 1): yield i def itersFloatValues(self, attributeName, firstValue=-1): """ Take the string of a given attribute and create a list of floats that will be used by the iteratioins. An special first value will be added to the list for iteration 0. """ valuesStr = self.getAttributeValue(attributeName) if valuesStr is None: raise Exception('None value for attribute: %s' % attributeName) return [firstValue] + getFloatListFromValues( valuesStr, length=self.numberOfIterations.get()) def itersBoolValues(self, attributeName, firstValue=False): """ Take the string of a given attribute and create a list of booleans that will be used by the iteratioins. An special first value will be added to the list for iteration 0. """ valuesStr = self.getAttributeValue(attributeName) if valuesStr is None: raise Exception('None value for attribute: %s' % attributeName) return [firstValue] + getBoolListFromValues( valuesStr, length=self.numberOfIterations.get()) def itersStringValues(self, attributeName, firstValue='c1'): """ Take the string of a given attribute and create a list of strings that will be used by the iteratioins. An special first value will be added to the list for iteration 0. """ valuesStr = self.getAttributeValue(attributeName) if valuesStr is None: raise Exception('None value for attribute: %s' % attributeName) return [firstValue] + getStringListFromValues( valuesStr, length=self.numberOfIterations.get()) def _getBlockFileName(self, blockName, blockNumber, filename, length=None): l = length or self.FILENAMENUMBERLENGTH return blockName + str(blockNumber).zfill(l) + '@' + filename def _getExpImagesFileName(self, filename): return self.blockWithAllExpImages + '@' + filename def _getRefBlockFileName(self, ctfBlName, ctfBlNumber, refBlName, refBlNumber, filename, length=None): l = length or self.FILENAMENUMBERLENGTH return ctfBlName + str(ctfBlNumber).zfill(l) + '_' + refBlName + str( refBlNumber).zfill(l) + '@' + filename def _getFourierMaxFrequencyOfInterest(self, iterN, refN): """ Read the corresponding resolution metadata and return the desired resolution. """ md = xmipp.MetaData( self._getFileName('resolutionXmdMax', iter=iterN, ref=refN)) return md.getValue(xmipp.MDL_RESOLUTION_FREQREAL, md.firstObject()) def calculateDeviationsStep(self, it): """ Calculate both angles and shifts devitations for all iterations """ SL = xmipp.SymList() mdIter = xmipp.MetaData() #for it in self.allIters(): mdIter.clear() SL.readSymmetryFile(self._symmetry[it]) md1 = xmipp.MetaData(self.docFileInputAngles[it]) md2 = xmipp.MetaData(self.docFileInputAngles[it - 1]) #ignore disabled, md1.removeDisabled() md2.removeDisabled() #first metadata file may not have shiftx and shifty if not md2.containsLabel(xmipp.MDL_SHIFT_X): md2.addLabel(xmipp.MDL_SHIFT_X) md2.addLabel(xmipp.MDL_SHIFT_Y) md2.fillConstant(xmipp.MDL_SHIFT_X, 0.) md2.fillConstant(xmipp.MDL_SHIFT_Y, 0.) oldLabels = [ xmipp.MDL_ANGLE_ROT, xmipp.MDL_ANGLE_TILT, xmipp.MDL_ANGLE_PSI, xmipp.MDL_SHIFT_X, xmipp.MDL_SHIFT_Y ] newLabels = [ xmipp.MDL_ANGLE_ROT2, xmipp.MDL_ANGLE_TILT2, xmipp.MDL_ANGLE_PSI2, xmipp.MDL_SHIFT_X2, xmipp.MDL_SHIFT_Y2 ] md2.renameColumn(oldLabels, newLabels) md2.addLabel(xmipp.MDL_SHIFT_X_DIFF) md2.addLabel(xmipp.MDL_SHIFT_Y_DIFF) md2.addLabel(xmipp.MDL_SHIFT_DIFF) mdIter.join1(md1, md2, xmipp.MDL_IMAGE, xmipp.INNER_JOIN) SL.computeDistance(mdIter, False, False, False) xmipp.activateMathExtensions() #operate in sqlite shiftXLabel = xmipp.label2Str(xmipp.MDL_SHIFT_X) shiftX2Label = xmipp.label2Str(xmipp.MDL_SHIFT_X2) shiftXDiff = xmipp.label2Str(xmipp.MDL_SHIFT_X_DIFF) shiftYLabel = xmipp.label2Str(xmipp.MDL_SHIFT_Y) shiftY2Label = xmipp.label2Str(xmipp.MDL_SHIFT_Y2) shiftYDiff = xmipp.label2Str(xmipp.MDL_SHIFT_Y_DIFF) shiftDiff = xmipp.label2Str(xmipp.MDL_SHIFT_DIFF) #timeStr = str(dtBegin) operateString = shiftXDiff + "=" + shiftXLabel + "-" + shiftX2Label operateString += "," + shiftYDiff + "=" + shiftYLabel + "-" + shiftY2Label mdIter.operate(operateString) operateString = shiftDiff+"=sqrt(" \ +shiftXDiff+"*"+shiftXDiff+"+" \ +shiftYDiff+"*"+shiftYDiff+");" mdIter.operate(operateString) iterFile = self._mdDevitationsFn(it) mdIter.write(iterFile, xmipp.MD_APPEND) self._setLastIter(it) def _mdDevitationsFn(self, it): mdFn = self._getPath('deviations.xmd') return "iter_%03d@" % it + mdFn def _setLastIter(self, iterN): self._lastIter.set(iterN) self._store(self._lastIter) def getLastIter(self): return self._lastIter.get() def _fillParticlesFromIter(self, partSet, iteration): print("_fillParticlesFromIter") import pyworkflow.em.metadata as md imgSet = self.inputParticles.get() imgFn = "all_exp_images@" + self._getFileName( 'docfileInputAnglesIters', iter=iteration, ref=1) partSet.copyInfo(imgSet) partSet.setAlignmentProj() partSet.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows( imgFn, sortByLabel=md.MDL_ITEM_ID)) def _createItemMatrix(self, item, row): from pyworkflow.em.packages.xmipp3.convert import createItemMatrix import pyworkflow.em as em createItemMatrix(item, row, align=em.ALIGN_PROJ) def _getIterParticles(self, it, clean=False): import pyworkflow.em as em """ Return a classes .sqlite file for this iteration. If the file doesn't exists, it will be created by converting from this iteration data.star file. """ dataParticles = self._getFileName('particlesScipion', iter=it) if clean: cleanPath(dataParticles) if not exists(dataParticles): partSet = em.SetOfParticles(filename=dataParticles) self._fillParticlesFromIter(partSet, it) partSet.write() partSet.close() else: partSet = em.SetOfParticles(filename=dataParticles) imgSet = self.inputParticles.get() partSet.copyInfo(imgSet) partSet.setAlignmentProj() return partSet
class ProtAnnotateMembranes(EMProtocol): """ Manual annotation tool for segmented membranes """ _label = 'annotate segmented membranes' _devStatus = BETA def __init__(self, **kwargs): EMProtocol.__init__(self, **kwargs) self._objectsToGo = Integer() self._provider = None self._tomoList = None def _defineParams(self, form): form.addSection(label='Input') form.addParam( 'inputTomoMasks', PointerParam, label="Input Tomo Masks", important=True, pointerClass='SetOfTomoMasks', allowsNull=False, help= 'Select the Tomogram Masks (segmented tomograms) for the membrane annotation.' ) # --------------------------- INSERT steps functions ---------------------- def _insertAllSteps(self): self._initialize() self._insertFunctionStep(self.runMembraneAnnotator, interactive=True) # --------------------------- STEPS functions ----------------------------- def runMembraneAnnotator(self): # There are still some objects which haven't been annotated --> launch GUI self._getAnnotationStatus() if self._objectsToGo.get() > 0: MembAnnotatorDialog(None, self._getExtraPath(), provider=self._provider, prot=self) # All the objetcs have been annotated --> create output objects self._getAnnotationStatus() if self._objectsToGo.get() == 0: print("\n==> Generating the outputs") labelledSet = self._genOutputSetOfTomoMasks() self._defineOutputs(outputSetofTomoMasks=labelledSet) self._store() # --------------------------- INFO functions ----------------------------------- def _summary(self): summary = [] objects2go = self._objectsToGo.get() if objects2go is not None: if objects2go > 0: summary.append( '*%i* remaining segmentations to be annotated.' % objects2go) else: summary.append( 'All segmentations have been already annotated.') return summary # --------------------------- UTIL functions ----------------------------------- def _initialize(self): self._tomoList = [ tomo.clone() for tomo in self.inputTomoMasks.get().iterItems() ] self._provider = MembAnnotatorProvider(self._tomoList, self._getExtraPath(), 'membAnnotator') self._getAnnotationStatus() def _getAnnotationStatus(self): """Check if all the tomo masks have been annotated and store current status in a text file""" doneTomes = [ self._provider.getObjectInfo(tomo)['tags'] == 'done' for tomo in self._tomoList ] self._objectsToGo.set(len(self._tomoList) - sum(doneTomes)) def _getCurrentTomoMaskFile(self, inTomoFile): baseName = removeBaseExt(inTomoFile) return glob.glob(self._getExtraPath(baseName + '_materials.mrc'))[0] def _genOutputSetOfTomoMasks(self): tomoMaskSet = SetOfTomoMasks.create(self._getPath(), template='tomomasks%s.sqlite', suffix='annotated') inTomoSet = self.inputTomoMasks.get() tomoMaskSet.copyInfo(inTomoSet) counter = 1 for inTomo in inTomoSet.iterItems(): tomoMask = TomoMask() inTomoFile = inTomo.getVolName() tomoMask.copyInfo(inTomo) tomoMask.setLocation( (counter, self._getCurrentTomoMaskFile(inTomoFile))) tomoMask.setVolName(inTomoFile) tomoMaskSet.append(tomoMask) counter += 1 return tomoMaskSet