class XmippProtParticlePickingPairs(ProtParticlePicking, XmippProtocol): """ Picks particles in a set of untilted-tilted pairs of micrographs. """ _label = 'tilt pairs particle picking' def __init__(self, **args): ProtParticlePicking.__init__(self, **args) # The following attribute is only for testing self.importFolder = String(args.get('importFolder', None)) #--------------- DEFINE param functions --------------------------------- def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputMicrographsTiltedPair', params.PointerParam, pointerClass='MicrographsTiltPair', label="Micrographs tilt pair", help='Select the MicrographsTiltPair ') form.addParam('memory', params.FloatParam, default=2, label='Memory to use (In Gb)', expertLevel=2) #----------- INSERT steps functions ---------------------------------- def _insertAllSteps(self): """ The Particle Picking process is realized for a pair of set of micrographs """ self.micsFn = self._getPath('input_micrographs.xmd') # Convert input into xmipp Metadata format self._insertFunctionStep('convertInputStep') # Launch Particle Picking GUI if not self.importFolder.hasValue(): self._insertFunctionStep('launchParticlePickGUIStep', interactive=True) else: # This is only used for test purposes self._insertFunctionStep('_importFromFolderStep') #------------------- STEPS functions ----------------------------------- def convertInputStep(self): micTiltPairs = self.inputMicrographsTiltedPair.get() # Get the converted input micrographs in Xmipp format convert.writeSetOfMicrographsPairs(micTiltPairs.getUntilted(), micTiltPairs.getTilted(), self.micsFn) def launchParticlePickGUIStep(self): process = launchTiltPairPickerGUI(self.micsFn, self._getExtraPath(), self, memory='%dg' % self.memory.get()) process.wait() def _importFromFolderStep(self): """ This function will copy Xmipp .pos files for simulating a particle picking run...this is only for testing purposes. """ extraDir = self._getExtraPath() for f in pwutils.getFiles(self.importFolder.get()): pwutils.copyFile(f, extraDir) self.registerCoords(extraDir, readFromExtra=True) #--------------------------- INFO functions -------------------------------------------- def _citations(self): return [] #--------------------------- UTILS functions ------------------------------------------- def __str__(self): """ String representation of a Particle Picking Tilt run """ outputs = self.getOutputsSize() if outputs == 0: msg = "No particles picked yet." elif outputs == 1: picked = self.getCoords().getSize() mics = self.inputMicrographsTiltedPair.get().getTilted().getSize() msg = "Number of particles picked: %d " % picked msg += "(from %d micrographs)" % mics else: msg = 'Number of outputs: %d' % outputs return msg def getInputMicrographs(self): return self.inputMicrographsTiltedPair.get().getTilted() def getCoords(self): return self.getCoordsTiltPair() def _summary(self): summary = [] if self.getInputMicrographs() is not None: summary.append("Number of input micrographs: %d" % self.getInputMicrographs().getSize()) if self.getOutputsSize() >= 1: for key, output in self.iterOutputAttributes(CoordinatesTiltPair): summary.append("*%s:*" % key) summary.append(" Particles pairs picked: %d" % output.getSize()) summary.append(" Particle size: %d \n" % output.getBoxSize()) else: summary.append("Output tilpairs not ready yet.") return summary def __getOutputSuffix(self): maxCounter = -1 for attrName, _ in self.iterOutputAttributes(CoordinatesTiltPair): suffix = attrName.replace('outputCoordinatesTiltPair', '') try: counter = int(suffix) except: counter = 1 # when there is not number assume 1 maxCounter = max(counter, maxCounter) return str(maxCounter + 1) if maxCounter > 0 else '' # empty if not outputs def _getBoxSize(self): """ Redefine this function to set a specific box size to the output coordinates untilted and tilted. """ return None def _readCoordinates(self, coordsDir, suffix=''): micTiltPairs = self.inputMicrographsTiltedPair.get() uSuffix = 'Untilted' + suffix tSuffix = 'Tilted' + suffix uSet = micTiltPairs.getUntilted() tSet = micTiltPairs.getTilted() # Create Untilted and Tilted SetOfCoordinates uCoordSet = self._createSetOfCoordinates(uSet, suffix=uSuffix) convert.readSetOfCoordinates(coordsDir, uSet, uCoordSet) uCoordSet.write() tCoordSet = self._createSetOfCoordinates(tSet, suffix=tSuffix) convert.readSetOfCoordinates(coordsDir, tSet, tCoordSet) tCoordSet.write() boxSize = self._getBoxSize() if boxSize: uCoordSet.setBoxSize(boxSize) tCoordSet.setBoxSize(boxSize) return uCoordSet, tCoordSet def _readAngles(self, micsFn, suffix=''): # Read Angles from input micrographs anglesSet = self._createSetOfAngles(suffix=suffix) convert.readAnglesFromMicrographs(micsFn, anglesSet) anglesSet.write() return anglesSet def registerCoords(self, coordsDir, store=True, readFromExtra=False): micTiltPairs = self.inputMicrographsTiltedPair.get() suffix = self.__getOutputSuffix() uCoordSet, tCoordSet = self._readCoordinates(coordsDir, suffix) if readFromExtra: micsFn = self._getExtraPath('input_micrographs.xmd') else: micsFn = self._getPath('input_micrographs.xmd') anglesSet = self._readAngles(micsFn, suffix) # Create CoordinatesTiltPair object outputset = self._createCoordinatesTiltPair(micTiltPairs, uCoordSet, tCoordSet, anglesSet, suffix) summary = self.getSummary(outputset) outputset.setObjComment(summary) outputName = 'outputCoordinatesTiltPair' + suffix outputs = {outputName: outputset} self._defineOutputs(**outputs) self._defineSourceRelation(self.inputMicrographsTiltedPair, outputset) if store: self._store()
class XmippProtParticlePickingPairs(ProtParticlePicking, XmippProtocol): """ Picks particles in a set of untilted-tilted pairs of micrographs. """ _label = 'tilt pairs particle picking' def __init__(self, **args): ProtParticlePicking.__init__(self, **args) # The following attribute is only for testing self.importFolder = String(args.get('importFolder', None)) #--------------- DEFINE param functions --------------------------------- def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputMicrographsTiltedPair', params.PointerParam, pointerClass='MicrographsTiltPair', label="Micrographs tilt pair", help='Select the MicrographsTiltPair ') #----------- INSERT steps functions ---------------------------------- def _insertAllSteps(self): """ The Particle Picking process is realized for a pair of set of micrographs """ self.micsFn = self._getPath('input_micrographs.xmd') # Convert input into xmipp Metadata format self._insertFunctionStep('convertInputStep') # Launch Particle Picking GUI if not self.importFolder.hasValue(): self._insertFunctionStep('launchParticlePickGUIStep', interactive=True) else: # This is only used for test purposes self._insertFunctionStep('_importFromFolderStep') #------------------- STEPS functions ----------------------------------- def convertInputStep(self): micTiltPairs = self.inputMicrographsTiltedPair.get() # Get the converted input micrographs in Xmipp format convert.writeSetOfMicrographsPairs(micTiltPairs.getUntilted(), micTiltPairs.getTilted(), self.micsFn) def launchParticlePickGUIStep(self): process = launchTiltPairPickerGUI(self.micsFn, self._getExtraPath(), self) process.wait() def _importFromFolderStep(self): """ This function will copy Xmipp .pos files for simulating a particle picking run...this is only for testing purposes. """ extraDir = self._getExtraPath() for f in pwutils.getFiles(self.importFolder.get()): pwutils.copyFile(f, extraDir) self.registerCoords(extraDir, readFromExtra=True) #--------------------------- INFO functions -------------------------------------------- def _citations(self): return [] #--------------------------- UTILS functions ------------------------------------------- def __str__(self): """ String representation of a Particle Picking Tilt run """ outputs = self.getOutputsSize() if outputs == 0: msg = "No particles picked yet." elif outputs == 1: picked = self.getCoords().getSize() mics = self.inputMicrographsTiltedPair.get().getTilted().getSize() msg = "Number of particles picked: %d " % picked msg += "(from %d micrographs)" % mics else: msg = 'Number of outputs: %d' % outputs return msg def getInputMicrographs(self): return self.inputMicrographsTiltedPair.get().getTilted() def getCoords(self): return self.getCoordsTiltPair() def _summary(self): summary = [] if self.getInputMicrographs() is not None: summary.append("Number of input micrographs: %d" % self.getInputMicrographs().getSize()) if self.getOutputsSize() >= 1: for key, output in self.iterOutputAttributes(CoordinatesTiltPair): summary.append("*%s:*" % key) summary.append(" Particles pairs picked: %d" % output.getSize()) summary.append(" Particle size: %d \n" % output.getBoxSize()) else: summary.append("Output tilpairs not ready yet.") return summary def __getOutputSuffix(self): maxCounter = -1 for attrName, _ in self.iterOutputAttributes(CoordinatesTiltPair): suffix = attrName.replace('outputCoordinatesTiltPair', '') try: counter = int(suffix) except: counter = 1 # when there is not number assume 1 maxCounter = max(counter, maxCounter) return str(maxCounter+1) if maxCounter > 0 else '' # empty if not outputs def _getBoxSize(self): """ Redefine this function to set a specific box size to the output coordinates untilted and tilted. """ return None def _readCoordinates(self, coordsDir, suffix=''): micTiltPairs = self.inputMicrographsTiltedPair.get() uSuffix = 'Untilted' + suffix tSuffix = 'Tilted' + suffix uSet = micTiltPairs.getUntilted() tSet = micTiltPairs.getTilted() # Create Untilted and Tilted SetOfCoordinates uCoordSet = self._createSetOfCoordinates(uSet, suffix=uSuffix) convert.readSetOfCoordinates(coordsDir, uSet, uCoordSet) uCoordSet.write() tCoordSet = self._createSetOfCoordinates(tSet, suffix=tSuffix) convert.readSetOfCoordinates(coordsDir, tSet, tCoordSet) tCoordSet.write() boxSize = self._getBoxSize() if boxSize: uCoordSet.setBoxSize(boxSize) tCoordSet.setBoxSize(boxSize) return uCoordSet, tCoordSet def _readAngles(self, micsFn, suffix=''): # Read Angles from input micrographs anglesSet = self._createSetOfAngles(suffix=suffix) convert.readAnglesFromMicrographs(micsFn, anglesSet) anglesSet.write() return anglesSet def registerCoords(self, coordsDir, store=True, readFromExtra=False): micTiltPairs = self.inputMicrographsTiltedPair.get() suffix = self.__getOutputSuffix() uCoordSet, tCoordSet = self._readCoordinates(coordsDir, suffix) if readFromExtra: micsFn = self._getExtraPath('input_micrographs.xmd') else: micsFn = self._getPath('input_micrographs.xmd') anglesSet = self._readAngles(micsFn, suffix) # Create CoordinatesTiltPair object outputset = self._createCoordinatesTiltPair(micTiltPairs, uCoordSet, tCoordSet, anglesSet, suffix) summary = self.getSummary(outputset) outputset.setObjComment(summary) outputName = 'outputCoordinatesTiltPair' + suffix outputs = {outputName: outputset} self._defineOutputs(**outputs) self._defineSourceRelation(self.inputMicrographsTiltedPair, outputset) if store: self._store()
class QueueSystemConfig(OrderedObject): def __init__(self, **kwargs): OrderedObject.__init__(self, **kwargs) self.name = String() # Number of cores from which the queue is mandatory # 0 means no mandatory at all # 1 will force to launch all jobs through the queue self.mandatory = Integer() self.queues = None # List for queue configurations self.submitCommand = String() # Allow to change the prefix of submission scripts # we used by default the ID.job, but in some clusters # the job script should start by a letter self.submitPrefix = String() self.checkCommand = String() self.cancelCommand = String() self.submitTemplate = String() self.jobDoneRegex = String() def hasName(self): return self.name.hasValue() def hasValue(self): return self.hasName() and len(self.queues) def getName(self): return self.name.get() def getMandatory(self): return self.mandatory.get() def getSubmitTemplate(self): return self.submitTemplate.get() def getSubmitCommand(self): return self.submitCommand.get() def getCheckCommand(self): return self.checkCommand.get() def getCancelCommand(self): return self.cancelCommand.get() def getQueues(self): return self.queues def setName(self, name): self.name.set(name) def setMandatory(self, mandatory): # This condition is to be backward compatible # when mandatory was a boolean # now it should use the number of CPU # that should force to use the queue if mandatory in ['False', 'false']: mandatory = 0 elif mandatory in ['True', 'true']: mandatory = 1 self.mandatory.set(mandatory) def setSubmitTemplate(self, submitTemplate): self.submitTemplate.set(submitTemplate) def setSubmitCommand(self, submitCommand): self.submitCommand.set(submitCommand) def setCheckCommand(self, checkCommand): self.checkCommand.set(checkCommand) def setCancelCommand(self, cancelCommand): self.cancelCommand.set(cancelCommand) def setJobDoneRegex(self, jobDoneRegex): self.jobDoneRegex.set(jobDoneRegex) def setQueues(self, queues): self.queues = queues def getQueueConfig(self, objId): if objId is not None and self.queues is not None: for queueConfig in self.queues: if objId == queueConfig.getObjId(): return queueConfig return None
class XmippProtParticlePicking(ProtParticlePicking, XmippProtocol): """ Picks particles in a set of micrographs either manually or in a supervised mode. """ _label = 'manual-picking (step 1)' def __init__(self, **args): ProtParticlePicking.__init__(self, **args) # The following attribute is only for testing self.importFolder = String(args.get('importFolder', None)) #--------------------------- DEFINE param functions ------------------------ def _defineParams(self, form): ProtParticlePicking._defineParams(self, form) form.addParam('saveDiscarded', BooleanParam, default=False, label='Save discarded particles', help='Generates an output with ' 'the manually discarded particles.') form.addParam('doInteractive', BooleanParam, default=True, label='Run in interactive mode', expertLevel=LEVEL_ADVANCED, help='If YES, you can pick particles in differents sessions.\n' 'If NO, once an outputCoordinates is created, ' 'the protocol finishes. \n' '(the last can be useful when other protocol ' 'waits until this finish -internal scheduled-)') #--------------------------- INSERT steps functions ------------------------ def _insertAllSteps(self): """The Particle Picking process is realized for a set of micrographs""" # Get pointer to input micrographs self.inputMics = self.inputMicrographs.get() micFn = self.inputMics.getFileName() # Launch Particle Picking GUI if not self.importFolder.hasValue(): self._insertFunctionStep('launchParticlePickGUIStep', micFn, interactive=self.doInteractive) else: # This is only used for test purposes self._insertFunctionStep('_importFromFolderStep') # Insert step to create output objects self._insertFunctionStep('createOutputStep') def launchParticlePickGUIStep(self, micFn): # Launch the particle picking GUI extraDir = self._getExtraPath() process = launchSupervisedPickerGUI(micFn, extraDir, self) process.wait() # generate the discarded output only if there is a good output if self.saveDiscarded and exists(self._getPath('coordinates.sqlite')): self.createDiscardedStep() coordSet = self.getCoords() if coordSet: boxSize = Integer(coordSet.getBoxSize()) self._defineOutputs(boxsize=boxSize) self._defineSourceRelation(self.inputMicrographs.get(), boxSize) def _importFromFolderStep(self): """ This function will copy Xmipp .pos files for simulating a particle picking run...this is only for testing purposes. """ for f in getFiles(self.importFolder.get()): copyFile(f, self._getExtraPath()) def createOutputStep(self): posDir = self._getExtraPath() coordSet = self._createSetOfCoordinates(self.inputMics) readSetOfCoordinates(posDir, self.inputMics, coordSet) self._defineOutputs(outputCoordinates=coordSet) self._defineSourceRelation(self.inputMicrographs, coordSet) boxSize = Integer(coordSet.getBoxSize()) self._defineOutputs(boxsize=boxSize) self._defineSourceRelation(self.inputMicrographs.get(), boxSize) def createDiscardedStep(self): posDir = self._getExtraPath() suffixRoot = self._ProtParticlePicking__getOutputSuffix() suffix = '' if suffixRoot=='2' or suffixRoot=='' \ else str(int(suffixRoot)-1) coordSetDisc = self._createSetOfCoordinates(self.inputMics, suffix='Discarded'+suffix) readSetOfCoordinates(posDir, self.inputMics, coordSetDisc, readDiscarded=True) if coordSetDisc.getSize()>0: outputName = 'outputDiscardedCoordinates' + suffix outputs = {outputName: coordSetDisc} self._defineOutputs(**outputs) self._defineSourceRelation(self.inputMicrographs, coordSetDisc) #--------------------------- INFO functions -------------------------------- def _citations(self): return ['Abrishami2013'] #--------------------------- UTILS functions ------------------------------- def __str__(self): """ String representation of a Supervised Picking run """ if not hasattr(self, 'outputCoordinates'): msg = "No particles picked yet." else: picked = 0 # Get the number of picked particles of the last coordinates set for key, output in self.iterOutputAttributes(EMObject): picked = output.getSize() msg = "%d particles picked (from %d micrographs)" % \ (picked, self.inputMicrographs.get().getSize()) return msg def _methods(self): if self.getOutputsSize() > 0: return ProtParticlePicking._methods(self) else: return [self._getTmpMethods()] def _getTmpMethods(self): """ Return the message when there is not output generated yet. We will read the Xmipp .pos files and other configuration files. """ configfile = join(self._getExtraPath(), 'config.xmd') existsConfig = exists(configfile) msg = '' if existsConfig: md = emlib.MetaData('properties@' + configfile) configobj = md.firstObject() pickingState = md.getValue(emlib.MDL_PICKING_STATE, configobj) particleSize = md.getValue(emlib.MDL_PICKING_PARTICLE_SIZE, configobj) isAutopick = pickingState != "Manual" manualParts = md.getValue(emlib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj) autoParts = md.getValue(emlib.MDL_PICKING_AUTOPARTICLES_SIZE, configobj) if manualParts is None: manualParts = 0 if autoParts is None: autoParts = 0 msg = 'User picked %d particles ' % (autoParts + manualParts) msg += 'with a particle size of %d.' % particleSize if isAutopick: msg += "Automatic picking was used ([Abrishami2013]). " msg += "%d particles were picked automatically " % autoParts msg += "and %d manually." % manualParts return msg def _summary(self): if self.getOutputsSize() > 0: return ProtParticlePicking._summary(self) else: return [self._getTmpSummary()] def _getTmpSummary(self): summary = [] configfile = join(self._getExtraPath(), 'config.xmd') existsConfig = exists(configfile) if existsConfig: md = emlib.MetaData('properties@' + configfile) configobj = md.firstObject() pickingState = md.getValue(emlib.MDL_PICKING_STATE, configobj) particleSize = md.getValue(emlib.MDL_PICKING_PARTICLE_SIZE, configobj) activeMic = md.getValue(emlib.MDL_MICROGRAPH, configobj) isAutopick = pickingState != "Manual" manualParticlesSize = md.getValue(emlib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj) autoParticlesSize = md.getValue(emlib.MDL_PICKING_AUTOPARTICLES_SIZE, configobj) summary.append("Manual particles picked: %d"%manualParticlesSize) summary.append("Particle size:%d" %(particleSize)) autopick = "Yes" if isAutopick else "No" summary.append("Autopick: " + autopick) if isAutopick: summary.append("Automatic particles picked: %d"%autoParticlesSize) summary.append("Last micrograph: " + activeMic) return "\n".join(summary) def getCoordsDir(self): return self._getExtraPath()