def test_readDM4(self): """ Check we can read dm4 files (using EMAN) """ micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4') ih = ImageHandler() # Check that we can read the dimensions of the dm4 file: EXPECTED_SIZE = (7676, 7420, 1, 1) self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE) # We could even convert to an mrc file: outSuffix = pwutils.replaceBaseExt(micFn, 'mrc') outFn = join('/tmp', outSuffix) print "Converting: \n%s -> %s" % (micFn, outFn) ih.convert(micFn, outFn) self.assertTrue(os.path.exists(outFn)) self.assertTrue(pwutils.getFileSize(outFn) > 0) # Check dimensions are still the same: self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE) # Clean up tmp files pwutils.cleanPath(outFn)
def volumeToNGL(volPath): """ We need an mrc map to be loaded into the ngl viewer""" from pyworkflow.em.convert import ImageHandler # return getResource('reference.mrc') # Clean volPath in case it comes with :mrc, ... nglVolume = volPath.split(":")[0] # If it ends in mrc if nglVolume.endswith(".mrc"): return nglVolume else: # Add mrc nglVolume += ".mrc" # If the ngl volume already exists if os.path.exists(nglVolume): return nglVolume else: # Convert input volumes ih = ImageHandler() ih.convert(volPath, nglVolume) return nglVolume
def convertInputStep(self, particlesId, volId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ inputParticles = self.inputParticles.get() inputVolume = self.inputVolume.get() writeSetOfParticles(inputParticles, self._getExpParticlesFn()) img = ImageHandler() img.convert(inputVolume, self._getInputVolFn()) if self._useSeveralClasses(): # Scale particles Xdim = inputParticles.getXDim() Ts = inputParticles.getSamplingRate() newTs = self.targetResolution.get() * 0.4 newTs = max(Ts, newTs) newXdim = Xdim * Ts / newTs self.runJob("xmipp_image_resize", "-i %s -o %s --save_metadata_stack %s --fourier %d" % (self._getExpParticlesFn(), self._getTmpPath('scaled_particles.stk'), self._getTmpPath('scaled_particles.xmd'), newXdim)) # Scale volume Xdim = inputVolume.getXDim() if Xdim != newXdim: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (self._getInputVolFn(), newXdim), numberOfMpi=1)
def convertInputStep(self, particlesId, volId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ inputParticles = self.inputParticles.get() inputVolume = self.inputVolume.get() writeSetOfParticles(inputParticles, self._getExpParticlesFn()) img = ImageHandler() img.convert(inputVolume, self._getInputVolFn()) if self._useSeveralClasses(): # Scale particles Xdim = inputParticles.getXDim() Ts = inputParticles.getSamplingRate() newTs = self.targetResolution.get() * 0.4 newTs = max(Ts, newTs) newXdim = long(Xdim * Ts / newTs) self.writeInfoField(self._getExtraPath(), "sampling", xmippLib.MDL_SAMPLINGRATE, newTs) self.writeInfoField(self._getExtraPath(), "size", xmippLib.MDL_XSIZE, newXdim) self.runJob("xmipp_image_resize", "-i %s -o %s --save_metadata_stack %s --fourier %d" % (self._getExpParticlesFn(), self._getTmpPath('scaled_particles.stk'), self._getTmpPath('scaled_particles.xmd'), newXdim)) # Scale volume Xdim = inputVolume.getXDim() if Xdim != newXdim: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (self._getInputVolFn(), newXdim), numberOfMpi=1)
def prepareMask(self,maskObject,fnMask,TsMaskOut,XdimOut): img=ImageHandler() img.convert(maskObject, fnMask) self.runJob('xmipp_image_resize',"-i %s --factor %f"%(fnMask,maskObject.getSamplingRate()/TsMaskOut),numberOfMpi=1) maskXdim, _, _, _ =img.getDimensions((1,fnMask)) if XdimOut!=maskXdim: self.runJob('xmipp_transform_window',"-i %s --size %d"%(fnMask,XdimOut),numberOfMpi=1) self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1)
def _particlesToEmx(emxData, partSet, micSet=None, **kwargs): """ Write a SetOfMicrograph as expected in EMX format Params: micSet: input set of micrographs filename: the EMX file where to store the micrographs information. micSet: micrographs set associated with the particles **kwargs: writeImages: if set to False, only coordinates are exported. imagesStack: if passed all images will be output into a single stack file. imagesPrefix: used when not imagesStack is passed. A different stack will be created per micrograph. """ writeImages = kwargs.get('writeImages', True) imagesPrefix = kwargs.get('imagesPrefix', None) micDict = {} # Use singleMic for count all particles to be written to a single stack imagesStack = kwargs.get('imagesStack', None) singleMic = Micrograph() singleMic.setFileName(imagesStack) singleMic.counter = pwobj.Integer(0) def _getMicKey(particle): coord = particle.getCoordinate() if coord is None or coord.getMicName() is None: return '%05d' % particle.getMicId() else: return pwutils.removeExt(coord.getMicName()) def _getLocation(particle): if imagesStack is not None: mic = singleMic else: micKey = _getMicKey(particle) if micKey not in micDict: mic = Micrograph() mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey)) mic.counter = pwobj.Integer(0) micDict[micKey] = mic else: mic = micDict[micKey] # Count one more particle assigned to this micrograph mic.counter.increment() return (mic.counter.get(), mic.getFileName()) ih = ImageHandler() partAlign = partSet.getAlignment() for particle in partSet: if writeImages: newLoc = _getLocation(particle) ih.convert(particle, newLoc) localFn = basename(newLoc[1]) particle.setLocation(newLoc[0], localFn) emxObj = _particleToEmx(emxData, particle, micSet, partAlign) else: emxObj = _coordinateToEmx(emxData, particle, micSet) emxData.addObject(emxObj)
def projectInitialVolume(self): fnOutputInitVolume=self._getTmpPath("initialVolume.vol") img = ImageHandler() img.convert(self.initialVolume.get(), fnOutputInitVolume) self.runJob("xmipp_image_resize","-i %s --dim %d %d"%(fnOutputInitVolume,self.Xdim2,self.Xdim2)) fnGallery=self._getTmpPath('gallery_InitialVolume.stk') fnOutputReducedClass = self._getExtraPath("reducedClasses.xmd") self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\ %(fnOutputInitVolume,fnGallery,self.angularSampling.get(),self.symmetryGroup.get(),fnOutputReducedClass))
def _runBeforePreWhitening(self): prot = self.form.protocol # Convert input volumes ih = ImageHandler() ih.convert(prot.inputVolume.get(), join(self.workingDir, 'volume1.map')) if prot.useSplitVolume: ih.convert(prot.splitVolume.get(), join(self.workingDir, 'volume2.map')) self.results = prot.runResmap(self.workingDir, wizardMode=True)
def convertInputStep(self, volLocation1, volLocation2=None): """ Convert input volume to .mrc as expected by ResMap. Params: volLocation1: a tuple containing index and filename of the input volume. volLocation2: if not None, a tuple like volLocation1 for the split volume. """ ih = ImageHandler() ih.convert(volLocation1, self._getPath('volume1.map')) if volLocation2 is not None: ih.convert(volLocation2, self._getPath('volume2.map'))
def exportVolumeStep(self): #create directory if needed dirName = self.filesPath.get() try: os.makedirs(dirName) except OSError: if not os.path.isdir(dirName): raise ih = ImageHandler() ih.convert(self.exportVolume.get().getLocation(), os.path.join(dirName, self.VOLUMENAME))
def projectInitialVolume(self): fnOutputInitVolume = self._getTmpPath("initialVolume.vol") img = ImageHandler() img.convert(self.initialVolume.get(), fnOutputInitVolume) self.runJob( "xmipp_image_resize", "-i %s --dim %d %d" % (fnOutputInitVolume, self.Xdim2, self.Xdim2)) fnGallery = self._getTmpPath('gallery_InitialVolume.stk') fnOutputReducedClass = self._getExtraPath("reducedClasses.xmd") self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\ %(fnOutputInitVolume,fnGallery,self.angularSampling.get(),self.symmetryGroup.get(),fnOutputReducedClass))
def convertInputStep(self, particlesId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ writeSetOfParticles(self.inputParticles.get(), self._getPath('input_particles.xmd')) if self.doWiener.get(): params = ' -i %s' % self._getPath('input_particles.xmd') params += ' -o %s' % self._getExtraPath( 'corrected_ctf_particles.stk') params += ' --save_metadata_stack %s' % self._getExtraPath( 'corrected_ctf_particles.xmd') params += ' --pad %s' % self.padding_factor.get() params += ' --wc %s' % self.wiener_constant.get() params += ' --sampling_rate %s' % self.inputParticles.get( ).getSamplingRate() if self.inputParticles.get().isPhaseFlipped(): params += ' --phase_flipped ' if self.correctEnvelope: params += ' --correct_envelope ' nproc = self.numberOfMpi.get() nT = self.numberOfThreads.get() self.runJob('xmipp_ctf_correct_wiener2d', params) newTs, newXdim = self._getModifiedSizeAndSampling() if self.doWiener.get(): params = ' -i %s' % self._getExtraPath( 'corrected_ctf_particles.xmd') else: params = ' -i %s' % self._getPath('input_particles.xmd') params += ' -o %s' % self._getExtraPath('scaled_particles.stk') params += ' --save_metadata_stack %s' % self._getExtraPath( 'scaled_particles.xmd') params += ' --dim %d' % newXdim self.runJob('xmipp_image_resize', params) from pyworkflow.em.convert import ImageHandler img = ImageHandler() img.convert(self.inputVolumes.get(), self._getExtraPath("volume.vol")) Xdim = self.inputVolumes.get().getDim()[0] if Xdim != newXdim: self.runJob("xmipp_image_resize","-i %s --dim %d"%\ (self._getExtraPath("volume.vol"), newXdim), numberOfMpi=1)
def _beforePreWhitening(protocol, dir): from pyworkflow.em.convert import ImageHandler # Convert input volumes ih = ImageHandler() inputVolume = protocol.inputVolume.get() path = join(dir, 'volume1.map') print path ih.convert(inputVolume, path) if protocol.useSplitVolume: ih.convert(protocol.splitVolume.get(), join(dir, 'volume2.map')) return protocol.runResmap(dir, wizardMode=True)
def convertInputStep(self, particlesId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ writeSetOfParticles(self.inputParticles.get(), self._getPath('input_particles.xmd')) if self.doWiener.get(): params = ' -i %s' % self._getPath('input_particles.xmd') params += ' -o %s' % self._getExtraPath('corrected_ctf_particles.stk') params += ' --save_metadata_stack %s' % self._getExtraPath('corrected_ctf_particles.xmd') params += ' --pad %s' % self.padding_factor.get() params += ' --wc %s' % self.wiener_constant.get() params += ' --sampling_rate %s' % self.inputParticles.get().getSamplingRate() if self.inputParticles.get().isPhaseFlipped(): params += ' --phase_flipped ' if self.correctEnvelope: params += ' --correct_envelope ' nproc = self.numberOfMpi.get() nT=self.numberOfThreads.get() self.runJob('xmipp_ctf_correct_wiener2d', params) newTs, newXdim = self._getModifiedSizeAndSampling() if self.doWiener.get(): params = ' -i %s' % self._getExtraPath('corrected_ctf_particles.xmd') else : params = ' -i %s' % self._getPath('input_particles.xmd') params += ' -o %s' % self._getExtraPath('scaled_particles.stk') params += ' --save_metadata_stack %s' % self._getExtraPath('scaled_particles.xmd') params += ' --fourier %d' % newXdim self.runJob('xmipp_image_resize',params) from pyworkflow.em.convert import ImageHandler img = ImageHandler() img.convert(self.inputVolumes.get(), self._getExtraPath("volume.vol")) Xdim = self.inputVolumes.get().getDim()[0] if Xdim!=newXdim: self.runJob("xmipp_image_resize","-i %s --dim %d"%\ (self._getExtraPath("volume.vol"), newXdim), numberOfMpi=1)
def convertStep(self, imgsFn): from convert import writeSetOfClasses2D, writeSetOfParticles imgSet = self.inputSet.get() if isinstance(imgSet, SetOfClasses2D): writeSetOfClasses2D(imgSet, self.imgsFn, writeParticles=True) else: writeSetOfParticles(imgSet, self.imgsFn) from pyworkflow.em.convert import ImageHandler img = ImageHandler() fnVol = self._getTmpPath("volume.vol") img.convert(self.inputVolume.get(), fnVol) xdim=self.inputVolume.get().getDim()[0] if xdim!=self._getDimensions(): self.runJob("xmipp_image_resize","-i %s --dim %d"%(fnVol,self._getDimensions()))
def convertStep(self, imgsFn): from ..convert import writeSetOfClasses2D, writeSetOfParticles imgSet = self.inputSet.get() if isinstance(imgSet, SetOfClasses2D): writeSetOfClasses2D(imgSet, self.imgsFn, writeParticles=True) else: writeSetOfParticles(imgSet, self.imgsFn) from pyworkflow.em.convert import ImageHandler img = ImageHandler() fnVol = self._getTmpPath("volume.vol") img.convert(self.inputVolume.get(), fnVol) xdim=self.inputVolume.get().getDim()[0] if xdim!=self._getDimensions(): self.runJob("xmipp_image_resize","-i %s --dim %d"%(fnVol,self._getDimensions()),numberOfMpi=1)
def prepareMask(self, maskObject, fnMask, TsMaskOut, XdimOut): img = ImageHandler() img.convert(maskObject, fnMask) self.runJob('xmipp_image_resize', "-i %s --factor %f" % (fnMask, maskObject.getSamplingRate() / TsMaskOut), numberOfMpi=1) maskXdim, _, _, _ = img.getDimensions((1, fnMask)) if XdimOut != maskXdim: self.runJob('xmipp_transform_window', "-i %s --size %d" % (fnMask, XdimOut), numberOfMpi=1) self.runJob('xmipp_transform_threshold', "-i %s --select below 0.5 --substitute binarize" % fnMask, numberOfMpi=1)
def convertStep(self, imgsFn): from xmipp3.convert import writeSetOfParticles imgSet = self.inputSet.get() writeSetOfParticles(imgSet, self.imgsFn) from pyworkflow.em.convert import ImageHandler img = ImageHandler() fnVol = self._getTmpPath("volume.vol") img.convert(self.inputVolume.get(), fnVol) xdim = self.inputVolume.get().getDim()[0] imgXdim = imgSet.getDim()[0] if xdim != imgXdim: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (fnVol, imgXdim), numberOfMpi=1)
def convertStep(self): inputParts = self.inputParticles.get() writeSetOfParticles(inputParts, self.imgsInputFn) #Resizing inputs ih = ImageHandler() ih.convert(self.inputReference.get(), self.fnRefVol) XdimRef = self.inputReference.get().getDim()[0] ih.convert(self.inputVolume.get(), self.fnInputVol) XdimInput = self.inputVolume.get().getDim()[0] if XdimRef != XdimInput: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (self.fnRefVol, XdimInput), numberOfMpi=1)
def prime2DStep(self): partFile = self._getExtraPath("particles.mrc") SamplingRate = self.inputParticles.get().getSamplingRate() kV = self.inputParticles.get().getAcquisition().getVoltage() partitions = 1 partName = os.path.basename(partFile) partName = os.path.splitext(partName)[0] tmpDir = self._getTmpPath(partName) makePath(tmpDir) paramsOri = 'prg=print_project_field oritype=ptcl2D > oritab.txt' paramsImp = 'prg=import_particles cs=2.7 ctf=no fraca=0.1 kv=%f smpd=%f stk=%s' % ( kV, SamplingRate, os.path.abspath(partFile)) paramsC2D = ' prg=cluster2D msk=%d ncls=%d nparts=%d nthr=%d' % ( self.mask.get(), self.clusters.get(), partitions, self.numberOfThreads.get()) if self.maxIter.get() > 0: paramsC2D = paramsC2D + (' maxits=%d' % self.maxIter.get()) self.runJob(simple.Plugin.sim_exec(), 'prg=new_project projname=temp', cwd=os.path.abspath(tmpDir), env=simple.Plugin.getEnviron()) self.runJob(simple.Plugin.sim_exec(), paramsImp, cwd=os.path.abspath(tmpDir) + '/temp', env=simple.Plugin.getEnviron()) self.runJob(simple.Plugin.distr_exec(), paramsC2D, cwd=os.path.abspath(tmpDir) + '/temp', env=simple.Plugin.getEnviron()) self.runJob(simple.Plugin.sim_exec(), paramsOri, cwd=os.path.abspath(tmpDir) + '/temp', env=simple.Plugin.getEnviron()) #Move output files to ExtraPath and rename them properly lastIter = self.getLastIteration(tmpDir) os.remove(os.path.abspath(self._getExtraPath("particles.mrc"))) mvRoot1 = os.path.join(tmpDir + '/temp/2_cluster2D', "cavgs_iter%03d.mrc" % lastIter) mvRoot2 = os.path.join(tmpDir + '/temp', "oritab.txt") # moveFile(mvRoot1, self._getExtraPath(partName + "_cavgs_final.mrc")) ih = ImageHandler() ih.convert(mvRoot1, self._getExtraPath(partName + "_cavgs_final.mrcs")) moveFile(mvRoot2, self._getExtraPath(partName + "_oritab.txt")) cleanPath(tmpDir)
def _insertAllSteps(self): self._params = {} # diameter must be passed in Armstrongs and therefore should be converted self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate() # self._params['num-slices'] = self.numberSizes.get() # self._params['size-range'] = self.sizeRange.get() self._params['apix'] = self.inputMicrographs.get().getSamplingRate() self._params['thresh'] = self.threshold.get() # self._params['max-thresh'] = self.maxThreshold.get() # self._params['max-area'] = self.maxArea.get() # self._params['max-peaks'] = self.maxPeaks.get() args = "" for par, val in self._params.iteritems(): args += " --%s=%s" % (par, str(val)) if self.invert: args += " --invert" args += " " + self.extraParams.get('') deps = [] # Store all steps ids, final step createOutput depends on all of them ih = ImageHandler() for mic in self.inputMicrographs.get(): # Create micrograph folder micName = mic.getFileName() micDir = self._getTmpPath(removeBaseExt(micName)) makePath(micDir) # If needed convert micrograph to mrc format, otherwise link it if getExt(micName) != ".mrc": fnMicBase = replaceBaseExt(micName, 'mrc') inputMic = join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = join(micDir, basename(micName)) createLink(micName, inputMic) # Insert step to execute program stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args) deps.append(stepId) self._insertFinalSteps(deps)
def _insertAllSteps(self): self._params = {} # diameter must be passed in Angstrongs and therefore should be converted self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate() # self._params['num-slices'] = self.numberSizes.get() # self._params['size-range'] = self.sizeRange.get() self._params['apix'] = self.inputMicrographs.get().getSamplingRate() self._params['thresh'] = self.threshold.get() # self._params['max-thresh'] = self.maxThreshold.get() # self._params['max-area'] = self.maxArea.get() # self._params['max-peaks'] = self.maxPeaks.get() args = "" for par, val in self._params.iteritems(): args += " --%s=%s" % (par, str(val)) if self.invert: args += " --invert" args += " " + self.extraParams.get('') deps = [] # Store all steps ids, final step createOutput depends on all of them ih = ImageHandler() for mic in self.inputMicrographs.get(): # Create micrograph folder micName = mic.getFileName() micDir = self._getTmpPath(removeBaseExt(micName)) makePath(micDir) # If needed convert micrograph to mrc format, otherwise link it if getExt(micName) != ".mrc": fnMicBase = replaceBaseExt(micName, 'mrc') inputMic = join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = join(micDir, basename(micName)) createLink(micName, inputMic) # Insert step to execute program stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args) deps.append(stepId) self._insertFinalSteps(deps)
def generateSplittedVolumes(self): inputParticles = self.directionalClasses.get() Xdim = inputParticles.getDimensions()[0] fnMask = "" if self.mask.hasValue(): fnMask = self._getExtraPath("mask.vol") img=ImageHandler() img.convert(self.mask.get(), fnMask) self.runJob('xmipp_image_resize',"-i %s --dim %d"%(fnMask,Xdim),numberOfMpi=1) self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1) args="-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f"%\ (self._getExtraPath("directionalClasses.xmd"),self._getExtraPath("split"),self.Nrec.get(),self.Nsamples.get(), self.symmetryGroup.get(), self.alpha.get()) if fnMask!="": args+=" --mask binary_file %s"%fnMask self.runJob("xmipp_classify_first_split",args)
def splitVolumeStep(self): newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE) newXdim = self.readInfoField(self._getExtraPath(),"size",xmippLib.MDL_XSIZE) fnMask = "" if self.mask.hasValue(): fnMask = self._getExtraPath("mask.vol") img=ImageHandler() img.convert(self.mask.get(), fnMask) self.runJob('xmipp_image_resize',"-i %s --dim %d"%(fnMask,newXdim),numberOfMpi=1) self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1) args="-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f"%\ (self._getDirectionalClassesFn(),self._getExtraPath("split"),self.Nrec.get(),self.Nsamples.get(), self.symmetryGroup.get(), self.alpha.get()) if fnMask!="": args+=" --mask binary_file %s"%fnMask self.runJob("xmipp_classify_first_split",args,numberOfMpi=1)
def writeSetOfImages(imgSet, stackFn, selFn): """ This function will write a SetOfMicrographs as a Spider stack and selfile. Params: imgSet: the SetOfMicrograph instance. stackFn: the filename where to write the stack. selFn: the filename of the Spider selection file. """ ih = ImageHandler() doc = SpiderDocFile(selFn, 'w+') for i, img in enumerate(imgSet): ih.convert(img, (i+1, stackFn)) doc.writeValues(i+1) doc.close() convertEndian(stackFn, imgSet.getSize())
def prepareReferences(self,fnDir,TsCurrent,targetResolution): print "Preparing references to sampling rate=",TsCurrent fnMask='' newXdim=self.readInfoField(fnDir,"size",xmipp.MDL_XSIZE) if self.nextMask.hasValue(): fnMask=join(fnDir,"mask.vol") self.prepareMask(self.nextMask.get(), fnMask, TsCurrent, newXdim) fnReferenceVol=join(fnDir,"volumeRef.vol") img=ImageHandler() img.convert(self.inputVolume.get(), fnReferenceVol) Xdim=self.inputVolume.get().getDim()[0] if Xdim!=newXdim: self.runJob("xmipp_image_resize","-i %s --fourier %d"%(fnReferenceVol,newXdim),numberOfMpi=1) if fnMask!='': self.runJob('xmipp_image_operate','-i %s --mult %s'%(fnReferenceVol,fnMask),numberOfMpi=1)
def test_convertMicrographs(self): """ Convert micrograhs to different formats. EMAN2 required for .img """ micFn = self.dataset.getFile('micrographs/BPV_1386.mrc') outSuffix = pwutils.replaceBaseExt(micFn, 'img') ih = ImageHandler() outFn = join('/tmp', outSuffix) print "Converting: \n%s -> %s" % (micFn, outFn) ih.convert(micFn, outFn) self.assertTrue(os.path.exists(outFn)) self.assertTrue(pwutils.getFileSize(outFn) > 0) pwutils.cleanPath(outFn) pwutils.cleanPath(outFn.replace('.img', '.hed'))
def _micrographsToEmx(emxData, micSet, emxDir, ctfSet=None, writeData=True): """ Write a SetOfMicrograph as expected in EMX format (xml file) Params: micSet: input set of micrographs filename: the EMX file where to store the micrographs information. """ ih = ImageHandler() for mic in micSet: if writeData: loc = mic.getLocation() fnMicBase = pwutils.replaceBaseExt(loc[1], 'mrc') newLoc = join(emxDir, fnMicBase) ih.convert(loc, newLoc) mic.setLocation(NO_INDEX, fnMicBase) if ctfSet: mic.setCTF(ctfSet[mic.getObjId()]) emxMic = _micrographToEmx(mic) emxData.addObject(emxMic)
def _particlesToEmx(emxData, partSet, stackFn=None, micSet=None): """ Write a SetOfMicrograph as expected in EMX format Params: micSet: input set of micrographs filename: the EMX file where to store the micrographs information. """ ih = ImageHandler() partAlign = partSet.getAlignment() for i, particle in enumerate(partSet): if stackFn: print ("stackFn-----------------------------------------") loc = particle.getLocation() newLoc = (i+1, stackFn) ih.convert(loc, newLoc) newFn = basename(stackFn) particle.setLocation(i+1, newFn) emxObj = _particleToEmx(emxData, particle, micSet, partAlign) else: emxObj = _coordinateToEmx(emxData, particle, micSet) emxData.addObject(emxObj)
def _computeRightPreview(self): """ This function should compute the right preview using the self.lastObj that was selected """ from pyworkflow.em.packages.xmipp3 import locationToXmipp # Copy image to filter to Tmp project folder inputPath = os.path.join("Tmp", "bsoft_filter_input.spi") outputPath = os.path.join("Tmp", "bsoft_filter_output.spi") cleanPath(inputPath, outputPath) ih = ImageHandler() ih.convert(self.lastObj.getLocation(), inputPath) self.protocolParent.runFilter(inputPath, outputPath) # Get output image and update filtered image img = ih._img img.read(outputPath) self.rightImage = img self.updateFilteredImage()
def prepareReferences(self, fnDir, TsCurrent, targetResolution): print "Preparing references to sampling rate=", TsCurrent fnMask = '' newXdim = self.readInfoField(fnDir, "size", xmipp.MDL_XSIZE) if self.nextMask.hasValue(): fnMask = join(fnDir, "mask.vol") self.prepareMask(self.nextMask.get(), fnMask, TsCurrent, newXdim) fnReferenceVol = join(fnDir, "volumeRef.vol") img = ImageHandler() img.convert(self.inputVolume.get(), fnReferenceVol) Xdim = self.inputVolume.get().getDim()[0] if Xdim != newXdim: self.runJob("xmipp_image_resize", "-i %s --fourier %d" % (fnReferenceVol, newXdim), numberOfMpi=1) if fnMask != '': self.runJob('xmipp_image_operate', '-i %s --mult %s' % (fnReferenceVol, fnMask), numberOfMpi=1)
def _computeRightPreview(self): """ This function should compute the right preview using the self.lastObj that was selected """ from pyworkflow.em.packages.xmipp3 import locationToXmipp # Copy image to filter to Tmp project folder outputName = os.path.join("Tmp", "filtered_particle") outputPath = outputName + ".spi" cleanPath(outputPath) outputLoc = (1, outputPath) ih = ImageHandler() ih.convert(self.lastObj.getLocation(), outputLoc) outputLocSpiStr = locationToSpider(1, outputName) pars = {} pars["filterType"] = self.protocolParent.filterType.get() pars["filterMode"] = self.protocolParent.filterMode.get() pars["usePadding"] = self.protocolParent.usePadding.get() pars["op"] = "FQ" if self.protocolParent.filterType <= FILTER_SPACE_REAL: pars['filterRadius'] = self.getRadius() else: pars['lowFreq'] = self.getLowFreq() pars['highFreq'] = self.getHighFreq() if self.protocolParent.filterType == FILTER_FERMI: pars['temperature'] = self.getTemperature() filter_spider(outputLocSpiStr, outputLocSpiStr, **pars) # Get output image and update filtered image img = ImageHandler()._img locXmippStr = locationToXmipp(1, outputPath) img.read(locXmippStr) self.rightImage = img self.updateFilteredImage()
def _computeRightPreview(self): """ This function should compute the right preview using the self.lastObj that was selected """ from pyworkflow.em.packages.xmipp3 import locationToXmipp # Copy image to filter to Tmp project folder outputName = os.path.join("Tmp", "filtered_particle") outputPath = outputName + ".spi" cleanPath(outputPath) outputLoc = (1, outputPath) ih = ImageHandler() ih.convert(self.lastObj.getLocation(), outputLoc) outputLocSpiStr = locationToSpider(1, outputName) pars = {} pars["filterType"] = self.protocolParent.filterType.get() pars["filterMode"] = self.protocolParent.filterMode.get() pars["usePadding"] = self.protocolParent.usePadding.get() pars["op"] = "FQ" if self.protocolParent.filterType <= FILTER_FERMI: pars['filterRadius'] = self.getRadius() else: pars['lowFreq'] = self.getLowFreq() pars['highFreq'] = self.getHighFreq() if self.protocolParent.filterType == FILTER_FERMI: pars['temperature'] = self.getTemperature() filter_spider(outputLocSpiStr, outputLocSpiStr, **pars) # Get output image and update filtered image img = ImageHandler()._img locXmippStr = locationToXmipp(1, outputPath) img.read(locXmippStr) self.rightImage = img self.updateFilteredImage()
def _pickMicrograph(self, mic, radius): micFn = mic.getFileName() micDir = self._getMicDir(micFn) fnMicBase = pwutils.replaceBaseExt(micFn, 'mrc') fnMicCfg = pwutils.replaceBaseExt(micFn, 'cfg') fnMicFull = os.path.join(micDir, fnMicBase) fnPosBase = self._getMicPosFn(micFn) # Convert micrographs to mrc (uint8) as required by ETHAN program ih = ImageHandler() ih.convert(micFn, fnMicFull, md.DT_UCHAR) # Create a configuration file to be used by ETHAN with the parameters # selected by the user self.writeConfigFile(os.path.join(micDir, fnMicCfg)) # Run ethan program with the required arguments program = self.getProgram() args = "%s %s %s %s" % (radius, fnMicBase, fnPosBase, fnMicCfg) self.runJob(program, args, cwd=micDir) # Clean temporary micrograph pwutils.cleanPath(fnMicFull)
def _pickMicrograph(self, mic, args): # Prepare mic folder and convert if needed micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) ih = ImageHandler() # If needed convert micrograph to mrc format, otherwise link it if pwutils.getExt(micName) != ".mrc": fnMicBase = pwutils.replaceBaseExt(micName, 'mrc') inputMic = os.path.join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = os.path.join(micDir, os.path.basename(micName)) pwutils.createLink(micName, inputMic) # Program to execute and it arguments program = "ApDogPicker.py" outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt")) args += " --image=%s --outfile=%s" % (inputMic, outputFile) self.runJob(program, args)
def test_readCompressedTIF(self): """ Check we can read tif files """ micFn = self.dsFormat.getFile('c3-adp-se-xyz-0228_200.tif') ih = ImageHandler() # Check that we can read the dimensions of the dm4 file: EXPECTED_SIZE = (7676, 7420, 1, 38) self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE) # We could even convert to an mrc file: outSuffix = pwutils.replaceBaseExt(micFn, 'mrc') outFn = join('/tmp', outSuffix) print "Converting: \n%s -> %s" % ((1, micFn), outFn) ih.convert((1, micFn), outFn) self.assertTrue(os.path.exists(outFn)) self.assertTrue(pwutils.getFileSize(outFn) > 0) self.assertEqual(ih.getDimensions(outFn), (7676, 7420, 1, 1)) # Clean up tmp files pwutils.cleanPath(outFn)
def _pickMicrograph(self, mic, args): # Prepare mic folder and convert if needed micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) ih = ImageHandler() # If needed convert micrograph to mrc format, otherwise link it if pwutils.getExt(micName) != ".mrc": fnMicBase = pwutils.replaceBaseExt(micName, 'mrc') inputMic = os.path.join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = os.path.join(micDir, os.path.basename(micName)) pwutils.createLink(micName, inputMic) # Program to execute and it arguments program = "ApDogPicker.py" outputFile = self._getExtraPath(pwutils.replaceBaseExt( inputMic, "txt")) args += " --image=%s --outfile=%s" % (inputMic, outputFile) self.runJob(program, args)
def _runBeforePreWhitening(self): prot = self.form.protocol # Convert input volumes ih = ImageHandler() if prot.useSplitVolume: ih.convert(prot.volumeHalf1.get(), join(self.workingDir, 'volume1.map')) ih.convert(prot.volumeHalf2.get(), join(self.workingDir, 'volume2.map')) else: ih.convert(prot.inputVolume.get(), join(self.workingDir, 'volume1.map')) self.results = prot.runResmap(self.workingDir, wizardMode=True)
class CustomMaskDialog(ImagePreviewDialog): def _beforePreview(self): imgLocation = self.protocolParent.inputImage.get().getLocation() self.dim = ImageHandler().getDimensions(imgLocation)[0] self.lastObj = None self.rightPreviewLabel = "Final mask" self.message = "Generating mask..." self.ih = ImageHandler() self.rightImage = self.ih.createImage() def _createPreview(self, frame): """ Should be implemented by subclasses to create the items preview. """ self._previews = [] for i, label in enumerate(MASKRESULT_LABELS): self.previewLabel = label previewFrame = tk.Frame(frame) ImagePreviewDialog._createPreview(self, previewFrame) self._previews.append(self.preview) # store all previews created previewFrame.grid(row=i/4, column=i%4) def _itemSelected(self, obj): self.lastObj = obj dialog.FlashMessage(self, self.message, func=self._computeRightPreview) def _createVarWidgets(self, parent, varName, varLabel, row, col): var = tk.StringVar() self._vars[varName] = var var.set(self.protocolParent.getAttributeValue(varName)) varLabel = tk.Label(parent, text=varLabel) varLabel.grid(row=row, column=col*2, padx=5, pady=5) varEntry = tk.Entry(parent, width=10, textvariable=var) varEntry.grid(row=row, column=col*2+1, padx=5, pady=5) def _createControls(self, frame): self._vars = {} inputFrame = tk.Frame(frame) inputFrame.grid(row=0, column=0) for i, varName in enumerate(CUSTOMMASK_VARS): self._createVarWidgets(inputFrame, varName, CUSTOMMASK_VARS[varName], i%2, i/2) previewBtn = HotButton(frame, text='Preview', command=self._computeRightPreview) previewBtn.grid(row=1, column=1, padx=5, pady=5) def getVarValue(self, varName): return self._vars[varName].get() def _computeRightPreview(self, e=None): """ This function should compute the right preview using the self.lastObj that was selected """ prot = self.protocolParent # short notation tmp = prot.getProject().getTmpPath() ext = prot.getExt() # Convert input image to spider imgPrefix = 'inputImage' imgName = '%s.%s' % (imgPrefix, ext) imgFn = os.path.join(tmp, imgName) self.ih.convert(self.lastObj, (1, imgFn)) runCustomMaskScript(self.getVarValue('filterRadius1'), self.getVarValue('sdFactor'), self.getVarValue('filterRadius2'), self.getVarValue('maskThreshold'), workingDir=tmp, ext=ext, inputImage=imgPrefix+'@1') for i, preview in enumerate(self._previews): if i == 0: self.rightImage.read(imgFn) else: self.rightImage.read('%d@%s/stkmask.%s' % (i, tmp, ext)) preview.updateData(self.rightImage.getData())
class ProtRelion2Autopick(ProtParticlePickingAuto, ProtRelionBase): """ This Relion protocol uses the 'relion_autopick' program to pick particles from micrographs, either using templates or gaussian blobs. The picking with this protocol is divided in three steps: 1) Run with 'Optimize' option for several (less than 30) micrographs. 2) Execute the wizard to refine the picking parameters. 3) Run with 'Pick all' option to pick particles from all micrographs. The first steps will use internally the option '--write-fom-maps' to write to disk the FOM maps. The expensive part of this calculation is to calculate a probability-based figure-of-merit (related to the cross-correlation coefficient between each rotated reference and all positions in the micrographs. That's why it is only done in an small subset of the micrographs, where one should use representative micrographs for the entire data set, e.g. a high and a low-defocus one, and/or with thin or thick ice. Step 2 uses a much cheaper peak-detection algorithm that uses the threshold and minimum distance parameters. """ _label = 'auto-picking' @classmethod def isDisabled(cls): return not isVersion2() # -------------------------- DEFINE param functions ------------------------ def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputMicrographs', params.PointerParam, pointerClass='SetOfMicrographs', label='Input micrographs', important=True, help='Select the input micrographs. ' 'If using the *Optimize* mode, just a subset of ' 'micrographs are used to compute the FOM maps. ' 'If in *Compute* mode, all micrographs will be ' 'auto-picked.') form.addParam('ctfRelations', params.RelationParam, relationName=RELATION_CTF, attributeName='getInputMicrographs', label='CTF estimation', help='Choose some CTF estimation related to the ' 'input micrographs.') form.addParam('runType', params.EnumParam, default=RUN_OPTIMIZE, choices=['Optimize params', 'Pick all micrographs'], display=params.EnumParam.DISPLAY_LIST, label='Run type: ', help='Usually, first you should use the *Optimize* mode ' 'to compute the FOM maps for a few micrographs and ' 'use them to tune the picking parameters using the ' 'wizard. After that you can run the job in *Compute*' ' mode and auto-pick all the micrographs. ') group = form.addGroup('Micrographs for optimization', condition='runType==%d' % RUN_OPTIMIZE) group.addParam('micrographsSelection', params.EnumParam, default=MICS_AUTO, choices=['automatic selection', 'input subset'], display=params.EnumParam.DISPLAY_HLIST, label='Choose micrographs by', help='If you choose "automatic selection", you only ' 'need to provide the number of microgrphs to use ' 'and that number will be selected to cover the ' 'defocus range. ') group.addParam('micrographsNumber', params.IntParam, default='10', condition='micrographsSelection==%d' % MICS_AUTO, label='Micrographs for optimization:', help='Select the number of micrographs that you want' 'to be used for the parameters optimization. ') group.addParam('micrographsSubset', params.PointerParam, condition='micrographsSelection==%d' % MICS_SUBSET, pointerClass='SetOfMicrographs', label='Subset of micrographs', help='Choose as input a subset of micrographs that ' 'you have previously selected. ' '(Probably covering the defocus range).') # From Relion 2.+, it can be picked with gaussian blobs, so we # need to add these parameters refCondition = 'referencesType==%s' % REF_AVERAGES group = form.addGroup('References') group.addParam('referencesType', params.EnumParam, choices=['References', 'Gaussian blobs'], default=REF_AVERAGES, display=params.EnumParam.DISPLAY_HLIST, label='References type', help='You may select "Gaussian blobs" to be used as ' 'references. The preferred way to autopick is ' 'by providing 2D references images that were ' 'by 2D classification. \n' 'The Gaussian blob references may be useful to ' 'kickstart a new data set.') group.addParam('gaussianPeak', params.FloatParam, default=0.1, condition='referencesType==%s' % REF_BLOBS, label='Gaussian peak value', help='The peak value of the Gaussian blob. ' 'Weaker data will need lower values.') group.addParam('inputReferences', params.PointerParam, pointerClass='SetOfAverages', condition=refCondition, label='Input references', important=True, help='Input references (SetOfAverages) for auto-pick. \n\n' 'Note that the absolute greyscale needs to be correct, \n' 'so only use images with proper normalization.') group.addParam('particleDiameter', params.IntParam, default=-1, label='Mask diameter (A)', help='Diameter of the circular mask that will be applied ' 'around the templates in Angstroms. When set to a ' 'negative value, this value is estimated ' 'automatically from the templates themselves.') form.addSection('References') form.addParam('lowpassFilterRefs', params.IntParam, default=20, condition=refCondition, label='Lowpass filter references (A)', help='Lowpass filter that will be applied to the ' 'references before template matching. \n' 'Do NOT use very high-resolution templates to ' 'search your micrographs. \n' 'The signal will be too weak at high resolution ' 'anyway, and you may find Einstein from noise...') form.addParam('highpassFilterMics', params.IntParam, default=-1, label='Highpass filter (A)', help='Highpass filter that will be applied to the ' 'micrographs. This may be useful to get rid of ' 'background ramps due to uneven ice distributions. ' 'Give a negative value to skip the highpass ' 'filter. Useful values are often in the range ' 'of 200-400 Angstroms.') form.addParam('angularSampling', params.IntParam, default=5, label='Angular sampling (deg)', help='Angular sampling in degrees for exhaustive searches ' 'of the in-plane rotations for all references.') form.addParam('refsHaveInvertedContrast', params.BooleanParam, default=True, label='References have inverted contrast?', help='Set to Yes to indicate that the reference have ' 'inverted contrast with respect to the particles ' 'in the micrographs.') form.addParam('refsCtfCorrected', params.BooleanParam, default=True, condition=refCondition, label='Are References CTF corrected?', help='Set to Yes if the references were created with ' 'CTF-correction inside RELION.\n' 'If set to Yes, the input micrographs should contain ' 'the CTF information.') form.addParam('ignoreCTFUntilFirstPeak', params.BooleanParam, condition=refCondition, default=False, expertLevel=params.LEVEL_ADVANCED, label='Ignore CTFs until first peak?', help='Set this to Yes, only if this option was also used ' 'to generate the references.') form.addSection('Autopicking') group = form.addGroup('Autopick') group.addParam('pickingThreshold', params.FloatParam, default=0.25, label='Picking threshold:', help='Use lower thresholds to pick more particles ' '(and more junk probably)') group.addParam('interParticleDistance', params.IntParam, default=-1, label='Minimum inter-particle distance (A):', help='Particles closer together than this distance \n' 'will be consider to be a single cluster. \n' 'From each cluster, only one particle will be ' 'picked.') group.addParam('maxStddevNoise', params.FloatParam, default=1.1, label='Maximum stddev noise:', help='This is useful to prevent picking in carbon areas, ' 'or areas with big contamination features. Peaks in ' 'areas where the background standard deviation in ' 'the normalized micrographs is higher than this ' 'value will be ignored. Useful values are probably ' 'in the range 1.0 to 1.2. Set to -1 to switch off ' 'the feature to eliminate peaks due to high ' 'background standard deviations.') group = form.addGroup('Computing') group.addParam('shrinkFactor', params.FloatParam, default=0, validators=[params.Range(0, 1, "value should be " "between 0 and 1. ")], label='Shrink factor', help='This is useful to speed up the calculations, ' 'and to make them less memory-intensive. The ' 'micrographs will be downscaled (shrunk) to ' 'calculate the cross-correlations, and peak ' 'searching will be done in the downscaled FOM ' 'maps. When set to 0, the micrographs will de ' 'downscaled to the lowpass filter of the ' 'references, a value between 0 and 1 will ' 'downscale the micrographs by that factor. ' 'Note that the results will not be exactly ' 'the same when you shrink micrographs!') group.addParam('doGpu', params.BooleanParam, default=True, label='Use GPU acceleration?', help='If set to Yes, the job will try to use GPU ' 'acceleration.') group.addParam('gpusToUse', params.StringParam, default='', label='Which GPUs to use:', condition='doGpu', help='This argument is not necessary. If left empty, ' 'the job itself will try to allocate available GPU ' 'resources. You can override the default ' 'allocation by providing a list of which GPUs ' '(0,1,2,3, etc) to use. MPI-processes are ' 'separated by ":", threads by ",". ' 'For example: "0,0:1,1:0,0:1,1"') form.addParam('extraParams', params.StringParam, default='', label='Additional arguments:', help='In this box command-line arguments may be provided ' 'that are not generated by the GUI. This may be ' 'useful for testing developmental options and/or ' 'expert use of the program. \n' 'The command "relion_autopick" will print a list ' 'of possible options.') form.addSection('Helix') form.addParam('fomLabel', params.LabelParam, important=True, label='Helix processing is not implemented still.') self._defineStreamingParams(form) form.addParallelSection(threads=0, mpi=4) # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): self.inputStreaming = self.getInputMicrographs().isStreamOpen() if self.inputStreaming and not self.isRunOptimize(): # If the input is in streaming, follow the base class policy # about inserting new steps and discovery new input/output ProtParticlePickingAuto._insertAllSteps(self) self.createOutputStep = self._doNothing else: # If not in streaming, then we will just insert a single step to # pick all micrographs at once since it is much faster self._insertInitialSteps() self._insertFunctionStep('_pickMicrographsFromStar', self._getPath('input_micrographs.star'), *self._getPickArgs()) self._insertFunctionStep('createOutputStep') # Disable streaming functions: self._insertFinalSteps = self._doNothing self._stepsCheck = self._doNothing def _insertInitialSteps(self): # Convert the input micrographs and references to # the required Relion star files inputRefs = self.getInputReferences() refId = inputRefs.strId() if self.useInputReferences() else 'Gaussian' convertId = self._insertFunctionStep('convertInputStep', self.getInputMicrographs().strId(), refId, self.runType.get()) return [convertId] def _doNothing(self, *args): pass # used to avoid some streaming functions def _loadInputList(self): """ This function is re-implemented in this protocol, because it have a SetOfCTF as input, so for streaming, we only want to report those micrographs for which the CTF is ready. """ micDict, micClose = self._loadMics(self.getInputMicrographs()) ctfDict, ctfClosed = self._loadCTFs(self.ctfRelations.get()) # Remove the micrographs that have not CTF # and set the CTF property for those who have it for micKey, mic in micDict.iteritems(): if micKey in ctfDict: mic.setCTF(ctfDict[micKey]) else: del micDict[micKey] # Return the updated micDict and the closed status return micDict, micClose and ctfClosed # -------------------------- STEPS functions ------------------------------ def convertInputStep(self, micsId, refsId, runType): # runType is passed as parameter to force a re-execute of this step # if there is a change in the type self._ih = ImageHandler() # used to convert micrographs # Match ctf information against the micrographs self.ctfDict = {} if self.ctfRelations.get() is not None: for ctf in self.ctfRelations.get(): self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone() micStar = self._getPath('input_micrographs.star') writeSetOfMicrographs(self.getMicrographList(), micStar, alignType=ALIGN_NONE, preprocessImageRow=self._preprocessMicrographRow) if self.useInputReferences(): writeReferences(self.getInputReferences(), self._getPath('input_references'), useBasename=True) # FIXME: (JMRT-20180523) The following code does not seems to work # here it has been worked around by changing the name of the wizard # output but this seems to reflect a deeper problem of deleting # already existing output objects in a protocol. Maybe when updating # from run.db to project.sqlite? # Clean up if previously created the outputMicrographs and Coordinates # in the wizard - optimization run # if self.hasAttribute('outputMicrographs'): # self._deleteChild('outputMicrographs', self.outputMicrographs) # if self.hasAttribute('outputCoordinates'): # self._deleteChild('outputCoordinates', self.outputCoordinates) # self._store() def getAutopickParams(self): # Return the autopicking parameters except for the interative ones: # - threshold # - minDistance # - maxStd params = ' --pickname autopick' params += ' --odir ""' params += ' --particle_diameter %d' % self.particleDiameter params += ' --angpix %0.3f' % self.getInputMicrographs().getSamplingRate() params += ' --shrink %0.3f' % self.shrinkFactor if self.doGpu: params += ' --gpu "%s"' % self.gpusToUse # Now in Relion2.0 autopick can use gassian blobs if self.useInputReferences(): params += ' --ref input_references.star' ps = self.getInputReferences().getSamplingRate() params += ' --angpix_ref %0.3f' % ps else: # Gaussian blobs params += ' --ref gauss --gauss_max %0.3f' % self.gaussianPeak if self.refsHaveInvertedContrast: params += ' --invert' if self.refsCtfCorrected: params += ' --ctf' params += ' --ang %d' % self.angularSampling # Negative values for filters means no-filter if self.lowpassFilterRefs > 0: params += ' --lowpass %d' % self.lowpassFilterRefs if self.highpassFilterMics > 0: params += ' --highpass %d' % self.highpassFilterMics # Add extra params is any params += ' %s' % self.extraParams return params def _getPickArgs(self): basicArgs = self.getAutopickParams() threshold = self.pickingThreshold.get() interDist = self.interParticleDistance.get() fomParam = ' --write_fom_maps' if self.isRunOptimize() else '' return [basicArgs, threshold, interDist, fomParam] def _pickMicrographsFromStar(self, micStarFile, params, threshold, minDistance, fom): """ Launch the 'relion_autopick' for micrographs in the inputStarFile. If the input set of complete, the star file will contain all the micrographs. If working in streaming, it will be only one micrograph. """ params += ' --i %s' % relpath(micStarFile, self.getWorkingDir()) params += ' --threshold %0.3f ' % threshold params += ' --min_distance %0.3f %s' % (minDistance, fom) program = self._getProgram('relion_autopick') self.runJob(program, params, cwd=self.getWorkingDir()) def _pickMicrograph(self, mic, params, threshold, minDistance, fom): """ This method should be invoked only when working in streaming mode. """ micRow = md.Row() self._preprocessMicrographRow(mic, micRow) micrographToRow(mic, micRow) self._postprocessMicrographRow(mic, micRow) self._pickMicrographsFromStar(self._getMicStarFile(mic), params, threshold, minDistance, fom) def _pickMicrographList(self, micList, params, threshold, minDistance, fom): micStar = self._getPath('input_micrographs_%s-%s.star' % (micList[0].strId(), micList[-1].strId())) writeSetOfMicrographs(micList, micStar, alignType=ALIGN_NONE, preprocessImageRow=self._preprocessMicrographRow) self._pickMicrographsFromStar(micStar, params, threshold, minDistance, fom) def _createSetOfCoordinates(self, micSet, suffix=''): """ Override this method to set the box size. """ coordSet = ProtParticlePickingAuto._createSetOfCoordinates(self, micSet, suffix=suffix) coordSet.setBoxSize(self.getBoxSize()) return coordSet def readCoordsFromMics(self, workingDir, micList, coordSet): """ Parse back the output star files and populate the SetOfCoordinates. """ template = self._getExtraPath("%s_autopick.star") starFiles = [template % pwutils.removeBaseExt(mic.getFileName()) for mic in micList] readSetOfCoordinates(coordSet, starFiles, micList) # -------------------------- STEPS functions ------------------------------- def autopickStep(self, micStarFile, params, threshold, minDistance, maxStddevNoise, fom): """ This method is used from the wizard to optimize the parameters. """ self._pickMicrographsFromStar(micStarFile, params, threshold, minDistance, maxStddevNoise, fom) def createOutputStep(self): micSet = self.getInputMicrographs() outputCoordinatesName = 'outputCoordinates' outputSuffix = '' # If in optimization phase, let's create a subset of the micrographs if self.isRunOptimize(): outputSuffix = '_subset' outputCoordinatesName = 'outputCoordinatesSubset' micSubSet = self._createSetOfMicrographs(suffix=outputSuffix) micSubSet.copyInfo(micSet) # Use previously written star file for reading the subset of micrographs, for row in md.iterRows(self._getPath('input_micrographs.star')): mic = micSet[row.getValue('rlnImageId')] micSubSet.append(mic) self._defineOutputs(outputMicrographsSubset=micSubSet) self._defineTransformRelation(self.getInputMicrographsPointer(), micSubSet) micSet = micSubSet coordSet = self._createSetOfCoordinates(micSet) template = self._getExtraPath("%s_autopick.star") starFiles = [template % pwutils.removeBaseExt(mic.getFileName()) for mic in micSet] readSetOfCoordinates(coordSet, starFiles, micSet) self._defineOutputs(**{outputCoordinatesName: coordSet}) self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet) # -------------------------- INFO functions -------------------------------- def _validate(self): errors = [] self.validatePackageVersion('RELION_HOME', errors) if self.useInputReferences(): if self.particleDiameter > self.getInputDimA(): errors.append('Particle diameter (%d) can not be greater than ' 'size (%d)' % (self.particleDiameter, self.getInputDimA())) if self.getInputReferences().isOddX(): errors.append("Relion only works with even values for the " "average dimensions!") else: if self.particleDiameter <= 0: errors.append('When using Gaussian blobs, you need to specify ' 'the particles diameter manually. ') if self.ctfRelations.get() is None and self.refsCtfCorrected: errors.append("References CTF corrected parameter must be set to " "False or set ctf relations.") errors.extend(self._validateMicSelection()) return errors def _validateMicSelection(self): """ Validate the cases when selecting a subset of micrographs to optimize. """ inputMics = self.getInputMicrographs() inputCTFs = self.ctfRelations.get() if self.isRunOptimize(): if self.micrographsSelection == MICS_AUTO: n = self.micrographsNumber.get() if n < 3 or n > min(30, inputMics.getSize()): return ['Number of micrographs should be between 3 and ' 'min(30, input_size)'] else: micSubset = self.micrographsSubset.get() if micSubset is None: return ['Select the subset of micrographs'] def missing(mic): micId = mic.getObjId() return inputMics[micId] is None or inputCTFs[micId] is None if any(missing(mic) for mic in micSubset): return ['Some selected micrograph IDs are missing from the ' 'input micrographs or CTFs.'] return [] def _warnings(self): if not self.isRunOptimize(): if not hasattr(self, 'wizardExecuted'): return ['It seems that you have not executed the wizard to ' 'optimize the picking parameters. \n' 'Do you want to launch the whole picking anyway?'] return [] def _summary(self): summary = [] return summary def _methods(self): methodsMsgs = [] if self.getOutputsSize() > 0: output = self.getCoords() methodsMsgs.append("%s: User picked %d particles with a particle " "size of %d px." % (self.getObjectTag(output), output.getSize(), output.getBoxSize())) else: methodsMsgs.append(Message.TEXT_NO_OUTPUT_CO) return methodsMsgs # -------------------------- UTILS functions ------------------------------- def useInputReferences(self): return self.referencesType == REF_AVERAGES def isRunOptimize(self): return self.runType == RUN_OPTIMIZE def getInputDimA(self): """ Return the dimension of input references in A. """ inputRefs = self.getInputReferences() if inputRefs is None: return None else: return inputRefs.getXDim() * inputRefs.getSamplingRate() def getBoxSize(self): """ Return a reasonable box-size in pixels. """ inputRefs = self.getInputReferences() inputMics = self.getInputMicrographs() micsSampling = inputMics.getSamplingRate() if inputRefs is None: boxSize = int(self.particleDiameter.get() * 1.25 / micsSampling) else: # Scale boxsize if the pixel size of the references is not the same # of the micrographs scale = inputRefs.getSamplingRate() / micsSampling boxSize = int(inputRefs.getXDim() * scale) if boxSize % 2 == 1: boxSize += 1 # Use even box size for relion return boxSize def getInputReferences(self): return self.inputReferences.get() def getInputMicrographsPointer(self): return self.inputMicrographs def getInputMicrographs(self): return self.getInputMicrographsPointer().get() def getMicrographList(self): """ Return the list of micrographs (either a subset or the full set) that will be used for optimizing the parameters or the picking. """ # Use all micrographs only when going for the full picking inputMics = self.getInputMicrographs() if not self.isRunOptimize(): return inputMics if self.micrographsSelection == MICS_AUTO: mics = getSubsetByDefocus(self.ctfRelations.get(), inputMics, self.micrographsNumber.get()) else: # Subset selection mics = [mic.clone() for mic in self.micrographsSubset.get()] return mics def getCoordsDir(self): return self._getTmpPath('xmipp_coordinates') def _writeXmippCoords(self, coordSet): micSet = self.getInputMicrographs() coordPath = self._getTmpPath('xmipp_coordinates') pwutils.cleanPath(coordPath) pwutils.makePath(coordPath) import pyworkflow.em.packages.xmipp3 as xmipp3 micPath = micSet.getFileName() xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False) return micPath, coordPath def writeXmippOutputCoords(self): return self._writeXmippCoords(self.outputCoordinates) def writeXmippCoords(self): """ Write the SetOfCoordinates as expected by Xmipp to be displayed with its GUI. """ micSet = self.getInputMicrographs() coordSet = self._createSetOfCoordinates(micSet) coordSet.setBoxSize(self.getBoxSize()) starFiles = [self._getExtraPath(pwutils.removeBaseExt(mic.getFileName()) + '_autopick.star') for mic in micSet] readSetOfCoordinates(coordSet, starFiles) return self._writeXmippCoords(coordSet) def _preprocessMicrographRow(self, img, imgRow): # Temporarly convert the few micrographs to tmp and make sure # they are in 'mrc' format # Get basename and replace extension by 'mrc' newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc') newPath = self._getExtraPath(newName) # If the micrographs are in 'mrc' format just create a link # if not, convert to 'mrc' if img.getFileName().endswith('mrc'): pwutils.createLink(img.getFileName(), newPath) else: self._ih.convert(img, newPath) # The command will be launched from the working dir # so, let's make the micrograph path relative to that img.setFileName(os.path.join('extra', newName)) # JMRT: The following is not needed since it is set when loading CTF # if self.ctfRelations.get() is not None: # img.setCTF(self.ctfDict[img.getMicName()]) def _postprocessMicrographRow(self, img, imgRow): imgRow.writeToFile(self._getMicStarFile(img)) def _getMicStarFile(self, mic): micBase = pwutils.replaceBaseExt(mic.getFileName(), 'star') return self._getExtraPath(micBase)
def _processMovie(self, movie): movId = movie.getObjId() x, y, n = movie.getDim() iniFrame, lastFrame, _ = movie.getFramesRange() frame0, frameN = self._getRange(movie) boxSize = self.boxSize.get() if movie.hasAlignment() and self.applyAlignment: shiftX, shiftY = movie.getAlignment().getShifts() # lists. else: shiftX = [0] * (lastFrame - iniFrame + 1) shiftY = shiftX stkIndex = 0 movieStk = self._getMovieName(movie, '.stk') movieMdFile = self._getMovieName(movie, '.xmd') movieMd = md.MetaData() frameMd = md.MetaData() frameMdImages = md.MetaData() frameRow = md.Row() if self._hasCoordinates(movie): imgh = ImageHandler() for frame in range(frame0, frameN + 1): indx = frame - iniFrame frameName = self._getFnRelated('frameMic', movId, frame) frameMdFile = self._getFnRelated('frameMdFile', movId, frame) coordinatesName = self._getFnRelated('frameCoords', movId, frame) frameImages = self._getFnRelated('frameImages', movId, frame) frameStk = self._getFnRelated('frameStk', movId, frame) self._writeXmippPosFile(movie, coordinatesName, shiftX[indx], shiftY[indx]) self.info("Writing frame: %s" % frameName) # TODO: there is no need to write the frame and then operate # the input of the first operation should be the movie movieName = imgh.fixXmippVolumeFileName(movie) imgh.convert((frame, movieName), frameName) if self.doRemoveDust: self.info("Removing Dust") self._runNoDust(frameName) self.info("Extracting particles") args = '-i %(frameName)s --pos %(coordinatesName)s ' \ '-o %(frameImages)s --Xdim %(boxSize)d' % locals() if self.doInvert: args += " --invert" if self.doBorders: args += " --fillBorders" args += " --downsampling %f " % self.getBoxScale() self.runJob('xmipp_micrograph_scissor', args) cleanPath(frameName) self.info("Combining particles into one stack.") frameMdImages.read(frameMdFile) frameMd.read('particles@%s' % coordinatesName) frameMd.merge(frameMdImages) for objId in frameMd: stkIndex += 1 frameRow.readFromMd(frameMd, objId) location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE)) newLocation = (stkIndex, movieStk) imgh.convert(location, newLocation) # Fix the name to be accessible from the Project directory # so we know that the movie stack file will be moved # to final particles folder newImageName = '%d@%s' % newLocation frameRow.setValue(md.MDL_IMAGE, newImageName) frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movId)) frameRow.setValue(md.MDL_MICROGRAPH, str(movId)) frameRow.setValue(md.MDL_FRAME_ID, long(frame)) frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID)) frameRow.writeToMd(movieMd, movieMd.addObject()) movieMd.addItemId() movieMd.write(movieMdFile) cleanPath(frameStk) if self.doNormalize: numberOfFrames = frameN - frame0 + 1 self._runNormalize(movieStk, numberOfFrames)
class ProtRelion2Autopick(ProtParticlePickingAuto, ProtRelionBase): """ This Relion protocol uses the 'relion_autopick' program to pick particles from micrographs, either using templates or gaussian blobs. The picking with this protocol is divided in three steps: 1) Run with 'Optimize' option for several (less than 30) micrographs. 2) Execute the wizard to refine the picking parameters. 3) Run with 'Pick all' option to pick particles from all micrographs. The first steps will use internally the option '--write-fom-maps' to write to disk the FOM maps. The expensive part of this calculation is to calculate a probability-based figure-of-merit (related to the cross-correlation coefficient between each rotated reference and all positions in the micrographs. That's why it is only done in an small subset of the micrographs, where one should use representative micrographs for the entire data set, e.g. a high and a low-defocus one, and/or with thin or thick ice. Step 2 uses a much cheaper peak-detection algorithm that uses the threshold and minimum distance parameters. """ _label = 'auto-picking' @classmethod def isDisabled(cls): return not (isVersion2() or isVersion3()) # -------------------------- DEFINE param functions ------------------------ def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputMicrographs', params.PointerParam, pointerClass='SetOfMicrographs', label='Input micrographs', important=True, help='Select the input micrographs. ' 'If using the *Optimize* mode, just a subset of ' 'micrographs are used to compute the FOM maps. ' 'If in *Compute* mode, all micrographs will be ' 'auto-picked.') form.addParam('ctfRelations', params.RelationParam, relationName=RELATION_CTF, attributeName='getInputMicrographs', label='CTF estimation', help='Choose some CTF estimation related to the ' 'input micrographs.') form.addParam('runType', params.EnumParam, default=RUN_OPTIMIZE, choices=['Optimize params', 'Pick all micrographs'], display=params.EnumParam.DISPLAY_LIST, label='Run type: ', help='Usually, first you should use the *Optimize* mode ' 'to compute the FOM maps for a few micrographs and ' 'use them to tune the picking parameters using the ' 'wizard. After that you can run the job in *Compute*' ' mode and auto-pick all the micrographs. ') group = form.addGroup('Micrographs for optimization', condition='runType==%d' % RUN_OPTIMIZE) group.addParam('micrographsSelection', params.EnumParam, default=MICS_AUTO, choices=['automatic selection', 'input subset'], display=params.EnumParam.DISPLAY_HLIST, label='Choose micrographs by', help='If you choose "automatic selection", you only ' 'need to provide the number of microgrphs to use ' 'and that number will be selected to cover the ' 'defocus range. ') group.addParam('micrographsNumber', params.IntParam, default='10', condition='micrographsSelection==%d' % MICS_AUTO, label='Micrographs for optimization:', help='Select the number of micrographs that you want' 'to be used for the parameters optimization. ') group.addParam('micrographsSubset', params.PointerParam, condition='micrographsSelection==%d' % MICS_SUBSET, pointerClass='SetOfMicrographs', label='Subset of micrographs', help='Choose as input a subset of micrographs that ' 'you have previously selected. ' '(Probably covering the defocus range).') # From Relion 2.+, it can be picked with gaussian blobs, so we # need to add these parameters refCondition = 'referencesType==%s' % REF_AVERAGES group = form.addGroup('References') group.addParam('referencesType', params.EnumParam, choices=['References', 'Gaussian blobs'], default=REF_AVERAGES, display=params.EnumParam.DISPLAY_HLIST, label='References type', help='You may select "Gaussian blobs" to be used as ' 'references. The preferred way to autopick is ' 'by providing 2D references images that were ' 'obtained by 2D classification. \n' 'The Gaussian blob references may be useful to ' 'kickstart a new data set.') group.addParam('gaussianPeak', params.FloatParam, default=0.1, condition='referencesType==%s' % REF_BLOBS, label='Gaussian peak value', help='The peak value of the Gaussian blob. ' 'Weaker data will need lower values.') pointerClassStr = 'SetOfAverages' # In Relion 3 it is also possible to pass a volume as reference for # autopicking if isVersion3(): pointerClassStr += ",Volume" group.addParam( 'inputReferences', params.PointerParam, pointerClass='SetOfAverages', condition=refCondition, label='Input references', important=True, help='Input references (SetOfAverages) for auto-pick. \n\n' 'Note that the absolute greyscale needs to be correct, \n' 'so only use images with proper normalization. ' 'From Relion 3.0 it is also possible to provide a ' '3D volume which projections will be used as ' 'references.') group.addParam( 'particleDiameter', params.IntParam, default=-1, label='Mask diameter (A)', help='Diameter of the circular mask that will be applied ' 'around the templates in Angstroms. When set to a ' 'negative value, this value is estimated ' 'automatically from the templates themselves.') form.addSection('References') form.addParam('lowpassFilterRefs', params.IntParam, default=20, condition=refCondition, label='Lowpass filter references (A)', help='Lowpass filter that will be applied to the ' 'references before template matching. \n' 'Do NOT use very high-resolution templates to ' 'search your micrographs. \n' 'The signal will be too weak at high resolution ' 'anyway, and you may find Einstein from noise...') form.addParam('highpassFilterMics', params.IntParam, default=-1, label='Highpass filter (A)', help='Highpass filter that will be applied to the ' 'micrographs. This may be useful to get rid of ' 'background ramps due to uneven ice distributions. ' 'Give a negative value to skip the highpass ' 'filter. Useful values are often in the range ' 'of 200-400 Angstroms.') form.addParam( 'angularSampling', params.IntParam, default=5, label='Angular sampling (deg)', help='Angular sampling in degrees for exhaustive searches ' 'of the in-plane rotations for all references.') form.addParam('refsHaveInvertedContrast', params.BooleanParam, default=True, label='References have inverted contrast?', help='Set to Yes to indicate that the reference have ' 'inverted contrast with respect to the particles ' 'in the micrographs.') form.addParam('refsCtfCorrected', params.BooleanParam, default=True, condition=refCondition, label='Are References CTF corrected?', help='Set to Yes if the references were created with ' 'CTF-correction inside RELION.\n' 'If set to Yes, the input micrographs should contain ' 'the CTF information.') form.addParam( 'ignoreCTFUntilFirstPeak', params.BooleanParam, condition=refCondition, default=False, expertLevel=params.LEVEL_ADVANCED, label='Ignore CTFs until first peak?', help='Set this to Yes, only if this option was also used ' 'to generate the references.') form.addSection('Autopicking') group = form.addGroup('Autopick') group.addParam('pickingThreshold', params.FloatParam, default=0.25, label='Picking threshold:', help='Use lower thresholds to pick more particles ' '(and more junk probably)') group.addParam('interParticleDistance', params.IntParam, default=-1, label='Minimum inter-particle distance (A):', help='Particles closer together than this distance \n' 'will be consider to be a single cluster. \n' 'From each cluster, only one particle will be ' 'picked.') group.addParam( 'maxStddevNoise', params.FloatParam, default=1.1, label='Maximum stddev noise:', help='This is useful to prevent picking in carbon areas, ' 'or areas with big contamination features. Peaks in ' 'areas where the background standard deviation in ' 'the normalized micrographs is higher than this ' 'value will be ignored. Useful values are probably ' 'in the range 1.0 to 1.2. Set to -1 to switch off ' 'the feature to eliminate peaks due to high ' 'background standard deviations.') group = form.addGroup('Computing') group.addParam('shrinkFactor', params.FloatParam, default=0, validators=[ params.Range(0, 1, "value should be " "between 0 and 1. ") ], label='Shrink factor', help='This is useful to speed up the calculations, ' 'and to make them less memory-intensive. The ' 'micrographs will be downscaled (shrunk) to ' 'calculate the cross-correlations, and peak ' 'searching will be done in the downscaled FOM ' 'maps. When set to 0, the micrographs will de ' 'downscaled to the lowpass filter of the ' 'references, a value between 0 and 1 will ' 'downscale the micrographs by that factor. ' 'Note that the results will not be exactly ' 'the same when you shrink micrographs!') group.addParam('doGpu', params.BooleanParam, default=True, label='Use GPU acceleration?', help='If set to Yes, the job will try to use GPU ' 'acceleration.') group.addParam('gpusToUse', params.StringParam, default='', label='Which GPUs to use:', condition='doGpu', help='This argument is not necessary. If left empty, ' 'the job itself will try to allocate available GPU ' 'resources. You can override the default ' 'allocation by providing a list of which GPUs ' '(0,1,2,3, etc) to use. MPI-processes are ' 'separated by ":", threads by ",". ' 'For example: "0,0:1,1:0,0:1,1"') form.addParam( 'extraParams', params.StringParam, default='', label='Additional arguments:', help='In this box command-line arguments may be provided ' 'that are not generated by the GUI. This may be ' 'useful for testing developmental options and/or ' 'expert use of the program. \n' 'The command "relion_autopick" will print a list ' 'of possible options.') form.addSection('Helix') form.addParam('fomLabel', params.LabelParam, important=True, label='Helix processing is not implemented still.') self._defineStreamingParams(form) form.addParallelSection(threads=0, mpi=4) # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): self.inputStreaming = self.getInputMicrographs().isStreamOpen() if self.inputStreaming and not self.isRunOptimize(): # If the input is in streaming, follow the base class policy # about inserting new steps and discovery new input/output ProtParticlePickingAuto._insertAllSteps(self) self.createOutputStep = self._doNothing else: # If not in streaming, then we will just insert a single step to # pick all micrographs at once since it is much faster self._insertInitialSteps() self._insertFunctionStep('_pickMicrographsFromStar', self._getPath('input_micrographs.star'), *self._getPickArgs()) self._insertFunctionStep('createOutputStep') # Disable streaming functions: self._insertFinalSteps = self._doNothing self._stepsCheck = self._doNothing def _insertInitialSteps(self): # Convert the input micrographs and references to # the required Relion star files inputRefs = self.getInputReferences() refId = inputRefs.strId() if self.useInputReferences() else 'Gaussian' convertId = self._insertFunctionStep( 'convertInputStep', self.getInputMicrographs().strId(), refId, self.runType.get()) return [convertId] def _doNothing(self, *args): pass # used to avoid some streaming functions def _loadInputList(self): """ This function is re-implemented in this protocol, because it have a SetOfCTF as input, so for streaming, we only want to report those micrographs for which the CTF is ready. """ micDict, micClose = self._loadMics(self.getInputMicrographs()) ctfDict, ctfClosed = self._loadCTFs(self.ctfRelations.get()) # Remove the micrographs that have not CTF # and set the CTF property for those who have it for micKey, mic in micDict.iteritems(): if micKey in ctfDict: mic.setCTF(ctfDict[micKey]) else: del micDict[micKey] # Return the updated micDict and the closed status return micDict, micClose and ctfClosed # -------------------------- STEPS functions ------------------------------ def convertInputStep(self, micsId, refsId, runType): # runType is passed as parameter to force a re-execute of this step # if there is a change in the type self._ih = ImageHandler() # used to convert micrographs # Match ctf information against the micrographs self.ctfDict = {} if self.ctfRelations.get() is not None: for ctf in self.ctfRelations.get(): self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone() micStar = self._getPath('input_micrographs.star') writeSetOfMicrographs(self.getMicrographList(), micStar, alignType=ALIGN_NONE, preprocessImageRow=self._preprocessMicrographRow) if self.useInputReferences(): writeReferences(self.getInputReferences(), self._getPath('input_references'), useBasename=True) # FIXME: (JMRT-20180523) The following code does not seems to work # here it has been worked around by changing the name of the wizard # output but this seems to reflect a deeper problem of deleting # already existing output objects in a protocol. Maybe when updating # from run.db to project.sqlite? # Clean up if previously created the outputMicrographs and Coordinates # in the wizard - optimization run # if self.hasAttribute('outputMicrographs'): # self._deleteChild('outputMicrographs', self.outputMicrographs) # if self.hasAttribute('outputCoordinates'): # self._deleteChild('outputCoordinates', self.outputCoordinates) # self._store() def getAutopickParams(self): # Return the autopicking parameters except for the interative ones: # - threshold # - minDistance # - maxStd params = ' --pickname autopick' params += ' --odir ""' params += ' --particle_diameter %d' % self.particleDiameter params += ' --angpix %0.3f' % self.getInputMicrographs( ).getSamplingRate() params += ' --shrink %0.3f' % self.shrinkFactor if self.doGpu: params += ' --gpu "%s"' % self.gpusToUse # Now in Relion2.0 autopick can use gassian blobs if self.useInputReferences(): params += ' --ref input_references.star' ps = self.getInputReferences().getSamplingRate() params += ' --angpix_ref %0.3f' % ps else: # Gaussian blobs params += ' --ref gauss --gauss_max %0.3f' % self.gaussianPeak if self.refsHaveInvertedContrast: params += ' --invert' if self.refsCtfCorrected: params += ' --ctf' params += ' --ang %d' % self.angularSampling # Negative values for filters means no-filter if self.lowpassFilterRefs > 0: params += ' --lowpass %d' % self.lowpassFilterRefs if self.highpassFilterMics > 0: params += ' --highpass %d' % self.highpassFilterMics # Add extra params is any params += ' %s' % self.extraParams return params def _getPickArgs(self): basicArgs = self.getAutopickParams() threshold = self.pickingThreshold.get() interDist = self.interParticleDistance.get() fomParam = ' --write_fom_maps' if self.isRunOptimize() else '' return [basicArgs, threshold, interDist, fomParam] def _pickMicrographsFromStar(self, micStarFile, params, threshold, minDistance, fom): """ Launch the 'relion_autopick' for micrographs in the inputStarFile. If the input set of complete, the star file will contain all the micrographs. If working in streaming, it will be only one micrograph. """ params += ' --i %s' % relpath(micStarFile, self.getWorkingDir()) params += ' --threshold %0.3f ' % threshold params += ' --min_distance %0.3f %s' % (minDistance, fom) program = self._getProgram('relion_autopick') self.runJob(program, params, cwd=self.getWorkingDir()) def _pickMicrograph(self, mic, params, threshold, minDistance, fom): """ This method should be invoked only when working in streaming mode. """ micRow = md.Row() self._preprocessMicrographRow(mic, micRow) micrographToRow(mic, micRow) self._postprocessMicrographRow(mic, micRow) self._pickMicrographsFromStar(self._getMicStarFile(mic), params, threshold, minDistance, fom) def _pickMicrographList(self, micList, params, threshold, minDistance, fom): micStar = self._getPath('input_micrographs_%s-%s.star' % (micList[0].strId(), micList[-1].strId())) writeSetOfMicrographs(micList, micStar, alignType=ALIGN_NONE, preprocessImageRow=self._preprocessMicrographRow) self._pickMicrographsFromStar(micStar, params, threshold, minDistance, fom) def _createSetOfCoordinates(self, micSet, suffix=''): """ Override this method to set the box size. """ coordSet = ProtParticlePickingAuto._createSetOfCoordinates( self, micSet, suffix=suffix) coordSet.setBoxSize(self.getBoxSize()) return coordSet def readCoordsFromMics(self, workingDir, micList, coordSet): """ Parse back the output star files and populate the SetOfCoordinates. """ template = self._getExtraPath("%s_autopick.star") starFiles = [ template % pwutils.removeBaseExt(mic.getFileName()) for mic in micList ] readSetOfCoordinates(coordSet, starFiles, micList) # -------------------------- STEPS functions ------------------------------- def autopickStep(self, micStarFile, params, threshold, minDistance, maxStddevNoise, fom): """ This method is used from the wizard to optimize the parameters. """ self._pickMicrographsFromStar(micStarFile, params, threshold, minDistance, maxStddevNoise, fom) def createOutputStep(self): micSet = self.getInputMicrographs() outputCoordinatesName = 'outputCoordinates' outputSuffix = '' # If in optimization phase, let's create a subset of the micrographs if self.isRunOptimize(): outputSuffix = '_subset' outputCoordinatesName = 'outputCoordinatesSubset' micSubSet = self._createSetOfMicrographs(suffix=outputSuffix) micSubSet.copyInfo(micSet) # Use previously written star file for reading the subset of micrographs, for row in md.iterRows(self._getPath('input_micrographs.star')): mic = micSet[row.getValue('rlnImageId')] micSubSet.append(mic) self._defineOutputs(outputMicrographsSubset=micSubSet) self._defineTransformRelation(self.getInputMicrographsPointer(), micSubSet) micSet = micSubSet coordSet = self._createSetOfCoordinates(micSet) template = self._getExtraPath("%s_autopick.star") starFiles = [ template % pwutils.removeBaseExt(mic.getFileName()) for mic in micSet ] readSetOfCoordinates(coordSet, starFiles, micSet) self._defineOutputs(**{outputCoordinatesName: coordSet}) self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet) # -------------------------- INFO functions -------------------------------- def _validate(self): errors = [] self.validatePackageVersion('RELION_HOME', errors) if self.useInputReferences(): if self.particleDiameter > self.getInputDimA(): errors.append('Particle diameter (%d) can not be greater than ' 'size (%d)' % (self.particleDiameter, self.getInputDimA())) if self.getInputReferences().isOddX(): errors.append("Relion only works with even values for the " "average dimensions!") else: if self.particleDiameter <= 0: errors.append('When using Gaussian blobs, you need to specify ' 'the particles diameter manually. ') if self.ctfRelations.get() is None and self.refsCtfCorrected: errors.append("References CTF corrected parameter must be set to " "False or set ctf relations.") errors.extend(self._validateMicSelection()) return errors def _validateMicSelection(self): """ Validate the cases when selecting a subset of micrographs to optimize. """ inputMics = self.getInputMicrographs() inputCTFs = self.ctfRelations.get() if self.isRunOptimize(): if self.micrographsSelection == MICS_AUTO: n = self.micrographsNumber.get() if n < 3 or n > min(30, inputMics.getSize()): return [ 'Number of micrographs should be between 3 and ' 'min(30, input_size)' ] else: micSubset = self.micrographsSubset.get() if micSubset is None: return ['Select the subset of micrographs'] def missing(mic): micId = mic.getObjId() return inputMics[micId] is None or inputCTFs[micId] is None if any(missing(mic) for mic in micSubset): return [ 'Some selected micrograph IDs are missing from the ' 'input micrographs or CTFs.' ] return [] def _warnings(self): if not self.isRunOptimize(): if not hasattr(self, 'wizardExecuted'): return [ 'It seems that you have not executed the wizard to ' 'optimize the picking parameters. \n' 'Do you want to launch the whole picking anyway?' ] return [] def _summary(self): summary = [] return summary def _methods(self): methodsMsgs = [] if self.getOutputsSize() > 0: output = self.getCoords() methodsMsgs.append("%s: User picked %d particles with a particle " "size of %d px." % (self.getObjectTag(output), output.getSize(), output.getBoxSize())) else: methodsMsgs.append(Message.TEXT_NO_OUTPUT_CO) return methodsMsgs # -------------------------- UTILS functions ------------------------------- def useInputReferences(self): return self.referencesType == REF_AVERAGES def isRunOptimize(self): return self.runType == RUN_OPTIMIZE def getInputDimA(self): """ Return the dimension of input references in A. """ inputRefs = self.getInputReferences() if inputRefs is None: return None else: return inputRefs.getXDim() * inputRefs.getSamplingRate() def getBoxSize(self): """ Return a reasonable box-size in pixels. """ inputRefs = self.getInputReferences() inputMics = self.getInputMicrographs() micsSampling = inputMics.getSamplingRate() if inputRefs is None: boxSize = int(self.particleDiameter.get() * 1.25 / micsSampling) else: # Scale boxsize if the pixel size of the references is not the same # of the micrographs scale = inputRefs.getSamplingRate() / micsSampling boxSize = int(inputRefs.getXDim() * scale) if boxSize % 2 == 1: boxSize += 1 # Use even box size for relion return boxSize def getInputReferences(self): return self.inputReferences.get() def getInputMicrographsPointer(self): return self.inputMicrographs def getInputMicrographs(self): return self.getInputMicrographsPointer().get() def getMicrographList(self): """ Return the list of micrographs (either a subset or the full set) that will be used for optimizing the parameters or the picking. """ # Use all micrographs only when going for the full picking inputMics = self.getInputMicrographs() if not self.isRunOptimize(): return inputMics if self.micrographsSelection == MICS_AUTO: mics = getSubsetByDefocus(self.ctfRelations.get(), inputMics, self.micrographsNumber.get()) else: # Subset selection mics = [mic.clone() for mic in self.micrographsSubset.get()] return mics def getCoordsDir(self): return self._getTmpPath('xmipp_coordinates') def _writeXmippCoords(self, coordSet): micSet = self.getInputMicrographs() coordPath = self._getTmpPath('xmipp_coordinates') pwutils.cleanPath(coordPath) pwutils.makePath(coordPath) import pyworkflow.em.packages.xmipp3 as xmipp3 micPath = micSet.getFileName() xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False) return micPath, coordPath def writeXmippOutputCoords(self): return self._writeXmippCoords(self.outputCoordinates) def writeXmippCoords(self): """ Write the SetOfCoordinates as expected by Xmipp to be displayed with its GUI. """ micSet = self.getInputMicrographs() coordSet = self._createSetOfCoordinates(micSet) coordSet.setBoxSize(self.getBoxSize()) starFiles = [ self._getExtraPath( pwutils.removeBaseExt(mic.getFileName()) + '_autopick.star') for mic in micSet ] readSetOfCoordinates(coordSet, starFiles) return self._writeXmippCoords(coordSet) def _preprocessMicrographRow(self, img, imgRow): # Temporarly convert the few micrographs to tmp and make sure # they are in 'mrc' format # Get basename and replace extension by 'mrc' newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc') newPath = self._getExtraPath(newName) # If the micrographs are in 'mrc' format just create a link # if not, convert to 'mrc' if img.getFileName().endswith('mrc'): pwutils.createLink(img.getFileName(), newPath) else: self._ih.convert(img, newPath) # The command will be launched from the working dir # so, let's make the micrograph path relative to that img.setFileName(os.path.join('extra', newName)) # JMRT: The following is not needed since it is set when loading CTF # if self.ctfRelations.get() is not None: # img.setCTF(self.ctfDict[img.getMicName()]) def _postprocessMicrographRow(self, img, imgRow): imgRow.writeToFile(self._getMicStarFile(img)) def _getMicStarFile(self, mic): micBase = pwutils.replaceBaseExt(mic.getFileName(), 'star') return self._getExtraPath(micBase)
def _processMovie(self, movieId, movieName, movieFolder, shifts): ###pasar shifts movieName = os.path.join(movieFolder, movieName) boxSize = self.boxSize.get() # Read movie dimensions to iterate through each frame imgh = ImageHandler() x, y, z, n = imgh.getDimensions(movieName) first = self.firstFrame.get() if first <= 1: first = 1 last = self.lastFrame.get() if last <= 0 or last >= n: last = n numberOfFrames = last - first + 1 if shifts is None: frames = max(z, n) shifts = [0] * (2 * frames) stkIndex = 0 movieStk = self._getMovieName(movieId, '.stk') movieMdFile = self._getMovieName(movieId, '.xmd') movieMd = md.MetaData() frameMd = md.MetaData() frameMdImages = md.MetaData() frameRow = md.Row() for frame in range(first, last + 1): # Get the frame shifts index = frame - first shiftX = shifts[2 * index] shiftY = shifts[2 * index + 1] frameRoot = os.path.join(movieFolder, 'frame_%02d' % frame) frameName = frameRoot + '.mrc' frameMdFile = frameRoot + '.xmd' framePosFile = frameRoot + '_coordinates.xmd' coordinatesName = frameRoot + '_coordinates.xmd' hasCoordinates = self._writeXmippPosFile(movieId, movieName, coordinatesName, shiftX, shiftY) if hasCoordinates: self.info("Writing frame: %s" % frameName) #TODO: there is no need to write the frame and then operate #the input of the first operation should be the movie imgh.convert(tuple([frame, movieName]), frameName) if self.doRemoveDust: self.info("Removing Dust") self._runNoDust(frameName) self.info("Extracting particles") frameImages = frameRoot + '_images' args = '-i %(frameName)s --pos %(coordinatesName)s ' \ '-o %(frameRoot)s --Xdim %(boxSize)d' % locals() if self.doInvert: args += " --invert" args += " --downsampling %f " % self.factor self.runJob('xmipp_micrograph_scissor', args) cleanPath(frameName) frameStk = frameRoot + '.stk' self.info("Combining particles into one stack.") frameMdImages.read(frameMdFile) frameMd.read('particles@%s' % framePosFile) frameMd.merge(frameMdImages) for objId in frameMd: stkIndex += 1 frameRow.readFromMd(frameMd, objId) location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE)) newLocation = (stkIndex, movieStk) imgh.convert(location, newLocation) # Fix the name to be accesible from the Project directory # so we know that the movie stack file will be moved # to final particles folder newImageName = '%d@%s' % newLocation frameRow.setValue(md.MDL_IMAGE, newImageName) frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movieId)) frameRow.setValue(md.MDL_MICROGRAPH, str(movieId)) frameRow.setValue(md.MDL_FRAME_ID, long(frame)) frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID)) frameRow.writeToMd(movieMd, movieMd.addObject()) movieMd.addItemId() movieMd.write(movieMdFile) cleanPath(frameStk) if self.doNormalize: self._runNormalize(movieStk, numberOfFrames)
def _processMovie(self, movieId, movieName, movieFolder, shifts):###pasar shifts movieName = os.path.join(movieFolder, movieName) boxSize = self.boxSize.get() # Read movie dimensions to iterate through each frame imgh = ImageHandler() x, y, z, n = imgh.getDimensions(movieName) first = self.firstFrame.get() if first <= 1: first = 1 last = self.lastFrame.get() if last <= 0 or last >= n: last = n numberOfFrames = last - first + 1 stkIndex = 0 movieStk = self._getMovieName(movieId, '.stk') movieMdFile = self._getMovieName(movieId, '.xmd') movieMd = md.MetaData() frameMd = md.MetaData() frameMdImages = md.MetaData() frameRow = md.Row() for frame in range(first, last+1): # Get the frame shifts index = frame - first shiftX = shifts[2*index] shiftY = shifts[2*index+1] frameRoot = os.path.join(movieFolder, 'frame_%02d' % frame) frameName = frameRoot + '.mrc' frameMdFile = frameRoot + '.xmd' framePosFile = frameRoot + '_coordinates.xmd' coordinatesName = frameRoot + '_coordinates.xmd' hasCoordinates = self._writeXmippPosFile(movieId, movieName, coordinatesName, shiftX, shiftY) if hasCoordinates: self.info("Writing frame: %s" % frameName) #TODO: there is no need to write the frame and then operate #the input of the first operation should be the movie imgh.convert(tuple([frame, movieName]), frameName) if self.doRemoveDust: self.info("Removing Dust") self._runNoDust(frameName) self.info("Extracting particles") frameImages = frameRoot + '_images' args = '-i %(frameName)s --pos %(coordinatesName)s ' \ '-o %(frameRoot)s --Xdim %(boxSize)d' % locals() if self.doInvert: args += " --invert" args += " --downsampling %f " % self.factor self.runJob('xmipp_micrograph_scissor', args) cleanPath(frameName) frameStk = frameRoot + '.stk' self.info("Combining particles into one stack.") frameMdImages.read(frameMdFile) frameMd.read('particles@%s' % framePosFile) frameMd.merge(frameMdImages) for objId in frameMd: stkIndex += 1 frameRow.readFromMd(frameMd, objId) location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE)) newLocation = (stkIndex, movieStk) imgh.convert(location, newLocation) # Fix the name to be accesible from the Project directory # so we know that the movie stack file will be moved # to final particles folder newImageName = '%d@%s' % newLocation frameRow.setValue(md.MDL_IMAGE, newImageName) frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movieId)) frameRow.setValue(md.MDL_MICROGRAPH, str(movieId)) frameRow.setValue(md.MDL_FRAME_ID, long(frame)) frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID)) frameRow.writeToMd(movieMd, movieMd.addObject()) movieMd.addItemId() movieMd.write(movieMdFile) cleanPath(frameStk) if self.doNormalize: self._runNormalize(movieStk, numberOfFrames)
def _processMovie(self, movie): movId = movie.getObjId() x, y, n = movie.getDim() iniFrame, lastFrame, _ = movie.getFramesRange() frame0, frameN = self._getRange(movie) boxSize = self.boxSize.get() if movie.hasAlignment() and self.applyAlignment: shiftX, shiftY = movie.getAlignment().getShifts() # lists. else: shiftX = [0] * (lastFrame-iniFrame+1) shiftY = shiftX stkIndex = 0 movieStk = self._getMovieName(movie, '.stk') movieMdFile = self._getMovieName(movie, '.xmd') movieMd = md.MetaData() frameMd = md.MetaData() frameMdImages = md.MetaData() frameRow = md.Row() if self._hasCoordinates(movie): imgh = ImageHandler() for frame in range(frame0, frameN+1): indx = frame-iniFrame frameName = self._getFnRelated('frameMic',movId, frame) frameMdFile = self._getFnRelated('frameMdFile',movId, frame) coordinatesName = self._getFnRelated('frameCoords',movId, frame) frameImages = self._getFnRelated('frameImages',movId, frame) frameStk = self._getFnRelated('frameStk', movId, frame) self._writeXmippPosFile(movie, coordinatesName, shiftX[indx], shiftY[indx]) self.info("Writing frame: %s" % frameName) # TODO: there is no need to write the frame and then operate # the input of the first operation should be the movie movieName = imgh.fixXmippVolumeFileName(movie) imgh.convert((frame, movieName), frameName) if self.doRemoveDust: self.info("Removing Dust") self._runNoDust(frameName) self.info("Extracting particles") args = '-i %(frameName)s --pos %(coordinatesName)s ' \ '-o %(frameImages)s --Xdim %(boxSize)d' % locals() if self.doInvert: args += " --invert" if self.doBorders: args += " --fillBorders" args += " --downsampling %f " % self.getBoxScale() self.runJob('xmipp_micrograph_scissor', args) cleanPath(frameName) self.info("Combining particles into one stack.") frameMdImages.read(frameMdFile) frameMd.read('particles@%s' % coordinatesName) frameMd.merge(frameMdImages) for objId in frameMd: stkIndex += 1 frameRow.readFromMd(frameMd, objId) location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE)) newLocation = (stkIndex, movieStk) imgh.convert(location, newLocation) # Fix the name to be accessible from the Project directory # so we know that the movie stack file will be moved # to final particles folder newImageName = '%d@%s' % newLocation frameRow.setValue(md.MDL_IMAGE, newImageName) frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movId)) frameRow.setValue(md.MDL_MICROGRAPH, str(movId)) frameRow.setValue(md.MDL_FRAME_ID, long(frame)) frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID)) frameRow.writeToMd(movieMd, movieMd.addObject()) movieMd.addItemId() movieMd.write(movieMdFile) cleanPath(frameStk) if self.doNormalize: numberOfFrames = frameN - frame0 + 1 self._runNormalize(movieStk, numberOfFrames)
def runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName): from pyworkflow.em.convert import ImageHandler img = ImageHandler() img.convert(reconstructedFilteredVolume, maskedFileName)