def convertInputStep(self, particlesId, volId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ inputParticles = self.inputParticles.get() inputVolume = self.inputVolume.get() writeSetOfParticles(inputParticles, self._getExpParticlesFn()) img = ImageHandler() img.convert(inputVolume, self._getInputVolFn()) if self._useSeveralClasses(): # Scale particles Xdim = inputParticles.getXDim() Ts = inputParticles.getSamplingRate() newTs = self.targetResolution.get() * 0.4 newTs = max(Ts, newTs) newXdim = long(Xdim * Ts / newTs) self.writeInfoField(self._getExtraPath(), "sampling", xmippLib.MDL_SAMPLINGRATE, newTs) self.writeInfoField(self._getExtraPath(), "size", xmippLib.MDL_XSIZE, newXdim) self.runJob("xmipp_image_resize", "-i %s -o %s --save_metadata_stack %s --fourier %d" % (self._getExpParticlesFn(), self._getTmpPath('scaled_particles.stk'), self._getTmpPath('scaled_particles.xmd'), newXdim)) # Scale volume Xdim = inputVolume.getXDim() if Xdim != newXdim: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (self._getInputVolFn(), newXdim), numberOfMpi=1)
def convertInputStep(self, classesFn): inputSet = self.inputSet.get() if isinstance(inputSet, SetOfClasses2D): writeSetOfClasses2D(inputSet, classesFn) else: writeSetOfParticles(inputSet, classesFn)
def convertInputStep(self, *dataDicts): def __getSetOfParticlesFromFname(fname): if fname == self._getExtraPath("inputTrueParticlesSet.xmd"): return self.inTrueSetOfParticles.get() elif fname == self._getExtraPath("predictSetOfParticles.xmd"): return self.predictSetOfParticles.get() elif fname == self._getExtraPath("testTrueParticlesSet.xmd"): return self.testPosSetOfParticles.get() elif fname == self._getExtraPath("testFalseParticlesSet.xmd"): return self.testNegSetOfParticles.get() else: matchOjb = re.match( self._getExtraPath("negativeSet_(\d+).xmd"), fname) if matchOjb: num = matchOjb.group(1) return self.__dict__["negativeSet_%s" % num].get() else: raise ValueError("Error, unexpected fname") if ((not self.doContinue.get() or self.keepTraining.get()) and self.nEpochs.get() > 0): assert not self.inTrueSetOfParticles.get() is None, \ "Positive particles must be provided for training if nEpochs!=0" for dataDict in dataDicts: if not dataDict is None: for fnameParticles in sorted(dataDict): setOfParticles = __getSetOfParticlesFromFname( fnameParticles) writeSetOfParticles(setOfParticles, fnameParticles)
def convertInputStep(self, particlesId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ writeSetOfParticles(self.partSet, self._getMdParticles())
def eliminationStep(self, fnInputMd): self.inputImages = self.inputParticles.get() partsFile = self.inputImages.getFileName() self.partsSet = SetOfParticles(filename=partsFile) self.partsSet.loadAllProperties() self.streamClosed = self.partsSet.isStreamClosed() if self.check == None: writeSetOfParticles(self.partsSet, fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation') else: writeSetOfParticles(self.partsSet, fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation', where='creation>"' + str(self.check) + '"') for p in self.partsSet.iterItems(orderBy='creation', direction='DESC'): self.check = p.getObjCreation() break self.partsSet.close() self.lenPartsSet = len(self.partsSet) print("os.path.exists(fnInputMd)): %s" % os.path.exists(fnInputMd)) args = "-i %s -o %s -e %s -t %f" % ( fnInputMd, self.fnOutputMd, self.fnElimMd, self.threshold.get()) if self.addFeatures: args += " --addFeatures" if self.useDenoising: args += " --useDenoising -d %f" % self.denoising.get() self.runJob("xmipp_image_eliminate_empty_particles", args) cleanPath(fnInputMd)
def eliminationStep(self, fnInputMd): self.preparePartsSet() self.inputImages.loadAllProperties() self.streamClosed = self.inputImages.isStreamClosed() if self.check == None: writeSetOfParticles(self.inputImages, fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation') else: writeSetOfParticles(self.inputImages, fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation', where='creation>"' + str(self.check) + '"') idsToCheck = [] for p in self.inputImages.iterItems(orderBy='creation', direction='ASC'): self.check = p.getObjCreation() idsToCheck.append(p.getObjId()) self.inputImages.close() self.lenPartsSet = len(self.inputImages) self.rejectByPopulation(idsToCheck) args = "-i %s -o %s -e %s -t %f" % ( fnInputMd, self.fnOutputMd, self.fnElimMd, self.threshold.get()) if self.addFeatures: args += " --addFeatures" if self.useDenoising: args += " --useDenoising -d %f" % self.denoising.get() self.runJob("xmipp_image_eliminate_empty_particles", args) os.remove(fnInputMd)
def _checkNewOutput(self): if getattr(self, 'finished', False): return self.finished = self.streamClosed and \ self.outputSize == self.inputSize streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN newData = os.path.exists(self.fnOutputMd) lastToClose = self.finished and hasattr(self, 'outputParticles') if newData or lastToClose: outSet = self._loadOutputSet(SetOfParticles, 'outputParticles.sqlite') if newData: partsSet = self._createSetOfParticles() readSetOfParticles(self.fnOutputMd, partsSet) outSet.copyItems(partsSet) for item in partsSet: self._calculateSummaryValues(item) self._store() writeSetOfParticles(outSet.iterItems(orderBy='_xmipp_zScore'), self._getPath("images.xmd"), alignType=ALIGN_NONE) cleanPath(self.fnOutputMd) self._updateOutputSet('outputParticles', outSet, streamMode) if self.finished: # Unlock createOutputStep if finished all jobs outputStep = self._getFirstJoinStep() if outputStep and outputStep.isWaiting(): outputStep.setStatus(cons.STATUS_NEW)
def initializeStep(self): """ Create paths where data will be saved """ for i in range(len(self.inputCoordinatesTrue)): makePath(self._getExtraPath(self.PRE_PROC_MICs_PATH % i)) if self.doTesting.get() and self.testTrueSetOfParticles.get( ) and self.testFalseSetOfParticles.get(): writeSetOfParticles(self.testTrueSetOfParticles.get(), self._getExtraPath("testTrueParticlesSet.xmd")) writeSetOfParticles( self.testFalseSetOfParticles.get(), self._getExtraPath("testFalseParticlesSet.xmd")) if self.addTrainingData.get() == self.ADD_TRAIN_CUST: if self.trainTrueSetOfParticles.get(): writeSetOfParticles( self.trainTrueSetOfParticles.get(), self._getExtraPath("trainTrueParticlesSet.xmd")) if self.trainFalseSetOfParticles.get(): writeSetOfParticles( self.trainFalseSetOfParticles.get(), self._getExtraPath("trainFalseParticlesSet.xmd")) elif self.addTrainingData.get() == self.ADD_TRAIN_MODEL: writeSetOfParticles(self.retrieveTrainSets(), self._getTmpPath("addNegTrainParticles.xmd"))
def eliminationStep(self, stepId): """ Common code for particles and classes/averages """ fnInputMd = self.fnInputMd % stepId partsSet = self.prepareImages() if self.check == None: # if no previous, get all writeSetOfParticles(partsSet, fnInputMd, alignType=ALIGN_NONE, orderBy='creation') else: # if previous, take the last ones writeSetOfParticles(partsSet, fnInputMd, alignType=ALIGN_NONE, orderBy='creation', where='creation>"' + str(self.check) + '"') # special use of partSet before closing it self.specialBehavoir(partsSet) args = "-i %s -o %s -e %s -t %f" % ( fnInputMd, self.fnOutputMd, self.fnElimMd, self.threshold.get()) if self.addFeatures: args += " --addFeatures" if self.useDenoising: args += " --useDenoising -d %f" % self.denoising.get() self.runJob("xmipp_image_eliminate_empty_particles", args)
def classifyStep(self): inputImgs = self._getInputFn() writeSetOfParticles(self.listOfParticles, inputImgs, alignType=ALIGN_NONE) for p in self.listOfParticles: partId = p.getObjId() self.htAlreadyProcessed.pushItem(partId) self.lastDate = p.getObjCreation() self._saveCreationTimeFile(self.lastDate) # Calling program xmipp_cuda_correlation outImgs, clasesOut = self._getOutputsFn() self._params = { 'imgsRef': self.imgsRef, 'imgsExp': inputImgs, 'outputFile': outImgs, 'keepBest': self.keepBest.get(), 'maxshift': self.maximumShift, 'outputClassesFile': clasesOut, } args = ('-i_ref %(imgsRef)s -i_exp %(imgsExp)s -o %(outputFile)s ' '--keep_best %(keepBest)d --maxShift %(maxshift)d ' '--simplifiedMd --classify %(outputClassesFile)s') self.runJob("xmipp_cuda_correlation", args % self._params)
def convertAveragesStep(self): if self.useAsRef == REF_CLASSES: writeSetOfClasses2D(self.inputRefs.get(), self.imgsRef, writeParticles=True) else: writeSetOfParticles(self.inputRefs.get(), self.imgsRef)
def convertInputStep(self): """ Read the input metadatata. """ # Get the converted input micrographs in Xmipp format inputSet = self.input.get() inputPath = self._getExtraPath('inputSet') fnSet = inputPath + '.xmd' writeSetOfParticles(inputSet, fnSet)
def convertInputStep(self, atomsFn): # Write the modes metadata taking into account the selection self.writeModesMetaData() # Write a metadata with the normal modes information # to launch the nma alignment programs writeSetOfParticles(self.inputParticles.get(), self.imgsFn) # Copy the atoms file to current working dir copyFile(atomsFn, self.atomsFn)
def convertInputStep(self, inputId): """ Write the input images as a Xmipp metadata file. """ writeSetOfParticles(self.inputParticles.get(), self._getFileName('input_particles')) # If input references, also convert to xmipp metadata if not self.doGenerateReferences: writeSetOfParticles(self.inputReferences.get(), self._getFileName('input_references'))
def convertInputStep(self): """ convert if necessary""" # By default the prepocess protocol will ignore geometry # info and apply the operation on the binary data only. # then the new location (index, filename) is the most # common property to update in the single items. writeSetOfParticles(self.inputParticles.get(), self.inputFn, alignType=ALIGN_NONE)
def convertInputStep(self, particlesId1, particlesId2): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ writeSetOfParticles(self.inputParticles1.get(), self._getExtraPath("angles1.xmd")) writeSetOfParticles(self.inputParticles2.get(), self._getExtraPath("angles2.xmd"))
def convertInputStep(self): """ convert to Xmipp image model""" writeSetOfParticles(self.inputParticles.get(), self.inputFn, alignType=ALIGN_NONE) if self.inputParticles2.get() is not None: writeSetOfParticles(self.inputParticles2.get(), self._getSecondSetFn(), alignType=ALIGN_NONE)
def convertSetStep(self): writeSetOfParticles(self.inputParticles.get(), self.imgsExp, alignType=ALIGN_NONE) if self.useReferenceImages: if isinstance(self.referenceImages.get(), SetOfClasses2D): writeSetOfClasses2D(self.referenceImages.get(), self.refSet, writeParticles=False) else: writeSetOfParticles(self.referenceImages.get(), self.refSet, alignType=ALIGN_NONE)
def projectionStep(self, changeInserts): partSet = self.inputImages.get() imgsFn = self._getTmpPath('input_images.xmd') if partSet.getSize() > 200: newPartSet = self._getRandomSubset(partSet, 200) else: newPartSet = partSet writeSetOfParticles(newPartSet, imgsFn) if not partSet.hasAlignmentProj(): if not self.useGpu.get(): params = { 'imgsFn': imgsFn, 'dir': self._getTmpPath(), 'vols': self.inputFn, 'symmetryGroup': self.sigSymGroup.get(), } sigArgs = '-i %(imgsFn)s --initvolumes %(vols)s --odir %(dir)s' \ ' --sym %(symmetryGroup)s --alpha0 0.005 --dontReconstruct ' \ % params self.runJob("xmipp_reconstruct_significant", sigArgs) else: fnGallery = self._getExtraPath('gallery.stk') fnGalleryMd = self._getExtraPath('gallery.doc') angleStep = 5 args = "-i %s -o %s --sampling_rate %f --sym %s --min_tilt_angle 0 --max_tilt_angle 180 " % \ (self.inputFn, fnGallery, angleStep, self.sigSymGroup.get()) self.runJob("xmipp_angular_project_library", args, numberOfMpi=min(self.numberOfMpi.get(), 24)) count = 0 GpuListCuda = '' if self.useQueueForSteps() or self.useQueue(): GpuList = os.environ["CUDA_VISIBLE_DEVICES"] GpuList = GpuList.split(",") for elem in GpuList: GpuListCuda = GpuListCuda + str(count) + ' ' count += 1 else: GpuListAux = '' for elem in self.getGpuList(): GpuListCuda = GpuListCuda + str(count) + ' ' GpuListAux = GpuListAux + str(elem) + ',' count += 1 os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux fnAngles = 'images_iter001_00.xmd' args = '-i %s -r %s -o %s --odir %s --keepBestN 1 --dev %s ' % ( imgsFn, fnGalleryMd, fnAngles, self._getTmpPath(), GpuListCuda) self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
def convertStep(self): """convert input to proper format and dimensions if necessary""" imgSet = self.inputSet.get() writeSetOfParticles(imgSet, self._getExtraPath('input_imgs.xmd')) img = emlib.image.ImageHandler() fnVol = self._getExtraPath("volume.vol") img.convert(self.inputVolume.get(), fnVol) xDimVol = self.inputVolume.get().getDim()[0] xDimImg = imgSet.getDim()[0] if xDimVol != xDimImg: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (fnVol, xDimImg), numberOfMpi=1)
def convertInputStep(self, particlesId): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ writeSetOfParticles(self.inputParticles.get(), self._getPath('input_particles.xmd')) if self.doWiener.get(): params = self.INPUTARG % self._getPath('input_particles.xmd') params += self.OUTPUTARG % self._getExtraPath( 'corrected_ctf_particles.stk') params += ' --save_metadata_stack %s' % self._getExtraPath( 'corrected_ctf_particles.xmd') params += ' --pad %s' % self.padding_factor.get() params += ' --wc %s' % self.wiener_constant.get() params += ' --sampling_rate %s' % self.inputParticles.get().getSamplingRate() if self.inputParticles.get().isPhaseFlipped(): params += ' --phase_flipped ' if self.correctEnvelope: params += ' --correct_envelope ' nproc = self.numberOfMpi.get() nT = self.numberOfThreads.get() self.runJob('xmipp_ctf_correct_wiener2d', params) newTs, newXdim = self._getModifiedSizeAndSampling() if self.doWiener.get(): params = self.INPUTARG % self._getExtraPath('corrected_ctf_particles.xmd') else : params = self.INPUTARG % self._getPath('input_particles.xmd') params += self.OUTPUTARG % self._getExtraPath('scaled_particles.stk') params += ' --save_metadata_stack %s' % self._getExtraPath('scaled_particles.xmd') params += ' --fourier %d' % newXdim self.runJob('xmipp_image_resize',params) from pwem.emlib.image import ImageHandler img = ImageHandler() img.convert(self.inputVolumes.get(), self._getFileName("volume")) Xdim = self.inputVolumes.get().getDim()[0] if Xdim != newXdim: self.runJob("xmipp_image_resize", "-i %s --dim %d" % \ (self._getFileName("volume"), newXdim), numberOfMpi=1)
def createOutputStep(self, fnOutputParts, inputSet, inputVol): volTransformMatrix = np.matrix(inputVol.getTransform().getMatrix()) outputSet = self._createSetOfParticles() for part in inputSet.iterItems(): partTransformMat = part.getTransform().getMatrix() partTransformMatrix = np.matrix(partTransformMat) newTransformMatrix = volTransformMatrix * partTransformMatrix part.getTransform().setMatrix(newTransformMatrix) outputSet.append(part) outputSet.copyInfo(inputSet) self._defineOutputs(outputParticles=outputSet) writeSetOfParticles(outputSet, fnOutputParts)
def convertInputStep(self, particlesId, classesId): writeSetOfParticles(self.inputParticles.get(), self._getFileName('input_particles'), alignType=em.ALIGN_NONE) if not self.randomInitialization: if isinstance(self.initialClasses.get(), SetOfClasses2D): writeSetOfClasses2D(self.initialClasses.get(), self._getFileName('input_references'), writeParticles=False) else: writeSetOfParticles(self.initialClasses.get(), self._getFileName('input_references'))
def convertInputStep(self, imagesMd): # It is unusual to create the output in the convertInputStep, # but just to avoid reading twice the sqlite with particles inputSet = self.inputNmaDimred.get().getInputParticles() partSet = self._createSetOfParticles() partSet.copyInfo(inputSet) tmpSet = SetOfParticles(filename=self.sqliteFile.get()) partSet.appendFromImages(tmpSet) # Register outputs partSet.setAlignmentProj() self._defineOutputs(outputParticles=partSet) self._defineTransformRelation(inputSet, partSet) writeSetOfParticles(partSet, imagesMd)
def convertStep(self): inputParts = self.inputParticles.get() writeSetOfParticles(inputParts, self.imgsInputFn) #Resizing inputs ih = ImageHandler() ih.convert(self.inputReference.get(), self.fnRefVol) XdimRef = self.inputReference.get().getDim()[0] ih.convert(self.inputVolume.get(), self.fnInputVol) XdimInput = self.inputVolume.get().getDim()[0] if XdimRef != XdimInput: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (self.fnRefVol, XdimInput), numberOfMpi=1)
def convertStep(self, imgsFn): from xmipp3.convert import writeSetOfParticles imgSet = self.inputSet.get() writeSetOfParticles(imgSet, self.imgsFn) from pyworkflow.em.convert import ImageHandler img = ImageHandler() fnVol = self._getTmpPath("volume.vol") img.convert(self.inputVolume.get(), fnVol) xdim = self.inputVolume.get().getDim()[0] imgXdim = imgSet.getDim()[0] if xdim != imgXdim: self.runJob("xmipp_image_resize", "-i %s --dim %d" % (fnVol, imgXdim), numberOfMpi=1)
def convertInputStep(self, inputParticlesId): fnDir=self._getExtraPath() writeSetOfParticles(self.inputParticles.get(),self.imgsFn) # Choose the target sampling rate TsOrig=self.inputParticles.get().getSamplingRate() TsCurrent=max(TsOrig,self.targetResolution.get()/3) Xdim=self.inputParticles.get().getDimensions()[0] newXdim=long(round(Xdim*TsOrig/TsCurrent)) if newXdim<40: newXdim=long(40) TsCurrent=Xdim*(TsOrig/newXdim) print "Preparing images to sampling rate=",TsCurrent self.writeInfoField(fnDir,"size",xmippLib.MDL_XSIZE,newXdim) self.writeInfoField(fnDir,"sampling",xmippLib.MDL_SAMPLINGRATE,TsCurrent) # Prepare particles fnNewParticles=join(fnDir,"images.stk") if newXdim!=Xdim: self.runJob("xmipp_image_resize","-i %s -o %s --fourier %d"%(self.imgsFn,fnNewParticles,newXdim), numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get()) else: self.runJob("xmipp_image_convert","-i %s -o %s --save_metadata_stack %s"%(self.imgsFn,fnNewParticles,join(fnDir,"images.xmd")), numberOfMpi=1) R=self.particleRadius.get() if R<=0: R=self.inputParticles.get().getDimensions()[0]/2 R=min(round(R*TsOrig/TsCurrent*1.1),newXdim/2) self.runJob("xmipp_transform_mask","-i %s --mask circular -%d"%(fnNewParticles,R),numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get()) # Prepare mask imgHandler=ImageHandler() if self.nextMask.hasValue(): self.convertInputVolume(imgHandler, self.nextMask.get(), getImageLocation(self.nextMask.get()), join(fnDir,"mask.vol"), TsCurrent, newXdim) # Prepare references i=0 for vol in self.inputVolumes.get(): fnVol=join(fnDir,"volume%03d.vol"%i) self.convertInputVolume(imgHandler, vol, getImageLocation(vol), fnVol, TsCurrent, newXdim) self.runJob("xmipp_image_operate","-i %s --mult 0 -o %s"%(fnVol,join(fnDir,"volume%03d_speed.vol"%i)),numberOfMpi=1) i+=1 xmippLib.MetaData().write("best@"+self._getExtraPath("swarm.xmd")) # Empty write to guarantee this block is the first one xmippLib.MetaData().write("bestByVolume@"+self._getExtraPath("swarm.xmd"),xmippLib.MD_APPEND) # Empty write to guarantee this block is the second one
def denoiseImages(self, inputId, inputClassesId): # We start preparing writing those elements we're using as input to keep them untouched imagesMd = self._getPath('images.xmd') writeSetOfParticles(self.inputParticles.get(), imagesMd) classesMd = self._getPath('classes.xmd') writeSetOfClasses2D(self.inputClasses.get(), classesMd) fnRoot = self._getExtraPath('pca') fnRootDenoised = self._getExtraPath('imagesDenoised') args = '-i Particles@%s --oroot %s --eigenvectors %d --maxImages %d' % ( imagesMd, fnRoot, self.maxPCABases.get(), self.maxClasses.get()) self.runJob("xmipp_image_rotational_pca", args) N = min(self.maxPCABases.get(), self.PCABases2Project.get()) args='-i %s -o %s.stk --save_metadata_stack %s.xmd --basis %s.stk %d'\ % (imagesMd, fnRootDenoised, fnRootDenoised, fnRoot, N) self.runJob("xmipp_transform_filter", args) self.outputMd = String('%s.stk' % fnRootDenoised)
def _insertNewPartsSteps(self, insertedDict, inputParticles): deps = [] writeSetOfParticles([m.clone() for m in inputParticles], self._getExtraPath("allDone.xmd"), alignType=ALIGN_NONE) writeSetOfParticles([ m.clone() for m in inputParticles if int(m.getObjId()) not in insertedDict ], self._getExtraPath("newDone.xmd"), alignType=ALIGN_NONE) stepId = \ self._insertFunctionStep('kmeansClassifyStep', self._getExtraPath("newDone.xmd"), prerequisites=[]) deps.append(stepId) for part in inputParticles: if part.getObjId() not in insertedDict: insertedDict[part.getObjId()] = stepId return deps
def _loadInput(self): self.lastCheck = datetime.now() partsFile = self.inputParticles.get().getFileName() inPartsSet = SetOfParticles(filename=partsFile) inPartsSet.loadAllProperties() check = None for p in inPartsSet.iterItems(orderBy='creation', direction='DESC'): check = p.getObjCreation() break if self.check is None: writeSetOfParticles(inPartsSet, self.fnInputMd, alignType=ALIGN_NONE, orderBy='creation') else: writeSetOfParticles(inPartsSet, self.fnInputMd, alignType=ALIGN_NONE, orderBy='creation', where='creation>"' + str(self.check) + '"') writeSetOfParticles(inPartsSet, self.fnInputOldMd, alignType=ALIGN_NONE, orderBy='creation', where='creation<"' + str(self.check) + '"') self.check = check streamClosed = inPartsSet.isStreamClosed() inputSize = inPartsSet.getSize() inPartsSet.close() return inputSize, streamClosed