def createOutputStep(self): cleanPattern(self._getExtraPath('*.stk')) cleanPattern(self._getExtraPath('projectionsCudaCorr*')) inputParticles = self.inputSet.get() fnOutputParticles = self._getExtraPath('outConesParticles.xmd') outputSetOfParticles = self._createSetOfParticles() outputSetOfParticles.copyInfo(inputParticles) outputSetOfParticles.setAlignmentProj() Xdim = inputParticles.getXDim() newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) Ts = readInfoField(self._getExtraPath(), "sampling", emlib.MDL_SAMPLINGRATE) if newXdim != Xdim: self.scaleFactor = Ts / inputParticles.getSamplingRate() self.iterMd = md.iterRows(fnOutputParticles, emlib.MDL_ITEM_ID) self.lastRow = next(self.iterMd) outputSetOfParticles.copyItems(inputParticles, updateItemCallback=self._updateItem) else: readSetOfParticles(fnOutputParticles, outputSetOfParticles) self._defineOutputs(outputParticles=outputSetOfParticles)
def predictStep(self, gpuId): if not exists(self._getConePrediction()): # if self.useQueueForSteps() or self.useQueue(): # myStr = os.environ["CUDA_VISIBLE_DEVICES"] # else: # myStr = self.gpuList.get() # numGPU = myStr.split(',') # numGPU = numGPU[0] # print("Predict", myStr, numGPU) # sys.stdout.flush() mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc)) self.numCones = mdNumCones.size() imgsOutStk = self._getExtraPath('images_out_filtered.stk') imgsOutXmd = self._getExtraPath('images_out_filtered.xmd') self.runJob("xmipp_transform_filter", " -i %s -o %s " "--save_metadata_stack %s " "--keep_input_columns " "--fourier low_pass %f " % (self.imgsFn, imgsOutStk, imgsOutXmd, 0.15), numberOfMpi=self.myMPI.get()) numMax = int(self.numConesSelected) newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) args = "%s %s %d %d %d " % (imgsOutXmd, self._getExtraPath(), newXdim, self.numCones, numMax) #args += " %(GPU)s" args += " %s "%(gpuId) self.runJob("xmipp_cone_deepalign_predict", args, numberOfMpi=1, env=self.getCondaEnv())
def alignmentStep(self): imgsFn = self._getFileName('imgsFn') fnVol = self._getFileName('fnVol') fnOut = self._getFileName('fnOut') fnVolMask = self._getFileName('fnVolMask') fnOutDir = self._getFileName('fnOutDir') Ts = readInfoField(self._getExtraPath(), "sampling", md.MDL_SAMPLINGRATE) params = ' -i %s --ref %s -o %s --optimizeDeformation --optimizeDefocus ' \ '--l1 %d --l2 %d --max_shift %f --max_angular_change %f --sampling %f ' \ ' --max_resolution %f --odir %s --resume --regularization %f' %\ (imgsFn, fnVol, fnOut, self.l1.get(), self.l2.get(), self.maxShift, self.maxAngular, Ts, self.maxResolution, fnOutDir, self.regularization.get()) if self.optimizeAlignment.get(): params += ' --optimizeAlignment' if self.ignoreCTF.get(): params += ' --ignoreCTF' if self.inputParticles.get().isPhaseFlipped(): params += ' --phaseFlipped' if self.inputVolumeMask.get(): params += ' --mask %s' % fnVolMask if self.useGpu.get(): params += ' --device %d' % self.getGpuList()[0] program = 'xmipp_cuda_angular_sph_alignment' self.runJob(program, params) else: program = 'xmipp_angular_sph_alignment' self.runJob(program, params, numberOfMpi=self.numberOfMpi.get())
def convertStep(self): if self.modelPretrain.get() is True: fnPreProtocol = self.pretrainedModels.get()._getExtraPath() preXDim = readInfoField(fnPreProtocol, "size", emlib.MDL_XSIZE) self.inputTrainSet = self.pretrainedModels.get().inputTrainSet from ..convert import writeSetOfParticles inputParticles = self.inputSet.get() writeSetOfParticles(inputParticles, self.imgsFn) Ts = inputParticles.getSamplingRate() row = getFirstRow(self.imgsFn) hasCTF = row.containsLabel(emlib.MDL_CTF_DEFOCUSU) or emlib.containsLabel( emlib.MDL_CTF_MODEL) fnCorrected = self.imgsFn self._correctWiener(hasCTF, self.imgsFn, Ts) Xdim = inputParticles.getXDim() newTs = self.targetResolution.get() * 1.0 / 3.0 newTs = max(Ts, newTs) self.newXdim = int(preXDim) if self.modelPretrain.get() else int( float(Xdim * Ts / newTs)) self.firstMaxShift = int(round(self.newXdim / 10)) writeInfoField(self._getExtraPath(), "sampling", emlib.MDL_SAMPLINGRATE, newTs) writeInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE, self.newXdim) self._resize(Xdim, fnCorrected, 'scaled_particles', self.imgsFn) self._removeCorrectedParticles() fnVol = self._getTmpPath(self._fnVolumeVol) self._ih.convert(self.inputVolume.get(), fnVol) Xdim = self.inputVolume.get().getDim()[0] if Xdim != self.newXdim: self.runJob("xmipp_image_resize", "-i %s --fourier %d" % (fnVol, self.newXdim), numberOfMpi=self.myMPI.get()) inputTrain = self.inputTrainSet.get() writeSetOfParticles(inputTrain, self.trainImgsFn) row = getFirstRow(self.trainImgsFn) hasCTF = row.containsLabel(emlib.MDL_CTF_DEFOCUSU) or emlib.containsLabel( emlib.MDL_CTF_MODEL) fnCorrected = self.trainImgsFn self._correctWiener(hasCTF, self.trainImgsFn, Ts) self._resize(Xdim, fnCorrected, 'scaled_train_particles', self.trainImgsFn) self._removeCorrectedParticles()
def splitVolumeStep(self): newTs = readInfoField(self._getExtraPath(), "sampling", emlib.MDL_SAMPLINGRATE) newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) fnMask = "" if self.mask.hasValue(): fnMask = self._getExtraPath("mask.vol") img = ImageHandler() img.convert(self.mask.get(), fnMask) self.runJob('xmipp_image_resize', "-i %s --dim %d" % (fnMask, newXdim), numberOfMpi=1) self.runJob('xmipp_transform_threshold', "-i %s --select below 0.5 --substitute binarize" % fnMask, numberOfMpi=1) args = "-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f" % \ (self._getDirectionalClassesFn(), self._getExtraPath("split"), self.Nrec.get(), self.Nsamples.get(), self.symmetryGroup.get(), self.alpha.get()) if fnMask != "": args += " --mask binary_file %s" % fnMask self.runJob("xmipp_classify_first_split", args, numberOfMpi=1)
def _coneStep(self, gpuId, idx, modelFn): fnLabels = self._getExtraPath('labels.txt') fileLabels = open(fnLabels, "r") expSet = self._getProjectionsExp(self.numCones) if not exists(expSet): for n in range(1, self.numCones): if exists(self._getProjectionsExp(self.numCones - n)): expSet = self._getProjectionsExp(self.numCones - n) break newFnLabels = self._getExtraPath('labels%d.txt' % idx) newFileLabels = open(newFnLabels, "w") lines = fileLabels.readlines() for line in lines: if line == str(idx - 1) + '\n': newFileLabels.write('1\n') else: newFileLabels.write('0\n') newFileLabels.close() fileLabels.close() newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) fnLabels = self._getExtraPath('labels%d.txt' % idx) print("Training region ", idx, " in GPU ", gpuId) sys.stdout.flush() try: args = "%s %s %s %s %d %d %d %d " % ( expSet, fnLabels, self._getExtraPath(), modelFn+'_aux', self.numEpochs, newXdim, 2, self.batchSize.get()) #args += " %(GPU)s" args += " %s " % (gpuId) #args += " %s " %(int(idx % totalGpu)) self.runJob("xmipp_cone_deepalign", args, numberOfMpi=1, env=self.getCondaEnv()) except Exception as e: raise Exception( "ERROR: Please, if you are suffering memory problems, " "check the target resolution to work with lower dimensions.") moveFile(self._getExtraPath(modelFn + '_aux.h5'), self._getExtraPath(modelFn + '.h5'))
def _processRows(self, label, fnExp, mdIn, mdExp, Nrepeats, fileLabels): newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) maxPsi = 180 maxShift = round(newXdim / 10) for row in iterRows(mdIn): fnImg = row.getValue(emlib.MDL_IMAGE) myRow = row I = emlib.Image(fnImg) Xdim, Ydim, _, _ = I.getDimensions() Xdim2 = Xdim / 2 Ydim2 = Ydim / 2 if Nrepeats == 0: myRow.addToMd(mdExp) idx += 1 fileLabels.write(str(label - 1) + '\n') else: for i in range(Nrepeats): psiDeg = np.random.uniform(-maxPsi, maxPsi) psi = psiDeg * math.pi / 180.0 deltaX = np.random.uniform(-maxShift, maxShift) deltaY = np.random.uniform(-maxShift, maxShift) c = math.cos(psi) s = math.sin(psi) M = np.float32([[c, s, (1 - c) * Xdim2 - s * Ydim2 + deltaX], [-s, c, s * Xdim2 + (1 - c) * Ydim2 + deltaY]]) newFn = ('%06d@' % idx) + fnExp[:-3] + 'stk' self._ih.applyTransform( fnImg, newFn, M, (Ydim, Xdim), doWrap=True) myRow.setValue(emlib.MDL_IMAGE, newFn) myRow.setValue(emlib.MDL_ANGLE_PSI, psiDeg) myRow.setValue(emlib.MDL_SHIFT_X, deltaX) myRow.setValue(emlib.MDL_SHIFT_Y, deltaY) myRow.addToMd(mdExp) idx += 1 fileLabels.write(str(label - 1) + '\n')
def projectStep(self, numProj, iniRot, endRot, iniTilt, endTilt, fn, idx): newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) fnVol = self._getTmpPath(self._fnVolumeVol) uniformProjectionsStr = """ # XMIPP_STAR_1 * data_block1 _dimensions2D '%d %d' _projRotRange '%d %d %d' _projRotRandomness random _projRotNoise '0' _projTiltRange '%d %d 1' _projTiltRandomness random _projTiltNoise '0' _projPsiRange '0 0 1' _projPsiRandomness random _projPsiNoise '0' _noisePixelLevel '0' _noiseCoord '0' """ % (newXdim, newXdim, iniRot, endRot, numProj, iniTilt, endTilt) fnParams = self._getExtraPath("uniformProjections%d.xmd" % idx) fh = open(fnParams, "w") fh.write(uniformProjectionsStr) fh.close() fnProjs = self._getExtraPath(fn + "%d.stk" % idx) self.runJob("xmipp_phantom_project", "-i %s -o %s --method fourier 1 0.5 " "--params %s" % (fnVol, fnProjs, fnParams), numberOfMpi=1) fnProjsXmd=fnProjs[:-3]+'xmd' self.runJob("xmipp_metadata_utilities", "-i %s --fill ref lineal 1 1 " % (fnProjsXmd), numberOfMpi=1) cleanPattern(self._getExtraPath('uniformProjections*'))
def createOutputStep(self): inputParticles = self.inputParticles.get() if not self._useSeveralClasses(): newTs = inputParticles.getSamplingRate() else: newTs = readInfoField(self._getExtraPath(), "sampling", emlib.MDL_SAMPLINGRATE) self.mdClasses = emlib.MetaData(self._getDirectionalClassesFn()) self.mdImages = emlib.MetaData(self._getDirectionalImagesFn()) classes2D = self._createSetOfClasses2D(inputParticles) classes2D.getImages().setSamplingRate(newTs) self.averageSet = self._createSetOfAverages() self.averageSet.copyInfo(inputParticles) self.averageSet.setAlignmentProj() self.averageSet.setSamplingRate(newTs) # Let's use a SetMdIterator because it should be less particles # in the metadata produced than in the input set iterator = md.SetMdIterator(self.mdImages, sortByLabel=md.MDL_ITEM_ID, updateItemCallback=self._updateParticle, skipDisabled=True) fnHomogeneous = self._getExtraPath("images_homogeneous.xmd") if exists(fnHomogeneous): homogeneousSet = self._createSetOfParticles() homogeneousSet.copyInfo(inputParticles) homogeneousSet.setSamplingRate(newTs) homogeneousSet.setAlignmentProj() self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID) self.lastRow = next(self.iterMd) homogeneousSet.copyItems( inputParticles, updateItemCallback=self._updateHomogeneousItem) self._defineOutputs(outputHomogeneous=homogeneousSet) self._defineSourceRelation(self.inputParticles, homogeneousSet) classes2D.classifyItems(updateItemCallback=iterator.updateItem, updateClassCallback=self._updateClass) self._defineOutputs(outputClasses=classes2D) self._defineOutputs(outputAverages=self.averageSet) self._defineSourceRelation(self.inputParticles, classes2D) self._defineSourceRelation(self.inputParticles, self.averageSet) if self.splitVolume and self.directionalClasses.get() > 1: volumesSet = self._createSetOfVolumes() volumesSet.setSamplingRate(newTs) for i in range(2): fnVol = self._getExtraPath("split_v%d.vol" % (i + 1)) fnMrc = self._getExtraPath("split_v%d.mrc" % (i + 1)) self.runJob("xmipp_image_convert", "-i %s -o %s -t vol" % (fnVol, fnMrc), numberOfMpi=1) self.runJob("xmipp_image_header", "-i %s --sampling_rate %f" % (fnMrc, newTs), numberOfMpi=1) cleanPath(fnVol) vol = Volume() vol.setLocation(1, fnMrc) volumesSet.append(vol) self._defineOutputs(outputVolumes=volumesSet) self._defineSourceRelation(inputParticles, volumesSet)
def refineAnglesStep(self): fnTmpDir = self._getTmpPath() fnDirectional = self._getDirectionalClassesFn() inputParticles = self.inputParticles.get() newTs = readInfoField(self._getExtraPath(), "sampling", emlib.MDL_SAMPLINGRATE) newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE) # Generate projections fnGallery = join(fnTmpDir, "gallery.stk") fnGalleryMd = join(fnTmpDir, "gallery.doc") fnVol = self._getInputVolFn() args = "-i %s -o %s --sampling_rate %f --sym %s" % \ (fnVol, fnGallery, 5.0, self.symmetryGroup) args += " --compute_neighbors --angular_distance -1 --experimental_images %s" % fnDirectional self.runJob("xmipp_angular_project_library", args, numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get()) # Global angular assignment maxShift = 0.15 * newXdim fnAngles = join(fnTmpDir, "angles_iter001_00.xmd") if not self.useGpu.get(): args = '-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0' % \ (fnDirectional, fnGalleryMd, maxShift, fnTmpDir) self.runJob('xmipp_reconstruct_significant', args, numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get()) else: count = 0 GpuListCuda = '' if self.useQueueForSteps() or self.useQueue(): GpuList = os.environ["CUDA_VISIBLE_DEVICES"] GpuList = GpuList.split(",") for elem in GpuList: GpuListCuda = GpuListCuda + str(count) + ' ' count += 1 else: GpuList = ' '.join([str(elem) for elem in self.getGpuList()]) GpuListAux = '' for elem in self.getGpuList(): GpuListCuda = GpuListCuda + str(count) + ' ' GpuListAux = GpuListAux + str(elem) + ',' count += 1 os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux args = '-i %s -r %s -o %s --dev %s ' % (fnDirectional, fnGalleryMd, fnAngles, GpuListCuda) self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1) self.runJob("xmipp_metadata_utilities", "-i %s --operate drop_column ref" % fnAngles, numberOfMpi=1) self.runJob("xmipp_metadata_utilities", "-i %s --set join %s ref2" % (fnAngles, fnDirectional), numberOfMpi=1) # Local angular assignment fnAnglesLocalStk = self._getPath("directional_local_classes.stk") args = "-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 " % \ (fnAngles, fnAnglesLocalStk, newTs, newXdim / 2, 2, fnVol, self.targetResolution) args += " --optimizeShift --max_shift %f" % maxShift args += " --optimizeAngles --max_angular_change %f" % self.angularDistance self.runJob("xmipp_angular_continuous_assign2", args, numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get()) moveFile(self._getPath("directional_local_classes.xmd"), self._getDirectionalClassesFn()) cleanPattern(self._getExtraPath("direction_*"))
def createOutputStep(self): inputParticles = self.inputParticles.get() # if not self._useSeveralClasses(): # newTs = inputParticles.getSamplingRate() # else: # newTs = readInfoField(self._getExtraPath(), "sampling", # xmipp.MDL_SAMPLINGRATE) self.mdClasses = emlib.MetaData(self._getDirectionalClassesFn()) self.mdImages = emlib.MetaData(self._getDirectionalImagesFn()) origTs = inputParticles.getSamplingRate() lastTs = readInfoField(self._getExtraPath(), "sampling", emlib.MDL_SAMPLINGRATE) if origTs != lastTs: newXdim = inputParticles.getXDim() self.runJob( "xmipp_image_resize", "-i %s -o %s --save_metadata_stack %s --fourier %d" % (self._getDirectionalClassesFn(), self._getPath("aux_directional_local_classes.stk"), self._getPath("aux_directional_classes.xmd"), newXdim), numberOfMpi=1) from shutil import copy copy(self._getPath("aux_directional_local_classes.stk"), self._getPath("directional_local_classes.stk")) copy(self._getPath("aux_directional_classes.xmd"), self._getPath("directional_classes.xmd")) cleanPattern(self._getPath("aux_directional*")) classes2D = self._createSetOfClasses2D(inputParticles) #classes2D.getImages().setSamplingRate(newTs) classes2D.getImages().setSamplingRate(origTs) self.averageSet = self._createSetOfAverages() self.averageSet.copyInfo(inputParticles) self.averageSet.setAlignmentProj() #self.averageSet.setSamplingRate(newTs) self.averageSet.setSamplingRate(origTs) # Let's use a SetMdIterator because it should be less particles # in the metadata produced than in the input set iterator = md.SetMdIterator(self.mdImages, sortByLabel=md.MDL_ITEM_ID, updateItemCallback=self._updateParticle, skipDisabled=True) fnHomogeneous = self._getExtraPath("images_homogeneous.xmd") if exists(fnHomogeneous): if origTs != lastTs: newXdim = inputParticles.getXDim() self.runJob("xmipp_image_resize", "-i %s --dim %d" % (fnHomogeneous, newXdim), numberOfMpi=1) homogeneousSet = self._createSetOfParticles() homogeneousSet.copyInfo(inputParticles) #homogeneousSet.getImages().setSamplingRate(newTs) homogeneousSet.getImages().setSamplingRate(origTs) homogeneousSet.setAlignmentProj() self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID) self.lastRow = next(self.iterMd) homogeneousSet.copyItems( inputParticles, updateItemCallback=self._updateHomogeneousItem) self._defineOutputs(outputHomogeneous=homogeneousSet) self._defineSourceRelation(self.inputParticles, homogeneousSet) #AJ testing #AJ por que desaparece una clase que tiene imagenes asignadas listRefId = [] for row in md.iterRows(self.mdClasses, emlib.MDL_REF2): refId = row.getValue(emlib.MDL_REF2, row.getObjId()) if len(listRefId) > 0 and refId != listRefId[-1] + 1: whereEnd = listRefId[-1] + 1 for i in range(refId - whereEnd): rowNew = row rowNew.setValue(emlib.MDL_REF2, listRefId[-1] + i + 1) rowNew.setValue(emlib.MDL_IMAGE, 'None') rowNew.setValue(emlib.MDL_IMAGE1, 'None') rowNew.addToMd(self.mdClasses) listRefId.append(listRefId[-1] + i + 1) listRefId.append(refId) else: listRefId.append(refId) self.mdClasses.write(self._getDirectionalClassesFn()) self.mdClasses = emlib.MetaData(self._getDirectionalClassesFn()) #END AJ classes2D.classifyItems(updateItemCallback=iterator.updateItem, updateClassCallback=self._updateClass) self._defineOutputs(outputClasses=classes2D) self._defineOutputs(outputAverages=self.averageSet) self._defineSourceRelation(self.inputParticles, classes2D) self._defineSourceRelation(self.inputParticles, self.averageSet) if self.splitVolume and self.directionalClasses.get() > 1: volumesSet = self._createSetOfVolumes() #volumesSet.setSamplingRate(newTs) volumesSet.setSamplingRate(origTs) for i in range(2): vol = Volume() fnVol = self._getExtraPath("split%d.vol" % (i + 1)) if origTs != lastTs: newXdim = inputParticles.getXDim() self.runJob("xmipp_image_resize", "-i %s --dim %d" % (fnVol, newXdim), numberOfMpi=1) fnMrc = self._getExtraPath("split%d.mrc" % (i + 1)) self.runJob("xmipp_image_convert", "-i %s -o %s -t vol" % (fnVol, fnMrc), numberOfMpi=1) self.runJob("xmipp_image_header", "-i %s --sampling_rate %f" % (fnMrc, origTs), numberOfMpi=1) cleanPath(fnVol) vol.setLocation(1, fnMrc) volumesSet.append(vol) self._defineOutputs(outputVolumes=volumesSet) self._defineSourceRelation(inputParticles, volumesSet)