def getCorrThresh(log,WorkingDir,NRansac,CorrThresh): corrVector = [] fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd") mdCorr= MetaData() for n in range(NRansac): fnRoot=os.path.join("ransac%05d"%n) fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd") md=MetaData(fnAngles) for objId in md: corr = md.getValue(MDL_MAXCC, objId) corrVector.append(corr) objIdCorr = mdCorr.addObject() mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr) mdCorr.write("correlations@"+fnCorr,MD_APPEND) mdCorr= MetaData() sortedCorrVector = sorted(corrVector) indx = int(floor(CorrThresh*(len(sortedCorrVector)-1))) #With the line below commented the percentil is not used for the threshold and is used the value introduced in the form #CorrThresh = sortedCorrVector[indx]# objId = mdCorr.addObject() mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId) mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND) print "Correlation threshold: "+str(CorrThresh)
def scoreFinalVolumes(log,WorkingDir,NumVolumes): threshold=getCCThreshold(WorkingDir) mdOut=MetaData() for n in range(NumVolumes): fnRoot=os.path.join(WorkingDir,'volumeProposed%05d'%n) fnAssignment=fnRoot+".xmd" if exists(fnAssignment): runJob(log,"xmipp_metadata_utilities","-i %s --fill weight constant 1"%fnAssignment) MDassignment=MetaData(fnAssignment) sum=0 thresholdedSum=0 N=0 minCC=2 for id in MDassignment: cc=MDassignment.getValue(MDL_MAXCC,id) sum+=cc thresholdedSum+=cc-threshold if cc<minCC: minCC=cc N+=1 avg=sum/N id=mdOut.addObject() mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id) mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id) mdOut.setValue(MDL_VOLUME_SCORE_SUM_TH,float(thresholdedSum),id) mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id) mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id) mdOut.write(os.path.join(WorkingDir,"volumesProposed.xmd"))
def computeAtomShifts(log,WorkingDir,NumberOfModes): fnOutDir=os.path.join(WorkingDir,"extra/distanceProfiles") createDir(log,fnOutDir) maxShift=[] maxShiftMode=[] for n in range(7,NumberOfModes+1): fhIn=open(os.path.join(WorkingDir,"modes/vec."+str(n))) md=MetaData() atomCounter=0; for line in fhIn: coord=line.split() x=float(coord[0]) y=float(coord[1]) z=float(coord[2]) d=math.sqrt(x*x+y*y+z*z) if n==7: maxShift.append(d) maxShiftMode.append(7) else: if d>maxShift[atomCounter]: maxShift[atomCounter]=d maxShiftMode[atomCounter]=n atomCounter+=1 md.setValue(MDL_NMA_ATOMSHIFT,d,md.addObject()) md.write(os.path.join(fnOutDir,"vec"+str(n)+".xmd")) fhIn.close() md=MetaData() for i in range(len(maxShift)): id=md.addObject() md.setValue(MDL_NMA_ATOMSHIFT,maxShift[i],id) md.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec."+str(maxShiftMode[i]+1)),id) md.write(os.path.join(WorkingDir,'extra','maxAtomShifts.xmd'))
class RowMetaData(): """ This class is a wrapper for MetaData in row mode. Where only one object is used. """ def __init__(self, filename=None): self._md = MetaData() self._md.setColumnFormat(False) self._id = self._md.addObject() if filename: self.read(filename) def setValue(self, label, value): self._md.setValue(label, value, self._id) def getValue(self, label): return self._md.getValue(label, self._id) def write(self, filename, mode=MD_APPEND): self._md.write(filename, mode) def read(self, filename): self._md.read(filename) self._md.setColumnFormat(False) self._id = self._md.firstObject() def containsLabel(self, label): return self._md.containsLabel(label) def __str__(self): return str(self._md)
class RowMetaData(): """ This class is a wrapper for MetaData in row mode. Where only one object is used. """ def __init__(self, filename=None): self._md = MetaData() self._md.setColumnFormat(False) self._id = self._md.addObject() if filename: self.read(filename) def setValue(self, label, value): self._md.setValue(label, value, self._id) def getValue(self, label): return self._md.getValue(label, self._id) def write(self, filename, mode=MD_APPEND): self._md.write(filename, mode) def read(self, filename): self._md.read(filename) self._md.setColumnFormat(False) self._id = self._md.firstObject() def containsLabel(self, label): return self._md.containsLabel(label) def __str__(self): return str(self._md)
def projMatchStep(self, volume, angularSampling, symmetryGroup, images, fnAngles, Xdim): from pyworkflow.utils.path import cleanPath # Generate gallery of projections fnGallery = self._getExtraPath("gallery.stk") self.runJob( "xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s" % (volume, fnGallery, angularSampling, symmetryGroup, images), ) # Assign angles self.runJob( "xmipp_angular_projection_matching", "-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 --search5d_shift %s --search5d_step %s --append" % (images, fnAngles, fnGallery, str(Xdim / 2), str(int(Xdim / 10)), str(int(Xdim / 25))), ) cleanPath(self._getExtraPath("gallery_sampling.xmd")) cleanPath(self._getExtraPath("gallery_angles.doc")) cleanPath(self._getExtraPath("gallery.doc")) # Write angles in the original file and sort MD = MetaData(fnAngles) for id in MD: galleryReference = MD.getValue(xmipp.MDL_REF, id) MD.setValue(xmipp.MDL_IMAGE_REF, "%05d@%s" % (galleryReference + 1, fnGallery), id) MD.write(fnAngles)
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images): from numpy import array, dot fnOut = "classes_aligned@" + fn MDin = MetaData(images) MDout = MetaData() n = 1 hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL) for i in MDin: fnImg = MDin.getValue(xmipp.MDL_IMAGE, i) fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i) maxCC = MDin.getValue(xmipp.MDL_MAXCC, i) rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i) tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i) psi = -1.0 * MDin.getValue(xmipp.MDL_ANGLE_PSI, i) flip = MDin.getValue(xmipp.MDL_FLIP, i) if flip: psi = -psi eulerMatrix = Euler_angles2matrix(0.0, 0.0, psi) x = MDin.getValue(xmipp.MDL_SHIFT_X, i) y = MDin.getValue(xmipp.MDL_SHIFT_Y, i) shift = array([x, y, 0]) shiftOut = dot(eulerMatrix, shift) [x, y, z] = shiftOut if flip: x = -x id = MDout.addObject() MDout.setValue(xmipp.MDL_IMAGE, fnImg, id) MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id) MDout.setValue(xmipp.MDL_IMAGE1, "%05d@%s" % (n, self._getExtraPath("diff.stk")), id) if hasCTF: fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i) MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id) MDout.setValue(xmipp.MDL_MAXCC, maxCC, id) MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id) MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id) MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id) MDout.setValue(xmipp.MDL_SHIFT_X, x, id) MDout.setValue(xmipp.MDL_SHIFT_Y, y, id) MDout.setValue(xmipp.MDL_FLIP, flip, id) MDout.setValue(xmipp.MDL_ENABLED, 1, id) n += 1 MDout.write(fnOut, xmipp.MD_APPEND) # Actually create the differences img = Image() imgRef = Image() if hasCTF and volumeIsCTFCorrected: Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject()) for i in MDout: img.readApplyGeo(MDout, i) imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i)) if hasCTF and volumeIsCTFCorrected: fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i) imgRef.applyCTF(fnCTF, Ts) img.convert2DataType(DT_DOUBLE) imgDiff = img - imgRef imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images): from numpy import array, dot fnOut = 'classes_aligned@' + fn MDin = MetaData(images) MDout = MetaData() n = 1 hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL) for i in MDin: fnImg = MDin.getValue(xmipp.MDL_IMAGE, i) fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i) maxCC = MDin.getValue(xmipp.MDL_MAXCC, i) rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i) tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i) psi = -1. * MDin.getValue(xmipp.MDL_ANGLE_PSI, i) flip = MDin.getValue(xmipp.MDL_FLIP, i) if flip: psi = -psi eulerMatrix = Euler_angles2matrix(0., 0., psi) x = MDin.getValue(xmipp.MDL_SHIFT_X, i) y = MDin.getValue(xmipp.MDL_SHIFT_Y, i) shift = array([x, y, 0]) shiftOut = dot(eulerMatrix, shift) [x, y, z] = shiftOut if flip: x = -x id = MDout.addObject() MDout.setValue(xmipp.MDL_IMAGE, fnImg, id) MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id) MDout.setValue(xmipp.MDL_IMAGE1, "%05d@%s" % (n, self._getExtraPath("diff.stk")), id) if hasCTF: fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i) MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id) MDout.setValue(xmipp.MDL_MAXCC, maxCC, id) MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id) MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id) MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id) MDout.setValue(xmipp.MDL_SHIFT_X, x, id) MDout.setValue(xmipp.MDL_SHIFT_Y, y, id) MDout.setValue(xmipp.MDL_FLIP, flip, id) MDout.setValue(xmipp.MDL_ENABLED, 1, id) n += 1 MDout.write(fnOut, xmipp.MD_APPEND) # Actually create the differences img = Image() imgRef = Image() if hasCTF and volumeIsCTFCorrected: Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject()) for i in MDout: img.readApplyGeo(MDout, i) imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i)) if hasCTF and volumeIsCTFCorrected: fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i) imgRef.applyCTF(fnCTF, Ts) img.convert2DataType(DT_DOUBLE) imgDiff = img - imgRef imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
def projMatchStep(self, volume, angularSampling, symmetryGroup, images, fnAngles, Xdim): from pyworkflow.utils.path import cleanPath # Generate gallery of projections fnGallery = self._getExtraPath('gallery.stk') if volume.endswith('.mrc'): volume += ":mrc" self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\ % (volume, fnGallery, angularSampling, symmetryGroup, images)) # Assign angles self.runJob("xmipp_angular_projection_matching", "-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 --search5d_shift %s --search5d_step %s --append"\ % (images, fnAngles, fnGallery, str(Xdim/2), str(int(Xdim/10)), str(int(Xdim/25)))) cleanPath(self._getExtraPath('gallery_sampling.xmd')) cleanPath(self._getExtraPath('gallery_angles.doc')) cleanPath(self._getExtraPath('gallery.doc')) # Write angles in the original file and sort MD = MetaData(fnAngles) for id in MD: galleryReference = MD.getValue(xmipp.MDL_REF, id) MD.setValue(xmipp.MDL_IMAGE_REF, "%05d@%s" % (galleryReference + 1, fnGallery), id) MD.write(fnAngles)
def createOutput(self): import glob levelFiles = glob.glob(self._getExtraPath("results_classes_level*.xmd")) if levelFiles: levelFiles.sort() lastLevelFile = levelFiles[-1] setOfClasses = self._createSetOfClassesVol() setOfClasses.setImages(self.inputVolumes.get()) readSetOfClassesVol(setOfClasses, lastLevelFile) self._defineOutputs(outputClasses=setOfClasses) self._defineSourceRelation(self.inputVolumes, self.outputClasses) if self.generateAligned.get(): setOfVolumes = self._createSetOfVolumes() fnAligned = self._getExtraPath("results_aligned.xmd") self.runJob( "xmipp_metadata_selfile_create", "-p %s -o %s -s" % (self._getExtraPath("results_aligned.stk"), fnAligned), ) md = MetaData(fnAligned) md.addItemId() md.write(fnAligned) readSetOfVolumes(fnAligned, setOfVolumes) volumeList = self.inputVolumes.get() setOfVolumes.setSamplingRate(volumeList.getSamplingRate()) self._defineOutputs(alignedVolumes=setOfVolumes) self._defineTransformRelation(self.inputVolumes, self.alignedVolumes)
def getCorrThresh(log,WorkingDir,WorkingDirStructure,NRansac,CorrThresh): corrVector = [] fnCorr=os.path.join(WorkingDirStructure,"correlations.xmd") mdCorr= MetaData() for n in range(NRansac): fnRoot=os.path.join("ransac%05d"%n) fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd") if os.path.exists(fnAngles): md=MetaData(fnAngles) for objId in md: corr = md.getValue(MDL_MAXCC, objId) corrVector.append(corr) objIdCorr = mdCorr.addObject() mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr) mdCorr.write("correlations@"+fnCorr,MD_APPEND) mdCorr= MetaData() sortedCorrVector = sorted(corrVector) indx = int(floor(CorrThresh*(len(sortedCorrVector)-1))) objId = mdCorr.addObject() mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId) mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND) print "Correlation threshold: "+str(CorrThresh)
def createSubset(log, inputFile, inputFileLabel, subsetFile, subsetFileLabel, outputFile): mdInputFile = MetaData(inputFile) mdSubsetFile = MetaData(subsetFile) mdOutputFile = MetaData() print inputFile, inputFileLabel, subsetFile, subsetFileLabel, outputFile mdOutputFile.join2( mdInputFile , mdSubsetFile , str2Label(inputFileLabel) , str2Label(subsetFileLabel) , INNER_JOIN) mdOutputFile.write(outputFile)
def rewriteClassBlock(log,WorkingDir,ExtraDir): fnClassMetadata=os.path.join(ExtraDir,"kerdensom_classes.xmd") fnClass="classes@%s"%fnClassMetadata fnClassStack=os.path.join(ExtraDir,"classes.stk") mD = MetaData(fnClass) counter = 1 for id in mD: mD.setValue(MDL_IMAGE,"%06d@%s"%(counter,fnClassStack),id) counter += 1 mD.write(fnClass,MD_APPEND) createLink(log,fnClassMetadata,os.path.join(WorkingDir,"classes.xmd")) createLink(log,os.path.join(ExtraDir,"kerdensom_images.xmd"),os.path.join(WorkingDir,"images.xmd"))
def reconstructClass( log, WorkingDir, ExtraDir, ClassNameIn, ClassNameOut, ClassImage, CenterMaxShift, ThinObject, SkipTiltedTranslations, ClassVolumeOut, ReconstructAdditionalParams, DoLowPassFilter, LowPassFilter, ): # If class image doesn't exists, generate it by averaging if len(ClassImage) == 0: classRootOut = ClassNameOut.replace(".xmd", "") + "_" params = "-i %(ClassNameIn)s --save_image_stats %(classRootOut)s -o %(ExtraDir)s/stats.xmd" % locals() runJob(log, "xmipp_image_statistics", params) ClassImage = classRootOut + "average.xmp" deleteFile(log, classRootOut + "stddev.xmp") deleteFile(log, "%(ExtraDir)s/stats.xmd" % locals()) params = "-i %(ClassNameIn)s -o %(ClassNameOut)s --ref %(ClassImage)s --max_shift %(CenterMaxShift)d" % locals() if ThinObject: params += " --do_stretch" if SkipTiltedTranslations: params += " --do_not_align_tilted" runJob(log, "xmipp_image_align_tilt_pairs", params) params = "-i %(ClassNameOut)s -o %(ClassVolumeOut)s %(ReconstructAdditionalParams)s" % locals() runJob(log, "xmipp_reconstruct_art", params) if exists(ClassVolumeOut): mdFn = join(WorkingDir, "volumes.xmd") md = MetaData() if exists(mdFn): md.read(mdFn) objId = md.addObject() md.setValue(MDL_IMAGE, ClassVolumeOut, objId) if DoLowPassFilter: filteredVolume = ClassVolumeOut.replace(".vol", "_filtered.vol") params = "-i %(ClassVolumeOut)s -o %(filteredVolume)s --fourier low_pass %(LowPassFilter)f" % locals() runJob(log, "xmipp_transform_filter", params) objId = md.addObject() md.setValue(MDL_IMAGE, filteredVolume, objId) md.write(mdFn)
def sortImages(log, ImagesFn, rejectionMethod='None', maxZscore=3, percentage=5): md = MetaData(ImagesFn) if not md.isEmpty(): args="" if rejectionMethod=='MaxZscore': args+=" --zcut "+str(maxZscore) elif rejectionMethod=='Percentage': args+=" --percent "+str(percentage) runJob(log, "xmipp_image_sort_by_statistics","-i %(ImagesFn)s --addToInput" % locals()+args) md.read(ImagesFn) # Should have ZScore label after runJob md.sort(MDL_ZSCORE) md.write(ImagesFn)
def sortClasses(log,ExtraDir,Nproc,suffix): if Nproc==1: Nproc=2 if Nproc>8: Nproc=8 for filename in glob.glob(os.path.join(ExtraDir,"level_??/level_classes%s.xmd"%suffix)): level=int(re.search('level_(\d\d)',filename).group(1)) fnRoot=os.path.join(ExtraDir,"level_%02d/level_classes%s_sorted"%(level,suffix)) params= "-i classes@"+filename+" --oroot "+fnRoot runJob(log,"xmipp_image_sort",params,Nproc) mD=MetaData(fnRoot+".xmd") mD.write("classes_sorted@"+filename,MD_APPEND) deleteFile(log,fnRoot+".xmd")
def qualifyModes(log,WorkingDir,NumberOfModes,StructureType,CollectivityThreshold): currentDir=os.getcwd() changeDir(log,WorkingDir) fnVec=glob.glob("modes/vec.*") if len(fnVec)<NumberOfModes: fhWarning=open("warnings.xmd",'w') fhWarning.write(redStr("There are only "+str(len(fnVec))+" modes instead of "+str(NumberOfModes)+". Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks and for pseudoatomic normal mode analysis 3 times the number of pseudoatoms. However, the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between these two numbers, consider increasing cut-off distance.")+"\n") fhWarning.close() fnDiag="diagrtb.eigenfacs" if StructureType=="EM": runJob(log,"nma_reformatForElNemo.sh","%d"%NumberOfModes) fnDiag="diag_arpack.eigenfacs" runJob(log,"echo","%s | nma_check_modes"%fnDiag) deleteFile(log,fnDiag) fh=open("Chkmod.res") MDout=MetaData() collectivityList=[] for n in range(NumberOfModes): line=fh.readline() collectivity=float(line.split()[1]) collectivityList.append(collectivity) id=MDout.addObject() MDout.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec.%d"%(n+1)),id) MDout.setValue(MDL_ORDER,long(n+1),id) if n>=6: MDout.setValue(MDL_ENABLED,1,id) else: MDout.setValue(MDL_ENABLED,-1,id) MDout.setValue(MDL_NMA_COLLECTIVITY,collectivity,id) if collectivity<CollectivityThreshold: MDout.setValue(MDL_ENABLED,-1,id) fh.close() idxSorted=[i[0] for i in sorted(enumerate(collectivityList), key=lambda x:x[1])] score=[0]*NumberOfModes for i in range(NumberOfModes): score[i]+=i+1 score[idxSorted[i]]+=NumberOfModes-i i=0 for id in MDout: score_i=float(score[i])/(2.0*NumberOfModes) MDout.setValue(MDL_NMA_SCORE,score_i,id) i+=1 MDout.write("modes.xmd") deleteFile(log,"Chkmod.res") changeDir(log,currentDir)
def createAcquisition(log,InputFile,WorkingDir,DoResize,NewSize): fnAcqIn = findAcquisitionInfo(InputFile) if not fnAcqIn is None: fnAcqOut = getProtocolFilename('acquisition', WorkingDir=WorkingDir) if not DoResize: createLink(log, fnAcqIn, fnAcqOut) else: md = MetaData(fnAcqIn) id = md.firstObject() Ts = md.getValue(MDL_SAMPLINGRATE, id) (Xdim, Ydim, Zdim, Ndim, _) = MetaDataInfo(InputFile) downsampling = float(Xdim)/NewSize; md.setValue(MDL_SAMPLINGRATE,Ts*downsampling,id) md.write(getProtocolFilename('acquisition', WorkingDir=WorkingDir))
def writeImagesMd(log, md, ImportAll, SubsetMode, Nsubset, imagesFn, imagesStk, DoCopy): if not ImportAll: if SubsetMode=="Random particles": mdaux=MetaData() mdaux.randomize(md) else: mdaux=MetaData(md) md.selectPart(mdaux, 0, Nsubset) md.addItemId() md.write(imagesFn) if DoCopy: runJob(log,"xmipp_image_convert","-i %(imagesFn)s -o %(imagesStk)s" % locals()) md = MetaData(imagesFn) mdStk = MetaData(imagesStk) md.merge(mdStk) # Change MDL_IMAGE to point to new stack md.write(imagesFn)
def coocurenceMatrix(log,RemainingClasses,WorkingDirStructure,NumVolumes,nI,CorePercentile,CorrThresh): import numpy mdRemaining = MetaData(RemainingClasses) Nimgs=mdRemaining.size() allNames=mdRemaining.getColumnValues(MDL_IMAGE) matrixTotal = numpy.zeros([Nimgs,Nimgs]) for n in range(NumVolumes): fnBase='proposedVolume%05d'%n fnRoot=os.path.join(WorkingDirStructure,fnBase) md = MetaData(fnRoot+".xmd") size = md.size() num=[] corr=[] for objId in md: name = md.getValue(MDL_IMAGE, objId) if name in allNames: num.append(allNames.index(name)) corr.append(md.getValue(MDL_MAXCC, objId)) else: print "Cannot find ",name if size!=len(num): print "Error when processing: ",fnRoot+".xmd" aaa matrix = numpy.zeros([Nimgs,Nimgs]) for i in range(size): for j in range(size): matrix[num[i],num[j]]=((corr[i]+corr[j])/2) #numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix_%05d.txt'%n), matrix) matrixTotal=matrixTotal+matrix matrixTotal=matrixTotal/NumVolumes numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix.txt'),matrixTotal) largestComponent=procCoocurenceMatrix(matrixTotal,CorePercentile,CorrThresh) md = MetaData() for idx in largestComponent: id=md.addObject() md.setValue(MDL_IMAGE,allNames[idx],id) md.write(os.path.join(WorkingDirStructure+"_core","imagesCore.xmd")) if md.size()==0: print "There are no images in the core" aaa
def joinMetaDatas(log,InputFile,UntiltedMetadata,TiltedMetadata): mDInput=MetaData(InputFile) mDstack = MetaData(UntiltedMetadata) mDstack.removeLabel(MDL_MICROGRAPH) mDstack.removeLabel(MDL_XCOOR) mDstack.removeLabel(MDL_YCOOR) mdPart1=MetaData() mdPart1.join2(mDstack,mDInput,MDL_IMAGE_ORIGINAL,MDL_IMAGE,INNER_JOIN) mdPart1.removeLabel(MDL_IMAGE_ORIGINAL) mDstack = MetaData(TiltedMetadata) mDstack.removeLabel(MDL_MICROGRAPH) mDstack.removeLabel(MDL_XCOOR) mDstack.removeLabel(MDL_YCOOR) mDstack.renameColumn(MDL_IMAGE,MDL_IMAGE_TILTED) mdJoined=MetaData() mdJoined.join2(mDstack,mdPart1,MDL_IMAGE_ORIGINAL,MDL_IMAGE_TILTED,INNER_JOIN) mdJoined.removeLabel(MDL_IMAGE_ORIGINAL) mdJoined.write(InputFile)
def wizardTiltPairs(gui, var): dirMicrographs = gui.getVarValue('DirMicrographs') extMicrographs = gui.getVarValue('ExtMicrographs') resultFilename = var.getTkValue() uList = [] tList = [] from os.path import basename, dirname from xmipp import MDL_MICROGRAPH, MDL_MICROGRAPH_TILTED if exists(resultFilename): md = MetaData(resultFilename) for id in md: tPath = md.getValue(MDL_MICROGRAPH_TILTED, id) tList.append(basename(tPath)) uPath = md.getValue(MDL_MICROGRAPH, id) uList.append(basename(uPath)) prefix = dirname(uPath) # This assumes that all micrograph are in the same folder else: if len(resultFilename) == 0: resultFilename = "tilted_pairs.xmd" micrographs = glob(join(dirMicrographs, extMicrographs)) micrographs.sort() for i, m in enumerate(micrographs): m = basename(m) if i % 2 == 0: tList.append(m) else: uList.append(m) prefix = dirMicrographs from protlib_gui_ext import showTiltPairsDialog results = showTiltPairsDialog((uList, tList), gui.master) if results: var.setTkValue(resultFilename) uList, tList = results md = MetaData() for u, t in zip(uList, tList): id = md.addObject() md.setValue(MDL_MICROGRAPH, join(prefix,u), id) md.setValue(MDL_MICROGRAPH_TILTED, join(prefix,t), id) md.write(resultFilename)
def evaluateVolumes(log,WorkingDir,NRansac): fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd") fnCorr = 'corrThreshold@'+fnCorr mdCorr= MetaData(fnCorr) objId = mdCorr.firstObject() CorrThresh = mdCorr.getValue(MDL_WEIGHT,objId) for n in range(NRansac): fnRoot=os.path.join("ransac%05d"%n) fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd") md=MetaData(fnAngles) numInliers=0 for objId in md: corr = md.getValue(MDL_MAXCC, objId) if (corr >= CorrThresh) : numInliers = numInliers+corr md= MetaData() objId = md.addObject() md.setValue(MDL_WEIGHT,float(numInliers),objId) md.write("inliers@"+fnAngles,MD_APPEND)
def findCenter(log, HowCenter, Selfile, ExtraDir, SpectraInnerRadius, SpectraOuterRadius): if HowCenter == 'Minimize first harmonic': if SpectraOuterRadius + 20 > 100: R3 = SpectraOuterRadius + (100 - SpectraOuterRadius) / 2 R4 = 100 else: R3 = SpectraOuterRadius + 10 R4 = SpectraOuterRadius + 20 runJob(log, 'xmipp_image_find_center', '-i ' + Selfile + \ ' --oroot ' + ExtraDir + "/center2d" + \ ' --r1 ' + str(SpectraInnerRadius) + \ ' --r2 ' + str(SpectraOuterRadius) + \ ' --r3 ' + str(R3) + ' --r4 ' + str(R4)) elif HowCenter == 'Use the middle of the image': from xmipp import MetaDataInfo dims = MetaDataInfo(Selfile) MD = MetaData() id = MD.addObject() MD.setValue(MDL_X, float(dims[0] / 2), id) MD.setValue(MDL_Y, float(dims[1] / 2), id) MD.write(join(ExtraDir, "center2d_center.xmd"))
def run(self): oldCtfdat = self.getParam('-i') oroot = self.getParam('--oroot') md = MetaData() md.readPlain(oldCtfdat, "image CTFModel") ctfparamsDict = {} mode = MD_OVERWRITE for objId in md: oldCtfparam = md.getValue(MDL_CTF_MODEL, objId) if oldCtfparam in ctfparamsDict: newCtfparam = ctfparamsDict[oldCtfparam] else: block = splitext(basename(oldCtfparam))[0] newCtfparam = "%(block)s@%(oroot)s.ctfparam" % locals() md2 = convertCtfparam(oldCtfparam) md2.write(newCtfparam, mode) ctfparamsDict[oldCtfparam] = newCtfparam mode = MD_APPEND md.setValue(MDL_CTF_MODEL, newCtfparam, objId) md.write("%s.ctfdat" % oroot)
def gatherPairs(log, WorkingDir, ExtraDir, ClassSelection, ClassFile, ExtractDir, PickingDir): mdImages = getImagesMd(ExtractDir) fnMicrographs = getProtocolFilename("micrographs", WorkingDir=ExtractDir) mdTiltAngles = MetaData("micrographPairs@" + fnMicrographs) mdU = getImagesMd(ExtractDir, "untilted") mdUAux = MetaData() mdUAux.setColumnValues(MDL_IMAGE, mdU.getColumnValues(MDL_IMAGE)) mdUAux.setColumnValues(MDL_MICROGRAPH, mdU.getColumnValues(MDL_MICROGRAPH)) mdT = getImagesMd(ExtractDir, "tilted") mdTAux = MetaData() mdTAux.setColumnValues(MDL_IMAGE_TILTED, mdT.getColumnValues(MDL_IMAGE)) mdTAux.setColumnValues(MDL_MICROGRAPH_TILTED, mdT.getColumnValues(MDL_MICROGRAPH)) fnExtractImages = os.path.join(ExtractDir, "images.xmd") mdJoin1 = MetaData() mdJoin2 = MetaData() mdJoin3 = MetaData() mdJoin4 = MetaData() fixedMdClass = MetaData() fnOut = join(ExtraDir, "rct_classes.xmd") for classNo in ClassSelection: MDclass = MetaData(getClassBlock(classNo, ClassFile)) if not MDclass.containsLabel(MDL_IMAGE_TILTED): if MDclass.containsLabel(MDL_IMAGE_ORIGINAL): fixedMdClass.join2(MDclass, mdImages, MDL_IMAGE_ORIGINAL, MDL_IMAGE, INNER_JOIN) else: fixedMdClass.join1(MDclass, mdImages, MDL_IMAGE, INNER_JOIN) else: fixedMdClass = MDclass mdJoin1.join1(fixedMdClass, mdImages, MDL_IMAGE, LEFT_JOIN) mdJoin2.join1(mdJoin1, mdUAux, MDL_IMAGE, LEFT_JOIN) mdJoin3.join1(mdJoin2, mdTAux, MDL_IMAGE_TILTED, LEFT_JOIN) mdJoin4.join1(mdJoin3, mdTiltAngles, MDL_MICROGRAPH, LEFT_JOIN) fnClassOut = getClassBlock(classNo, fnOut) mdJoin4.write(fnClassOut, MD_APPEND)
def scoreFinalVolumes(log,WorkingDir,NumVolumes): mdOut=MetaData() for n in range(NumVolumes): fnRoot=os.path.join(WorkingDir,'Structure00000_core_extended','proposedVolume%05d'%n) fnAssignment=fnRoot+".xmd" if exists(fnAssignment): MDassignment=MetaData(fnAssignment) sum=0 N=0 minCC=2 for id in MDassignment: cc=MDassignment.getValue(MDL_MAXCC,id) sum+=cc if cc<minCC: minCC=cc N+=1 avg=sum/N id=mdOut.addObject() mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id) mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id) mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id) mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id) mdOut.write(os.path.join(WorkingDir,"proposedVolumes.xmd"))
def evaluateDeformationsStep(self): N = self.inputStructures.get().getSize() import numpy distances=numpy.zeros([N,N]) for volCounter in range(1,N+1): pdb1=open(self._getPath('pseudoatoms_%02d.pdb'%volCounter)).readlines() for volCounter2 in range(1,N+1): if volCounter!=volCounter2: davg=0. Navg=0. pdb2=open(self._getExtraPath('alignment_%02d_%02d.pdb'%(volCounter,volCounter2))).readlines() for i in range(len(pdb1)): line1=pdb1[i] if line1.startswith("ATOM"): line2=pdb2[i] x1=float(line1[30:37]) y1=float(line1[38:45]) z1=float(line1[46:53]) x2=float(line2[30:37]) y2=float(line2[38:45]) z2=float(line2[46:53]) dx=x1-x2 dy=y1-y2 dz=z1-z2 d=math.sqrt(dx*dx+dy*dy+dz*dz) davg+=d Navg+=1 if Navg>0: davg/=Navg distances[volCounter-1,volCounter2-1]=davg distances=0.5*(distances+numpy.transpose(distances)) numpy.savetxt(self._getPath('distances.txt'),distances) distances1D=numpy.mean(distances,axis=0) print("Average distance to rest of volumes=",distances1D) imin=numpy.argmin(distances1D) print("The volume in the middle is pseudoatoms_%02d.pdb"%(imin+1)) createLink(self._getPath("pseudoatoms_%02d.pdb"%(imin+1)),self._getPath("pseudoatoms.pdb")) createLink(self._getPath("modes_%02d.xmd"%(imin+1)),self._getPath("modes.xmd")) createLink(self._getExtraPath("pseudoatoms_%02d_distance.hist"%(imin+1)),self._getExtraPath("pseudoatoms_distance.hist")) # Measure range minDisplacement= 1e38*numpy.ones([self.numberOfModes.get(),1]) maxDisplacement=-1e38*numpy.ones([self.numberOfModes.get(),1]) mdNMA=MetaData(self._getPath("modes.xmd")) for volCounter in range(1,N+1): if volCounter!=imin+1: md=MetaData(self._getExtraPath("alignment_%02d_%02d.xmd"%(imin+1,volCounter))) displacements=md.getValue(MDL_NMA, md.firstObject()) idx1=0 idx2=0 for idRow in mdNMA: if mdNMA.getValue(MDL_ENABLED,idRow)==1: minDisplacement[idx2]=min(minDisplacement[idx2],displacements[idx1]) maxDisplacement[idx2]=max(maxDisplacement[idx2],displacements[idx1]) idx1+=1 else: minDisplacement[idx2]=0 maxDisplacement[idx2]=0 idx2+=1 idx2=0 for idRow in mdNMA: mdNMA.setValue(MDL_NMA_MINRANGE,float(minDisplacement[idx2]),idRow) mdNMA.setValue(MDL_NMA_MAXRANGE,float(maxDisplacement[idx2]),idRow) idx2+=1 mdNMA.write(self._getPath("modes.xmd")) # Create output volCounter=0 for inputStructure in self.inputStructures.get(): if volCounter==imin: print("The corresponding volume is %s"%(getImageLocation(inputStructure))) finalStructure=inputStructure break volCounter+=1 pdb = PdbFile(self._getPath('pseudoatoms.pdb'), pseudoatoms=True) self._defineOutputs(outputPdb=pdb) modes = NormalModes(filename=self._getPath('modes.xmd')) self._defineOutputs(outputModes=modes) self._defineSourceRelation(finalStructure, self.outputPdb) self._defineSourceRelation(self.outputPdb, self.outputModes)
def createEmptyMicrographSel(log, fnOut): md = MetaData() md.setValue(MDL_IMAGE,"ImportedImages", md.addObject()) md.write(fnOut)
def createAcquisitionMd(log, samplingRate, fnOut): md = MetaData() md.setValue(MDL_SAMPLINGRATE, float(samplingRate), md.addObject()) md.write(fnOut)
def copyImages(log,InputFile,OutputStack): fnOutputMetadata=OutputStack.replace(".stk",".xmd") runJob(log,"xmipp_image_convert","-i %(InputFile)s -o %(OutputStack)s --track_origin --save_metadata_stack %(fnOutputMetadata)s --keep_input_columns" % locals()) mDstack = MetaData(fnOutputMetadata) mDstack.removeLabel(MDL_ZSCORE) mDstack.write(fnOutputMetadata)
def createAcquisition(log,WorkingDir,Ts): md=MetaData() id=md.addObject() md.setValue(MDL_SAMPLINGRATE,float(Ts),id) md.write(os.path.join(WorkingDir,"acquisition_info.xmd"))
def writeToFile(self, fn): md = MetaData() self.writeToMd(md, md.addObject()) md.write(fn)
def evaluateDeformationsStep(self): N = self.inputStructures.get().getSize() import numpy distances = numpy.zeros([N, N]) for volCounter in range(1, N + 1): pdb1 = open(self._getPath('pseudoatoms_%02d.pdb' % volCounter)).readlines() for volCounter2 in range(1, N + 1): if volCounter != volCounter2: davg = 0. Navg = 0. pdb2 = open( self._getExtraPath( 'alignment_%02d_%02d.pdb' % (volCounter, volCounter2))).readlines() for i in range(len(pdb1)): line1 = pdb1[i] if line1.startswith("ATOM"): line2 = pdb2[i] x1 = float(line1[30:37]) y1 = float(line1[38:45]) z1 = float(line1[46:53]) x2 = float(line2[30:37]) y2 = float(line2[38:45]) z2 = float(line2[46:53]) dx = x1 - x2 dy = y1 - y2 dz = z1 - z2 d = math.sqrt(dx * dx + dy * dy + dz * dz) davg += d Navg += 1 if Navg > 0: davg /= Navg distances[volCounter - 1, volCounter2 - 1] = davg distances = 0.5 * (distances + numpy.transpose(distances)) numpy.savetxt(self._getPath('distances.txt'), distances) distances1D = numpy.mean(distances, axis=0) print("Average distance to rest of volumes=", distances1D) imin = numpy.argmin(distances1D) print("The volume in the middle is pseudoatoms_%02d.pdb" % (imin + 1)) createLink(self._getPath("pseudoatoms_%02d.pdb" % (imin + 1)), self._getPath("pseudoatoms.pdb")) createLink(self._getPath("modes_%02d.xmd" % (imin + 1)), self._getPath("modes.xmd")) createLink( self._getExtraPath("pseudoatoms_%02d_distance.hist" % (imin + 1)), self._getExtraPath("pseudoatoms_distance.hist")) # Measure range minDisplacement = 1e38 * numpy.ones([self.numberOfModes.get(), 1]) maxDisplacement = -1e38 * numpy.ones([self.numberOfModes.get(), 1]) mdNMA = MetaData(self._getPath("modes.xmd")) for volCounter in range(1, N + 1): if volCounter != imin + 1: md = MetaData( self._getExtraPath("alignment_%02d_%02d.xmd" % (imin + 1, volCounter))) displacements = md.getValue(MDL_NMA, md.firstObject()) idx1 = 0 idx2 = 0 for idRow in mdNMA: if mdNMA.getValue(MDL_ENABLED, idRow) == 1: minDisplacement[idx2] = min(minDisplacement[idx2], displacements[idx1]) maxDisplacement[idx2] = max(maxDisplacement[idx2], displacements[idx1]) idx1 += 1 else: minDisplacement[idx2] = 0 maxDisplacement[idx2] = 0 idx2 += 1 idx2 = 0 for idRow in mdNMA: mdNMA.setValue(MDL_NMA_MINRANGE, float(minDisplacement[idx2]), idRow) mdNMA.setValue(MDL_NMA_MAXRANGE, float(maxDisplacement[idx2]), idRow) idx2 += 1 mdNMA.write(self._getPath("modes.xmd")) # Create output volCounter = 0 for inputStructure in self.inputStructures.get(): if volCounter == imin: print("The corresponding volume is %s" % (getImageLocation(inputStructure))) finalStructure = inputStructure break volCounter += 1 pdb = PdbFile(self._getPath('pseudoatoms.pdb'), pseudoatoms=True) self._defineOutputs(outputPdb=pdb) modes = NormalModes(filename=self._getPath('modes.xmd')) self._defineOutputs(outputModes=modes) self._defineSourceRelation(self.inputStructures, self.outputPdb)
def writeToFile(self, fn): md = MetaData() self.writeToMd(md, md.addObject()) md.write(fn)