def getCorrThresh(log,WorkingDir,NRansac,CorrThresh): corrVector = [] fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd") mdCorr= MetaData() for n in range(NRansac): fnRoot=os.path.join("ransac%05d"%n) fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd") md=MetaData(fnAngles) for objId in md: corr = md.getValue(MDL_MAXCC, objId) corrVector.append(corr) objIdCorr = mdCorr.addObject() mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr) mdCorr.write("correlations@"+fnCorr,MD_APPEND) mdCorr= MetaData() sortedCorrVector = sorted(corrVector) indx = int(floor(CorrThresh*(len(sortedCorrVector)-1))) #With the line below commented the percentil is not used for the threshold and is used the value introduced in the form #CorrThresh = sortedCorrVector[indx]# objId = mdCorr.addObject() mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId) mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND) print "Correlation threshold: "+str(CorrThresh)
def computeAtomShifts(log,WorkingDir,NumberOfModes): fnOutDir=os.path.join(WorkingDir,"extra/distanceProfiles") createDir(log,fnOutDir) maxShift=[] maxShiftMode=[] for n in range(7,NumberOfModes+1): fhIn=open(os.path.join(WorkingDir,"modes/vec."+str(n))) md=MetaData() atomCounter=0; for line in fhIn: coord=line.split() x=float(coord[0]) y=float(coord[1]) z=float(coord[2]) d=math.sqrt(x*x+y*y+z*z) if n==7: maxShift.append(d) maxShiftMode.append(7) else: if d>maxShift[atomCounter]: maxShift[atomCounter]=d maxShiftMode[atomCounter]=n atomCounter+=1 md.setValue(MDL_NMA_ATOMSHIFT,d,md.addObject()) md.write(os.path.join(fnOutDir,"vec"+str(n)+".xmd")) fhIn.close() md=MetaData() for i in range(len(maxShift)): id=md.addObject() md.setValue(MDL_NMA_ATOMSHIFT,maxShift[i],id) md.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec."+str(maxShiftMode[i]+1)),id) md.write(os.path.join(WorkingDir,'extra','maxAtomShifts.xmd'))
def createMetaDataFromPattern(pattern, isStack=False, label="image"): ''' Create a metadata from files matching pattern''' import glob files = glob.glob(pattern) files.sort() from xmipp import MetaData, FileName, getImageSize, MDL_ENABLED, str2Label label = str2Label(label) #Check for label value mD = MetaData() inFile = FileName() nSize = 1 for file in files: fileAux=file if isStack: if file.endswith(".mrc"): fileAux=file+":mrcs" x, x, x, nSize = getImageSize(fileAux) if nSize != 1: counter = 1 for jj in range(nSize): inFile.compose(counter, fileAux) objId = mD.addObject() mD.setValue(label, inFile, objId) mD.setValue(MDL_ENABLED, 1, objId) counter += 1 else: objId = mD.addObject() mD.setValue(label, fileAux, objId) mD.setValue(MDL_ENABLED, 1, objId) return mD
def getCorrThresh(log,WorkingDir,WorkingDirStructure,NRansac,CorrThresh): corrVector = [] fnCorr=os.path.join(WorkingDirStructure,"correlations.xmd") mdCorr= MetaData() for n in range(NRansac): fnRoot=os.path.join("ransac%05d"%n) fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd") if os.path.exists(fnAngles): md=MetaData(fnAngles) for objId in md: corr = md.getValue(MDL_MAXCC, objId) corrVector.append(corr) objIdCorr = mdCorr.addObject() mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr) mdCorr.write("correlations@"+fnCorr,MD_APPEND) mdCorr= MetaData() sortedCorrVector = sorted(corrVector) indx = int(floor(CorrThresh*(len(sortedCorrVector)-1))) objId = mdCorr.addObject() mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId) mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND) print "Correlation threshold: "+str(CorrThresh)
def reconstructClass( log, WorkingDir, ExtraDir, ClassNameIn, ClassNameOut, ClassImage, CenterMaxShift, ThinObject, SkipTiltedTranslations, ClassVolumeOut, ReconstructAdditionalParams, DoLowPassFilter, LowPassFilter, ): # If class image doesn't exists, generate it by averaging if len(ClassImage) == 0: classRootOut = ClassNameOut.replace(".xmd", "") + "_" params = "-i %(ClassNameIn)s --save_image_stats %(classRootOut)s -o %(ExtraDir)s/stats.xmd" % locals() runJob(log, "xmipp_image_statistics", params) ClassImage = classRootOut + "average.xmp" deleteFile(log, classRootOut + "stddev.xmp") deleteFile(log, "%(ExtraDir)s/stats.xmd" % locals()) params = "-i %(ClassNameIn)s -o %(ClassNameOut)s --ref %(ClassImage)s --max_shift %(CenterMaxShift)d" % locals() if ThinObject: params += " --do_stretch" if SkipTiltedTranslations: params += " --do_not_align_tilted" runJob(log, "xmipp_image_align_tilt_pairs", params) params = "-i %(ClassNameOut)s -o %(ClassVolumeOut)s %(ReconstructAdditionalParams)s" % locals() runJob(log, "xmipp_reconstruct_art", params) if exists(ClassVolumeOut): mdFn = join(WorkingDir, "volumes.xmd") md = MetaData() if exists(mdFn): md.read(mdFn) objId = md.addObject() md.setValue(MDL_IMAGE, ClassVolumeOut, objId) if DoLowPassFilter: filteredVolume = ClassVolumeOut.replace(".vol", "_filtered.vol") params = "-i %(ClassVolumeOut)s -o %(filteredVolume)s --fourier low_pass %(LowPassFilter)f" % locals() runJob(log, "xmipp_transform_filter", params) objId = md.addObject() md.setValue(MDL_IMAGE, filteredVolume, objId) md.write(mdFn)
def scoreFinalVolumes(log,WorkingDir,NumVolumes): threshold=getCCThreshold(WorkingDir) mdOut=MetaData() for n in range(NumVolumes): fnRoot=os.path.join(WorkingDir,'volumeProposed%05d'%n) fnAssignment=fnRoot+".xmd" if exists(fnAssignment): runJob(log,"xmipp_metadata_utilities","-i %s --fill weight constant 1"%fnAssignment) MDassignment=MetaData(fnAssignment) sum=0 thresholdedSum=0 N=0 minCC=2 for id in MDassignment: cc=MDassignment.getValue(MDL_MAXCC,id) sum+=cc thresholdedSum+=cc-threshold if cc<minCC: minCC=cc N+=1 avg=sum/N id=mdOut.addObject() mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id) mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id) mdOut.setValue(MDL_VOLUME_SCORE_SUM_TH,float(thresholdedSum),id) mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id) mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id) mdOut.write(os.path.join(WorkingDir,"volumesProposed.xmd"))
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images): from numpy import array, dot fnOut = "classes_aligned@" + fn MDin = MetaData(images) MDout = MetaData() n = 1 hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL) for i in MDin: fnImg = MDin.getValue(xmipp.MDL_IMAGE, i) fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i) maxCC = MDin.getValue(xmipp.MDL_MAXCC, i) rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i) tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i) psi = -1.0 * MDin.getValue(xmipp.MDL_ANGLE_PSI, i) flip = MDin.getValue(xmipp.MDL_FLIP, i) if flip: psi = -psi eulerMatrix = Euler_angles2matrix(0.0, 0.0, psi) x = MDin.getValue(xmipp.MDL_SHIFT_X, i) y = MDin.getValue(xmipp.MDL_SHIFT_Y, i) shift = array([x, y, 0]) shiftOut = dot(eulerMatrix, shift) [x, y, z] = shiftOut if flip: x = -x id = MDout.addObject() MDout.setValue(xmipp.MDL_IMAGE, fnImg, id) MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id) MDout.setValue(xmipp.MDL_IMAGE1, "%05d@%s" % (n, self._getExtraPath("diff.stk")), id) if hasCTF: fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i) MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id) MDout.setValue(xmipp.MDL_MAXCC, maxCC, id) MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id) MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id) MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id) MDout.setValue(xmipp.MDL_SHIFT_X, x, id) MDout.setValue(xmipp.MDL_SHIFT_Y, y, id) MDout.setValue(xmipp.MDL_FLIP, flip, id) MDout.setValue(xmipp.MDL_ENABLED, 1, id) n += 1 MDout.write(fnOut, xmipp.MD_APPEND) # Actually create the differences img = Image() imgRef = Image() if hasCTF and volumeIsCTFCorrected: Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject()) for i in MDout: img.readApplyGeo(MDout, i) imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i)) if hasCTF and volumeIsCTFCorrected: fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i) imgRef.applyCTF(fnCTF, Ts) img.convert2DataType(DT_DOUBLE) imgDiff = img - imgRef imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images): from numpy import array, dot fnOut = 'classes_aligned@' + fn MDin = MetaData(images) MDout = MetaData() n = 1 hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL) for i in MDin: fnImg = MDin.getValue(xmipp.MDL_IMAGE, i) fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i) maxCC = MDin.getValue(xmipp.MDL_MAXCC, i) rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i) tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i) psi = -1. * MDin.getValue(xmipp.MDL_ANGLE_PSI, i) flip = MDin.getValue(xmipp.MDL_FLIP, i) if flip: psi = -psi eulerMatrix = Euler_angles2matrix(0., 0., psi) x = MDin.getValue(xmipp.MDL_SHIFT_X, i) y = MDin.getValue(xmipp.MDL_SHIFT_Y, i) shift = array([x, y, 0]) shiftOut = dot(eulerMatrix, shift) [x, y, z] = shiftOut if flip: x = -x id = MDout.addObject() MDout.setValue(xmipp.MDL_IMAGE, fnImg, id) MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id) MDout.setValue(xmipp.MDL_IMAGE1, "%05d@%s" % (n, self._getExtraPath("diff.stk")), id) if hasCTF: fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i) MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id) MDout.setValue(xmipp.MDL_MAXCC, maxCC, id) MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id) MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id) MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id) MDout.setValue(xmipp.MDL_SHIFT_X, x, id) MDout.setValue(xmipp.MDL_SHIFT_Y, y, id) MDout.setValue(xmipp.MDL_FLIP, flip, id) MDout.setValue(xmipp.MDL_ENABLED, 1, id) n += 1 MDout.write(fnOut, xmipp.MD_APPEND) # Actually create the differences img = Image() imgRef = Image() if hasCTF and volumeIsCTFCorrected: Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject()) for i in MDout: img.readApplyGeo(MDout, i) imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i)) if hasCTF and volumeIsCTFCorrected: fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i) imgRef.applyCTF(fnCTF, Ts) img.convert2DataType(DT_DOUBLE) imgDiff = img - imgRef imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
class RowMetaData(): """ This class is a wrapper for MetaData in row mode. Where only one object is used. """ def __init__(self, filename=None): self._md = MetaData() self._md.setColumnFormat(False) self._id = self._md.addObject() if filename: self.read(filename) def setValue(self, label, value): self._md.setValue(label, value, self._id) def getValue(self, label): return self._md.getValue(label, self._id) def write(self, filename, mode=MD_APPEND): self._md.write(filename, mode) def read(self, filename): self._md.read(filename) self._md.setColumnFormat(False) self._id = self._md.firstObject() def containsLabel(self, label): return self._md.containsLabel(label) def __str__(self): return str(self._md)
class RowMetaData(): """ This class is a wrapper for MetaData in row mode. Where only one object is used. """ def __init__(self, filename=None): self._md = MetaData() self._md.setColumnFormat(False) self._id = self._md.addObject() if filename: self.read(filename) def setValue(self, label, value): self._md.setValue(label, value, self._id) def getValue(self, label): return self._md.getValue(label, self._id) def write(self, filename, mode=MD_APPEND): self._md.write(filename, mode) def read(self, filename): self._md.read(filename) self._md.setColumnFormat(False) self._id = self._md.firstObject() def containsLabel(self, label): return self._md.containsLabel(label) def __str__(self): return str(self._md)
def qualifyModes(log,WorkingDir,NumberOfModes,StructureType,CollectivityThreshold): currentDir=os.getcwd() changeDir(log,WorkingDir) fnVec=glob.glob("modes/vec.*") if len(fnVec)<NumberOfModes: fhWarning=open("warnings.xmd",'w') fhWarning.write(redStr("There are only "+str(len(fnVec))+" modes instead of "+str(NumberOfModes)+". Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks and for pseudoatomic normal mode analysis 3 times the number of pseudoatoms. However, the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between these two numbers, consider increasing cut-off distance.")+"\n") fhWarning.close() fnDiag="diagrtb.eigenfacs" if StructureType=="EM": runJob(log,"nma_reformatForElNemo.sh","%d"%NumberOfModes) fnDiag="diag_arpack.eigenfacs" runJob(log,"echo","%s | nma_check_modes"%fnDiag) deleteFile(log,fnDiag) fh=open("Chkmod.res") MDout=MetaData() collectivityList=[] for n in range(NumberOfModes): line=fh.readline() collectivity=float(line.split()[1]) collectivityList.append(collectivity) id=MDout.addObject() MDout.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec.%d"%(n+1)),id) MDout.setValue(MDL_ORDER,long(n+1),id) if n>=6: MDout.setValue(MDL_ENABLED,1,id) else: MDout.setValue(MDL_ENABLED,-1,id) MDout.setValue(MDL_NMA_COLLECTIVITY,collectivity,id) if collectivity<CollectivityThreshold: MDout.setValue(MDL_ENABLED,-1,id) fh.close() idxSorted=[i[0] for i in sorted(enumerate(collectivityList), key=lambda x:x[1])] score=[0]*NumberOfModes for i in range(NumberOfModes): score[i]+=i+1 score[idxSorted[i]]+=NumberOfModes-i i=0 for id in MDout: score_i=float(score[i])/(2.0*NumberOfModes) MDout.setValue(MDL_NMA_SCORE,score_i,id) i+=1 MDout.write("modes.xmd") deleteFile(log,"Chkmod.res") changeDir(log,currentDir)
def coocurenceMatrix(log,RemainingClasses,WorkingDirStructure,NumVolumes,nI,CorePercentile,CorrThresh): import numpy mdRemaining = MetaData(RemainingClasses) Nimgs=mdRemaining.size() allNames=mdRemaining.getColumnValues(MDL_IMAGE) matrixTotal = numpy.zeros([Nimgs,Nimgs]) for n in range(NumVolumes): fnBase='proposedVolume%05d'%n fnRoot=os.path.join(WorkingDirStructure,fnBase) md = MetaData(fnRoot+".xmd") size = md.size() num=[] corr=[] for objId in md: name = md.getValue(MDL_IMAGE, objId) if name in allNames: num.append(allNames.index(name)) corr.append(md.getValue(MDL_MAXCC, objId)) else: print "Cannot find ",name if size!=len(num): print "Error when processing: ",fnRoot+".xmd" aaa matrix = numpy.zeros([Nimgs,Nimgs]) for i in range(size): for j in range(size): matrix[num[i],num[j]]=((corr[i]+corr[j])/2) #numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix_%05d.txt'%n), matrix) matrixTotal=matrixTotal+matrix matrixTotal=matrixTotal/NumVolumes numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix.txt'),matrixTotal) largestComponent=procCoocurenceMatrix(matrixTotal,CorePercentile,CorrThresh) md = MetaData() for idx in largestComponent: id=md.addObject() md.setValue(MDL_IMAGE,allNames[idx],id) md.write(os.path.join(WorkingDirStructure+"_core","imagesCore.xmd")) if md.size()==0: print "There are no images in the core" aaa
def wizardTiltPairs(gui, var): dirMicrographs = gui.getVarValue('DirMicrographs') extMicrographs = gui.getVarValue('ExtMicrographs') resultFilename = var.getTkValue() uList = [] tList = [] from os.path import basename, dirname from xmipp import MDL_MICROGRAPH, MDL_MICROGRAPH_TILTED if exists(resultFilename): md = MetaData(resultFilename) for id in md: tPath = md.getValue(MDL_MICROGRAPH_TILTED, id) tList.append(basename(tPath)) uPath = md.getValue(MDL_MICROGRAPH, id) uList.append(basename(uPath)) prefix = dirname(uPath) # This assumes that all micrograph are in the same folder else: if len(resultFilename) == 0: resultFilename = "tilted_pairs.xmd" micrographs = glob(join(dirMicrographs, extMicrographs)) micrographs.sort() for i, m in enumerate(micrographs): m = basename(m) if i % 2 == 0: tList.append(m) else: uList.append(m) prefix = dirMicrographs from protlib_gui_ext import showTiltPairsDialog results = showTiltPairsDialog((uList, tList), gui.master) if results: var.setTkValue(resultFilename) uList, tList = results md = MetaData() for u, t in zip(uList, tList): id = md.addObject() md.setValue(MDL_MICROGRAPH, join(prefix,u), id) md.setValue(MDL_MICROGRAPH_TILTED, join(prefix,t), id) md.write(resultFilename)
def evaluateVolumes(log,WorkingDir,NRansac): fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd") fnCorr = 'corrThreshold@'+fnCorr mdCorr= MetaData(fnCorr) objId = mdCorr.firstObject() CorrThresh = mdCorr.getValue(MDL_WEIGHT,objId) for n in range(NRansac): fnRoot=os.path.join("ransac%05d"%n) fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd") md=MetaData(fnAngles) numInliers=0 for objId in md: corr = md.getValue(MDL_MAXCC, objId) if (corr >= CorrThresh) : numInliers = numInliers+corr md= MetaData() objId = md.addObject() md.setValue(MDL_WEIGHT,float(numInliers),objId) md.write("inliers@"+fnAngles,MD_APPEND)
def findCenter(log, HowCenter, Selfile, ExtraDir, SpectraInnerRadius, SpectraOuterRadius): if HowCenter == 'Minimize first harmonic': if SpectraOuterRadius + 20 > 100: R3 = SpectraOuterRadius + (100 - SpectraOuterRadius) / 2 R4 = 100 else: R3 = SpectraOuterRadius + 10 R4 = SpectraOuterRadius + 20 runJob(log, 'xmipp_image_find_center', '-i ' + Selfile + \ ' --oroot ' + ExtraDir + "/center2d" + \ ' --r1 ' + str(SpectraInnerRadius) + \ ' --r2 ' + str(SpectraOuterRadius) + \ ' --r3 ' + str(R3) + ' --r4 ' + str(R4)) elif HowCenter == 'Use the middle of the image': from xmipp import MetaDataInfo dims = MetaDataInfo(Selfile) MD = MetaData() id = MD.addObject() MD.setValue(MDL_X, float(dims[0] / 2), id) MD.setValue(MDL_Y, float(dims[1] / 2), id) MD.write(join(ExtraDir, "center2d_center.xmd"))
def scoreFinalVolumes(log,WorkingDir,NumVolumes): mdOut=MetaData() for n in range(NumVolumes): fnRoot=os.path.join(WorkingDir,'Structure00000_core_extended','proposedVolume%05d'%n) fnAssignment=fnRoot+".xmd" if exists(fnAssignment): MDassignment=MetaData(fnAssignment) sum=0 N=0 minCC=2 for id in MDassignment: cc=MDassignment.getValue(MDL_MAXCC,id) sum+=cc if cc<minCC: minCC=cc N+=1 avg=sum/N id=mdOut.addObject() mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id) mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id) mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id) mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id) mdOut.write(os.path.join(WorkingDir,"proposedVolumes.xmd"))
def writeToFile(self, fn): md = MetaData() self.writeToMd(md, md.addObject()) md.write(fn)
def writeToFile(self, fn): md = MetaData() self.writeToMd(md, md.addObject()) md.write(fn)
def createEmptyMicrographSel(log, fnOut): md = MetaData() md.setValue(MDL_IMAGE,"ImportedImages", md.addObject()) md.write(fnOut)
def createAcquisition(log,WorkingDir,Ts): md=MetaData() id=md.addObject() md.setValue(MDL_SAMPLINGRATE,float(Ts),id) md.write(os.path.join(WorkingDir,"acquisition_info.xmd"))
def createAcquisitionMd(log, samplingRate, fnOut): md = MetaData() md.setValue(MDL_SAMPLINGRATE, float(samplingRate), md.addObject()) md.write(fnOut)