def getCorrThresh(log,WorkingDir,NRansac,CorrThresh):
corrVector = []
fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd")
mdCorr= MetaData()
for n in range(NRansac):
fnRoot=os.path.join("ransac%05d"%n)
fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd")
md=MetaData(fnAngles)
for objId in md:
corr = md.getValue(MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr)
mdCorr.write("correlations@"+fnCorr,MD_APPEND)
mdCorr= MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(CorrThresh*(len(sortedCorrVector)-1)))
#With the line below commented the percentil is not used for the threshold and is used the value introduced in the form
#CorrThresh = sortedCorrVector[indx]#
objId = mdCorr.addObject()
mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId)
mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND)
print "Correlation threshold: "+str(CorrThresh)
def scoreFinalVolumes(log,WorkingDir,NumVolumes):
threshold=getCCThreshold(WorkingDir)
mdOut=MetaData()
for n in range(NumVolumes):
fnRoot=os.path.join(WorkingDir,'volumeProposed%05d'%n)
fnAssignment=fnRoot+".xmd"
if exists(fnAssignment):
runJob(log,"xmipp_metadata_utilities","-i %s --fill weight constant 1"%fnAssignment)
MDassignment=MetaData(fnAssignment)
sum=0
thresholdedSum=0
N=0
minCC=2
for id in MDassignment:
cc=MDassignment.getValue(MDL_MAXCC,id)
sum+=cc
thresholdedSum+=cc-threshold
if cc<minCC:
minCC=cc
N+=1
avg=sum/N
id=mdOut.addObject()
mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id)
mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id)
mdOut.setValue(MDL_VOLUME_SCORE_SUM_TH,float(thresholdedSum),id)
mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id)
mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id)
mdOut.write(os.path.join(WorkingDir,"volumesProposed.xmd"))
def computeAtomShifts(log,WorkingDir,NumberOfModes):
fnOutDir=os.path.join(WorkingDir,"extra/distanceProfiles")
createDir(log,fnOutDir)
maxShift=[]
maxShiftMode=[]
for n in range(7,NumberOfModes+1):
fhIn=open(os.path.join(WorkingDir,"modes/vec."+str(n)))
md=MetaData()
atomCounter=0;
for line in fhIn:
coord=line.split()
x=float(coord[0])
y=float(coord[1])
z=float(coord[2])
d=math.sqrt(x*x+y*y+z*z)
if n==7:
maxShift.append(d)
maxShiftMode.append(7)
else:
if d>maxShift[atomCounter]:
maxShift[atomCounter]=d
maxShiftMode[atomCounter]=n
atomCounter+=1
md.setValue(MDL_NMA_ATOMSHIFT,d,md.addObject())
md.write(os.path.join(fnOutDir,"vec"+str(n)+".xmd"))
fhIn.close()
md=MetaData()
for i in range(len(maxShift)):
id=md.addObject()
md.setValue(MDL_NMA_ATOMSHIFT,maxShift[i],id)
md.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec."+str(maxShiftMode[i]+1)),id)
md.write(os.path.join(WorkingDir,'extra','maxAtomShifts.xmd'))
class RowMetaData():
""" This class is a wrapper for MetaData in row mode.
Where only one object is used.
"""
def __init__(self, filename=None):
self._md = MetaData()
self._md.setColumnFormat(False)
self._id = self._md.addObject()
if filename:
self.read(filename)
def setValue(self, label, value):
self._md.setValue(label, value, self._id)
def getValue(self, label):
return self._md.getValue(label, self._id)
def write(self, filename, mode=MD_APPEND):
self._md.write(filename, mode)
def read(self, filename):
self._md.read(filename)
self._md.setColumnFormat(False)
self._id = self._md.firstObject()
def containsLabel(self, label):
return self._md.containsLabel(label)
def __str__(self):
return str(self._md)
class RowMetaData():
""" This class is a wrapper for MetaData in row mode.
Where only one object is used.
"""
def __init__(self, filename=None):
self._md = MetaData()
self._md.setColumnFormat(False)
self._id = self._md.addObject()
if filename:
self.read(filename)
def setValue(self, label, value):
self._md.setValue(label, value, self._id)
def getValue(self, label):
return self._md.getValue(label, self._id)
def write(self, filename, mode=MD_APPEND):
self._md.write(filename, mode)
def read(self, filename):
self._md.read(filename)
self._md.setColumnFormat(False)
self._id = self._md.firstObject()
def containsLabel(self, label):
return self._md.containsLabel(label)
def __str__(self):
return str(self._md)
def projMatchStep(self, volume, angularSampling, symmetryGroup, images, fnAngles, Xdim):
from pyworkflow.utils.path import cleanPath
# Generate gallery of projections
fnGallery = self._getExtraPath("gallery.stk")
self.runJob(
"xmipp_angular_project_library",
"-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"
% (volume, fnGallery, angularSampling, symmetryGroup, images),
)
# Assign angles
self.runJob(
"xmipp_angular_projection_matching",
"-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 --search5d_shift %s --search5d_step %s --append"
% (images, fnAngles, fnGallery, str(Xdim / 2), str(int(Xdim / 10)), str(int(Xdim / 25))),
)
cleanPath(self._getExtraPath("gallery_sampling.xmd"))
cleanPath(self._getExtraPath("gallery_angles.doc"))
cleanPath(self._getExtraPath("gallery.doc"))
# Write angles in the original file and sort
MD = MetaData(fnAngles)
for id in MD:
galleryReference = MD.getValue(xmipp.MDL_REF, id)
MD.setValue(xmipp.MDL_IMAGE_REF, "%05d@%s" % (galleryReference + 1, fnGallery), id)
MD.write(fnAngles)
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images):
from numpy import array, dot
fnOut = "classes_aligned@" + fn
MDin = MetaData(images)
MDout = MetaData()
n = 1
hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL)
for i in MDin:
fnImg = MDin.getValue(xmipp.MDL_IMAGE, i)
fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i)
maxCC = MDin.getValue(xmipp.MDL_MAXCC, i)
rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i)
tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i)
psi = -1.0 * MDin.getValue(xmipp.MDL_ANGLE_PSI, i)
flip = MDin.getValue(xmipp.MDL_FLIP, i)
if flip:
psi = -psi
eulerMatrix = Euler_angles2matrix(0.0, 0.0, psi)
x = MDin.getValue(xmipp.MDL_SHIFT_X, i)
y = MDin.getValue(xmipp.MDL_SHIFT_Y, i)
shift = array([x, y, 0])
shiftOut = dot(eulerMatrix, shift)
[x, y, z] = shiftOut
if flip:
x = -x
id = MDout.addObject()
MDout.setValue(xmipp.MDL_IMAGE, fnImg, id)
MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id)
MDout.setValue(xmipp.MDL_IMAGE1, "%05d@%s" % (n, self._getExtraPath("diff.stk")), id)
if hasCTF:
fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i)
MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id)
MDout.setValue(xmipp.MDL_MAXCC, maxCC, id)
MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id)
MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id)
MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id)
MDout.setValue(xmipp.MDL_SHIFT_X, x, id)
MDout.setValue(xmipp.MDL_SHIFT_Y, y, id)
MDout.setValue(xmipp.MDL_FLIP, flip, id)
MDout.setValue(xmipp.MDL_ENABLED, 1, id)
n += 1
MDout.write(fnOut, xmipp.MD_APPEND)
# Actually create the differences
img = Image()
imgRef = Image()
if hasCTF and volumeIsCTFCorrected:
Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject())
for i in MDout:
img.readApplyGeo(MDout, i)
imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i))
if hasCTF and volumeIsCTFCorrected:
fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i)
imgRef.applyCTF(fnCTF, Ts)
img.convert2DataType(DT_DOUBLE)
imgDiff = img - imgRef
imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images):
from numpy import array, dot
fnOut = 'classes_aligned@' + fn
MDin = MetaData(images)
MDout = MetaData()
n = 1
hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL)
for i in MDin:
fnImg = MDin.getValue(xmipp.MDL_IMAGE, i)
fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i)
maxCC = MDin.getValue(xmipp.MDL_MAXCC, i)
rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i)
tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i)
psi = -1. * MDin.getValue(xmipp.MDL_ANGLE_PSI, i)
flip = MDin.getValue(xmipp.MDL_FLIP, i)
if flip:
psi = -psi
eulerMatrix = Euler_angles2matrix(0., 0., psi)
x = MDin.getValue(xmipp.MDL_SHIFT_X, i)
y = MDin.getValue(xmipp.MDL_SHIFT_Y, i)
shift = array([x, y, 0])
shiftOut = dot(eulerMatrix, shift)
[x, y, z] = shiftOut
if flip:
x = -x
id = MDout.addObject()
MDout.setValue(xmipp.MDL_IMAGE, fnImg, id)
MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id)
MDout.setValue(xmipp.MDL_IMAGE1,
"%05d@%s" % (n, self._getExtraPath("diff.stk")), id)
if hasCTF:
fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i)
MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id)
MDout.setValue(xmipp.MDL_MAXCC, maxCC, id)
MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id)
MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id)
MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id)
MDout.setValue(xmipp.MDL_SHIFT_X, x, id)
MDout.setValue(xmipp.MDL_SHIFT_Y, y, id)
MDout.setValue(xmipp.MDL_FLIP, flip, id)
MDout.setValue(xmipp.MDL_ENABLED, 1, id)
n += 1
MDout.write(fnOut, xmipp.MD_APPEND)
# Actually create the differences
img = Image()
imgRef = Image()
if hasCTF and volumeIsCTFCorrected:
Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject())
for i in MDout:
img.readApplyGeo(MDout, i)
imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i))
if hasCTF and volumeIsCTFCorrected:
fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i)
imgRef.applyCTF(fnCTF, Ts)
img.convert2DataType(DT_DOUBLE)
imgDiff = img - imgRef
imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
def projMatchStep(self, volume, angularSampling, symmetryGroup, images,
fnAngles, Xdim):
from pyworkflow.utils.path import cleanPath
# Generate gallery of projections
fnGallery = self._getExtraPath('gallery.stk')
if volume.endswith('.mrc'):
volume += ":mrc"
self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\
% (volume, fnGallery, angularSampling, symmetryGroup, images))
# Assign angles
self.runJob("xmipp_angular_projection_matching", "-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 --search5d_shift %s --search5d_step %s --append"\
% (images, fnAngles, fnGallery, str(Xdim/2), str(int(Xdim/10)), str(int(Xdim/25))))
cleanPath(self._getExtraPath('gallery_sampling.xmd'))
cleanPath(self._getExtraPath('gallery_angles.doc'))
cleanPath(self._getExtraPath('gallery.doc'))
# Write angles in the original file and sort
MD = MetaData(fnAngles)
for id in MD:
galleryReference = MD.getValue(xmipp.MDL_REF, id)
MD.setValue(xmipp.MDL_IMAGE_REF,
"%05d@%s" % (galleryReference + 1, fnGallery), id)
MD.write(fnAngles)
def createOutput(self):
import glob
levelFiles = glob.glob(self._getExtraPath("results_classes_level*.xmd"))
if levelFiles:
levelFiles.sort()
lastLevelFile = levelFiles[-1]
setOfClasses = self._createSetOfClassesVol()
setOfClasses.setImages(self.inputVolumes.get())
readSetOfClassesVol(setOfClasses, lastLevelFile)
self._defineOutputs(outputClasses=setOfClasses)
self._defineSourceRelation(self.inputVolumes, self.outputClasses)
if self.generateAligned.get():
setOfVolumes = self._createSetOfVolumes()
fnAligned = self._getExtraPath("results_aligned.xmd")
self.runJob(
"xmipp_metadata_selfile_create",
"-p %s -o %s -s" % (self._getExtraPath("results_aligned.stk"), fnAligned),
)
md = MetaData(fnAligned)
md.addItemId()
md.write(fnAligned)
readSetOfVolumes(fnAligned, setOfVolumes)
volumeList = self.inputVolumes.get()
setOfVolumes.setSamplingRate(volumeList.getSamplingRate())
self._defineOutputs(alignedVolumes=setOfVolumes)
self._defineTransformRelation(self.inputVolumes, self.alignedVolumes)
def getCorrThresh(log,WorkingDir,WorkingDirStructure,NRansac,CorrThresh):
corrVector = []
fnCorr=os.path.join(WorkingDirStructure,"correlations.xmd")
mdCorr= MetaData()
for n in range(NRansac):
fnRoot=os.path.join("ransac%05d"%n)
fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd")
if os.path.exists(fnAngles):
md=MetaData(fnAngles)
for objId in md:
corr = md.getValue(MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr)
mdCorr.write("correlations@"+fnCorr,MD_APPEND)
mdCorr= MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(CorrThresh*(len(sortedCorrVector)-1)))
objId = mdCorr.addObject()
mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId)
mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND)
print "Correlation threshold: "+str(CorrThresh)
def createSubset(log, inputFile, inputFileLabel, subsetFile, subsetFileLabel, outputFile):
mdInputFile = MetaData(inputFile)
mdSubsetFile = MetaData(subsetFile)
mdOutputFile = MetaData()
print inputFile, inputFileLabel, subsetFile, subsetFileLabel, outputFile
mdOutputFile.join2( mdInputFile
, mdSubsetFile
, str2Label(inputFileLabel)
, str2Label(subsetFileLabel)
, INNER_JOIN)
mdOutputFile.write(outputFile)
def rewriteClassBlock(log,WorkingDir,ExtraDir):
fnClassMetadata=os.path.join(ExtraDir,"kerdensom_classes.xmd")
fnClass="classes@%s"%fnClassMetadata
fnClassStack=os.path.join(ExtraDir,"classes.stk")
mD = MetaData(fnClass)
counter = 1
for id in mD:
mD.setValue(MDL_IMAGE,"%06d@%s"%(counter,fnClassStack),id)
counter += 1
mD.write(fnClass,MD_APPEND)
createLink(log,fnClassMetadata,os.path.join(WorkingDir,"classes.xmd"))
createLink(log,os.path.join(ExtraDir,"kerdensom_images.xmd"),os.path.join(WorkingDir,"images.xmd"))
def reconstructClass(
log,
WorkingDir,
ExtraDir,
ClassNameIn,
ClassNameOut,
ClassImage,
CenterMaxShift,
ThinObject,
SkipTiltedTranslations,
ClassVolumeOut,
ReconstructAdditionalParams,
DoLowPassFilter,
LowPassFilter,
):
# If class image doesn't exists, generate it by averaging
if len(ClassImage) == 0:
classRootOut = ClassNameOut.replace(".xmd", "") + "_"
params = "-i %(ClassNameIn)s --save_image_stats %(classRootOut)s -o %(ExtraDir)s/stats.xmd" % locals()
runJob(log, "xmipp_image_statistics", params)
ClassImage = classRootOut + "average.xmp"
deleteFile(log, classRootOut + "stddev.xmp")
deleteFile(log, "%(ExtraDir)s/stats.xmd" % locals())
params = "-i %(ClassNameIn)s -o %(ClassNameOut)s --ref %(ClassImage)s --max_shift %(CenterMaxShift)d" % locals()
if ThinObject:
params += " --do_stretch"
if SkipTiltedTranslations:
params += " --do_not_align_tilted"
runJob(log, "xmipp_image_align_tilt_pairs", params)
params = "-i %(ClassNameOut)s -o %(ClassVolumeOut)s %(ReconstructAdditionalParams)s" % locals()
runJob(log, "xmipp_reconstruct_art", params)
if exists(ClassVolumeOut):
mdFn = join(WorkingDir, "volumes.xmd")
md = MetaData()
if exists(mdFn):
md.read(mdFn)
objId = md.addObject()
md.setValue(MDL_IMAGE, ClassVolumeOut, objId)
if DoLowPassFilter:
filteredVolume = ClassVolumeOut.replace(".vol", "_filtered.vol")
params = "-i %(ClassVolumeOut)s -o %(filteredVolume)s --fourier low_pass %(LowPassFilter)f" % locals()
runJob(log, "xmipp_transform_filter", params)
objId = md.addObject()
md.setValue(MDL_IMAGE, filteredVolume, objId)
md.write(mdFn)
def sortImages(log, ImagesFn, rejectionMethod='None', maxZscore=3, percentage=5):
md = MetaData(ImagesFn)
if not md.isEmpty():
args=""
if rejectionMethod=='MaxZscore':
args+=" --zcut "+str(maxZscore)
elif rejectionMethod=='Percentage':
args+=" --percent "+str(percentage)
runJob(log, "xmipp_image_sort_by_statistics","-i %(ImagesFn)s --addToInput" % locals()+args)
md.read(ImagesFn) # Should have ZScore label after runJob
md.sort(MDL_ZSCORE)
md.write(ImagesFn)
def sortClasses(log,ExtraDir,Nproc,suffix):
if Nproc==1:
Nproc=2
if Nproc>8:
Nproc=8
for filename in glob.glob(os.path.join(ExtraDir,"level_??/level_classes%s.xmd"%suffix)):
level=int(re.search('level_(\d\d)',filename).group(1))
fnRoot=os.path.join(ExtraDir,"level_%02d/level_classes%s_sorted"%(level,suffix))
params= "-i classes@"+filename+" --oroot "+fnRoot
runJob(log,"xmipp_image_sort",params,Nproc)
mD=MetaData(fnRoot+".xmd")
mD.write("classes_sorted@"+filename,MD_APPEND)
deleteFile(log,fnRoot+".xmd")
def qualifyModes(log,WorkingDir,NumberOfModes,StructureType,CollectivityThreshold):
currentDir=os.getcwd()
changeDir(log,WorkingDir)
fnVec=glob.glob("modes/vec.*")
if len(fnVec)<NumberOfModes:
fhWarning=open("warnings.xmd",'w')
fhWarning.write(redStr("There are only "+str(len(fnVec))+" modes instead of "+str(NumberOfModes)+". Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks and for pseudoatomic normal mode analysis 3 times the number of pseudoatoms. However, the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between these two numbers, consider increasing cut-off distance.")+"\n")
fhWarning.close()
fnDiag="diagrtb.eigenfacs"
if StructureType=="EM":
runJob(log,"nma_reformatForElNemo.sh","%d"%NumberOfModes)
fnDiag="diag_arpack.eigenfacs"
runJob(log,"echo","%s | nma_check_modes"%fnDiag)
deleteFile(log,fnDiag)
fh=open("Chkmod.res")
MDout=MetaData()
collectivityList=[]
for n in range(NumberOfModes):
line=fh.readline()
collectivity=float(line.split()[1])
collectivityList.append(collectivity)
id=MDout.addObject()
MDout.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec.%d"%(n+1)),id)
MDout.setValue(MDL_ORDER,long(n+1),id)
if n>=6:
MDout.setValue(MDL_ENABLED,1,id)
else:
MDout.setValue(MDL_ENABLED,-1,id)
MDout.setValue(MDL_NMA_COLLECTIVITY,collectivity,id)
if collectivity<CollectivityThreshold:
MDout.setValue(MDL_ENABLED,-1,id)
fh.close()
idxSorted=[i[0] for i in sorted(enumerate(collectivityList), key=lambda x:x[1])]
score=[0]*NumberOfModes
for i in range(NumberOfModes):
score[i]+=i+1
score[idxSorted[i]]+=NumberOfModes-i
i=0
for id in MDout:
score_i=float(score[i])/(2.0*NumberOfModes)
MDout.setValue(MDL_NMA_SCORE,score_i,id)
i+=1
MDout.write("modes.xmd")
deleteFile(log,"Chkmod.res")
changeDir(log,currentDir)
def createAcquisition(log,InputFile,WorkingDir,DoResize,NewSize):
fnAcqIn = findAcquisitionInfo(InputFile)
if not fnAcqIn is None:
fnAcqOut = getProtocolFilename('acquisition', WorkingDir=WorkingDir)
if not DoResize:
createLink(log, fnAcqIn, fnAcqOut)
else:
md = MetaData(fnAcqIn)
id = md.firstObject()
Ts = md.getValue(MDL_SAMPLINGRATE, id)
(Xdim, Ydim, Zdim, Ndim, _) = MetaDataInfo(InputFile)
downsampling = float(Xdim)/NewSize;
md.setValue(MDL_SAMPLINGRATE,Ts*downsampling,id)
md.write(getProtocolFilename('acquisition', WorkingDir=WorkingDir))
def writeImagesMd(log, md, ImportAll, SubsetMode, Nsubset, imagesFn, imagesStk, DoCopy):
if not ImportAll:
if SubsetMode=="Random particles":
mdaux=MetaData()
mdaux.randomize(md)
else:
mdaux=MetaData(md)
md.selectPart(mdaux, 0, Nsubset)
md.addItemId()
md.write(imagesFn)
if DoCopy:
runJob(log,"xmipp_image_convert","-i %(imagesFn)s -o %(imagesStk)s" % locals())
md = MetaData(imagesFn)
mdStk = MetaData(imagesStk)
md.merge(mdStk) # Change MDL_IMAGE to point to new stack
md.write(imagesFn)
def coocurenceMatrix(log,RemainingClasses,WorkingDirStructure,NumVolumes,nI,CorePercentile,CorrThresh):
import numpy
mdRemaining = MetaData(RemainingClasses)
Nimgs=mdRemaining.size()
allNames=mdRemaining.getColumnValues(MDL_IMAGE)
matrixTotal = numpy.zeros([Nimgs,Nimgs])
for n in range(NumVolumes):
fnBase='proposedVolume%05d'%n
fnRoot=os.path.join(WorkingDirStructure,fnBase)
md = MetaData(fnRoot+".xmd")
size = md.size()
num=[]
corr=[]
for objId in md:
name = md.getValue(MDL_IMAGE, objId)
if name in allNames:
num.append(allNames.index(name))
corr.append(md.getValue(MDL_MAXCC, objId))
else:
print "Cannot find ",name
if size!=len(num):
print "Error when processing: ",fnRoot+".xmd"
aaa
matrix = numpy.zeros([Nimgs,Nimgs])
for i in range(size):
for j in range(size):
matrix[num[i],num[j]]=((corr[i]+corr[j])/2)
#numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix_%05d.txt'%n), matrix)
matrixTotal=matrixTotal+matrix
matrixTotal=matrixTotal/NumVolumes
numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix.txt'),matrixTotal)
largestComponent=procCoocurenceMatrix(matrixTotal,CorePercentile,CorrThresh)
md = MetaData()
for idx in largestComponent:
id=md.addObject()
md.setValue(MDL_IMAGE,allNames[idx],id)
md.write(os.path.join(WorkingDirStructure+"_core","imagesCore.xmd"))
if md.size()==0:
print "There are no images in the core"
aaa
def joinMetaDatas(log,InputFile,UntiltedMetadata,TiltedMetadata):
mDInput=MetaData(InputFile)
mDstack = MetaData(UntiltedMetadata)
mDstack.removeLabel(MDL_MICROGRAPH)
mDstack.removeLabel(MDL_XCOOR)
mDstack.removeLabel(MDL_YCOOR)
mdPart1=MetaData()
mdPart1.join2(mDstack,mDInput,MDL_IMAGE_ORIGINAL,MDL_IMAGE,INNER_JOIN)
mdPart1.removeLabel(MDL_IMAGE_ORIGINAL)
mDstack = MetaData(TiltedMetadata)
mDstack.removeLabel(MDL_MICROGRAPH)
mDstack.removeLabel(MDL_XCOOR)
mDstack.removeLabel(MDL_YCOOR)
mDstack.renameColumn(MDL_IMAGE,MDL_IMAGE_TILTED)
mdJoined=MetaData()
mdJoined.join2(mDstack,mdPart1,MDL_IMAGE_ORIGINAL,MDL_IMAGE_TILTED,INNER_JOIN)
mdJoined.removeLabel(MDL_IMAGE_ORIGINAL)
mdJoined.write(InputFile)
def wizardTiltPairs(gui, var):
dirMicrographs = gui.getVarValue('DirMicrographs')
extMicrographs = gui.getVarValue('ExtMicrographs')
resultFilename = var.getTkValue()
uList = []
tList = []
from os.path import basename, dirname
from xmipp import MDL_MICROGRAPH, MDL_MICROGRAPH_TILTED
if exists(resultFilename):
md = MetaData(resultFilename)
for id in md:
tPath = md.getValue(MDL_MICROGRAPH_TILTED, id)
tList.append(basename(tPath))
uPath = md.getValue(MDL_MICROGRAPH, id)
uList.append(basename(uPath))
prefix = dirname(uPath) # This assumes that all micrograph are in the same folder
else:
if len(resultFilename) == 0:
resultFilename = "tilted_pairs.xmd"
micrographs = glob(join(dirMicrographs, extMicrographs))
micrographs.sort()
for i, m in enumerate(micrographs):
m = basename(m)
if i % 2 == 0:
tList.append(m)
else:
uList.append(m)
prefix = dirMicrographs
from protlib_gui_ext import showTiltPairsDialog
results = showTiltPairsDialog((uList, tList), gui.master)
if results:
var.setTkValue(resultFilename)
uList, tList = results
md = MetaData()
for u, t in zip(uList, tList):
id = md.addObject()
md.setValue(MDL_MICROGRAPH, join(prefix,u), id)
md.setValue(MDL_MICROGRAPH_TILTED, join(prefix,t), id)
md.write(resultFilename)
def evaluateVolumes(log,WorkingDir,NRansac):
fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd")
fnCorr = 'corrThreshold@'+fnCorr
mdCorr= MetaData(fnCorr)
objId = mdCorr.firstObject()
CorrThresh = mdCorr.getValue(MDL_WEIGHT,objId)
for n in range(NRansac):
fnRoot=os.path.join("ransac%05d"%n)
fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd")
md=MetaData(fnAngles)
numInliers=0
for objId in md:
corr = md.getValue(MDL_MAXCC, objId)
if (corr >= CorrThresh) :
numInliers = numInliers+corr
md= MetaData()
objId = md.addObject()
md.setValue(MDL_WEIGHT,float(numInliers),objId)
md.write("inliers@"+fnAngles,MD_APPEND)
def findCenter(log, HowCenter, Selfile, ExtraDir, SpectraInnerRadius, SpectraOuterRadius):
if HowCenter == 'Minimize first harmonic':
if SpectraOuterRadius + 20 > 100:
R3 = SpectraOuterRadius + (100 - SpectraOuterRadius) / 2
R4 = 100
else:
R3 = SpectraOuterRadius + 10
R4 = SpectraOuterRadius + 20
runJob(log, 'xmipp_image_find_center',
'-i ' + Selfile + \
' --oroot ' + ExtraDir + "/center2d" + \
' --r1 ' + str(SpectraInnerRadius) + \
' --r2 ' + str(SpectraOuterRadius) + \
' --r3 ' + str(R3) + ' --r4 ' + str(R4))
elif HowCenter == 'Use the middle of the image':
from xmipp import MetaDataInfo
dims = MetaDataInfo(Selfile)
MD = MetaData()
id = MD.addObject()
MD.setValue(MDL_X, float(dims[0] / 2), id)
MD.setValue(MDL_Y, float(dims[1] / 2), id)
MD.write(join(ExtraDir, "center2d_center.xmd"))
def run(self):
oldCtfdat = self.getParam('-i')
oroot = self.getParam('--oroot')
md = MetaData()
md.readPlain(oldCtfdat, "image CTFModel")
ctfparamsDict = {}
mode = MD_OVERWRITE
for objId in md:
oldCtfparam = md.getValue(MDL_CTF_MODEL, objId)
if oldCtfparam in ctfparamsDict:
newCtfparam = ctfparamsDict[oldCtfparam]
else:
block = splitext(basename(oldCtfparam))[0]
newCtfparam = "%(block)s@%(oroot)s.ctfparam" % locals()
md2 = convertCtfparam(oldCtfparam)
md2.write(newCtfparam, mode)
ctfparamsDict[oldCtfparam] = newCtfparam
mode = MD_APPEND
md.setValue(MDL_CTF_MODEL, newCtfparam, objId)
md.write("%s.ctfdat" % oroot)
def gatherPairs(log, WorkingDir, ExtraDir, ClassSelection, ClassFile, ExtractDir, PickingDir):
mdImages = getImagesMd(ExtractDir)
fnMicrographs = getProtocolFilename("micrographs", WorkingDir=ExtractDir)
mdTiltAngles = MetaData("micrographPairs@" + fnMicrographs)
mdU = getImagesMd(ExtractDir, "untilted")
mdUAux = MetaData()
mdUAux.setColumnValues(MDL_IMAGE, mdU.getColumnValues(MDL_IMAGE))
mdUAux.setColumnValues(MDL_MICROGRAPH, mdU.getColumnValues(MDL_MICROGRAPH))
mdT = getImagesMd(ExtractDir, "tilted")
mdTAux = MetaData()
mdTAux.setColumnValues(MDL_IMAGE_TILTED, mdT.getColumnValues(MDL_IMAGE))
mdTAux.setColumnValues(MDL_MICROGRAPH_TILTED, mdT.getColumnValues(MDL_MICROGRAPH))
fnExtractImages = os.path.join(ExtractDir, "images.xmd")
mdJoin1 = MetaData()
mdJoin2 = MetaData()
mdJoin3 = MetaData()
mdJoin4 = MetaData()
fixedMdClass = MetaData()
fnOut = join(ExtraDir, "rct_classes.xmd")
for classNo in ClassSelection:
MDclass = MetaData(getClassBlock(classNo, ClassFile))
if not MDclass.containsLabel(MDL_IMAGE_TILTED):
if MDclass.containsLabel(MDL_IMAGE_ORIGINAL):
fixedMdClass.join2(MDclass, mdImages, MDL_IMAGE_ORIGINAL, MDL_IMAGE, INNER_JOIN)
else:
fixedMdClass.join1(MDclass, mdImages, MDL_IMAGE, INNER_JOIN)
else:
fixedMdClass = MDclass
mdJoin1.join1(fixedMdClass, mdImages, MDL_IMAGE, LEFT_JOIN)
mdJoin2.join1(mdJoin1, mdUAux, MDL_IMAGE, LEFT_JOIN)
mdJoin3.join1(mdJoin2, mdTAux, MDL_IMAGE_TILTED, LEFT_JOIN)
mdJoin4.join1(mdJoin3, mdTiltAngles, MDL_MICROGRAPH, LEFT_JOIN)
fnClassOut = getClassBlock(classNo, fnOut)
mdJoin4.write(fnClassOut, MD_APPEND)
def scoreFinalVolumes(log,WorkingDir,NumVolumes):
mdOut=MetaData()
for n in range(NumVolumes):
fnRoot=os.path.join(WorkingDir,'Structure00000_core_extended','proposedVolume%05d'%n)
fnAssignment=fnRoot+".xmd"
if exists(fnAssignment):
MDassignment=MetaData(fnAssignment)
sum=0
N=0
minCC=2
for id in MDassignment:
cc=MDassignment.getValue(MDL_MAXCC,id)
sum+=cc
if cc<minCC:
minCC=cc
N+=1
avg=sum/N
id=mdOut.addObject()
mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id)
mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id)
mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id)
mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id)
mdOut.write(os.path.join(WorkingDir,"proposedVolumes.xmd"))
def evaluateDeformationsStep(self):
N = self.inputStructures.get().getSize()
import numpy
distances=numpy.zeros([N,N])
for volCounter in range(1,N+1):
pdb1=open(self._getPath('pseudoatoms_%02d.pdb'%volCounter)).readlines()
for volCounter2 in range(1,N+1):
if volCounter!=volCounter2:
davg=0.
Navg=0.
pdb2=open(self._getExtraPath('alignment_%02d_%02d.pdb'%(volCounter,volCounter2))).readlines()
for i in range(len(pdb1)):
line1=pdb1[i]
if line1.startswith("ATOM"):
line2=pdb2[i]
x1=float(line1[30:37])
y1=float(line1[38:45])
z1=float(line1[46:53])
x2=float(line2[30:37])
y2=float(line2[38:45])
z2=float(line2[46:53])
dx=x1-x2
dy=y1-y2
dz=z1-z2
d=math.sqrt(dx*dx+dy*dy+dz*dz)
davg+=d
Navg+=1
if Navg>0:
davg/=Navg
distances[volCounter-1,volCounter2-1]=davg
distances=0.5*(distances+numpy.transpose(distances))
numpy.savetxt(self._getPath('distances.txt'),distances)
distances1D=numpy.mean(distances,axis=0)
print("Average distance to rest of volumes=",distances1D)
imin=numpy.argmin(distances1D)
print("The volume in the middle is pseudoatoms_%02d.pdb"%(imin+1))
createLink(self._getPath("pseudoatoms_%02d.pdb"%(imin+1)),self._getPath("pseudoatoms.pdb"))
createLink(self._getPath("modes_%02d.xmd"%(imin+1)),self._getPath("modes.xmd"))
createLink(self._getExtraPath("pseudoatoms_%02d_distance.hist"%(imin+1)),self._getExtraPath("pseudoatoms_distance.hist"))
# Measure range
minDisplacement= 1e38*numpy.ones([self.numberOfModes.get(),1])
maxDisplacement=-1e38*numpy.ones([self.numberOfModes.get(),1])
mdNMA=MetaData(self._getPath("modes.xmd"))
for volCounter in range(1,N+1):
if volCounter!=imin+1:
md=MetaData(self._getExtraPath("alignment_%02d_%02d.xmd"%(imin+1,volCounter)))
displacements=md.getValue(MDL_NMA, md.firstObject())
idx1=0
idx2=0
for idRow in mdNMA:
if mdNMA.getValue(MDL_ENABLED,idRow)==1:
minDisplacement[idx2]=min(minDisplacement[idx2],displacements[idx1])
maxDisplacement[idx2]=max(maxDisplacement[idx2],displacements[idx1])
idx1+=1
else:
minDisplacement[idx2]=0
maxDisplacement[idx2]=0
idx2+=1
idx2=0
for idRow in mdNMA:
mdNMA.setValue(MDL_NMA_MINRANGE,float(minDisplacement[idx2]),idRow)
mdNMA.setValue(MDL_NMA_MAXRANGE,float(maxDisplacement[idx2]),idRow)
idx2+=1
mdNMA.write(self._getPath("modes.xmd"))
# Create output
volCounter=0
for inputStructure in self.inputStructures.get():
if volCounter==imin:
print("The corresponding volume is %s"%(getImageLocation(inputStructure)))
finalStructure=inputStructure
break
volCounter+=1
pdb = PdbFile(self._getPath('pseudoatoms.pdb'), pseudoatoms=True)
self._defineOutputs(outputPdb=pdb)
modes = NormalModes(filename=self._getPath('modes.xmd'))
self._defineOutputs(outputModes=modes)
self._defineSourceRelation(finalStructure, self.outputPdb)
self._defineSourceRelation(self.outputPdb, self.outputModes)
def createEmptyMicrographSel(log, fnOut):
md = MetaData()
md.setValue(MDL_IMAGE,"ImportedImages", md.addObject())
md.write(fnOut)
def createAcquisitionMd(log, samplingRate, fnOut):
md = MetaData()
md.setValue(MDL_SAMPLINGRATE, float(samplingRate), md.addObject())
md.write(fnOut)
def copyImages(log,InputFile,OutputStack):
fnOutputMetadata=OutputStack.replace(".stk",".xmd")
runJob(log,"xmipp_image_convert","-i %(InputFile)s -o %(OutputStack)s --track_origin --save_metadata_stack %(fnOutputMetadata)s --keep_input_columns" % locals())
mDstack = MetaData(fnOutputMetadata)
mDstack.removeLabel(MDL_ZSCORE)
mDstack.write(fnOutputMetadata)
def createAcquisition(log,WorkingDir,Ts):
md=MetaData()
id=md.addObject()
md.setValue(MDL_SAMPLINGRATE,float(Ts),id)
md.write(os.path.join(WorkingDir,"acquisition_info.xmd"))
def writeToFile(self, fn):
md = MetaData()
self.writeToMd(md, md.addObject())
md.write(fn)
def evaluateDeformationsStep(self):
N = self.inputStructures.get().getSize()
import numpy
distances = numpy.zeros([N, N])
for volCounter in range(1, N + 1):
pdb1 = open(self._getPath('pseudoatoms_%02d.pdb' %
volCounter)).readlines()
for volCounter2 in range(1, N + 1):
if volCounter != volCounter2:
davg = 0.
Navg = 0.
pdb2 = open(
self._getExtraPath(
'alignment_%02d_%02d.pdb' %
(volCounter, volCounter2))).readlines()
for i in range(len(pdb1)):
line1 = pdb1[i]
if line1.startswith("ATOM"):
line2 = pdb2[i]
x1 = float(line1[30:37])
y1 = float(line1[38:45])
z1 = float(line1[46:53])
x2 = float(line2[30:37])
y2 = float(line2[38:45])
z2 = float(line2[46:53])
dx = x1 - x2
dy = y1 - y2
dz = z1 - z2
d = math.sqrt(dx * dx + dy * dy + dz * dz)
davg += d
Navg += 1
if Navg > 0:
davg /= Navg
distances[volCounter - 1, volCounter2 - 1] = davg
distances = 0.5 * (distances + numpy.transpose(distances))
numpy.savetxt(self._getPath('distances.txt'), distances)
distances1D = numpy.mean(distances, axis=0)
print("Average distance to rest of volumes=", distances1D)
imin = numpy.argmin(distances1D)
print("The volume in the middle is pseudoatoms_%02d.pdb" % (imin + 1))
createLink(self._getPath("pseudoatoms_%02d.pdb" % (imin + 1)),
self._getPath("pseudoatoms.pdb"))
createLink(self._getPath("modes_%02d.xmd" % (imin + 1)),
self._getPath("modes.xmd"))
createLink(
self._getExtraPath("pseudoatoms_%02d_distance.hist" % (imin + 1)),
self._getExtraPath("pseudoatoms_distance.hist"))
# Measure range
minDisplacement = 1e38 * numpy.ones([self.numberOfModes.get(), 1])
maxDisplacement = -1e38 * numpy.ones([self.numberOfModes.get(), 1])
mdNMA = MetaData(self._getPath("modes.xmd"))
for volCounter in range(1, N + 1):
if volCounter != imin + 1:
md = MetaData(
self._getExtraPath("alignment_%02d_%02d.xmd" %
(imin + 1, volCounter)))
displacements = md.getValue(MDL_NMA, md.firstObject())
idx1 = 0
idx2 = 0
for idRow in mdNMA:
if mdNMA.getValue(MDL_ENABLED, idRow) == 1:
minDisplacement[idx2] = min(minDisplacement[idx2],
displacements[idx1])
maxDisplacement[idx2] = max(maxDisplacement[idx2],
displacements[idx1])
idx1 += 1
else:
minDisplacement[idx2] = 0
maxDisplacement[idx2] = 0
idx2 += 1
idx2 = 0
for idRow in mdNMA:
mdNMA.setValue(MDL_NMA_MINRANGE, float(minDisplacement[idx2]),
idRow)
mdNMA.setValue(MDL_NMA_MAXRANGE, float(maxDisplacement[idx2]),
idRow)
idx2 += 1
mdNMA.write(self._getPath("modes.xmd"))
# Create output
volCounter = 0
for inputStructure in self.inputStructures.get():
if volCounter == imin:
print("The corresponding volume is %s" %
(getImageLocation(inputStructure)))
finalStructure = inputStructure
break
volCounter += 1
pdb = PdbFile(self._getPath('pseudoatoms.pdb'), pseudoatoms=True)
self._defineOutputs(outputPdb=pdb)
modes = NormalModes(filename=self._getPath('modes.xmd'))
self._defineOutputs(outputModes=modes)
self._defineSourceRelation(self.inputStructures, self.outputPdb)
def writeToFile(self, fn):
md = MetaData()
self.writeToMd(md, md.addObject())
md.write(fn)
