def summary(self):
messages = []
messages.append("Input volume: [%s]" % self.InputVol)
if self.DoFSC:
messages.append("Reference volume: [%s]" % self.ReferenceVol)
fnFSC=self.workingDirPath("fsc.xmd")
if os.path.exists(fnFSC):
md = MetaData(fnFSC)
resolution = [md.getValue(MDL_RESOLUTION_FREQREAL, id) for id in md]
frc = [md.getValue(MDL_RESOLUTION_FRC, id) for id in md]
for i in range(len(frc)):
if frc[i]<0.5:
messages.append("Resolution FSC(=0.5) = %f"%resolution[i])
break
for i in range(len(frc)):
if frc[i]<0.143:
messages.append("Resolution FSC(=0.143)= %f"%resolution[i])
break
if self.DoSSNR:
fnSSNR=self.workingDirPath("ssnr.xmd")
if os.path.exists(fnSSNR):
md = MetaData(fnSSNR)
resolution = [md.getValue(MDL_RESOLUTION_FREQREAL, id) for id in md]
ssnr = [md.getValue(MDL_RESOLUTION_SSNR, id) for id in md]
for i in range(len(ssnr)):
if ssnr[i]<1:
messages.append("Resolution SSNR(=1) = %f"%resolution[i])
break
return messages
def wizardChooseSizeToExtract(gui, var):
from xmipp import MDL_PICKING_PARTICLE_SIZE, MDL_CTF_MODEL, MDL_SAMPLINGRATE, MDL_SAMPLINGRATE_ORIGINAL
from protlib_gui_ext import ListboxDialog
pickingRun = gui.getVarValue('PickingRun')
pickingProt = gui.project.getProtocolFromRunName(pickingRun)
fnConfig = pickingProt.getFilename("config")
if not exists(fnConfig):
showWarning("Warning", "No elements to select", parent=gui.master)
return
md = MetaData(fnConfig)
particleSize = md.getValue(MDL_PICKING_PARTICLE_SIZE, md.firstObject())
type = gui.getVarValue("DownsampleType")
mdAcquisition = MetaData(pickingProt.getFilename('acquisition'))
objId = mdAcquisition.firstObject()
tsOriginal = tsPicking = mdAcquisition.getValue(MDL_SAMPLINGRATE, objId)
if mdAcquisition.containsLabel(MDL_SAMPLINGRATE_ORIGINAL):
tsOriginal = mdAcquisition.getValue(MDL_SAMPLINGRATE_ORIGINAL, objId)
if type == "same as picking":
factor = 1
else:
factor = tsPicking / tsOriginal;
if type == "other":
try:
factor /= float(gui.getVarValue("DownsampleFactor"))
except Exception, e:
showWarning("Warning", "Please select valid downsample factor", parent=gui.master)
return
def runFineSearch(self, fnVol, dihedral, fnCoarse, fnFine, z0, zF, rot0, rotF, cylinderRadius, height):
md=MetaData(fnCoarse)
objId=md.firstObject()
rotInit=md.getValue(MDL_ANGLE_ROT,objId)
zInit=md.getValue(MDL_SHIFT_Z,objId)
args="-i %s --sym %s --localHelical %f %f -o %s -z %f %f 1 --rotHelical %f %f 1 "%(fnVol,self.getSymmetry(dihedral),
zInit,rotInit,fnFine,z0,zF,rot0,rotF)
if cylinderRadius>0:
args+=" --mask cylinder %d %d"%(-cylinderRadius,-height)
self.runJob('xmipp_volume_find_symmetry',args)
def runSymmetrize(self, fnVol, dihedral, fnParams, fnOut, cylinderRadius, height):
md=MetaData(fnParams)
objId=md.firstObject()
rot0=md.getValue(MDL_ANGLE_ROT,objId)
z0=md.getValue(MDL_SHIFT_Z,objId)
args="-i %s --sym %s --helixParams %f %f -o %s"%(fnVol,self.getSymmetry(dihedral),z0,rot0,fnOut)
self.runJob('xmipp_transform_symmetrize',args)
if cylinderRadius>0:
args="-i %s --mask cylinder %d %d"%(fnOut,-cylinderRadius,-height)
self.runJob('xmipp_transform_mask',args)
def _plotFSC(self, a, fnFSC):
md = MetaData(fnFSC)
resolution_inv = [md.getValue(MDL_RESOLUTION_FREQ, f) for f in md]
frc = [md.getValue(MDL_RESOLUTION_FRC, f) for f in md]
self.maxFrc = max(frc)
self.minInv = min(resolution_inv)
self.maxInv = max(resolution_inv)
a.plot(resolution_inv, frc)
a.xaxis.set_major_formatter(self._plotFormatter)
a.set_ylim([-0.1, 1.1])
def _plotFSC(self, a, fnFSC):
md = MetaData(fnFSC)
resolution_inv = [md.getValue(MDL_RESOLUTION_FREQ, f) for f in md]
frc = [md.getValue(MDL_RESOLUTION_FRC, f) for f in md]
self.maxFrc = max(frc)
self.minInv = min(resolution_inv)
self.maxInv = max(resolution_inv)
a.plot(resolution_inv, frc)
a.xaxis.set_major_formatter(self._plotFormatter)
a.set_ylim([-0.1, 1.1])
def fineSearch(log,WorkingDir,InputVolume,CylinderRadius,fnCoarse,Rot0, RotF, Z0, ZF,fnFine):
md=MetaData(fnCoarse)
id=md.firstObject()
rot0=md.getValue(MDL_ANGLE_ROT,id)
z0=md.getValue(MDL_SHIFT_Z,id)
args="-i %s --sym helical -z %f %f --rotHelical %f %f --localHelical %f %f -o %s"%(InputVolume,float(Z0),float(ZF),
float(Rot0),float(RotF),z0,rot0,fnFine)
if CylinderRadius>0:
[xdim,ydim,zdim,ndim]=getImageSize(InputVolume)
args+=" --mask cylinder %d %d"%(int(-CylinderRadius),int(-xdim))
runJob(log,'xmipp_volume_find_symmetry',args)
def getAngles(self, fnAngles=""):
if fnAngles=="":
if self.searchMode.get()==self.GLOBAL_SEARCH:
fnAngles = self._getExtraPath("coarse.xmd")
else:
fnAngles = self._getExtraPath("fine.xmd")
md = MetaData(fnAngles)
objId = md.firstObject()
rot0 = md.getValue(MDL_ANGLE_ROT, objId)
tilt0 = md.getValue(MDL_ANGLE_TILT, objId)
return (rot0,tilt0)
def symmetrize(log,WorkingDir,InputVolume,OutputVolume,CylinderRadius,fnFine):
md=MetaData(fnFine)
id=md.firstObject()
rot0=md.getValue(MDL_ANGLE_ROT,id)
z0=md.getValue(MDL_SHIFT_Z,id)
args="-i %s --sym helical --helixParams %f %f -o %s"%(InputVolume,z0,rot0,OutputVolume)
runJob(log,'xmipp_transform_symmetrize',args)
if CylinderRadius>0:
[xdim,ydim,zdim,ndim]=getImageSize(InputVolume)
args="-i %s --mask cylinder %d %d"%(OutputVolume,int(-CylinderRadius),int(-xdim))
runJob(log,'xmipp_transform_mask',args)
def createOutput(self):
volume = Volume()
volume.setFileName(self._getPath('volume_symmetrized.vol'))
#volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
volume.copyInfo(self.inputVolume.get())
self._defineOutputs(outputVolume=volume)
self._defineTransformRelation(self.inputVolume, self.outputVolume)
md = MetaData(self._getExtraPath('fineParams.xmd'))
objId = md.firstObject()
self._defineOutputs(deltaRot=pwobj.Float(md.getValue(MDL_ANGLE_ROT, objId)),
deltaZ=pwobj.Float(md.getValue(MDL_SHIFT_Z, objId)))
def runFineSearch(self, fnVol, dihedral, fnCoarse, fnFine, heightFraction, z0, zF, rot0, rotF, cylinderInnerRadius, cylinderOuterRadius, height, Ts):
md=MetaData(fnCoarse)
objId=md.firstObject()
rotInit=md.getValue(MDL_ANGLE_ROT,objId)
zInit=md.getValue(MDL_SHIFT_Z,objId)
args="-i %s --sym %s --heightFraction %f --localHelical %f %f -o %s -z %f %f 1 --rotHelical %f %f 1 --sampling %f"%(fnVol,self.getSymmetry(dihedral),heightFraction,
zInit,rotInit,fnFine,z0,zF,rot0,rotF,Ts)
if cylinderOuterRadius>0 and cylinderInnerRadius<0:
args+=" --mask cylinder %d %d"%(-cylinderOuterRadius,-height)
elif cylinderOuterRadius>0 and cylinderInnerRadius>0:
args+=" --mask tube %d %d %d"%(-cylinderInnerRadius,-cylinderOuterRadius,-height)
self.runJob('xmipp_volume_find_symmetry',args, numberOfMpi=1)
def summary(self):
message=[]
message.append("Input volume: [%s] "%self.InputVolume)
if os.path.exists(self.fnSym):
message.append("Symmetrized volume: [%s] "%self.fnSym)
if os.path.exists(self.fnFine):
md=MetaData(self.fnFine)
id=md.firstObject()
rot0=md.getValue(MDL_ANGLE_ROT,id)
z0=md.getValue(MDL_SHIFT_Z,id)
message.append("DeltaRot=%f"%rot0)
message.append("DeltaZ=%f"%z0)
return message
def createOutput(self):
volume = Volume()
volume.setFileName(self._getPath('volume_symmetrized.vol'))
#volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
volume.copyInfo(self.inputVolume.get())
self._defineOutputs(outputVolume=volume)
self._defineTransformRelation(self.inputVolume, self.outputVolume)
md = MetaData(self._getExtraPath('fineParams.xmd'))
objId = md.firstObject()
self._defineOutputs(deltaRot=pwobj.Float(
md.getValue(MDL_ANGLE_ROT, objId)),
deltaZ=pwobj.Float(md.getValue(MDL_SHIFT_Z,
objId)))
def defineSteps(self):
self.insertStep("createDir", path=self.ExtraDir)
pickingDir = getWorkingDirFromRunName(self.ExtractionRun)
self.insertStep("createLink2", filename="acquisition_info.xmd", dirSrc=pickingDir, dirDest=self.WorkingDir)
classSelection = getListFromRangeString(self.SelectedClasses)
fnRCTClasses = self.extraPath("rct_classes.xmd")
self.insertStep(
"gatherPairs",
verifyfiles=[fnRCTClasses],
WorkingDir=self.WorkingDir,
ExtraDir=self.ExtraDir,
ClassSelection=classSelection,
ClassFile=self.ClassifMd,
ExtractDir=self.ExtractDir,
PickingDir=pickingDir,
)
mdClasses = MetaData("classes@" + self.ClassifMd)
hasImage = mdClasses.containsLabel(MDL_IMAGE)
for classId in mdClasses:
classNo = mdClasses.getValue(MDL_REF, classId)
if classNo in classSelection:
# Locate class representative
classImage = ""
if hasImage:
classImage = mdClasses.getValue(MDL_IMAGE, classId)
classNameIn = "class%06d_images@%s" % (classNo, fnRCTClasses)
classNameOut = self.extraPath("rct_images_%06d.xmd" % classNo)
classVolumeOut = self.workingDirPath("rct_%06d.vol" % classNo)
self.insertParallelStep(
"reconstructClass",
verifyfiles=[classNameOut, classVolumeOut],
WorkingDir=self.WorkingDir,
ExtraDir=self.ExtraDir,
ClassNameIn=classNameIn,
ClassNameOut=classNameOut,
ClassImage=classImage,
CenterMaxShift=self.CenterMaxShift,
ThinObject=self.ThinObject,
SkipTiltedTranslations=self.SkipTiltedTranslations,
ClassVolumeOut=classVolumeOut,
ReconstructAdditionalParams=self.ReconstructAdditionalParams,
DoLowPassFilter=self.DoLowPassFilter,
LowPassFilter=self.LowPassFilter,
parent_step_id=XmippProjectDb.FIRST_STEP,
)
class RowMetaData():
""" This class is a wrapper for MetaData in row mode.
Where only one object is used.
"""
def __init__(self, filename=None):
self._md = MetaData()
self._md.setColumnFormat(False)
self._id = self._md.addObject()
if filename:
self.read(filename)
def setValue(self, label, value):
self._md.setValue(label, value, self._id)
def getValue(self, label):
return self._md.getValue(label, self._id)
def write(self, filename, mode=MD_APPEND):
self._md.write(filename, mode)
def read(self, filename):
self._md.read(filename)
self._md.setColumnFormat(False)
self._id = self._md.firstObject()
def containsLabel(self, label):
return self._md.containsLabel(label)
def __str__(self):
return str(self._md)
def __init__(self, scriptname, project):
XmippProtocol.__init__(self, protDict.ml2d.name, scriptname, project)
self.Import = 'from protocol_ml2d import *'
self.acquisionInfo = self.findAcquisitionInfo(self.ImgMd)
if not self.acquisionInfo is None:
md = MetaData(self.acquisionInfo)
self.SamplingRate = md.getValue(MDL_SAMPLINGRATE, md.firstObject())
def projMatchStep(self, volume, angularSampling, symmetryGroup, images, fnAngles, Xdim):
from pyworkflow.utils.path import cleanPath
# Generate gallery of projections
fnGallery = self._getExtraPath("gallery.stk")
self.runJob(
"xmipp_angular_project_library",
"-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"
% (volume, fnGallery, angularSampling, symmetryGroup, images),
)
# Assign angles
self.runJob(
"xmipp_angular_projection_matching",
"-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 --search5d_shift %s --search5d_step %s --append"
% (images, fnAngles, fnGallery, str(Xdim / 2), str(int(Xdim / 10)), str(int(Xdim / 25))),
)
cleanPath(self._getExtraPath("gallery_sampling.xmd"))
cleanPath(self._getExtraPath("gallery_angles.doc"))
cleanPath(self._getExtraPath("gallery.doc"))
# Write angles in the original file and sort
MD = MetaData(fnAngles)
for id in MD:
galleryReference = MD.getValue(xmipp.MDL_REF, id)
MD.setValue(xmipp.MDL_IMAGE_REF, "%05d@%s" % (galleryReference + 1, fnGallery), id)
MD.write(fnAngles)
def projMatchStep(self, volume, angularSampling, symmetryGroup, images,
fnAngles, Xdim):
from pyworkflow.utils.path import cleanPath
# Generate gallery of projections
fnGallery = self._getExtraPath('gallery.stk')
if volume.endswith('.mrc'):
volume += ":mrc"
self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\
% (volume, fnGallery, angularSampling, symmetryGroup, images))
# Assign angles
self.runJob("xmipp_angular_projection_matching", "-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 --search5d_shift %s --search5d_step %s --append"\
% (images, fnAngles, fnGallery, str(Xdim/2), str(int(Xdim/10)), str(int(Xdim/25))))
cleanPath(self._getExtraPath('gallery_sampling.xmd'))
cleanPath(self._getExtraPath('gallery_angles.doc'))
cleanPath(self._getExtraPath('gallery.doc'))
# Write angles in the original file and sort
MD = MetaData(fnAngles)
for id in MD:
galleryReference = MD.getValue(xmipp.MDL_REF, id)
MD.setValue(xmipp.MDL_IMAGE_REF,
"%05d@%s" % (galleryReference + 1, fnGallery), id)
MD.write(fnAngles)
def summary(self):
_, _, _, _, size = MetaDataInfo(self.ImgMd)
lines = ["Input images: [%s] (<%u>)" % (self.ImgMd, size)]
if self.DoMlf:
if self.DoCorrectAmplitudes:
suffix = "with CTF correction "
else:
suffix = "ignoring CTF effects "
lines.append("Using a ML in <Fourier-space> " + suffix)
if self.DoGenerateReferences:
lines.append("Number of references: <%d>" % self.NumberOfReferences)
else:
lines.append("Reference image(s): [%s]" % self.RefMd)
#logs = self.getFilename('iter_logs')
lastIter = lastIteration(self)
if lastIter > 0:#exists(logs):
logs = self.getFilename('iter_logs', iter=lastIter)
md = MetaData(logs)
id = md.lastObject()
#iteration = md.getValue(MDL_ITER, id)
lines.append("Last iteration: <%d>" % lastIter)
LL = md.getValue(MDL_LL, id)
lines.append("LogLikelihood: %f" % LL)
mdRefs = self.getFilename('iter_refs', iter=lastIter)
lines.append("Last classes: [%s]" % mdRefs)
return lines
def runFineSearch(self, fnVol, dihedral, fnCoarse, fnFine, heightFraction,
z0, zF, rot0, rotF, cylinderInnerRadius,
cylinderOuterRadius, height, Ts):
md = MetaData(fnCoarse)
objId = md.firstObject()
rotInit = md.getValue(MDL_ANGLE_ROT, objId)
zInit = md.getValue(MDL_SHIFT_Z, objId)
args = "-i %s --sym %s --heightFraction %f --localHelical %f %f -o %s -z %f %f 1 --rotHelical %f %f 1 --sampling %f" % (
fnVol, self.getSymmetry(dihedral), heightFraction, zInit, rotInit,
fnFine, z0, zF, rot0, rotF, Ts)
if cylinderOuterRadius > 0 and cylinderInnerRadius < 0:
args += " --mask cylinder %d %d" % (-cylinderOuterRadius, -height)
elif cylinderOuterRadius > 0 and cylinderInnerRadius > 0:
args += " --mask tube %d %d %d" % (-cylinderInnerRadius,
-cylinderOuterRadius, -height)
self.runJob('xmipp_volume_find_symmetry', args)
def launchProjSubPlots(self, selectedPlots):
''' Launch some plots for a Projection Matching protocol run '''
import numpy as np
_log = self.Log
xplotter=None
self._plot_count = 0
def doPlot(plotName):
return plotName in selectedPlots
file_name_tmp = join(self.CtfGroupDirectoryName, self.CtfGroupRootName) +'Info.xmd'
file_name = join(self.PrevRun.WorkingDir, file_name_tmp)
if exists(file_name):
auxMD = MetaData("numberGroups@"+file_name)
self.NumberOfCtfGroups = auxMD.getValue(MDL_COUNT,auxMD.firstObject())
else:
self.NumberOfCtfGroups = 1
if doPlot('DisplayReference'):
for indexCtf in range(1, self.NumberOfCtfGroups+1):
file_name = self.getFilename('SubtractedStack', ctf=indexCtf)+'ref'
if exists(file_name):
try:
runShowJ(file_name)
except Exception, e:
showError("Error launching java app", str(e), self.master)
def wizardBrowseCTF2(gui, var):
error = None
vList = ['LowResolCutoff', 'HighResolCutoff']
freqs = gui.getVarlistValue(vList)
importRunName = gui.getVarValue('ImportRun')
prot = gui.project.getProtocolFromRunName(importRunName)
path = prot.WorkingDir
if path and exists(path):
mdPath = prot.getFilename('micrographs')
if exists(mdPath):
from xmipp import MetaData, MDL_MICROGRAPH
md = MetaData(mdPath)
if md.size():
image = md.getValue(MDL_MICROGRAPH, md.firstObject())
if image:
filterExt = "*" + splitext(image)[1]
value = gui.getVarValue('DownsampleFactor')
results = wizardHelperSetDownsampling(gui, var, path, filterExt, value, freqs, md)
if results:
gui.setVarlistValue(vList, results[1:])
else:
error = "Not micrograph found in metadata <%s>" % mdPath
#gui.setVarValue('LowResolCutoff', results[1])
#gui.setVarValue('HighResolCutoff', results[2])
else:
error = "Micrograph metadata <%s> is empty" % mdPath
else:
error = "Micrograph metadata <%s> doesn't exists" % mdPath
else:
error = "Import run <%s> doesn't exists" % str(path)
if error:
showWarning("Select Downsampling Wizard", error, gui.master)
return None
else:
return results
def getCorrThresh(log,WorkingDir,NRansac,CorrThresh):
corrVector = []
fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd")
mdCorr= MetaData()
for n in range(NRansac):
fnRoot=os.path.join("ransac%05d"%n)
fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd")
md=MetaData(fnAngles)
for objId in md:
corr = md.getValue(MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr)
mdCorr.write("correlations@"+fnCorr,MD_APPEND)
mdCorr= MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(CorrThresh*(len(sortedCorrVector)-1)))
#With the line below commented the percentil is not used for the threshold and is used the value introduced in the form
#CorrThresh = sortedCorrVector[indx]#
objId = mdCorr.addObject()
mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId)
mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND)
print "Correlation threshold: "+str(CorrThresh)
def getBestVolumes(log,WorkingDir,NRansac,NumVolumes,UseAll):
volumes = []
inliers = []
for n in range(NRansac):
fnAngles = os.path.join(WorkingDir,"tmp/angles_ransac%05d"%n+".xmd")
md=MetaData("inliers@"+fnAngles)
numInliers=md.getValue(MDL_WEIGHT,md.firstObject())
volumes.append(fnAngles)
inliers.append(numInliers)
index = sorted(range(inliers.__len__()), key=lambda k: inliers[k])
fnBestAngles = ''
threshold=getCCThreshold(WorkingDir)
i=NRansac-1
indx = 0
while i>=0 and indx<NumVolumes:
fnBestAngles = volumes[index[i]]
fnBestAnglesOut=os.path.join(WorkingDir,"volumeProposed%05d"%indx+".xmd")
copyFile(log,fnBestAngles,fnBestAnglesOut)
print("Best volume "+str(indx)+" = "+fnBestAngles)
if not UseAll:
runJob(log,"xmipp_metadata_utilities","-i %s -o %s --query select \"maxCC>%f \" --mode append" %(fnBestAnglesOut,fnBestAnglesOut,threshold))
if getMdSize(fnBestAnglesOut) > 0:
indx += 1
else:
indx += 1
i -= 1
# Remove unnecessary files
for n in range(NRansac):
fnAngles = os.path.join(WorkingDir,"tmp/angles_ransac%05d"%n+".xmd")
deleteFile(log, fnAngles)
def scoreFinalVolumes(log,WorkingDir,NumVolumes):
threshold=getCCThreshold(WorkingDir)
mdOut=MetaData()
for n in range(NumVolumes):
fnRoot=os.path.join(WorkingDir,'volumeProposed%05d'%n)
fnAssignment=fnRoot+".xmd"
if exists(fnAssignment):
runJob(log,"xmipp_metadata_utilities","-i %s --fill weight constant 1"%fnAssignment)
MDassignment=MetaData(fnAssignment)
sum=0
thresholdedSum=0
N=0
minCC=2
for id in MDassignment:
cc=MDassignment.getValue(MDL_MAXCC,id)
sum+=cc
thresholdedSum+=cc-threshold
if cc<minCC:
minCC=cc
N+=1
avg=sum/N
id=mdOut.addObject()
mdOut.setValue(MDL_IMAGE,fnRoot+".vol",id)
mdOut.setValue(MDL_VOLUME_SCORE_SUM,float(sum),id)
mdOut.setValue(MDL_VOLUME_SCORE_SUM_TH,float(thresholdedSum),id)
mdOut.setValue(MDL_VOLUME_SCORE_MEAN,float(avg),id)
mdOut.setValue(MDL_VOLUME_SCORE_MIN,float(minCC),id)
mdOut.write(os.path.join(WorkingDir,"volumesProposed.xmd"))
def getCorrThresh(log,WorkingDir,WorkingDirStructure,NRansac,CorrThresh):
corrVector = []
fnCorr=os.path.join(WorkingDirStructure,"correlations.xmd")
mdCorr= MetaData()
for n in range(NRansac):
fnRoot=os.path.join("ransac%05d"%n)
fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd")
if os.path.exists(fnAngles):
md=MetaData(fnAngles)
for objId in md:
corr = md.getValue(MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(MDL_MAXCC,float(corr),objIdCorr)
mdCorr.write("correlations@"+fnCorr,MD_APPEND)
mdCorr= MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(CorrThresh*(len(sortedCorrVector)-1)))
objId = mdCorr.addObject()
mdCorr.setValue(MDL_WEIGHT,float(CorrThresh),objId)
mdCorr.write("corrThreshold@"+fnCorr,MD_APPEND)
print "Correlation threshold: "+str(CorrThresh)
def runSymmetrize(self, fnVol, dihedral, fnParams, fnOut, heightFraction, cylinderInnerRadius, cylinderOuterRadius, height, Ts):
md=MetaData(fnParams)
objId=md.firstObject()
rot0=md.getValue(MDL_ANGLE_ROT,objId)
z0=md.getValue(MDL_SHIFT_Z,objId)
args="-i %s --sym %s --helixParams %f %f --heightFraction %f -o %s --sampling %f"%(fnVol,self.getSymmetry(dihedral),z0,rot0,heightFraction,fnOut,Ts)
self.runJob('xmipp_transform_symmetrize',args,numberOfMpi=1)
doMask=False
if cylinderOuterRadius>0 and cylinderInnerRadius<0:
args="-i %s --mask cylinder %d %d"%(fnVol,-cylinderOuterRadius,-height)
doMask=True
elif cylinderOuterRadius>0 and cylinderInnerRadius>0:
args="-i %s --mask tube %d %d %d"%(fnVol,-cylinderInnerRadius,-cylinderOuterRadius,-height)
doMask=True
if doMask:
self.runJob('xmipp_transform_mask',args,numberOfMpi=1)
class RowMetaData():
""" This class is a wrapper for MetaData in row mode.
Where only one object is used.
"""
def __init__(self, filename=None):
self._md = MetaData()
self._md.setColumnFormat(False)
self._id = self._md.addObject()
if filename:
self.read(filename)
def setValue(self, label, value):
self._md.setValue(label, value, self._id)
def getValue(self, label):
return self._md.getValue(label, self._id)
def write(self, filename, mode=MD_APPEND):
self._md.write(filename, mode)
def read(self, filename):
self._md.read(filename)
self._md.setColumnFormat(False)
self._id = self._md.firstObject()
def containsLabel(self, label):
return self._md.containsLabel(label)
def __str__(self):
return str(self._md)
def wizardDesignMask(gui, var):
selfile = gui.getVarValue('InSelFile')
##workingDir = getWorkingDirFromRunName(gui.getVarValue('RunName'))
from xmipp import MetaData, MDL_IMAGE
md = MetaData(selfile)
fnImg = md.getValue(MDL_IMAGE, md.firstObject())
#runShowJ(fnImg, extraParams="--mask_toolbar")
runMaskToolbar(fnImg)
def coocurenceMatrix(log,RemainingClasses,WorkingDirStructure,NumVolumes,nI,CorePercentile,CorrThresh):
import numpy
mdRemaining = MetaData(RemainingClasses)
Nimgs=mdRemaining.size()
allNames=mdRemaining.getColumnValues(MDL_IMAGE)
matrixTotal = numpy.zeros([Nimgs,Nimgs])
for n in range(NumVolumes):
fnBase='proposedVolume%05d'%n
fnRoot=os.path.join(WorkingDirStructure,fnBase)
md = MetaData(fnRoot+".xmd")
size = md.size()
num=[]
corr=[]
for objId in md:
name = md.getValue(MDL_IMAGE, objId)
if name in allNames:
num.append(allNames.index(name))
corr.append(md.getValue(MDL_MAXCC, objId))
else:
print "Cannot find ",name
if size!=len(num):
print "Error when processing: ",fnRoot+".xmd"
aaa
matrix = numpy.zeros([Nimgs,Nimgs])
for i in range(size):
for j in range(size):
matrix[num[i],num[j]]=((corr[i]+corr[j])/2)
#numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix_%05d.txt'%n), matrix)
matrixTotal=matrixTotal+matrix
matrixTotal=matrixTotal/NumVolumes
numpy.savetxt(os.path.join(WorkingDirStructure,'coocurrenceMatrix.txt'),matrixTotal)
largestComponent=procCoocurenceMatrix(matrixTotal,CorePercentile,CorrThresh)
md = MetaData()
for idx in largestComponent:
id=md.addObject()
md.setValue(MDL_IMAGE,allNames[idx],id)
md.write(os.path.join(WorkingDirStructure+"_core","imagesCore.xmd"))
if md.size()==0:
print "There are no images in the core"
aaa
def wizardTiltPairs(gui, var):
dirMicrographs = gui.getVarValue('DirMicrographs')
extMicrographs = gui.getVarValue('ExtMicrographs')
resultFilename = var.getTkValue()
uList = []
tList = []
from os.path import basename, dirname
from xmipp import MDL_MICROGRAPH, MDL_MICROGRAPH_TILTED
if exists(resultFilename):
md = MetaData(resultFilename)
for id in md:
tPath = md.getValue(MDL_MICROGRAPH_TILTED, id)
tList.append(basename(tPath))
uPath = md.getValue(MDL_MICROGRAPH, id)
uList.append(basename(uPath))
prefix = dirname(uPath) # This assumes that all micrograph are in the same folder
else:
if len(resultFilename) == 0:
resultFilename = "tilted_pairs.xmd"
micrographs = glob(join(dirMicrographs, extMicrographs))
micrographs.sort()
for i, m in enumerate(micrographs):
m = basename(m)
if i % 2 == 0:
tList.append(m)
else:
uList.append(m)
prefix = dirMicrographs
from protlib_gui_ext import showTiltPairsDialog
results = showTiltPairsDialog((uList, tList), gui.master)
if results:
var.setTkValue(resultFilename)
uList, tList = results
md = MetaData()
for u, t in zip(uList, tList):
id = md.addObject()
md.setValue(MDL_MICROGRAPH, join(prefix,u), id)
md.setValue(MDL_MICROGRAPH_TILTED, join(prefix,t), id)
md.write(resultFilename)
def evaluateVolumes(log,WorkingDir,NRansac):
fnCorr=os.path.join(WorkingDir,"extra/correlations.xmd")
fnCorr = 'corrThreshold@'+fnCorr
mdCorr= MetaData(fnCorr)
objId = mdCorr.firstObject()
CorrThresh = mdCorr.getValue(MDL_WEIGHT,objId)
for n in range(NRansac):
fnRoot=os.path.join("ransac%05d"%n)
fnAngles=os.path.join(WorkingDir,"tmp/angles_"+fnRoot+".xmd")
md=MetaData(fnAngles)
numInliers=0
for objId in md:
corr = md.getValue(MDL_MAXCC, objId)
if (corr >= CorrThresh) :
numInliers = numInliers+corr
md= MetaData()
objId = md.addObject()
md.setValue(MDL_WEIGHT,float(numInliers),objId)
md.write("inliers@"+fnAngles,MD_APPEND)
def runSymmetrize(self, fnVol, dihedral, fnParams, fnOut, heightFraction,
cylinderInnerRadius, cylinderOuterRadius, height, Ts):
md = MetaData(fnParams)
objId = md.firstObject()
rot0 = md.getValue(MDL_ANGLE_ROT, objId)
z0 = md.getValue(MDL_SHIFT_Z, objId)
args = "-i %s --sym %s --helixParams %f %f --heightFraction %f -o %s --sampling %f" % (
fnVol, self.getSymmetry(dihedral), z0, rot0, heightFraction, fnOut,
Ts)
self.runJob('xmipp_transform_symmetrize', args)
doMask = False
if cylinderOuterRadius > 0 and cylinderInnerRadius < 0:
args = "-i %s --mask cylinder %d %d" % (
fnVol, -cylinderOuterRadius, -height)
doMask = True
elif cylinderOuterRadius > 0 and cylinderInnerRadius > 0:
args = "-i %s --mask tube %d %d %d" % (
fnVol, -cylinderInnerRadius, -cylinderOuterRadius, -height)
doMask = True
if doMask:
self.runJob('xmipp_transform_mask', args)
def visualize(self):
from protlib_gui_figure import XmippArrayPlotter1D, XmippArrayPlotter2D, XmippArrayPlotter3D
components = self.DisplayRawDeformation.split()
dim = len(components)
if dim > 0:
modeList = []
modeNameList = []
# Get modes
MD = MetaData(self.Modesfile)
MD.removeDisabled()
for modeComponent in components:
mode = int(modeComponent)
if mode > MD.size():
from protlib_gui_ext import showWarning
showWarning("Warning", "You don't have so many modes", parent=self.master)
else:
mode -= 1
currentMode = 0
modeName = ""
for id in MD:
modeName = MD.getValue(MDL_NMA_MODEFILE, id)
currentMode += 1
if currentMode > mode:
break
modeNameList.append(modeName)
modeList.append(mode)
# Actually plot
if dim == 1:
XmippArrayPlotter1D(
self.extraPath("deformations.txt"),
modeList[0],
"Histogram for mode %s" % modeNameList[0],
"Deformation value",
"Number of images",
)
elif dim == 2:
XmippArrayPlotter2D(
self.extraPath("deformations.txt"), modeList[0], modeList[1], "", modeNameList[0], modeNameList[1]
)
elif dim == 3:
XmippArrayPlotter3D(
self.extraPath("deformations.txt"),
modeList[0],
modeList[1],
modeList[2],
"",
modeNameList[0],
modeNameList[1],
modeNameList[2],
)
def importImages(log, InputFile, WorkingDir, DoCopy, ImportAll, SubsetMode, Nsubset):
imagesFn = getImagesFilename(WorkingDir)
fnInput = FileName(InputFile)
md = MetaData(InputFile)
#check if micrographs or ctfparam exists
doMic=False
doCTFParam=False
if md.containsLabel(MDL_MICROGRAPH):
doMic = True
if md.containsLabel(MDL_CTF_MODEL):
doCTFParam = True
if fnInput.isMetaData():
inputRelativePath = dirname(relpath(InputFile, '.'))
projectPath = findProjectPath(InputFile)
for id in md:
imgFn = md.getValue(MDL_IMAGE, id)
imgNo, imgFn = splitFilename(imgFn)
imgFn = xmippRelpath(fixPath(imgFn, projectPath, inputRelativePath, '.'))
if imgNo != None:
imgFn = "%s@%s" % (imgNo, imgFn)
md.setValue(MDL_IMAGE, imgFn, id)
#micrograph
if doMic:
imgFn = md.getValue(MDL_MICROGRAPH, id)
imgNo, imgFn = splitFilename(imgFn)
imgFn = xmippRelpath(fixPath(imgFn, projectPath, inputRelativePath, '.'))
if imgNo != None:
imgFn = "%s@%s" % (imgNo, imgFn)
md.setValue(MDL_MICROGRAPH, imgFn, id)
#ctf param
if doCTFParam:
ctfFn = md.getValue(MDL_CTF_MODEL, id)
ctfFn = xmippRelpath(fixPath(ctfFn, projectPath, inputRelativePath, '.'))
md.setValue(MDL_CTF_MODEL, ctfFn, id)
outExt = '.stk'
else:
outExt = '.%s' % fnInput.getExtension()
imagesStk = imagesFn.replace('.xmd', outExt)
writeImagesMd(log, md, ImportAll, SubsetMode, Nsubset, imagesFn, imagesStk, DoCopy)
def defineSteps(self):
self.insertStep("createDir",verifyfiles=[self.ExtraDir],path=self.ExtraDir)
md = MetaData(self.Input['config'])
for objId in md:
self.particleSizeForAuto = md.getValue(MDL_PICKING_PARTICLE_SIZE, objId)
filesToImport = [self.Input[k] for k in self.keysToImport]
filesToImport += [self.PrevRun.getFilename(k, model=self.model) for k in ['training', 'pca', 'rotpca', 'svm', 'average', 'config', 'templates']]
self.insertImportOfFiles(filesToImport)
self.insertCopyFile(self.MicrographsMd, self.getFilename('micrographs'))
md = MetaData(self.MicrographsMd)
particleSize = self.particleSizeForAuto
modelRoot = self.extraPath(self.model)
for objId in md:
# Get micrograph path and name
path = md.getValue(MDL_MICROGRAPH, objId)
micrographName = removeBasenameExt(path)
proceed = True
if not self.anotherSet:
fnPos = self.PrevRun.getFilename('pos', micrograph=micrographName)
if xmippExists(fnPos):
blocks = getBlocksInMetaDataFile(fnPos)
copy = True
if 'header' in blocks:
mdheader = MetaData("header@" + fnPos)
state = mdheader.getValue(MDL_PICKING_MICROGRAPH_STATE, mdheader.firstObject())
if state == "Available":
copy = False
if copy:
# Copy manual .pos file of this micrograph
self.insertCopyFile(fnPos, self.getFilename('pos', micrograph=micrographName))
proceed = False
if proceed:
oroot = self.extraPath(micrographName)
cmd = "-i %(path)s --particleSize %(particleSize)d --model %(modelRoot)s --outputRoot %(oroot)s --mode autoselect" % locals()
if self.Fast:
cmd += " --fast "
self.insertParallelRunJobStep("xmipp_micrograph_automatic_picking", cmd)
def createAcquisition(log,InputFile,WorkingDir,DoResize,NewSize):
fnAcqIn = findAcquisitionInfo(InputFile)
if not fnAcqIn is None:
fnAcqOut = getProtocolFilename('acquisition', WorkingDir=WorkingDir)
if not DoResize:
createLink(log, fnAcqIn, fnAcqOut)
else:
md = MetaData(fnAcqIn)
id = md.firstObject()
Ts = md.getValue(MDL_SAMPLINGRATE, id)
(Xdim, Ydim, Zdim, Ndim, _) = MetaDataInfo(InputFile)
downsampling = float(Xdim)/NewSize;
md.setValue(MDL_SAMPLINGRATE,Ts*downsampling,id)
md.write(getProtocolFilename('acquisition', WorkingDir=WorkingDir))
def evaluateDeformationsStep(self):
N = self.inputStructures.get().getSize()
import numpy
distances = numpy.zeros([N, N])
for volCounter in range(1, N + 1):
pdb1 = open(self._getPath('pseudoatoms_%02d.pdb' %
volCounter)).readlines()
for volCounter2 in range(1, N + 1):
if volCounter != volCounter2:
davg = 0.
Navg = 0.
pdb2 = open(
self._getExtraPath(
'alignment_%02d_%02d.pdb' %
(volCounter, volCounter2))).readlines()
for i in range(len(pdb1)):
line1 = pdb1[i]
if line1.startswith("ATOM"):
line2 = pdb2[i]
x1 = float(line1[30:37])
y1 = float(line1[38:45])
z1 = float(line1[46:53])
x2 = float(line2[30:37])
y2 = float(line2[38:45])
z2 = float(line2[46:53])
dx = x1 - x2
dy = y1 - y2
dz = z1 - z2
d = math.sqrt(dx * dx + dy * dy + dz * dz)
davg += d
Navg += 1
if Navg > 0:
davg /= Navg
distances[volCounter - 1, volCounter2 - 1] = davg
distances = 0.5 * (distances + numpy.transpose(distances))
numpy.savetxt(self._getPath('distances.txt'), distances)
distances1D = numpy.mean(distances, axis=0)
print("Average distance to rest of volumes=", distances1D)
imin = numpy.argmin(distances1D)
print("The volume in the middle is pseudoatoms_%02d.pdb" % (imin + 1))
createLink(self._getPath("pseudoatoms_%02d.pdb" % (imin + 1)),
self._getPath("pseudoatoms.pdb"))
createLink(self._getPath("modes_%02d.xmd" % (imin + 1)),
self._getPath("modes.xmd"))
createLink(
self._getExtraPath("pseudoatoms_%02d_distance.hist" % (imin + 1)),
self._getExtraPath("pseudoatoms_distance.hist"))
# Measure range
minDisplacement = 1e38 * numpy.ones([self.numberOfModes.get(), 1])
maxDisplacement = -1e38 * numpy.ones([self.numberOfModes.get(), 1])
mdNMA = MetaData(self._getPath("modes.xmd"))
for volCounter in range(1, N + 1):
if volCounter != imin + 1:
md = MetaData(
self._getExtraPath("alignment_%02d_%02d.xmd" %
(imin + 1, volCounter)))
displacements = md.getValue(MDL_NMA, md.firstObject())
idx1 = 0
idx2 = 0
for idRow in mdNMA:
if mdNMA.getValue(MDL_ENABLED, idRow) == 1:
minDisplacement[idx2] = min(minDisplacement[idx2],
displacements[idx1])
maxDisplacement[idx2] = max(maxDisplacement[idx2],
displacements[idx1])
idx1 += 1
else:
minDisplacement[idx2] = 0
maxDisplacement[idx2] = 0
idx2 += 1
idx2 = 0
for idRow in mdNMA:
mdNMA.setValue(MDL_NMA_MINRANGE, float(minDisplacement[idx2]),
idRow)
mdNMA.setValue(MDL_NMA_MAXRANGE, float(maxDisplacement[idx2]),
idRow)
idx2 += 1
mdNMA.write(self._getPath("modes.xmd"))
# Create output
volCounter = 0
for inputStructure in self.inputStructures.get():
if volCounter == imin:
print("The corresponding volume is %s" %
(getImageLocation(inputStructure)))
finalStructure = inputStructure
break
volCounter += 1
pdb = PdbFile(self._getPath('pseudoatoms.pdb'), pseudoatoms=True)
self._defineOutputs(outputPdb=pdb)
modes = NormalModes(filename=self._getPath('modes.xmd'))
self._defineOutputs(outputModes=modes)
self._defineSourceRelation(self.inputStructures, self.outputPdb)
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images):
from numpy import array, dot
fnOut = 'classes_aligned@' + fn
MDin = MetaData(images)
MDout = MetaData()
n = 1
hasCTF = MDin.containsLabel(xmipp.MDL_CTF_MODEL)
for i in MDin:
fnImg = MDin.getValue(xmipp.MDL_IMAGE, i)
fnImgRef = MDin.getValue(xmipp.MDL_IMAGE_REF, i)
maxCC = MDin.getValue(xmipp.MDL_MAXCC, i)
rot = MDin.getValue(xmipp.MDL_ANGLE_ROT, i)
tilt = MDin.getValue(xmipp.MDL_ANGLE_TILT, i)
psi = -1. * MDin.getValue(xmipp.MDL_ANGLE_PSI, i)
flip = MDin.getValue(xmipp.MDL_FLIP, i)
if flip:
psi = -psi
eulerMatrix = Euler_angles2matrix(0., 0., psi)
x = MDin.getValue(xmipp.MDL_SHIFT_X, i)
y = MDin.getValue(xmipp.MDL_SHIFT_Y, i)
shift = array([x, y, 0])
shiftOut = dot(eulerMatrix, shift)
[x, y, z] = shiftOut
if flip:
x = -x
id = MDout.addObject()
MDout.setValue(xmipp.MDL_IMAGE, fnImg, id)
MDout.setValue(xmipp.MDL_IMAGE_REF, fnImgRef, id)
MDout.setValue(xmipp.MDL_IMAGE1,
"%05d@%s" % (n, self._getExtraPath("diff.stk")), id)
if hasCTF:
fnCTF = MDin.getValue(xmipp.MDL_CTF_MODEL, i)
MDout.setValue(xmipp.MDL_CTF_MODEL, fnCTF, id)
MDout.setValue(xmipp.MDL_MAXCC, maxCC, id)
MDout.setValue(xmipp.MDL_ANGLE_ROT, rot, id)
MDout.setValue(xmipp.MDL_ANGLE_TILT, tilt, id)
MDout.setValue(xmipp.MDL_ANGLE_PSI, psi, id)
MDout.setValue(xmipp.MDL_SHIFT_X, x, id)
MDout.setValue(xmipp.MDL_SHIFT_Y, y, id)
MDout.setValue(xmipp.MDL_FLIP, flip, id)
MDout.setValue(xmipp.MDL_ENABLED, 1, id)
n += 1
MDout.write(fnOut, xmipp.MD_APPEND)
# Actually create the differences
img = Image()
imgRef = Image()
if hasCTF and volumeIsCTFCorrected:
Ts = MDin.getValue(xmipp.MDL_SAMPLINGRATE, MDin.firstObject())
for i in MDout:
img.readApplyGeo(MDout, i)
imgRef.read(MDout.getValue(xmipp.MDL_IMAGE_REF, i))
if hasCTF and volumeIsCTFCorrected:
fnCTF = MDout.getValue(xmipp.MDL_CTF_MODEL, i)
imgRef.applyCTF(fnCTF, Ts)
img.convert2DataType(DT_DOUBLE)
imgDiff = img - imgRef
imgDiff.write(MDout.getValue(xmipp.MDL_IMAGE1, i))
def _iterAngles(self, fnAngles):
md = MetaData(fnAngles)
for objId in md:
rot = md.getValue(MDL_ANGLE_ROT, objId)
tilt = md.getValue(MDL_ANGLE_TILT, objId)
yield rot, tilt
