def runInitAngularReferenceFileStep(self):
'''Create Initial angular file. Either fill it with zeros or copy input'''
#NOTE: if using angles, self.selFileName file should contain angles info
md = emlib.MetaData(self.selFileName)
# Ensure this labels are always
md.addLabel(emlib.MDL_ANGLE_ROT)
md.addLabel(emlib.MDL_ANGLE_TILT)
md.addLabel(emlib.MDL_ANGLE_PSI)
expImages = self._getFileName('inputParticlesDoc')
ctfImages = self._getFileName('imageCTFpairs')
md.write(self._getExpImagesFileName(expImages))
blocklist = emlib.getBlocksInMetaDataFile(ctfImages)
mdCtf = emlib.MetaData()
mdAux = emlib.MetaData()
readLabels = [emlib.MDL_ITEM_ID, emlib.MDL_IMAGE]
for block in blocklist:
#read ctf block from ctf file
mdCtf.read(block + '@' + ctfImages, readLabels)
#add ctf columns to images file
mdAux.joinNatural(md, mdCtf)
# write block in images file with ctf info
mdCtf.write(block + '@' + expImages, emlib.MD_APPEND)
return [expImages]
def _storeSummaryInfo(self, numVolumes):
""" Store some information when the protocol finishes. """
msg1 = ''
msg2 = ''
for n in range(numVolumes):
fnBase = 'proposedVolume%05d' % n
fnRoot = self._getPath(fnBase + ".xmd")
if os.path.isfile(fnRoot):
md = emlib.MetaData(fnRoot)
size = md.size()
if (size < 5):
msg1 = "Num of inliers for model %d too small and equal to %d \n" % (n, size)
msg1 += "Decrease the value of Inlier Threshold parameter and run again \n"
fnRoot = self._getTmpPath("ransac00000.xmd")
if os.path.isfile(fnRoot):
md = emlib.MetaData(fnRoot)
size = md.size()
if (size < 5):
msg2 = "Num of random samples too small and equal to %d.\n" % size
msg2 += "If the option Dimensionality reduction is on, increase the number of grids per dimension (In this case we recommend to put Dimensionality reduction off).\n"
msg2 += "If the option Dimensionality reduction is off, increase the number of random samples.\n"
msg = msg1 + msg2
self.summaryVar.set(msg)
def getCorrThreshStep(self):
corrVector = []
fnCorr=self._getExtraPath("correlations.xmd")
mdCorr= emlib.MetaData()
for n in range(self.nRansac.get()):
fnRoot="ransac%05d"%n
fnAngles=self._getTmpPath("angles_"+fnRoot+".xmd")
md = emlib.MetaData(fnAngles)
for objId in md:
corr = md.getValue(emlib.MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(emlib.MDL_MAXCC,float(corr),objIdCorr)
mdCorr.write("correlations@"+fnCorr,emlib.MD_APPEND)
mdCorr= emlib.MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(self.corrThresh.get()*(len(sortedCorrVector)-1)))
#With the line below commented the percentil is not used for the threshold and is used the value introduced in the form
#CorrThresh = sortedCorrVector[indx]#
objId = mdCorr.addObject()
mdCorr.setValue(emlib.MDL_WEIGHT,self.corrThresh.get(),objId)
mdCorr.write("corrThreshold@"+fnCorr,emlib.MD_APPEND)
print("Correlation threshold: "+str(self.corrThresh.get()))
def runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling):
self._log.info("compute resolution 1")
#compute resolution
mdRsol = emlib.MetaData(resolIterMd)
mdResolOut = emlib.MetaData()
mdResolOut.importObjects(mdRsol,
emlib.MDValueLT(emlib.MDL_RESOLUTION_FRC, 0.5))
self._log.info("compute resolution 2")
if mdResolOut.size() == 0:
mdResolOut.clear()
mdResolOut.addObject()
id = mdResolOut.firstObject()
mdResolOut.setValue(emlib.MDL_RESOLUTION_FREQREAL, sampling * 2., id)
mdResolOut.setValue(emlib.MDL_RESOLUTION_FRC, 0.5, id)
else:
mdResolOut.sort()
id = mdResolOut.firstObject()
filterFrequence = mdResolOut.getValue(emlib.MDL_RESOLUTION_FREQREAL, id)
frc = mdResolOut.getValue(emlib.MDL_RESOLUTION_FRC, id)
md = emlib.MetaData()
id = md.addObject()
md.setColumnFormat(False)
md.setValue(emlib.MDL_RESOLUTION_FREQREAL, filterFrequence, id)
md.setValue(emlib.MDL_RESOLUTION_FRC, frc, id)
md.setValue(emlib.MDL_SAMPLINGRATE, sampling, id)
md.write(resolIterMaxMd, emlib.MD_APPEND)
def generateExpImagesStep(self, Nimgs, nameProj, nameExp, label):
fnProj = self._getExtraPath(nameProj + self._tempNumXmd % label)
fnExp = self._getExtraPath(nameExp + self._tempNumXmd % label)
fnLabels = self._getExtraPath('labels.txt')
mdIn = emlib.MetaData(fnProj)
mdExp = emlib.MetaData()
NimgsMd = mdIn.size()
Nrepeats = int(Nimgs / NimgsMd)
# if Nrepeats<10:
# Nrepeats=10
print("Nrepeats", Nrepeats)
if (label == 1 and exists(fnLabels)):
remove(fnLabels)
fileLabels = open(fnLabels, "a")
self._processRows(label, fnExp, mdIn, mdExp, Nrepeats, fileLabels)
mdExp.write(fnExp)
fileLabels.close()
if (label - 1) > 0:
labelPrev = -1
for n in range(1, label):
if exists(self._getExtraPath(nameExp + self._tempNumXmd % (label - n))):
labelPrev = label - n
break
if labelPrev != -1:
lastFnExp = self._getExtraPath(
nameExp + self._tempNumXmd % (labelPrev))
self.runJob("xmipp_metadata_utilities",
" -i %s --set union %s -o %s " %
(lastFnExp, fnExp, fnExp), numberOfMpi=1)
remove(fnProj)
def _insertNewCtfsSteps(self, newIDs1, newIDs2, insertedDict):
deps = []
newIDs = list(set(newIDs1).intersection(set(newIDs2)))
md1 = emlib.MetaData()
md2 = emlib.MetaData()
for ctfID in newIDs:
if ctfID not in insertedDict:
ctf1 = self.allCtf1.get(ctfID)
ctf2 = self.allCtf2.get(ctfID)
try:
self._ctfToMd(ctf1, md1)
self._ctfToMd(ctf2, md2)
self._freqResol[ctfID] = emlib.errorMaxFreqCTFs2D(md1, md2)
except TypeError as exc:
print("Error reading ctf for id:%s. %s" % (ctfID, exc))
self._freqResol[ctfID] = 9999
stepId = self._insertFunctionStep('selectCtfStep', ctfID,
prerequisites=[])
deps.append(stepId)
insertedDict[ctfID] = stepId
return deps
def scoreFinalVolumes(self):
threshold=self.getCCThreshold()
mdOut=emlib.MetaData()
for n in range(self.numVolumes.get()):
fnRoot=self._getPath('proposedVolume%05d'%n)
fnAssignment=fnRoot+".xmd"
if os.path.exists(fnAssignment):
self.runJob("xmipp_metadata_utilities","-i %s --fill weight constant 1"%fnAssignment)
MDassignment=emlib.MetaData(fnAssignment)
sum=0
thresholdedSum=0
N=0
minCC=2
for id in MDassignment:
cc=MDassignment.getValue(emlib.MDL_MAXCC,id)
sum+=cc
thresholdedSum+=cc-threshold
if cc<minCC:
minCC=cc
N+=1
if N>0:
avg=sum/N
else:
avg=0.0
id=mdOut.addObject()
mdOut.setValue(emlib.MDL_IMAGE,fnRoot+".mrc",id)
mdOut.setValue(emlib.MDL_VOLUME_SCORE_SUM,float(sum),id)
mdOut.setValue(emlib.MDL_VOLUME_SCORE_SUM_TH,float(thresholdedSum),id)
mdOut.setValue(emlib.MDL_VOLUME_SCORE_MEAN,float(avg),id)
mdOut.setValue(emlib.MDL_VOLUME_SCORE_MIN,float(minCC),id)
mdOut.write(self._getPath("proposedVolumes.xmd"))
def computeTrainingSet(self, nameTrain):
totalCones = self.generateConeCenters('coneCenters')
self.numCones = totalCones
fnCentersMd = self._getExtraPath(self._fnConeCenterDoc)
mdCones = emlib.MetaData(fnCentersMd)
auxList = []
mdTrain = emlib.MetaData(self.trainImgsFn)
mdList = []
for i in range(totalCones):
mdList.append(emlib.MetaData())
for row in iterRows(mdTrain):
rot = row.getValue(emlib.MDL_ANGLE_ROT)
tilt = row.getValue(emlib.MDL_ANGLE_TILT)
flip = row.getValue(emlib.MDL_FLIP)
if flip:
tilt = tilt + 180
if rot < 0:
rot = rot + 360
if tilt < 0:
tilt = tilt + 360
numCone = self.angularDistance(rot, tilt, mdCones)
mdCone = mdList[numCone - 1]
auxList.append(numCone - 1)
row.addToMd(mdCone)
for i in range(totalCones):
fnTrain = self._getExtraPath(nameTrain + self._tempNumXmd % (i + 1))
mdList[i].write(fnTrain)
def runExecuteCtfGroupsStep(self, **kwargs):
makePath(self.ctfGroupDirectory)
self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory)
# printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line
if not self.doCTFCorrection:
md = emlib.MetaData(self.selFileName)
block_name = self._getBlockFileName(ctfBlockName, 1,
self._getFileName('imageCTFpairs'))
md.write(block_name)
self._log.info("Written a single CTF group to file: '%s'" % block_name)
self.numberOfCtfGroups.set(1)
else:
self._log.info(
'*********************************************************************'
)
self._log.info('* Make CTF groups')
# remove all entries not present in sel file by
# join between selfile and metadatafile
mdCtfData = emlib.MetaData()
mdCtfData.read(self.ctfDatName)
mdSel = emlib.MetaData()
mdSel.read(self.selFileName)
mdCtfData.intersection(mdSel, emlib.MDL_IMAGE)
tmpCtfDat = self.ctfDatName
mdCtfData.write(tmpCtfDat)
args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s'
args += ' --sampling_rate %(sampling)s'
params = {
'tmpCtfDat': tmpCtfDat,
'ctffile': self._getFileName('ctfGroupBase') + ':stk',
'wiener': self.wienerConstant.get(),
'pad': self.paddingFactor.get(),
'sampling': self.resolSam
}
if self.inputParticles.get().isPhaseFlipped():
args += ' --phase_flipped '
if self.doAutoCTFGroup:
args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s'
params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get()
params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get()
else:
if exists(self.setOfDefocus.get()):
args += ' --split %(setOfDefocus)s'
params['setOfDefocus'] = self.setOfDefocus.get()
self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs)
auxMD = emlib.MetaData("numberGroups@" +
self._getFileName('cTFGroupSummary'))
self.numberOfCtfGroups.set(
auxMD.getValue(emlib.MDL_COUNT, auxMD.firstObject()))
self._store(self.numberOfCtfGroups)
def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images):
from numpy import array, dot
fnOut = 'classes_aligned@' + fn
MDin = emlib.MetaData(images)
MDout = emlib.MetaData()
n = 1
hasCTF = MDin.containsLabel(emlib.MDL_CTF_MODEL)
for i in MDin:
fnImg = MDin.getValue(emlib.MDL_IMAGE,i)
fnImgRef = MDin.getValue(emlib.MDL_IMAGE_REF,i)
maxCC = MDin.getValue(emlib.MDL_MAXCC,i)
rot = MDin.getValue(emlib.MDL_ANGLE_ROT,i)
tilt = MDin.getValue(emlib.MDL_ANGLE_TILT,i)
psi =-1.*MDin.getValue(emlib.MDL_ANGLE_PSI,i)
flip = MDin.getValue(emlib.MDL_FLIP,i)
if flip:
psi = -psi
eulerMatrix = emlib.Euler_angles2matrix(0., 0., psi)
x = MDin.getValue(emlib.MDL_SHIFT_X,i)
y = MDin.getValue(emlib.MDL_SHIFT_Y,i)
shift = array([x, y, 0])
shiftOut = dot(eulerMatrix, shift)
[x,y,z]= shiftOut
if flip:
x = -x
id = MDout.addObject()
MDout.setValue(emlib.MDL_IMAGE, fnImg, id)
MDout.setValue(emlib.MDL_IMAGE_REF, fnImgRef, id)
MDout.setValue(emlib.MDL_IMAGE1, "%05d@%s"%(n, self._getExtraPath("diff.stk")), id)
if hasCTF:
fnCTF = MDin.getValue(emlib.MDL_CTF_MODEL,i)
MDout.setValue(emlib.MDL_CTF_MODEL,fnCTF,id)
MDout.setValue(emlib.MDL_MAXCC, maxCC, id)
MDout.setValue(emlib.MDL_ANGLE_ROT, rot, id)
MDout.setValue(emlib.MDL_ANGLE_TILT, tilt, id)
MDout.setValue(emlib.MDL_ANGLE_PSI, psi, id)
MDout.setValue(emlib.MDL_SHIFT_X, x,id)
MDout.setValue(emlib.MDL_SHIFT_Y, y,id)
MDout.setValue(emlib.MDL_FLIP,flip,id)
MDout.setValue(emlib.MDL_ENABLED,1,id)
n+=1
MDout.write(fnOut,emlib.MD_APPEND)
# Actually create the differences
img = emlib.Image()
imgRef = emlib.Image()
if hasCTF and volumeIsCTFCorrected:
Ts = MDin.getValue(emlib.MDL_SAMPLINGRATE, MDin.firstObject())
for i in MDout:
img.readApplyGeo(MDout,i)
imgRef.read(MDout.getValue(emlib.MDL_IMAGE_REF,i))
if hasCTF and volumeIsCTFCorrected:
fnCTF = MDout.getValue(emlib.MDL_CTF_MODEL,i)
emlib.applyCTF(imgRef, fnCTF, Ts)
img.convert2DataType(emlib.DT_DOUBLE)
imgDiff = img-imgRef
imgDiff.write(MDout.getValue(emlib.MDL_IMAGE1,i))
def calculateDeviationsStep(self, it):
""" Calculate both angles and shifts devitations for all iterations
"""
SL = emlib.SymList()
mdIter = emlib.MetaData()
#for it in self.allIters():
mdIter.clear()
SL.readSymmetryFile(self._symmetry[it])
md1 = emlib.MetaData(self.docFileInputAngles[it])
md2 = emlib.MetaData(self.docFileInputAngles[it - 1])
#ignore disabled,
md1.removeDisabled()
md2.removeDisabled()
#first metadata file may not have shiftx and shifty
if not md2.containsLabel(emlib.MDL_SHIFT_X):
md2.addLabel(emlib.MDL_SHIFT_X)
md2.addLabel(emlib.MDL_SHIFT_Y)
md2.fillConstant(emlib.MDL_SHIFT_X, 0.)
md2.fillConstant(emlib.MDL_SHIFT_Y, 0.)
oldLabels = [
emlib.MDL_ANGLE_ROT, emlib.MDL_ANGLE_TILT, emlib.MDL_ANGLE_PSI,
emlib.MDL_SHIFT_X, emlib.MDL_SHIFT_Y
]
newLabels = [
emlib.MDL_ANGLE_ROT2, emlib.MDL_ANGLE_TILT2, emlib.MDL_ANGLE_PSI2,
emlib.MDL_SHIFT_X2, emlib.MDL_SHIFT_Y2
]
md2.renameColumn(oldLabels, newLabels)
md2.addLabel(emlib.MDL_SHIFT_X_DIFF)
md2.addLabel(emlib.MDL_SHIFT_Y_DIFF)
md2.addLabel(emlib.MDL_SHIFT_DIFF)
mdIter.join1(md1, md2, emlib.MDL_IMAGE, emlib.INNER_JOIN)
SL.computeDistance(mdIter, False, False, False)
emlib.activateMathExtensions()
#operate in sqlite
shiftXLabel = emlib.label2Str(emlib.MDL_SHIFT_X)
shiftX2Label = emlib.label2Str(emlib.MDL_SHIFT_X2)
shiftXDiff = emlib.label2Str(emlib.MDL_SHIFT_X_DIFF)
shiftYLabel = emlib.label2Str(emlib.MDL_SHIFT_Y)
shiftY2Label = emlib.label2Str(emlib.MDL_SHIFT_Y2)
shiftYDiff = emlib.label2Str(emlib.MDL_SHIFT_Y_DIFF)
shiftDiff = emlib.label2Str(emlib.MDL_SHIFT_DIFF)
#timeStr = str(dtBegin)
operateString = shiftXDiff + "=" + shiftXLabel + "-" + shiftX2Label
operateString += "," + shiftYDiff + "=" + shiftYLabel + "-" + shiftY2Label
mdIter.operate(operateString)
operateString = shiftDiff+"=sqrt(" \
+shiftXDiff+"*"+shiftXDiff+"+" \
+shiftYDiff+"*"+shiftYDiff+");"
mdIter.operate(operateString)
iterFile = self._mdDevitationsFn(it)
mdIter.write(iterFile, emlib.MD_APPEND)
self._setLastIter(it)
def correlationCudaStep(self, thIdx, gpuId, totalGpu):
mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc))
self.numCones = mdNumCones.size()
# Cuda Correlation step - creating the metadata
predCones = np.loadtxt(self._getConePrediction())
mdConeList = []
numMax = int(self.numConesSelected)
for i in range(self.numCones):
mdConeList.append(emlib.MetaData())
mdIn = emlib.MetaData(self.imgsFn)
for i in range(self.numCones):
idx = i + 1
if (idx % totalGpu) != thIdx:
continue
#modelFn = 'modelCone%d_aux' % idx
#f = open(join(self._getExtraPath(), modelFn+'.txt'),'r')
#mae = float(f.readline())
#f.close()
modelFn = 'modelCone%d' % idx
positions = []
for n in range(numMax):
posAux = np.where(predCones[:, (n * 2) + 1] == (i + 1))
positions = positions + (np.ndarray.tolist(posAux[0]))
if len(positions) > 0 and exists(self._getExtraPath(modelFn + '.h5')):
print("Classifying cone ", idx, "in GPU ", gpuId)
for pos in positions:
id = pos + 1
row = md.Row()
row.readFromMd(mdIn, id)
row.addToMd(mdConeList[i])
fnExpCone = self._getExtraPath('metadataCone%d.xmd' % (i + 1))
mdConeList[i].write(fnExpCone)
fnProjCone = self._getExtraPath('projectionsCudaCorr%d.xmd' % (i + 1))
self.runJob("xmipp_metadata_utilities", "-i %s --fill ref lineal 1 1 " % (fnProjCone), numberOfMpi=1)
fnOutCone = self._getOutCone(i + 1)
if not exists(self._getExtraPath(fnOutCone)):
params = ' -i %s' % fnExpCone
params += ' -r %s' % fnProjCone
params += ' -o %s' % self._getExtraPath(fnOutCone)
params += ' --dev %s '%(gpuId)
self.runJob("xmipp_cuda_align_significant", params, numberOfMpi=1)
def _getTmpMethods(self):
""" Return the message when there is not output generated yet.
We will read the Xmipp .pos files and other configuration files.
"""
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
msg = ''
if existsConfig:
md = emlib.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(emlib.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(emlib.MDL_PICKING_PARTICLE_SIZE, configobj)
isAutopick = pickingState != "Manual"
manualParts = md.getValue(emlib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
autoParts = md.getValue(emlib.MDL_PICKING_AUTOPARTICLES_SIZE, configobj)
if manualParts is None:
manualParts = 0
if autoParts is None:
autoParts = 0
msg = 'User picked %d particles ' % (autoParts + manualParts)
msg += 'with a particle size of %d.' % particleSize
if isAutopick:
msg += "Automatic picking was used ([Abrishami2013]). "
msg += "%d particles were picked automatically " % autoParts
msg += "and %d manually." % manualParts
return msg
def projMatchStep(self, volume, angularSampling, symmetryGroup, images, fnAngles, Xdim):
# Generate gallery of projections
fnGallery = self._getExtraPath('gallery.stk')
if volume.endswith('.mrc'):
volume+=":mrc"
self.runJob("xmipp_angular_project_library",
"-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline "
"--compute_neighbors --angular_distance -1 --experimental_images %s"
% (volume, fnGallery, angularSampling, symmetryGroup, images))
# Assign angles
self.runJob("xmipp_angular_projection_matching",
"-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift 1000 "
"--search5d_shift %s --search5d_step %s --append"
% (images, fnAngles, fnGallery, str(Xdim/2),
str(int(Xdim/10)), str(int(Xdim/25))))
cleanPath(self._getExtraPath('gallery_sampling.xmd'))
cleanPath(self._getExtraPath('gallery_angles.doc'))
cleanPath(self._getExtraPath('gallery.doc'))
# Write angles in the original file and sort
MD=emlib.MetaData(fnAngles)
for id in MD:
galleryReference = MD.getValue(emlib.MDL_REF,id)
MD.setValue(emlib.MDL_IMAGE_REF, "%05d@%s" % (galleryReference+1, fnGallery), id)
MD.write(fnAngles)
def _showPolarPlot(self, fnmd):
"""
It is called by _showDirectionalResolution
This function shows the angular distribution of the resolution
"""
md = emlib.MetaData(fnmd)
radius = md.getColumnValues(emlib.MDL_ANGLE_ROT)
azimuth = md.getColumnValues(emlib.MDL_ANGLE_TILT)
counts = md.getColumnValues(emlib.MDL_RESOLUTION_FRC)
# define binning
azimuths = np.radians(np.linspace(0, 360, 360))
zeniths = np.arange(0, 91, 1)
r, theta = np.meshgrid(zeniths, azimuths)
values = np.zeros((len(azimuths), len(zeniths)))
for i in range(0, len(azimuth)):
values[int(radius[i]), int(azimuth[i])] = counts[i]
# ------ Plot ------
stp = 0.1
lowlim = max(0.0, values.min())
highlim = values.max() + stp
fig, ax = plt.subplots(subplot_kw=dict(projection='polar'))
pc = plt.contourf(theta, r, values, np.arange(lowlim, highlim, stp))
plt.colorbar(pc)
plt.show()
def _pickMicrograph(self, mic, *args):
micPath = mic.getFileName()
# Get particle picking boxsize from the previous run
boxSize = self.particlePickingRun.outputCoordinates.getBoxSize()
modelRoot = self._getExtraPath('model')
micName = removeBaseExt(micPath)
proceed = True
if self.micsToPick == MICS_SAMEASPICKING:
basePos = replaceBaseExt(micPath, "pos")
fnPos = self.particlePickingRun._getExtraPath(basePos)
if exists(fnPos):
blocks = emlib.getBlocksInMetaDataFile(fnPos)
copy = True
if 'header' in blocks:
mdheader = emlib.MetaData("header@" + fnPos)
state = mdheader.getValue(
emlib.MDL_PICKING_MICROGRAPH_STATE,
mdheader.firstObject())
if state == "Available":
copy = False
if copy:
# Copy manual .pos file of this micrograph
copyFile(fnPos, self._getExtraPath(basename(fnPos)))
proceed = False
if proceed:
args = "-i %s " % micPath
args += "--particleSize %d " % boxSize
args += "--model %s " % modelRoot
args += "--outputRoot %s " % self._getExtraPath(micName)
args += "--mode autoselect --thr %d" % self.numberOfThreads
self.runJob("xmipp_micrograph_automatic_picking", args)
def test(self):
subset = self.newProtocol(ProtSubSet,
inputFullSet=self.protImportParts.outputParticles,
chooseAtRandom=True,
nElements=400)
self.launchProtocol(subset)
highres = self.newProtocol(XmippProtReconstructHighRes,
inputParticles=subset.outputParticles,
inputVolumes=self.protImportVol.outputVolume,
particleRadius=180,
symmetryGroup="i1",
nextResolutionCriterion=0.143,
alignmentMethod=XmippProtReconstructHighRes.AUTOMATIC_ALIGNMENT,
maximumTargetResolution="15 10 7",
numberOfMpi=8)
self.launchProtocol(highres)
self.assertIsNotNone(highres.outputParticles,
"There was a problem with Highres")
fnResolution = highres._getExtraPath("Iter005/iterInfo.xmd")
if not exists(fnResolution):
self.assertTrue(False, fnResolution + " does not exist")
else:
md = emlib.MetaData("resolution@" + fnResolution)
R = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, md.firstObject())
self.assertTrue(R < 10, "Resolution is not below 10A")
def getSummary(self, coordSet):
summary = []
summary.append("Previous run: %s" %
self.xmippParticlePicking.get().getNameId())
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = emlib.MetaData('properties@' + configfile)
configobj = md.firstObject()
def _get(label):
return md.getValue(label, configobj)
pickingState = _get(emlib.MDL_PICKING_STATE)
particleSize = _get(emlib.MDL_PICKING_PARTICLE_SIZE)
activeMic = _get(emlib.MDL_MICROGRAPH)
isAutopick = pickingState != "Manual"
manualParticlesSize = _get(emlib.MDL_PICKING_MANUALPARTICLES_SIZE)
autoParticlesSize = _get(emlib.MDL_PICKING_AUTOPARTICLES_SIZE)
summary.append("Manual particles picked: %s" % manualParticlesSize)
summary.append("Particle size:%d" % (particleSize))
autopick = "Yes" if isAutopick else "No"
summary.append("Autopick: " + autopick)
if isAutopick:
summary.append("Automatic particles picked: %s" %
autoParticlesSize)
summary.append("Last micrograph: " + activeMic)
return "\n".join(summary)
def _getMdString(self, filename, block=None):
md = emlib.MetaData()
if block:
md.read(block + '@' + filename)
else:
md.read(filename, 1)
labels = md.getActiveLabels()
msg = "Metadata items: *%d*\n" % md.getParsedLines()
msg += "Metadata labels: " + ''.join(
["\n - %s" % emlib.label2Str(l) for l in labels])
imgPath = None
for label in labels:
if emlib.labelIsImage(label):
imgPath = self._getImgPath(
filename, md.getValue(label, md.firstObject()))
if imgPath is None or not os.path.exists(imgPath):
imgPath = None
break
# If there is an image and is not too big
if imgPath and pwutils.getFileSize(imgPath) < (
pwem.Config.MAX_PREVIEW_FILE_SIZE * 1024 * 1024):
self._imgPreview = self._getImagePreview(imgPath)
self._imgInfo = self._getImageString(imgPath)
return msg
def _viewLocalDefocus(self, paramName=None):
"""display a 3D view of where the particles are placed in the micrograph taking as height the value estimated
for local defocues"""
views = []
fnDefoci = "%s" % (self.protocol._getExtraPath("micrographDefoci.xmd"))
if exists(fnDefoci):
try:
mdPoints = emlib.MetaData(
"mic_%d@%s" % (self.displayLocalDefocus.get(), fnDefoci))
x = mdPoints.getColumnValues(emlib.MDL_XCOOR)
y = mdPoints.getColumnValues(emlib.MDL_YCOOR)
defocusA = mdPoints.getColumnValues(emlib.MDL_CTF_DEFOCUSA)
residuals = mdPoints.getColumnValues(
emlib.MDL_CTF_DEFOCUS_RESIDUAL)
title = "Micrograph %d defocus" % self.displayLocalDefocus.get(
)
xplotter = XmippPlotter(windowTitle=title)
a = xplotter.createSubPlot(title, 'x', 'y', projection='3d')
a.set_zlabel('Defocus')
a.scatter(x, y, defocusA, c='r', marker='o')
a.scatter(x,
y,
np.asarray(defocusA) - np.asarray(residuals),
c='b',
marker='^')
legends = ['Avg. defocus', 'Adjusted defocus']
xplotter.showLegend(legends, loc=1)
views.append(xplotter)
except Exception as e:
print(e)
return views
def getBestVolumesStep(self):
volumes = []
inliers = []
for n in range(self.nRansac.get()):
fnAngles = self._getTmpPath("angles_ransac%05d"%n+".xmd")
md=emlib.MetaData("inliers@"+fnAngles)
numInliers=md.getValue(emlib.MDL_WEIGHT,md.firstObject())
volumes.append(fnAngles)
inliers.append(numInliers)
index = sorted(range(inliers.__len__()), key=lambda k: inliers[k])
fnBestAngles = ''
threshold=self.getCCThreshold()
i = self.nRansac.get()-1
indx = 0
while i >= 0 and indx < self.numVolumes:
fnBestAngles = volumes[index[i]]
fnBestAnglesOut = self._getPath("proposedVolume%05d"%indx+".xmd")
copyFile(fnBestAngles, fnBestAnglesOut)
self._log.info("Best volume %d = %s" % (indx, fnBestAngles))
if not self.useAll:
self.runJob("xmipp_metadata_utilities","-i %s -o %s --query select \"maxCC>%f \" --mode append" %(fnBestAnglesOut,fnBestAnglesOut,threshold))
if not isMdEmpty(fnBestAnglesOut):
indx += 1
else:
indx += 1
i -= 1
# Remove unnecessary files
for n in range(self.nRansac.get()):
fnAngles = self._getTmpPath("angles_ransac%05d"%n+".xmd")
cleanPath(fnAngles)
def predictStep(self, gpuId):
if not exists(self._getConePrediction()):
# if self.useQueueForSteps() or self.useQueue():
# myStr = os.environ["CUDA_VISIBLE_DEVICES"]
# else:
# myStr = self.gpuList.get()
# numGPU = myStr.split(',')
# numGPU = numGPU[0]
# print("Predict", myStr, numGPU)
# sys.stdout.flush()
mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc))
self.numCones = mdNumCones.size()
imgsOutStk = self._getExtraPath('images_out_filtered.stk')
imgsOutXmd = self._getExtraPath('images_out_filtered.xmd')
self.runJob("xmipp_transform_filter", " -i %s -o %s "
"--save_metadata_stack %s "
"--keep_input_columns "
"--fourier low_pass %f " %
(self.imgsFn, imgsOutStk, imgsOutXmd, 0.15), numberOfMpi=self.myMPI.get())
numMax = int(self.numConesSelected)
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
args = "%s %s %d %d %d " % (imgsOutXmd, self._getExtraPath(), newXdim, self.numCones, numMax)
#args += " %(GPU)s"
args += " %s "%(gpuId)
self.runJob("xmipp_cone_deepalign_predict", args, numberOfMpi=1, env=self.getCondaEnv())
def _appendRctImages(self, particles):
blockMd = "class%06d_images@%s" % (particles.getObjId(),
self.rctClassesFn)
classMd = emlib.MetaData()
partPairs = self.inputParticlesTiltPair.get()
uImages = partPairs.getUntilted()
tImages = partPairs.getTilted()
sangles = partPairs.getCoordsPair().getAngles()
micPairs = partPairs.getCoordsPair().getMicsPair()
uMics = micPairs.getUntilted()
tMics = micPairs.getTilted()
scaleFactor = uImages.getSamplingRate() / particles.getSamplingRate()
for img in particles:
imgId = img.getObjId()
uImg = uImages[imgId]
tImg = tImages[imgId]
if uImg is None or tImg is None:
print(">>> Warning, for id %d, tilted or untilted particle "
"was not found. Ignored." % imgId)
else:
objId = classMd.addObject()
pairRow = emlib.metadata.Row()
pairRow.setValue(emlib.MDL_IMAGE, getImageLocation(uImg))
uCoord = uImg.getCoordinate()
micId = uCoord.getMicId()
uMic = uMics[micId]
angles = sangles[micId]
pairRow.setValue(emlib.MDL_MICROGRAPH, uMic.getFileName())
pairRow.setValue(emlib.MDL_XCOOR, uCoord.getX())
pairRow.setValue(emlib.MDL_YCOOR, uCoord.getY())
pairRow.setValue(emlib.MDL_ENABLED, 1)
pairRow.setValue(emlib.MDL_ITEM_ID, int(imgId))
pairRow.setValue(emlib.MDL_REF, 1)
alignment = img.getTransform()
# Scale alignment by scaleFactor
alignment.scale(scaleFactor)
alignmentToRow(alignment, pairRow, alignType=ALIGN_2D)
pairRow.setValue(emlib.MDL_IMAGE_TILTED,
getImageLocation(tImg))
tMic = tMics[micId]
pairRow.setValue(emlib.MDL_MICROGRAPH_TILTED,
tMic.getFileName())
(angleY, angleY2, angleTilt) = angles.getAngles()
pairRow.setValue(emlib.MDL_ANGLE_Y, float(angleY))
pairRow.setValue(emlib.MDL_ANGLE_Y2, float(angleY2))
pairRow.setValue(emlib.MDL_ANGLE_TILT, float(angleTilt))
pairRow.writeToMd(classMd, objId)
classMd.write(blockMd, emlib.MD_APPEND)
def centerFirstHarmonicStep(self, imagesFn, outputCenter):
dims = emlib.MetaDataInfo(str(imagesFn))
md = emlib.MetaData()
objId = md.addObject()
md.setValue(emlib.MDL_X, float(dims[0] / 2), objId)
md.setValue(emlib.MDL_Y, float(dims[1] / 2), objId)
md.write(outputCenter)
return [outputCenter] # this file should exists after the step
def test_writeSetOfDefocusGroups(self):
#TODO: FIX THIS test according to the new SetOfDefocusGroup
return
#reference metadata
md = emlib.MetaData()
objId = md.addObject()
defocusGroupRow = metadata.Row()
defocusGroupRow.setValue(emlib.MDL_ENABLED, 1)
defocusGroupRow.setValue(emlib.MDL_CTF_GROUP, 1)
defocusGroupRow.setValue(emlib.MDL_MIN, 2000.)
defocusGroupRow.setValue(emlib.MDL_MAX, 2500.)
defocusGroupRow.setValue(emlib.MDL_AVG, 2100.)
defocusGroupRow.writeToMd(md, objId)
objId = md.addObject()
defocusGroupRow.setValue(emlib.MDL_ENABLED, 1)
defocusGroupRow.setValue(emlib.MDL_CTF_GROUP, 2)
defocusGroupRow.setValue(emlib.MDL_MIN, 3000.)
defocusGroupRow.setValue(emlib.MDL_MAX, 5500.)
defocusGroupRow.setValue(emlib.MDL_AVG, 5000.)
defocusGroupRow.writeToMd(md, objId)
#
fnScipion = self.getOutputPath("writeSetOfDefocusGroups.sqlite")
setOfDefocus = SetOfDefocusGroup(filename=fnScipion)
df = DefocusGroup()
df.setDefocusMin(2000.)
df.setDefocusMax(2500.)
df.setDefocusAvg(2100.)
setOfDefocus.append(df)
df.cleanObjId()
df.setDefocusMin(3000)
df.setDefocusMax(5500)
df.setDefocusAvg(5000)
setOfDefocus.append(df)
fnXmipp = self.getOutputPath("writeSetOfDefocusGroups.xmd")
fnScipion = self.getOutputPath("writeSetOfDefocusGroups2.xmd")
writeSetOfDefocusGroups(setOfDefocus, fnXmipp)
mdAux = emlib.MetaData(fnXmipp)
md.write(fnScipion)
print("Comparing metadatas: \n%s\n%s" % (fnXmipp, fnScipion))
self.assertEqual(md, mdAux, "test writeSetOfDefocusGroups fails")
def test_micrographsToMd(self):
""" Test the conversion of a SetOfMicrographs to Xmipp metadata. """
micSet = SetOfMicrographs(
filename=self.getOutputPath("micrographs.sqlite"))
n = 3
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
micSet.setAcquisition(acquisition)
micSet.setSamplingRate(1.)
mdXmipp = emlib.MetaData()
for i in range(n):
p = Micrograph()
file = self.dataset.getFile("mic%s" % (i + 1))
p.setLocation(file)
ctf = ctfs[i % 2]
p.setCTF(ctf)
micSet.append(p)
id = mdXmipp.addObject()
mdXmipp.setValue(emlib.MDL_ENABLED, 1, id)
mdXmipp.setValue(emlib.MDL_ITEM_ID, int(i + 1), id)
mdXmipp.setValue(emlib.MDL_MICROGRAPH, file, id)
# set CTFModel params
mdXmipp.setValue(emlib.MDL_CTF_DEFOCUSU, ctf.getDefocusU(), id)
mdXmipp.setValue(emlib.MDL_CTF_DEFOCUSV, ctf.getDefocusV(), id)
mdXmipp.setValue(emlib.MDL_CTF_DEFOCUS_ANGLE,
ctf.getDefocusAngle(), id)
# set Acquisition params
mdXmipp.setValue(emlib.MDL_CTF_Q0,
acquisition.getAmplitudeContrast(), id)
mdXmipp.setValue(emlib.MDL_CTF_CS,
acquisition.getSphericalAberration(), id)
mdXmipp.setValue(emlib.MDL_CTF_VOLTAGE, acquisition.getVoltage(),
id)
mdScipion = emlib.MetaData()
setOfMicrographsToMd(micSet, mdScipion)
writeSetOfMicrographs(micSet, self.getOutputPath("micrographs.xmd"))
self.assertEqual(mdScipion, mdXmipp, "metadata are not the same")
def __init__(self, itemClass):
""" Create new set, base on a Metadata.
itemClass: Class that represent the items.
A method .getFileName should be available to store the md.
Items contained in XmippSet are supposed to inherit from Row.
"""
self._itemClass = itemClass
# self._fileName = fileName
self._md = emlib.MetaData()
def test_particlesToMd(self):
""" Test the conversion of a SetOfParticles to Xmipp metadata. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
mdXmipp = emlib.MetaData()
imgSet.setAcquisition(acquisition)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
imgSet.append(p)
id = mdXmipp.addObject()
mdXmipp.setValue(emlib.MDL_ENABLED, 1, id)
mdXmipp.setValue(emlib.MDL_ITEM_ID, int(i + 1), id)
mdXmipp.setValue(emlib.MDL_IMAGE, locationToXmipp(i + 1, fn), id)
# set CTFModel params
mdXmipp.setValue(emlib.MDL_CTF_DEFOCUSU, ctf.getDefocusU(), id)
mdXmipp.setValue(emlib.MDL_CTF_DEFOCUSV, ctf.getDefocusV(), id)
mdXmipp.setValue(emlib.MDL_CTF_DEFOCUS_ANGLE,
ctf.getDefocusAngle(), id)
# set Acquisition params
mdXmipp.setValue(emlib.MDL_CTF_Q0,
acquisition.getAmplitudeContrast(), id)
mdXmipp.setValue(emlib.MDL_CTF_CS,
acquisition.getSphericalAberration(), id)
mdXmipp.setValue(emlib.MDL_CTF_VOLTAGE, acquisition.getVoltage(),
id)
mdScipion = emlib.MetaData()
setOfParticlesToMd(imgSet, mdScipion)
self.assertEqual(mdScipion, mdXmipp, "metadata are not the same")
def prepareImagesForTraining(self):
fnCentersMd = self._getExtraPath(self._fnConeCenterDoc)
mdCones = emlib.MetaData(fnCentersMd)
span = self.spanConesTilt.get()
counterCones = 0
for row in iterRows(mdCones):
rotCenter = row.getValue(emlib.MDL_ANGLE_ROT)
tiltCenter = row.getValue(emlib.MDL_ANGLE_TILT)
if rotCenter < 0:
rotCenter = rotCenter + 360
if tiltCenter < 0:
tiltCenter = tiltCenter + 360
iniRot = rotCenter - span
endRot = rotCenter + span
iniTilt = tiltCenter - span
endTilt = tiltCenter + span
if iniRot < 0:
iniRot = iniRot + 360
if iniTilt < 0:
iniTilt = iniTilt + 360
if endRot < 0:
endRot = endRot + 360
if endTilt < 0:
endTilt = endTilt + 360
mdProj = emlib.MetaData(self._getExtraPath('projections%d.xmd' % (counterCones + 1)))
sizeProj = mdProj.size()
if sizeProj > 0:
lastLabel = counterCones + 1
self.projectStep(300, iniRot, endRot, iniTilt, endTilt,
'projectionsCudaCorr', counterCones + 1)
if self.modelPretrain.get() is False:
self.generateExpImagesStep(10000, 'projections',
'projectionsExp',
counterCones + 1)
else:
remove(self._getExtraPath('projections%d.xmd' % (counterCones + 1)))
counterCones = counterCones + 1
if self.modelPretrain.get() is False:
fnToFilter = self._getProjectionsExp(lastLabel)
self.runJob("xmipp_transform_filter", " -i %s --fourier low_pass %f" %
(fnToFilter, 0.15), numberOfMpi=self.myMPI.get())
def correlationSignificantStep(self):
mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc))
self.numCones = mdNumCones.size()
# Cuda Correlation step - creating the metadata
predCones = np.loadtxt(self._getConePrediction())
mdConeList = []
numMax = int(self.numConesSelected)
for i in range(self.numCones):
mdConeList.append(emlib.MetaData())
mdIn = emlib.MetaData(self.imgsFn)
for i in range(self.numCones):
print("Classifying cone ", i + 1)
positions = []
for n in range(numMax):
posAux = np.where(predCones[:, (n * 2) + 1] == (i + 1))
positions = positions + (np.ndarray.tolist(posAux[0]))
if len(positions) > 0:
for pos in positions:
id = pos + 1
row = md.Row()
row.readFromMd(mdIn, id)
row.addToMd(mdConeList[i])
fnExpCone = self._getExtraPath('metadataCone%d.xmd' % (i + 1))
mdConeList[i].write(fnExpCone)
fnProjCone = self._getExtraPath('projectionsCudaCorr%d.xmd' % (i + 1))
fnOutCone = self._getOutCone(i + 1)
if not exists(self._getExtraPath(fnOutCone)):
# Correlation step - calling significant program
args = '-i %s --initgallery %s --odir %s --dontReconstruct --useForValidation %d ' \
'--dontCheckMirrors --maxShift 30' % (fnExpCone, fnProjCone, self._getExtraPath(), 1)
self.runJob('xmipp_reconstruct_significant', args,
numberOfMpi=self.myMPI.get())
copy(self._getExtraPath('images_significant_iter001_00.xmd'), self._getExtraPath(fnOutCone))
remove(self._getExtraPath('angles_iter001_00.xmd'))
remove(self._getExtraPath('images_significant_iter001_00.xmd'))