def getCorrThreshStep(self):
corrVector = []
fnCorr = self._getExtraPath("correlations.xmd")
mdCorr = xmipp.MetaData()
for n in range(self.nRansac.get()):
fnRoot = "ransac%05d" % n
fnAngles = self._getTmpPath("angles_" + fnRoot + ".xmd")
md = xmipp.MetaData(fnAngles)
for objId in md:
corr = md.getValue(xmipp.MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(xmipp.MDL_MAXCC, float(corr), objIdCorr)
mdCorr.write("correlations@" + fnCorr, xmipp.MD_APPEND)
mdCorr = xmipp.MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(self.corrThresh.get() * (len(sortedCorrVector) - 1)))
#With the line below commented the percentil is not used for the threshold and is used the value introduced in the form
#CorrThresh = sortedCorrVector[indx]#
objId = mdCorr.addObject()
mdCorr.setValue(xmipp.MDL_WEIGHT, self.corrThresh.get(), objId)
mdCorr.write("corrThreshold@" + fnCorr, xmipp.MD_APPEND)
print "Correlation threshold: " + str(self.corrThresh.get())
def createOutputStep(self):
self.mdClasses = xmipp.MetaData(self._getDirectionalClassesFn())
self.mdImages = xmipp.MetaData(self._getDirectionalImagesFn())
inputParticles = self.inputParticles.get()
classes2D = self._createSetOfClasses2D(inputParticles)
self.averageSet = self._createSetOfAverages()
self.averageSet.copyInfo(inputParticles)
self.averageSet.setAlignmentProj()
# Let's use a SetMdIterator because it could be less particles
# in the metadata produced than in the input set
iterator = md.SetMdIterator(self.mdImages,
sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self._updateParticle,
skipDisabled=True)
classes2D.classifyItems(updateItemCallback=iterator.updateItem,
updateClassCallback=self._updateClass)
self._defineOutputs(outputClasses=classes2D)
self._defineOutputs(outputAverages=self.averageSet)
self._defineSourceRelation(self.inputParticles, classes2D)
self._defineSourceRelation(self.inputParticles, self.averageSet)
def runInitAngularReferenceFileStep(self):
'''Create Initial angular file. Either fill it with zeros or copy input'''
#NOTE: if using angles, self.selFileName file should contain angles info
md = xmipp.MetaData(self.selFileName)
# Ensure this labels are always
md.addLabel(xmipp.MDL_ANGLE_ROT)
md.addLabel(xmipp.MDL_ANGLE_TILT)
md.addLabel(xmipp.MDL_ANGLE_PSI)
expImages = self._getFileName('inputParticlesDoc')
ctfImages = self._getFileName('imageCTFpairs')
md.write(self._getExpImagesFileName(expImages))
blocklist = xmipp.getBlocksInMetaDataFile(ctfImages)
mdCtf = xmipp.MetaData()
mdAux = xmipp.MetaData()
readLabels = [xmipp.MDL_ITEM_ID, xmipp.MDL_IMAGE]
for block in blocklist:
#read ctf block from ctf file
mdCtf.read(block + '@' + ctfImages, readLabels)
#add ctf columns to images file
mdAux.joinNatural(md, mdCtf)
# write block in images file with ctf info
mdCtf.write(block + '@' + expImages, xmipp.MD_APPEND)
return [expImages]
def scoreFinalVolumes(self):
threshold = self.getCCThreshold()
mdOut = xmipp.MetaData()
for n in range(self.numVolumes.get()):
fnRoot = self._getPath('proposedVolume%05d' % n)
fnAssignment = fnRoot + ".xmd"
if os.path.exists(fnAssignment):
self.runJob("xmipp_metadata_utilities",
"-i %s --fill weight constant 1" % fnAssignment)
MDassignment = xmipp.MetaData(fnAssignment)
sum = 0
thresholdedSum = 0
N = 0
minCC = 2
for id in MDassignment:
cc = MDassignment.getValue(xmipp.MDL_MAXCC, id)
sum += cc
thresholdedSum += cc - threshold
if cc < minCC:
minCC = cc
N += 1
if N > 0:
avg = sum / N
else:
avg = 0.0
id = mdOut.addObject()
mdOut.setValue(xmipp.MDL_IMAGE, str(fnRoot + ".vol"), id)
mdOut.setValue(xmipp.MDL_VOLUME_SCORE_SUM, float(sum), id)
mdOut.setValue(xmipp.MDL_VOLUME_SCORE_SUM_TH,
float(thresholdedSum), id)
mdOut.setValue(xmipp.MDL_VOLUME_SCORE_MEAN, float(avg), id)
mdOut.setValue(xmipp.MDL_VOLUME_SCORE_MIN, float(minCC), id)
mdOut.write(self._getPath("proposedVolumes.xmd"))
def writeSetOfClassesVol(classesVolSet, filename, classesBlock='classes'):
""" This function will write a SetOfClassesVol as Xmipp metadata.
Params:
classesVolSet: the SetOfClassesVol instance.
filename: the filename where to write the metadata.
"""
classFn = '%s@%s' % (classesBlock, filename)
classMd = xmipp.MetaData()
classMd.write(classFn) # Empty write to ensure the classes is the first block
classRow = XmippMdRow()
for classVol in classesVolSet:
classVolToRow(classVol, classRow)
classRow.writeToMd(classMd, classMd.addObject())
ref = class3D.getObjId()
imagesFn = 'class%06d_images@%s' % (ref, filename)
imagesMd = xmipp.MetaData()
imgRow = XmippMdRow()
for vol in classVol:
volumeToRow(vol, imgRow)
imgRow.writeToMd(imagesMd, imagesMd.addObject())
imagesMd.write(imagesFn, xmipp.MD_APPEND)
classMd.write(classFn, xmipp.MD_APPEND) # Empty write to ensure the classes is the first block
def computeAtomShiftsStep(self, numberOfModes):
fnOutDir = self._getExtraPath("distanceProfiles")
makePath(fnOutDir)
maxShift = []
maxShiftMode = []
for n in range(7, numberOfModes + 1):
fnVec = self._getPath("modes", "vec.%d" % n)
if exists(fnVec):
fhIn = open(fnVec)
md = xmipp.MetaData()
atomCounter = 0
for line in fhIn:
x, y, z = map(float, line.split())
d = math.sqrt(x * x + y * y + z * z)
if n == 7:
maxShift.append(d)
maxShiftMode.append(7)
else:
if d > maxShift[atomCounter]:
maxShift[atomCounter] = d
maxShiftMode[atomCounter] = n
atomCounter += 1
md.setValue(xmipp.MDL_NMA_ATOMSHIFT, d, md.addObject())
md.write(join(fnOutDir, "vec%d.xmd" % n))
fhIn.close()
md = xmipp.MetaData()
for i, _ in enumerate(maxShift):
fnVec = self._getPath("modes", "vec.%d" % (maxShiftMode[i] + 1))
if exists(fnVec):
objId = md.addObject()
md.setValue(xmipp.MDL_NMA_ATOMSHIFT, maxShift[i], objId)
md.setValue(xmipp.MDL_NMA_MODEFILE, fnVec, objId)
md.write(self._getExtraPath('maxAtomShifts.xmd'))
def _storeSummaryInfo(self, numVolumes):
""" Store some information when the protocol finishes. """
msg1 = ''
msg2 = ''
for n in range(numVolumes):
fnBase = 'proposedVolume%05d' % n
fnRoot = self._getPath(fnBase + ".xmd")
if os.path.isfile(fnRoot):
md = xmipp.MetaData(fnRoot)
size = md.size()
if (size < 5):
msg1 = "Num of inliers for model %d too small and equal to %d \n" % (
n, size)
msg1 += "Decrease the value of Inlier Threshold parameter and run again \n"
fnRoot = self._getTmpPath("ransac00000.xmd")
if os.path.isfile(fnRoot):
md = xmipp.MetaData(fnRoot)
size = md.size()
if (size < 5):
msg2 = "Num of random samples too small and equal to %d.\n" % size
msg2 += "If the option Dimensionality reduction is on, increase the number of grids per dimension (In this case we recommend to put Dimensionality reduction off).\n"
msg2 += "If the option Dimensionality reduction is off, increase the number of random samples.\n"
msg = msg1 + msg2
self.summaryVar.set(msg)
def writeSetOfClasses2D(classes2DSet, filename, classesBlock='classes', writeParticles=True):
""" This function will write a SetOfClasses2D as Xmipp metadata.
Params:
classes2DSet: the SetOfClasses2D instance.
filename: the filename where to write the metadata.
"""
classFn = '%s@%s' % (classesBlock, filename)
classMd = xmipp.MetaData()
classMd.write(classFn) # Empty write to ensure the classes is the first block
classRow = XmippMdRow()
for class2D in classes2DSet:
class2DToRow(class2D, classRow)
classRow.writeToMd(classMd, classMd.addObject())
if writeParticles:
ref = class2D.getObjId()
imagesFn = 'class%06d_images@%s' % (ref, filename)
imagesMd = xmipp.MetaData()
imgRow = XmippMdRow()
if class2D.getSize() > 0:
for img in class2D:
particleToRow(img, imgRow)
imgRow.writeToMd(imagesMd, imagesMd.addObject())
imagesMd.write(imagesFn, xmipp.MD_APPEND)
classMd.write(classFn, xmipp.MD_APPEND) # Empty write to ensure the classes is the first block
def runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling):
self._log.info("compute resolution 1")
#compute resolution
mdRsol = xmipp.MetaData(resolIterMd)
mdResolOut = xmipp.MetaData()
mdResolOut.importObjects(mdRsol, xmipp.MDValueLT(xmipp.MDL_RESOLUTION_FRC, 0.5))
self._log.info("compute resolution 2")
if mdResolOut.size()==0:
mdResolOut.clear()
mdResolOut.addObject()
id=mdResolOut.firstObject()
mdResolOut.setValue(xmipp.MDL_RESOLUTION_FREQREAL, sampling*2., id)
mdResolOut.setValue(xmipp.MDL_RESOLUTION_FRC, 0.5, id)
else:
mdResolOut.sort()
id = mdResolOut.firstObject()
filterFrequence = mdResolOut.getValue(xmipp.MDL_RESOLUTION_FREQREAL, id)
frc = mdResolOut.getValue(xmipp.MDL_RESOLUTION_FRC, id)
md = xmipp.MetaData()
id = md.addObject()
md.setColumnFormat(False)
md.setValue(xmipp.MDL_RESOLUTION_FREQREAL, filterFrequence, id)
md.setValue(xmipp.MDL_RESOLUTION_FRC, frc, id)
md.setValue(xmipp.MDL_SAMPLINGRATE, sampling, id)
md.write(resolIterMaxMd, xmipp.MD_APPEND)
def runExecuteCtfGroupsStep(self, **kwargs):
makePath(self.ctfGroupDirectory)
self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory)
# printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line
if not self.doCTFCorrection:
md = xmipp.MetaData(self.selFileName)
block_name = self._getBlockFileName(ctfBlockName, 1,
self._getFileName('imageCTFpairs'))
md.write(block_name)
self._log.info("Written a single CTF group to file: '%s'" % block_name)
self.numberOfCtfGroups.set(1)
else:
self._log.info(
'*********************************************************************'
)
self._log.info('* Make CTF groups')
# remove all entries not present in sel file by
# join between selfile and metadatafile
mdCtfData = xmipp.MetaData()
mdCtfData.read(self.ctfDatName)
mdSel = xmipp.MetaData()
mdSel.read(self.selFileName)
mdCtfData.intersection(mdSel, xmipp.MDL_IMAGE)
tmpCtfDat = self.ctfDatName
mdCtfData.write(tmpCtfDat)
args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s'
args += ' --sampling_rate %(sampling)s'
params = {
'tmpCtfDat': tmpCtfDat,
'ctffile': self._getFileName('ctfGroupBase') + ':stk',
'wiener': self.wienerConstant.get(),
'pad': self.paddingFactor.get(),
'sampling': self.resolSam
}
if self.inputParticles.get().isPhaseFlipped():
args += ' --phase_flipped '
if self.doAutoCTFGroup:
args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s'
params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get()
params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get()
else:
if exists(self.setOfDefocus.get()):
args += ' --split %(setOfDefocus)s'
params['setOfDefocus'] = self.setOfDefocus.get()
self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs)
auxMD = xmipp.MetaData("numberGroups@" +
self._getFileName('cTFGroupSummary'))
self.numberOfCtfGroups.set(
auxMD.getValue(xmipp.MDL_COUNT, auxMD.firstObject()))
self._store(self.numberOfCtfGroups)
def calculateDeviationsStep(self, it):
""" Calculate both angles and shifts devitations for all iterations
"""
SL = xmipp.SymList()
mdIter = xmipp.MetaData()
#for it in self.allIters():
mdIter.clear()
SL.readSymmetryFile(self._symmetry[it])
md1 = xmipp.MetaData(self.docFileInputAngles[it])
md2 = xmipp.MetaData(self.docFileInputAngles[it-1])
#ignore disabled,
md1.removeDisabled()
md2.removeDisabled()
#first metadata file may not have shiftx and shifty
if not md2.containsLabel(xmipp.MDL_SHIFT_X):
md2.addLabel(xmipp.MDL_SHIFT_X)
md2.addLabel(xmipp.MDL_SHIFT_Y)
md2.fillConstant(xmipp.MDL_SHIFT_X,0.)
md2.fillConstant(xmipp.MDL_SHIFT_Y,0.)
oldLabels=[xmipp.MDL_ANGLE_ROT,
xmipp.MDL_ANGLE_TILT,
xmipp.MDL_ANGLE_PSI,
xmipp.MDL_SHIFT_X,
xmipp.MDL_SHIFT_Y]
newLabels=[xmipp.MDL_ANGLE_ROT2,
xmipp.MDL_ANGLE_TILT2,
xmipp.MDL_ANGLE_PSI2,
xmipp.MDL_SHIFT_X2,
xmipp.MDL_SHIFT_Y2]
md2.renameColumn(oldLabels,newLabels)
md2.addLabel(xmipp.MDL_SHIFT_X_DIFF)
md2.addLabel(xmipp.MDL_SHIFT_Y_DIFF)
md2.addLabel(xmipp.MDL_SHIFT_DIFF)
mdIter.join1(md1, md2, xmipp.MDL_IMAGE, xmipp.INNER_JOIN)
SL.computeDistance(mdIter,False,False,False)
xmipp.activateMathExtensions()
#operate in sqlite
shiftXLabel = xmipp.label2Str(xmipp.MDL_SHIFT_X)
shiftX2Label = xmipp.label2Str(xmipp.MDL_SHIFT_X2)
shiftXDiff = xmipp.label2Str(xmipp.MDL_SHIFT_X_DIFF)
shiftYLabel = xmipp.label2Str(xmipp.MDL_SHIFT_Y)
shiftY2Label = xmipp.label2Str(xmipp.MDL_SHIFT_Y2)
shiftYDiff = xmipp.label2Str(xmipp.MDL_SHIFT_Y_DIFF)
shiftDiff = xmipp.label2Str(xmipp.MDL_SHIFT_DIFF)
#timeStr = str(dtBegin)
operateString = shiftXDiff+"="+shiftXLabel+"-"+shiftX2Label
operateString += "," + shiftYDiff+"="+shiftYLabel+"-"+shiftY2Label
mdIter.operate(operateString)
operateString = shiftDiff+"=sqrt(" \
+shiftXDiff+"*"+shiftXDiff+"+" \
+shiftYDiff+"*"+shiftYDiff+");"
mdIter.operate(operateString)
iterFile = self._mdDevitationsFn(it)
mdIter.write(iterFile,xmipp.MD_APPEND)
self._setLastIter(it)
def _showProjMatchLibAndClasses(self, paramName=None):
#map stack position with ref number
list = []
mdIn = xmipp.MetaData()
mdOut = xmipp.MetaData()
for ref3d in self._refsList:
for it in self._iterations:
convert_refno_to_stack_position = {}
file_name = self.protocol._getFileName(
'projectLibrarySampling', iter=it, ref=ref3d)
file_nameReferences = 'projectionDirections@' + file_name
if exists(file_name):
mdReferences = xmipp.MetaData(file_nameReferences)
mdReferencesSize = mdReferences.size()
for id in mdReferences:
convert_refno_to_stack_position[mdReferences.getValue(
xmipp.MDL_NEIGHBOR, id)] = id
file_nameAverages = self.protocol._getFileName(
'outClassesXmd', iter=it, ref=ref3d)
file_references = self.protocol._getFileName(
'projectLibraryStk', iter=it, ref=ref3d)
if exists(file_nameAverages):
mdIn.read(file_nameAverages)
mdOut.clear()
for i in mdIn:
ref2D = mdIn.getValue(xmipp.MDL_REF, i)
file_reference = xmipp.FileName()
file_reference.compose(
convert_refno_to_stack_position[ref2D],
file_references)
id1 = mdOut.addObject()
mdOut.setValue(xmipp.MDL_IMAGE,
mdIn.getValue(xmipp.MDL_IMAGE, i),
id1)
mdOut.setValue(xmipp.MDL_IMAGE2, file_reference,
id1)
if mdOut.size() == 0:
print "Empty metadata: ", file_nameReferences
else:
file_nameReferences = self.protocol._getFileName(
'projectLibrarySampling', iter=it, ref=ref3d)
sfn = createUniqueFileName(file_nameReferences)
file_nameReferences = 'projectionDirections@' + sfn
mdOut.merge(mdIn)
mdOut.write(file_nameReferences)
# ToDo: show the metadata in "metadata" form.
list.append(
self.createDataView(file_nameReferences))
else:
print "File %s does not exist" % file_name
return list
def _createPlot(self,
title,
xTitle,
yTitle,
fnOutput,
mdLabelX,
mdLabelY,
color='g',
figure=None):
xplotter = XmippPlotter(figure=figure)
xplotter.plot_title_fontsize = 11
ax = xplotter.createSubPlot(title, xTitle, yTitle, 1, 1)
ax.set_yscale('log')
ax.set_xscale('log')
#plot noise and related errorbar
fnOutputN = self.protocol._defineResultsNoiseName()
md = xmipp.MetaData(fnOutputN)
xValueN = md.getColumnValues(xmipp.MDL_COUNT)
yValueN = md.getColumnValues(xmipp.MDL_AVG)
plt.plot(xValueN,
yValueN,
'--',
color='r',
label='Aligned gaussian noise')
# putting error bar
md = xmipp.MetaData(fnOutputN)
yErrN = md.getColumnValues(xmipp.MDL_STDDEV)
xValueNe = md.getColumnValues(xmipp.MDL_COUNT)
yValueNe = md.getColumnValues(xmipp.MDL_AVG)
plt.errorbar(xValueNe, yValueNe, yErrN, fmt='o', color='k')
#plot real data-set
fnOutput = self.protocol._defineResultsName()
md = xmipp.MetaData(fnOutput)
xValue = md.getColumnValues(xmipp.MDL_COUNT)
yValue = md.getColumnValues(xmipp.MDL_AVG)
plt.plot(xValue, yValue, color='g', label='Aligned particles')
# putting error bar
md = xmipp.MetaData(fnOutput)
yErr = md.getColumnValues(xmipp.MDL_STDDEV)
xValue = md.getColumnValues(xmipp.MDL_COUNT)
yValue = md.getColumnValues(xmipp.MDL_AVG)
plt.errorbar(xValue, yValue, yErr, fmt='o')
plt.legend(loc='upper right', fontsize=11)
return xplotter
def createMetaDataFromPattern(pattern, isStack=False, label="image"):
''' Create a metadata from files matching pattern'''
import glob
files = glob.glob(pattern)
files.sort()
label = xmipp.str2Label(label) #Check for label value
mD = xmipp.MetaData()
inFile = xmipp.FileName()
nSize = 1
for file in files:
fileAux = file
if isStack:
if file.endswith(".mrc"):
fileAux = file + ":mrcs"
x, x, x, nSize = xmipp.getImageSize(fileAux)
if nSize != 1:
counter = 1
for jj in range(nSize):
inFile.compose(counter, fileAux)
objId = mD.addObject()
mD.setValue(label, inFile, objId)
mD.setValue(xmipp.MDL_ENABLED, 1, objId)
counter += 1
else:
objId = mD.addObject()
mD.setValue(label, fileAux, objId)
mD.setValue(xmipp.MDL_ENABLED, 1, objId)
return mD
def _loadTable(self, tableName):
if tableName:
mdFn = tableName + "@" + self._filename
else:
mdFn = self._filename
md = xmipp.MetaData(mdFn)
return self._convertMdToTable(md)
def findRow(md, label, value):
""" Query the metadata for a row with label=value.
Params:
md: metadata to query.
label: label to check value
value: value for equal condition
Returns:
XmippMdRow object of the row found.
None if no row is found with label=value
"""
mdQuery = xmipp.MetaData() # store result
mdQuery.importObjects(md, xmipp.MDValueEQ(label, value))
n = mdQuery.size()
if n == 0:
row = None
elif n == 1:
row = XmippMdRow()
row.readFromMd(mdQuery, mdQuery.firstObject())
else:
raise Exception(
"findRow: more than one row found matching the query %s = %s" %
(xmipp.label2Str(label), value))
return row
def __init__(self, filename=None):
self._md = xmipp.MetaData()
self._md.setColumnFormat(False)
self._id = self._md.addObject()
if filename:
self.read(filename)
def _getTmpSummary(self):
summary = []
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = xmipp.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(xmipp.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE,
configobj)
activeMic = md.getValue(xmipp.MDL_MICROGRAPH, configobj)
isAutopick = pickingState != "Manual"
manualParticlesSize = md.getValue(
xmipp.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
autoParticlesSize = md.getValue(
xmipp.MDL_PICKING_AUTOPARTICLES_SIZE, configobj)
summary.append("Manual particles picked: %d" % manualParticlesSize)
summary.append("Particle size:%d" % (particleSize))
autopick = "Yes" if isAutopick else "No"
summary.append("Autopick: " + autopick)
if isAutopick:
summary.append("Automatic particles picked: %d" %
autoParticlesSize)
summary.append("Last micrograph: " + activeMic)
return "\n".join(summary)
def _getTmpMethods(self):
""" Return the message when there is not output generated yet.
We will read the Xmipp .pos files and other configuration files.
"""
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
msg = ''
if existsConfig:
md = xmipp.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(xmipp.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE,
configobj)
isAutopick = pickingState != "Manual"
manualParts = md.getValue(xmipp.MDL_PICKING_MANUALPARTICLES_SIZE,
configobj)
autoParts = md.getValue(xmipp.MDL_PICKING_AUTOPARTICLES_SIZE,
configobj)
if manualParts is None:
manualParts = 0
if autoParts is None:
autoParts = 0
msg = 'User picked %d particles ' % (autoParts + manualParts)
msg += 'with a particle size of %d.' % particleSize
if isAutopick:
msg += "Automatic picking was used ([Abrishami2013]). "
msg += "%d particles were picked automatically " % autoParts
msg += "and %d manually." % manualParts
return msg
def _getFourierMaxFrequencyOfInterest(self, iterN, refN):
""" Read the corresponding resolution metadata and return the
desired resolution.
"""
md = xmipp.MetaData(
self._getFileName('resolutionXmdMax', iter=iterN, ref=refN))
return md.getValue(xmipp.MDL_RESOLUTION_FREQREAL, md.firstObject())
def getSummary(self, coordSet):
summary = []
summary.append("Previous run: %s" %
self.xmippParticlePicking.get().getNameId())
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = xmipp.MetaData('properties@' + configfile)
configobj = md.firstObject()
def _get(label):
return md.getValue(label, configobj)
pickingState = _get(xmipp.MDL_PICKING_STATE)
particleSize = _get(xmipp.MDL_PICKING_PARTICLE_SIZE)
activeMic = _get(xmipp.MDL_MICROGRAPH)
isAutopick = pickingState != "Manual"
manualParticlesSize = _get(xmipp.MDL_PICKING_MANUALPARTICLES_SIZE)
autoParticlesSize = _get(xmipp.MDL_PICKING_AUTOPARTICLES_SIZE)
summary.append("Manual particles picked: %s" % manualParticlesSize)
summary.append("Particle size:%d" % (particleSize))
autopick = "Yes" if isAutopick else "No"
summary.append("Autopick: " + autopick)
if isAutopick:
summary.append("Automatic particles picked: %s" %
autoParticlesSize)
summary.append("Last micrograph: " + activeMic)
return "\n".join(summary)
def _pickMicrograph(self, mic, *args):
micPath = mic.getFileName()
# Get particle picking boxsize from the previous run
boxSize = self.particlePickingRun.outputCoordinates.getBoxSize()
modelRoot = self._getExtraPath('model')
micName = removeBaseExt(micPath)
proceed = True
if self.micsToPick == MICS_SAMEASPICKING:
basePos = replaceBaseExt(micPath, "pos")
fnPos = self.particlePickingRun._getExtraPath(basePos)
if exists(fnPos):
blocks = xmipp.getBlocksInMetaDataFile(fnPos)
copy = True
if 'header' in blocks:
mdheader = xmipp.MetaData("header@" + fnPos)
state = mdheader.getValue(
xmipp.MDL_PICKING_MICROGRAPH_STATE,
mdheader.firstObject())
if state == "Available":
copy = False
if copy:
# Copy manual .pos file of this micrograph
copyFile(fnPos, self._getExtraPath(basename(fnPos)))
proceed = False
if proceed:
args = "-i %s " % micPath
args += "--particleSize %d " % boxSize
args += "--model %s " % modelRoot
args += "--outputRoot %s " % self._getExtraPath(micName)
args += "--mode autoselect --thr %d" % self.numberOfThreads
self.runJob("xmipp_micrograph_automatic_picking", args)
def test(self):
subset = self.newProtocol(
ProtSubSet,
inputFullSet=self.protImportParts.outputParticles,
chooseAtRandom=True,
nElements=400)
self.launchProtocol(subset)
highres = self.newProtocol(
XmippProtReconstructHighRes,
inputParticles=subset.outputParticles,
inputVolumes=self.protImportVol.outputVolume,
particleRadius=180,
symmetryGroup="i1",
alignmentMethod=XmippProtReconstructHighRes.AUTOMATIC_ALIGNMENT,
maximumTargetResolution="15 10 7",
numberOfMpi=8)
self.launchProtocol(highres)
self.assertIsNotNone(highres.outputParticles,
"There was a problem with Highres")
fnResolution = highres._getExtraPath("Iter005/iterInfo.xmd")
if not exists(fnResolution):
self.assertTrue(False, fnResolution + " does not exist")
else:
md = xmipp.MetaData("resolution@" + fnResolution)
R = md.getValue(xmipp.MDL_RESOLUTION_FREQREAL, md.firstObject())
self.assertTrue(R < 8, "Resolution is not below 8A")
def autopickMicrographStep(self, micPath):
# Get particle picking boxsize from the previous run
boxSize = self.particlePickingRun.outputCoordinates.getBoxSize()
modelRoot = self._getExtraPath('model')
micName = removeBaseExt(micPath)
proceed = True
if self.micsToPick == MICS_SAMEASPICKING:
fnPos = self.particlePickingRun._getExtraPath(replaceBaseExt(micPath, "pos"))
if exists(fnPos):
blocks = xmipp.getBlocksInMetaDataFile(fnPos)
copy = True
if 'header' in blocks:
mdheader = xmipp.MetaData("header@" + fnPos)
state = mdheader.getValue(xmipp.MDL_PICKING_MICROGRAPH_STATE,
mdheader.firstObject())
if state == "Available":
copy = False
if copy:
# Copy manual .pos file of this micrograph
copyFile(fnPos, self._getExtraPath(basename(fnPos)))
proceed = False
if proceed:
oroot = self._getExtraPath(micName)
thr = self.numberOfThreads.get()
args = "-i %(micPath)s --particleSize %(boxSize)d --model %(modelRoot)s " % locals()
args += " --outputRoot %(oroot)s --mode autoselect --thr %(thr)d" % locals()
# TODO: What is this?
# if self.Fast:
# args += " --fast "
self.runJob("xmipp_micrograph_automatic_picking", args)
def _writeFreqsMetaData(self, pattern, outputFn):
""" Glob and read frequencies either from reconstruction
or noise and write the proper metadata file.
"""
fnFreqs = sorted(glob.glob(self._getExtraPath(pattern)))
subset = 0
numberOfParticles = getFloatListFromValues(
self.numberOfParticles.get())
validationMd = xmipp.MetaData()
for fnFreq in fnFreqs:
print fnFreq
data = []
fnFreqOpen = open(fnFreq, "r")
for line in fnFreqOpen:
fields = line.split()
rowdata = map(float, fields)
data.extend(rowdata)
meanRes = (sum(data) / len(data))
data[:] = [(x - meanRes)**2 for x in data]
varRes = (sum(data) / (len(data) - 1))
stdRes = sqrt(varRes)
objId = validationMd.addObject()
validationMd.setValue(xmipp.MDL_COUNT,
long(numberOfParticles[subset]), objId)
validationMd.setValue(xmipp.MDL_AVG, meanRes, objId)
validationMd.setValue(xmipp.MDL_STDDEV, stdRes, objId)
subset += 1
validationMd.write(outputFn)
def readSetOfImages(filename, imgSet, rowToFunc, **kwargs):
"""read from Xmipp image metadata.
filename: The metadata filename where the image are.
imgSet: the SetOfParticles that will be populated.
rowToFunc: this function will be used to convert the row to Object
"""
imgMd = xmipp.MetaData(filename)
# By default remove disabled items from metadata
# be careful if you need to preserve the original number of items
if kwargs.get('removeDisabled', True):
imgMd.removeDisabled()
# If the type of alignment is not sent through the kwargs
# try to deduced from the metadata labels
if 'alignType' not in kwargs:
imgRow = rowFromMd(imgMd, imgMd.firstObject())
if _containsAny(imgRow, ALIGNMENT_DICT):
if imgRow.containsLabel(xmipp.MDL_ANGLE_TILT):
kwargs['alignType'] = ALIGN_PROJ
else:
kwargs['alignType'] = ALIGN_2D
else:
kwargs['alignType'] = ALIGN_NONE
if imgMd.size()>0:
for objId in imgMd:
imgRow = rowFromMd(imgMd, objId)
img = rowToFunc(imgRow, **kwargs)
imgSet.append(img)
imgSet.setHasCTF(img.hasCTF())
imgSet.setAlignment(kwargs['alignType'])
def movieCreatePlot(mic, saveFig):
import xmipp
meanX = []
meanY = []
figureSize = (8, 6)
#alignedMovie = mic.alignMetaData
md = xmipp.MetaData(mic.alignMetaData)
plotter = Plotter(*figureSize)
figure = plotter.getFigure()
preX = 0.0
preY = 0.0
meanX.append(0.0)
meanY.append(0.0)
ax = figure.add_subplot(111)
ax.grid()
ax.set_title('Cartesian representation')
ax.set_xlabel('Drift x (pixels)')
ax.set_ylabel('Drift y (pixels)')
ax.plot(0, 0, 'yo-')
for objId in md:
preX += md.getValue(xmipp.MDL_OPTICALFLOW_MEANX, objId)
preY += md.getValue(xmipp.MDL_OPTICALFLOW_MEANY, objId)
meanX.append(preX)
meanY.append(preY)
ax.plot(preX, preY, 'yo-')
ax.text(preX - 0.02, preY + 0.01, str(objId + 1))
ax.plot(np.asarray(meanX), np.asarray(meanY))
if saveFig:
plotter.savefig(mic.plotCart)
return plotter
def getDefaultBoxSize(self):
""" This function is used by the wizard to estimate the box size. """
boxSize = None
ci = self.getImportClass()
if hasattr(ci, 'getBoxSize'):
boxSize = ci.getBoxSize([f for f, _ in self.iterFiles()][0])
if boxSize is not None and self.scale != 1.:
boxSize = int(boxSize * self.scale.get())
return boxSize
boxSize = 100
importFrom = self.getImportFrom()
scale = self.scale.get()
if importFrom == ProtImportCoordinates.IMPORT_FROM_XMIPP:
configfile = join(self.filesPath.get(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = xmipp.MetaData('properties@' + configfile)
configobj = md.firstObject()
boxSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE,
configobj)
if importFrom == ProtImportCoordinates.IMPORT_FROM_EMAN:
# Read the boxSize from the e2boxercache/base.json
jsonFnbase = join(self.filesPath.get(), 'e2boxercache',
'base.json')
from pyworkflow.em.packages.eman2 import loadJson
jsonBoxDict = loadJson(jsonFnbase)
boxSize = int(jsonBoxDict["box_size"])
boxSize = (int)(boxSize * scale)
return boxSize
def _appendRctImages(self, particles):
blockMd = "class%06d_images@%s" % (particles.getObjId(),
self.rctClassesFn)
classMd = xmipp.MetaData()
uImages = self.inputParticlesTiltPair.get().getUntilted()
tImages = self.inputParticlesTiltPair.get().getTilted()
sangles = self.inputParticlesTiltPair.get().getCoordsPair().getAngles()
uMics = self.inputParticlesTiltPair.get().getCoordsPair().getMicsPair(
).getUntilted()
tMics = tImages.getCoordinates().getMicrographs()
scaleFactor = uImages.getSamplingRate() / particles.getSamplingRate()
for img in particles:
imgId = img.getObjId()
uImg = uImages[imgId]
tImg = tImages[imgId]
if uImg is None or tImg is None:
print ">>> Warning, for id %d, tilted or untilted particle was not found. Ignored." % imgId
else:
objId = classMd.addObject()
pairRow = XmippMdRow()
pairRow.setValue(xmipp.MDL_IMAGE, getImageLocation(uImg))
uCoord = uImg.getCoordinate()
micId = uCoord.getMicId()
uMic = uMics[micId]
angles = sangles[micId]
pairRow.setValue(xmipp.MDL_MICROGRAPH, uMic.getFileName())
pairRow.setValue(xmipp.MDL_XCOOR, uCoord.getX())
pairRow.setValue(xmipp.MDL_YCOOR, uCoord.getY())
pairRow.setValue(xmipp.MDL_ENABLED, 1)
pairRow.setValue(xmipp.MDL_ITEM_ID, long(imgId))
pairRow.setValue(xmipp.MDL_REF, 1)
alignment = img.getTransform()
# Scale alignment by scaleFactor
alignment.scale(scaleFactor)
alignmentToRow(alignment, pairRow, alignType=ALIGN_2D)
pairRow.setValue(xmipp.MDL_IMAGE_TILTED,
getImageLocation(tImg))
tMic = tMics[micId]
pairRow.setValue(xmipp.MDL_MICROGRAPH_TILTED,
tMic.getFileName())
(angleY, angleY2, angleTilt) = angles.getAngles()
pairRow.setValue(xmipp.MDL_ANGLE_Y, float(angleY))
pairRow.setValue(xmipp.MDL_ANGLE_Y2, float(angleY2))
pairRow.setValue(xmipp.MDL_ANGLE_TILT, float(angleTilt))
pairRow.writeToMd(classMd, objId)
classMd.write(blockMd, xmipp.MD_APPEND)
def centerFirstHarmonicStep(self, imagesFn, outputCenter):
dims = xmipp.MetaDataInfo(str(imagesFn))
md = xmipp.MetaData()
objId = md.addObject()
md.setValue(xmipp.MDL_X, float(dims[0] / 2), objId)
md.setValue(xmipp.MDL_Y, float(dims[1] / 2), objId)
md.write(outputCenter)
return [outputCenter] # this file should exists after the step