def runInitAngularReferenceFileStep(self):
'''Create Initial angular file. Either fill it with zeros or copy input'''
#NOTE: if using angles, self.selFileName file should contain angles info
md = xmippLib.MetaData(self.selFileName)
# Ensure this labels are always
md.addLabel(xmippLib.MDL_ANGLE_ROT)
md.addLabel(xmippLib.MDL_ANGLE_TILT)
md.addLabel(xmippLib.MDL_ANGLE_PSI)
expImages = self._getFileName('inputParticlesDoc')
ctfImages = self._getFileName('imageCTFpairs')
md.write(self._getExpImagesFileName(expImages))
blocklist = xmippLib.getBlocksInMetaDataFile(ctfImages)
mdCtf = xmippLib.MetaData()
mdAux = xmippLib.MetaData()
readLabels = [xmippLib.MDL_ITEM_ID, xmippLib.MDL_IMAGE]
for block in blocklist:
#read ctf block from ctf file
mdCtf.read(block + '@' + ctfImages, readLabels)
#add ctf columns to images file
mdAux.joinNatural(md, mdCtf)
# write block in images file with ctf info
mdCtf.write(block + '@' + expImages, xmippLib.MD_APPEND)
return [expImages]
def computeAtomShiftsStep(self, numberOfModes):
fnOutDir = self._getExtraPath("distanceProfiles")
makePath(fnOutDir)
maxShift=[]
maxShiftMode=[]
for n in range(7, numberOfModes+1):
fnVec = self._getPath("modes", "vec.%d" % n)
if exists(fnVec):
fhIn = open(fnVec)
md = xmippLib.MetaData()
atomCounter = 0
for line in fhIn:
x, y, z = map(float, line.split())
d = math.sqrt(x*x+y*y+z*z)
if n==7:
maxShift.append(d)
maxShiftMode.append(7)
else:
if d>maxShift[atomCounter]:
maxShift[atomCounter]=d
maxShiftMode[atomCounter]=n
atomCounter+=1
md.setValue(xmippLib.MDL_NMA_ATOMSHIFT,d,md.addObject())
md.write(join(fnOutDir,"vec%d.xmd" % n))
fhIn.close()
md = xmippLib.MetaData()
for i, _ in enumerate(maxShift):
fnVec = self._getPath("modes", "vec.%d" % (maxShiftMode[i]+1))
if exists(fnVec):
objId = md.addObject()
md.setValue(xmippLib.MDL_NMA_ATOMSHIFT, maxShift[i],objId)
md.setValue(xmippLib.MDL_NMA_MODEFILE, fnVec, objId)
md.write(self._getExtraPath('maxAtomShifts.xmd'))
def homogeneizeStep(self):
minClass = self.homogeneize.get()
fnNeighbours = self._getExtraPath("neighbours.xmd")
# Look for the block with the minimum number of images
if minClass==0:
minClass = 1e38
for block in xmippLib.getBlocksInMetaDataFile(fnNeighbours):
projNumber = int(block.split("_")[1])
fnDir=self._getExtraPath("direction_%d"%projNumber,"level_00","class_classes.xmd")
if exists(fnDir):
blockSize = md.getSize("class000001_images@"+fnDir)
if blockSize<minClass:
minClass=blockSize
# Construct the homogeneized metadata
mdAll = xmippLib.MetaData()
mdSubset=xmippLib.MetaData()
mdRandom=xmippLib.MetaData()
for block in xmippLib.getBlocksInMetaDataFile(fnNeighbours):
projNumber = int(block.split("_")[1])
fnDir=self._getExtraPath("direction_%d"%projNumber,"level_00","class_classes.xmd")
if exists(fnDir):
mdDirection = xmippLib.MetaData("class000001_images@"+fnDir)
mdRandom.randomize(mdDirection)
mdSubset.selectPart(mdRandom,0L,min(mdRandom.size(),minClass))
mdAll.unionAll(mdSubset)
mdAll.removeDuplicates(md.MDL_ITEM_ID)
mdAll.sort(md.MDL_ITEM_ID)
mdAll.fillConstant(md.MDL_PARTICLE_ID,1)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
mdAll.write(fnHomogeneous)
self.runJob("xmipp_metadata_utilities",'-i %s --operate modify_values "particleId=itemId"'%fnHomogeneous,numberOfMpi=1)
def getCorrThreshStep(self):
corrVector = []
fnCorr = self._getExtraPath("correlations.xmd")
mdCorr = xmippLib.MetaData()
for n in range(self.nRansac.get()):
fnRoot = "ransac%05d" % n
fnAngles = self._getTmpPath("angles_" + fnRoot + ".xmd")
md = xmippLib.MetaData(fnAngles)
for objId in md:
corr = md.getValue(xmippLib.MDL_MAXCC, objId)
corrVector.append(corr)
objIdCorr = mdCorr.addObject()
mdCorr.setValue(xmippLib.MDL_MAXCC, float(corr), objIdCorr)
mdCorr.write("correlations@" + fnCorr, xmippLib.MD_APPEND)
mdCorr = xmippLib.MetaData()
sortedCorrVector = sorted(corrVector)
indx = int(floor(self.corrThresh.get() * (len(sortedCorrVector) - 1)))
#With the line below commented the percentil is not used for the threshold and is used the value introduced in the form
#CorrThresh = sortedCorrVector[indx]#
objId = mdCorr.addObject()
mdCorr.setValue(xmippLib.MDL_WEIGHT, self.corrThresh.get(), objId)
mdCorr.write("corrThreshold@" + fnCorr, xmippLib.MD_APPEND)
print "Correlation threshold: " + str(self.corrThresh.get())
def scoreFinalVolumes(self):
threshold = self.getCCThreshold()
mdOut = xmippLib.MetaData()
for n in range(self.numVolumes.get()):
fnRoot = self._getPath('proposedVolume%05d' % n)
fnAssignment = fnRoot + ".xmd"
if os.path.exists(fnAssignment):
self.runJob("xmipp_metadata_utilities",
"-i %s --fill weight constant 1" % fnAssignment)
MDassignment = xmippLib.MetaData(fnAssignment)
sum = 0
thresholdedSum = 0
N = 0
minCC = 2
for id in MDassignment:
cc = MDassignment.getValue(xmippLib.MDL_MAXCC, id)
sum += cc
thresholdedSum += cc - threshold
if cc < minCC:
minCC = cc
N += 1
if N > 0:
avg = sum / N
else:
avg = 0.0
id = mdOut.addObject()
mdOut.setValue(xmippLib.MDL_IMAGE, fnRoot + ".vol", id)
mdOut.setValue(xmippLib.MDL_VOLUME_SCORE_SUM, float(sum), id)
mdOut.setValue(xmippLib.MDL_VOLUME_SCORE_SUM_TH,
float(thresholdedSum), id)
mdOut.setValue(xmippLib.MDL_VOLUME_SCORE_MEAN, float(avg), id)
mdOut.setValue(xmippLib.MDL_VOLUME_SCORE_MIN, float(minCC), id)
mdOut.write(self._getPath("proposedVolumes.xmd"))
def _storeSummaryInfo(self, numVolumes):
""" Store some information when the protocol finishes. """
msg1 = ''
msg2 = ''
for n in range(numVolumes):
fnBase = 'proposedVolume%05d' % n
fnRoot = self._getPath(fnBase + ".xmd")
if os.path.isfile(fnRoot):
md = xmippLib.MetaData(fnRoot)
size = md.size()
if (size < 5):
msg1 = "Num of inliers for model %d too small and equal to %d \n" % (
n, size)
msg1 += "Decrease the value of Inlier Threshold parameter and run again \n"
fnRoot = self._getTmpPath("ransac00000.xmd")
if os.path.isfile(fnRoot):
md = xmippLib.MetaData(fnRoot)
size = md.size()
if (size < 5):
msg2 = "Num of random samples too small and equal to %d.\n" % size
msg2 += "If the option Dimensionality reduction is on, increase the number of grids per dimension (In this case we recommend to put Dimensionality reduction off).\n"
msg2 += "If the option Dimensionality reduction is off, increase the number of random samples.\n"
msg = msg1 + msg2
self.summaryVar.set(msg)
def runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling):
self._log.info("compute resolution 1")
#compute resolution
mdRsol = xmippLib.MetaData(resolIterMd)
mdResolOut = xmippLib.MetaData()
mdResolOut.importObjects(
mdRsol, xmippLib.MDValueLT(xmippLib.MDL_RESOLUTION_FRC, 0.5))
self._log.info("compute resolution 2")
if mdResolOut.size() == 0:
mdResolOut.clear()
mdResolOut.addObject()
id = mdResolOut.firstObject()
mdResolOut.setValue(xmippLib.MDL_RESOLUTION_FREQREAL, sampling * 2.,
id)
mdResolOut.setValue(xmippLib.MDL_RESOLUTION_FRC, 0.5, id)
else:
mdResolOut.sort()
id = mdResolOut.firstObject()
filterFrequence = mdResolOut.getValue(xmippLib.MDL_RESOLUTION_FREQREAL, id)
frc = mdResolOut.getValue(xmippLib.MDL_RESOLUTION_FRC, id)
md = xmippLib.MetaData()
id = md.addObject()
md.setColumnFormat(False)
md.setValue(xmippLib.MDL_RESOLUTION_FREQREAL, filterFrequence, id)
md.setValue(xmippLib.MDL_RESOLUTION_FRC, frc, id)
md.setValue(xmippLib.MDL_SAMPLINGRATE, sampling, id)
md.write(resolIterMaxMd, xmippLib.MD_APPEND)
def runExecuteCtfGroupsStep(self, **kwargs):
makePath(self.ctfGroupDirectory)
self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory)
# printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line
if not self.doCTFCorrection:
md = xmippLib.MetaData(self.selFileName)
block_name = self._getBlockFileName(ctfBlockName, 1,
self._getFileName('imageCTFpairs'))
md.write(block_name)
self._log.info("Written a single CTF group to file: '%s'" % block_name)
self.numberOfCtfGroups.set(1)
else:
self._log.info(
'*********************************************************************'
)
self._log.info('* Make CTF groups')
# remove all entries not present in sel file by
# join between selfile and metadatafile
mdCtfData = xmippLib.MetaData()
mdCtfData.read(self.ctfDatName)
mdSel = xmippLib.MetaData()
mdSel.read(self.selFileName)
mdCtfData.intersection(mdSel, xmippLib.MDL_IMAGE)
tmpCtfDat = self.ctfDatName
mdCtfData.write(tmpCtfDat)
args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s'
args += ' --sampling_rate %(sampling)s'
params = {
'tmpCtfDat': tmpCtfDat,
'ctffile': self._getFileName('ctfGroupBase') + ':stk',
'wiener': self.wienerConstant.get(),
'pad': self.paddingFactor.get(),
'sampling': self.resolSam
}
if self.inputParticles.get().isPhaseFlipped():
args += ' --phase_flipped '
if self.doAutoCTFGroup:
args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s'
params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get()
params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get()
else:
if exists(self.setOfDefocus.get()):
args += ' --split %(setOfDefocus)s'
params['setOfDefocus'] = self.setOfDefocus.get()
self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs)
auxMD = xmippLib.MetaData("numberGroups@" +
self._getFileName('cTFGroupSummary'))
self.numberOfCtfGroups.set(
auxMD.getValue(xmippLib.MDL_COUNT, auxMD.firstObject()))
self._store(self.numberOfCtfGroups)
def calculateDeviationsStep(self, it):
""" Calculate both angles and shifts devitations for all iterations
"""
SL = xmippLib.SymList()
mdIter = xmippLib.MetaData()
#for it in self.allIters():
mdIter.clear()
SL.readSymmetryFile(self._symmetry[it])
md1 = xmippLib.MetaData(self.docFileInputAngles[it])
md2 = xmippLib.MetaData(self.docFileInputAngles[it - 1])
#ignore disabled,
md1.removeDisabled()
md2.removeDisabled()
#first metadata file may not have shiftx and shifty
if not md2.containsLabel(xmippLib.MDL_SHIFT_X):
md2.addLabel(xmippLib.MDL_SHIFT_X)
md2.addLabel(xmippLib.MDL_SHIFT_Y)
md2.fillConstant(xmippLib.MDL_SHIFT_X, 0.)
md2.fillConstant(xmippLib.MDL_SHIFT_Y, 0.)
oldLabels = [
xmippLib.MDL_ANGLE_ROT, xmippLib.MDL_ANGLE_TILT,
xmippLib.MDL_ANGLE_PSI, xmippLib.MDL_SHIFT_X, xmippLib.MDL_SHIFT_Y
]
newLabels = [
xmippLib.MDL_ANGLE_ROT2, xmippLib.MDL_ANGLE_TILT2,
xmippLib.MDL_ANGLE_PSI2, xmippLib.MDL_SHIFT_X2,
xmippLib.MDL_SHIFT_Y2
]
md2.renameColumn(oldLabels, newLabels)
md2.addLabel(xmippLib.MDL_SHIFT_X_DIFF)
md2.addLabel(xmippLib.MDL_SHIFT_Y_DIFF)
md2.addLabel(xmippLib.MDL_SHIFT_DIFF)
mdIter.join1(md1, md2, xmippLib.MDL_IMAGE, xmippLib.INNER_JOIN)
SL.computeDistance(mdIter, False, False, False)
xmippLib.activateMathExtensions()
#operate in sqlite
shiftXLabel = xmippLib.label2Str(xmippLib.MDL_SHIFT_X)
shiftX2Label = xmippLib.label2Str(xmippLib.MDL_SHIFT_X2)
shiftXDiff = xmippLib.label2Str(xmippLib.MDL_SHIFT_X_DIFF)
shiftYLabel = xmippLib.label2Str(xmippLib.MDL_SHIFT_Y)
shiftY2Label = xmippLib.label2Str(xmippLib.MDL_SHIFT_Y2)
shiftYDiff = xmippLib.label2Str(xmippLib.MDL_SHIFT_Y_DIFF)
shiftDiff = xmippLib.label2Str(xmippLib.MDL_SHIFT_DIFF)
#timeStr = str(dtBegin)
operateString = shiftXDiff + "=" + shiftXLabel + "-" + shiftX2Label
operateString += "," + shiftYDiff + "=" + shiftYLabel + "-" + shiftY2Label
mdIter.operate(operateString)
operateString = shiftDiff+"=sqrt(" \
+shiftXDiff+"*"+shiftXDiff+"+" \
+shiftYDiff+"*"+shiftYDiff+");"
mdIter.operate(operateString)
iterFile = self._mdDevitationsFn(it)
mdIter.write(iterFile, xmippLib.MD_APPEND)
self._setLastIter(it)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
if not self._useSeveralClasses():
newTs = inputParticles.getSamplingRate()
else:
newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE)
self.mdClasses = xmippLib.MetaData(self._getDirectionalClassesFn())
self.mdImages = xmippLib.MetaData(self._getDirectionalImagesFn())
classes2D = self._createSetOfClasses2D(inputParticles)
classes2D.getImages().setSamplingRate(newTs)
self.averageSet = self._createSetOfAverages()
self.averageSet.copyInfo(inputParticles)
self.averageSet.setAlignmentProj()
self.averageSet.setSamplingRate(newTs)
# Let's use a SetMdIterator because it should be less particles
# in the metadata produced than in the input set
iterator = md.SetMdIterator(self.mdImages, sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self._updateParticle,
skipDisabled=True)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
if exists(fnHomogeneous):
homogeneousSet = self._createSetOfParticles()
homogeneousSet.copyInfo(inputParticles)
homogeneousSet.setSamplingRate(newTs)
homogeneousSet.setAlignmentProj()
self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID)
self.lastRow = next(self.iterMd)
homogeneousSet.copyItems(inputParticles,
updateItemCallback=self._updateHomogeneousItem)
self._defineOutputs(outputHomogeneous=homogeneousSet)
self._defineSourceRelation(self.inputParticles, homogeneousSet)
classes2D.classifyItems(updateItemCallback=iterator.updateItem,
updateClassCallback=self._updateClass)
self._defineOutputs(outputClasses=classes2D)
self._defineOutputs(outputAverages=self.averageSet)
self._defineSourceRelation(self.inputParticles, classes2D)
self._defineSourceRelation(self.inputParticles, self.averageSet)
if self.splitVolume and self.directionalClasses.get()>1:
volumesSet = self._createSetOfVolumes()
volumesSet.setSamplingRate(newTs)
for i in range(2):
vol = Volume()
vol.setLocation(1, self._getExtraPath("split_v%d.vol"%(i+1)))
volumesSet.append(vol)
self._defineOutputs(outputVolumes=volumesSet)
self._defineSourceRelation(inputParticles, volumesSet)
def predictModel(self):
from keras.models import load_model
metadataPart = xmippLib.MetaData(
self._getExtraPath('resizedParticles.xmd'))
if self.model.get() == ITER_TRAIN:
metadataProj = xmippLib.MetaData(
self._getExtraPath('resizedProjections.xmd'))
img = xmippLib.Image()
dimMetadata = getMdSize(self._getExtraPath('resizedParticles.xmd'))
xmippLib.createEmptyFile(self._getExtraPath('particlesDenoised.stk'),
self.newXdim, self.newXdim, 1, dimMetadata)
mdNewParticles = md.MetaData()
self.groupParticles = 5000
if self.model.get() == ITER_PREDICT:
if self.modelPretrain:
model = self.ownModel.get()._getPath('ModelTrained.h5')
model = load_model(model)
else:
myModelfile = xmipp3.Plugin.getModel('deepDenoising',
'PretrainModel.h5')
model = load_model(myModelfile)
for num in range(1, dimMetadata, self.groupParticles):
self.noisyParticles = self.gan.extractInfoMetadata(
metadataPart, xmippLib.MDL_IMAGE, img, num,
self.groupParticles, -1)
if self.model.get() == ITER_TRAIN:
self.predict = self.gan.predict(self.Generator,
self.noisyParticles)
self.projections = self.gan.extractInfoMetadata(
metadataProj, xmippLib.MDL_IMAGE, img, num,
self.groupParticles, 1)
else:
self.predict = self.gan.predict(model, self.noisyParticles)
self.noisyParticles = self.gan.normalization(
self.noisyParticles, 1)
self.prepareMetadata(mdNewParticles, img, num)
mdNewParticles.write(
'particles@' + self._getExtraPath('particlesDenoised.xmd'),
xmippLib.MD_APPEND)
self.runJob(
"xmipp_transform_normalize", "-i %s --method NewXmipp "
"--background circle %d " %
(self._getExtraPath('particlesDenoised.stk'), self.newXdim / 2))
def createMetaDataFromPattern(pattern, isStack=False, label="image"):
''' Create a metadata from files matching pattern'''
import glob
files = glob.glob(pattern)
files.sort()
label = xmippLib.str2Label(label) #Check for label value
mD = xmippLib.MetaData()
inFile = xmippLib.FileName()
nSize = 1
for file in files:
fileAux = file
if isStack:
if file.endswith(".mrc"):
fileAux = file + ":mrcs"
x, x, x, nSize = xmippLib.getImageSize(fileAux)
if nSize != 1:
counter = 1
for jj in range(nSize):
inFile.compose(counter, fileAux)
objId = mD.addObject()
mD.setValue(label, inFile, objId)
mD.setValue(xmippLib.MDL_ENABLED, 1, objId)
counter += 1
else:
objId = mD.addObject()
mD.setValue(label, fileAux, objId)
mD.setValue(xmippLib.MDL_ENABLED, 1, objId)
return mD
def findRow(md, label, value):
""" Query the metadata for a row with label=value.
Params:
md: metadata to query.
label: label to check value
value: value for equal condition
Returns:
XmippMdRow object of the row found.
None if no row is found with label=value
"""
mdQuery = xmippLib.MetaData() # store result
mdQuery.importObjects(md, xmippLib.MDValueEQ(label, value))
n = mdQuery.size()
if n == 0:
row = None
elif n == 1:
row = XmippMdRow()
row.readFromMd(mdQuery, mdQuery.firstObject())
else:
raise Exception(
"findRow: more than one row found matching the query %s = %s" %
(xmippLib.label2Str(label), value))
return row
def __init__(self, filename=None):
self._md = xmippLib.MetaData()
self._md.setColumnFormat(False)
self._id = self._md.addObject()
if filename:
self.read(filename)
def test(self):
subset = self.newProtocol(ProtSubSet,
inputFullSet=self.protImportParts.outputParticles,
chooseAtRandom=True,
nElements=400)
self.launchProtocol(subset)
highres = self.newProtocol(XmippProtReconstructHighRes,
inputParticles=subset.outputParticles,
inputVolumes=self.protImportVol.outputVolume,
particleRadius=180,
symmetryGroup="i1",
alignmentMethod=XmippProtReconstructHighRes.AUTOMATIC_ALIGNMENT,
maximumTargetResolution="15 10 7",
numberOfMpi=8)
self.launchProtocol(highres)
self.assertIsNotNone(highres.outputParticles,
"There was a problem with Highres")
fnResolution = highres._getExtraPath("Iter005/iterInfo.xmd")
if not exists(fnResolution):
self.assertTrue(False, fnResolution + " does not exist")
else:
md = xmippLib.MetaData("resolution@" + fnResolution)
R = md.getValue(xmippLib.MDL_RESOLUTION_FREQREAL, md.firstObject())
# FIXME: Review HighRes! Before pluginization under 8A is achieved
self.assertTrue(R < 9, "Resolution is not below 9A")
def getSummary(self, coordSet):
summary = []
summary.append("Previous run: %s" %
self.xmippParticlePicking.get().getNameId())
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = xmippLib.MetaData('properties@' + configfile)
configobj = md.firstObject()
def _get(label):
return md.getValue(label, configobj)
pickingState = _get(xmippLib.MDL_PICKING_STATE)
particleSize = _get(xmippLib.MDL_PICKING_PARTICLE_SIZE)
activeMic = _get(xmippLib.MDL_MICROGRAPH)
isAutopick = pickingState != "Manual"
manualParticlesSize = _get(
xmippLib.MDL_PICKING_MANUALPARTICLES_SIZE)
autoParticlesSize = _get(xmippLib.MDL_PICKING_AUTOPARTICLES_SIZE)
summary.append("Manual particles picked: %s" % manualParticlesSize)
summary.append("Particle size:%d" % (particleSize))
autopick = "Yes" if isAutopick else "No"
summary.append("Autopick: " + autopick)
if isAutopick:
summary.append("Automatic particles picked: %s" %
autoParticlesSize)
summary.append("Last micrograph: " + activeMic)
return "\n".join(summary)
def _pickMicrograph(self, mic, *args):
micPath = mic.getFileName()
# Get particle picking boxsize from the previous run
boxSize = self.particlePickingRun.outputCoordinates.getBoxSize()
modelRoot = self._getExtraPath('model')
micName = removeBaseExt(micPath)
proceed = True
if self.micsToPick == MICS_SAMEASPICKING:
basePos = replaceBaseExt(micPath, "pos")
fnPos = self.particlePickingRun._getExtraPath(basePos)
if exists(fnPos):
blocks = xmippLib.getBlocksInMetaDataFile(fnPos)
copy = True
if 'header' in blocks:
mdheader = xmippLib.MetaData("header@" + fnPos)
state = mdheader.getValue(
xmippLib.MDL_PICKING_MICROGRAPH_STATE,
mdheader.firstObject())
if state == "Available":
copy = False
if copy:
# Copy manual .pos file of this micrograph
copyFile(fnPos, self._getExtraPath(basename(fnPos)))
proceed = False
if proceed:
args = "-i %s " % micPath
args += "--particleSize %d " % boxSize
args += "--model %s " % modelRoot
args += "--outputRoot %s " % self._getExtraPath(micName)
args += "--mode autoselect --thr %d" % self.numberOfThreads
self.runJob("xmipp_micrograph_automatic_picking", args)
def elasticAlignmentStep(self, nVoli, Ts, nVolj, fnAlignedVolj):
fnVolOut = self._getExtraPath('DeformedVolume_Vol_%d_To_Vol_%d' %
(nVolj, nVoli))
if os.path.exists(fnVolOut + ".pdb"):
return
makePath(self._getExtraPath("modes%d" % nVoli))
for i in range(self.numberOfModes.get() + 1):
if i == 0:
i += 1
copyFile(self._getPath("modes/vec.%d" % i),
self._getExtraPath("modes%d/vec.%d" % (nVoli, i)))
mdVol = xmippLib.MetaData()
fnOutMeta = self._getExtraPath('RigidAlignVol_%d_To_Vol_%d.xmd' %
(nVolj, nVoli))
mdVol.setValue(xmippLib.MDL_IMAGE, fnAlignedVolj, mdVol.addObject())
mdVol.write(fnOutMeta)
fnPseudo = self._getPath("pseudoatoms_%d.pdb" % nVoli)
fnModes = self._getPath("modes_%d.xmd" % nVoli)
fnDeform = self._getExtraPath('compDeformVol_%d_To_Vol_%d.xmd' %
(nVolj, nVoli))
sigma = Ts * self.pseudoAtomRadius.get()
fnPseudoOut = self._getExtraPath(
'PseudoatomsDeformedPDB_Vol_%d_To_Vol_%d.pdb' % (nVolj, nVoli))
self.runJob('xmipp_nma_alignment_vol',
"-i %s --pdb %s --modes %s --sampling_rate %s -o %s --fixed_Gaussian %s --opdb %s"%\
(fnOutMeta, fnPseudo, fnModes, Ts, fnDeform, sigma, fnPseudoOut))
self.runJob(
'xmipp_volume_from_pdb',
"-i %s -o %s --sampling %s --fixed_Gaussian %s" %
(fnPseudoOut, fnVolOut, Ts, sigma))
def _getFourierMaxFrequencyOfInterest(self, iterN, refN):
""" Read the corresponding resolution metadata and return the
desired resolution.
"""
md = xmippLib.MetaData(
self._getFileName('resolutionXmdMax', iter=iterN, ref=refN))
return md.getValue(xmippLib.MDL_RESOLUTION_FREQREAL, md.firstObject())
def extractTrainData(self, path, label, norm=-1):
metadata = xmippLib.MetaData(path)
Image = []
I = xmippLib.Image()
cont = 0
for itemId in metadata:
fn = metadata.getValue(label, itemId)
I.read(fn)
Imresize = I.getData()
if norm == -1:
Imnormalize = 2 * (Imresize - np.min(Imresize)) / (
np.max(Imresize) - np.min(Imresize)) - 1
elif norm == 0:
Imnormalize = Imresize
elif norm == 1:
Imnormalize = (Imresize - np.min(Imresize)) / (
np.max(Imresize) - np.min(Imresize))
else:
Imnormalize = (Imresize - np.mean(Imresize)) / np.std(Imresize)
Image.append(Imnormalize)
if cont > 3000:
break
cont += 1
Image = np.array(Image).astype('float')
Image = Image.reshape(len(Image), Image.shape[1], Image.shape[2], 1)
return Image
def _getTmpSummary(self):
summary = []
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = xmippLib.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(xmippLib.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(xmippLib.MDL_PICKING_PARTICLE_SIZE,
configobj)
activeMic = md.getValue(xmippLib.MDL_MICROGRAPH, configobj)
isAutopick = pickingState != "Manual"
manualParticlesSize = md.getValue(
xmippLib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
autoParticlesSize = md.getValue(
xmippLib.MDL_PICKING_AUTOPARTICLES_SIZE, configobj)
summary.append("Manual particles picked: %d" % manualParticlesSize)
summary.append("Particle size:%d" % (particleSize))
autopick = "Yes" if isAutopick else "No"
summary.append("Autopick: " + autopick)
if isAutopick:
summary.append("Automatic particles picked: %d" %
autoParticlesSize)
summary.append("Last micrograph: " + activeMic)
return "\n".join(summary)
def _getTmpMethods(self):
""" Return the message when there is not output generated yet.
We will read the Xmipp .pos files and other configuration files.
"""
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
msg = ''
if existsConfig:
md = xmippLib.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(xmippLib.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(xmippLib.MDL_PICKING_PARTICLE_SIZE,
configobj)
isAutopick = pickingState != "Manual"
manualParts = md.getValue(
xmippLib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
autoParts = md.getValue(xmippLib.MDL_PICKING_AUTOPARTICLES_SIZE,
configobj)
if manualParts is None:
manualParts = 0
if autoParts is None:
autoParts = 0
msg = 'User picked %d particles ' % (autoParts + manualParts)
msg += 'with a particle size of %d.' % particleSize
if isAutopick:
msg += "Automatic picking was used ([Abrishami2013]). "
msg += "%d particles were picked automatically " % autoParts
msg += "and %d manually." % manualParts
return msg
def _appendRctImages(self, particles):
blockMd = "class%06d_images@%s" % (particles.getObjId(),
self.rctClassesFn)
classMd = xmippLib.MetaData()
partPairs = self.inputParticlesTiltPair.get()
uImages = partPairs.getUntilted()
tImages = partPairs.getTilted()
sangles = partPairs.getCoordsPair().getAngles()
micPairs = partPairs.getCoordsPair().getMicsPair()
uMics = micPairs.getUntilted()
tMics = micPairs.getTilted()
scaleFactor = uImages.getSamplingRate() / particles.getSamplingRate()
for img in particles:
imgId = img.getObjId()
uImg = uImages[imgId]
tImg = tImages[imgId]
if uImg is None or tImg is None:
print(">>> Warning, for id %d, tilted or untilted particle "
"was not found. Ignored." % imgId)
else:
objId = classMd.addObject()
pairRow = XmippMdRow()
pairRow.setValue(xmippLib.MDL_IMAGE, getImageLocation(uImg))
uCoord = uImg.getCoordinate()
micId = uCoord.getMicId()
uMic = uMics[micId]
angles = sangles[micId]
pairRow.setValue(xmippLib.MDL_MICROGRAPH, uMic.getFileName())
pairRow.setValue(xmippLib.MDL_XCOOR, uCoord.getX())
pairRow.setValue(xmippLib.MDL_YCOOR, uCoord.getY())
pairRow.setValue(xmippLib.MDL_ENABLED, 1)
pairRow.setValue(xmippLib.MDL_ITEM_ID, long(imgId))
pairRow.setValue(xmippLib.MDL_REF, 1)
alignment = img.getTransform()
# Scale alignment by scaleFactor
alignment.scale(scaleFactor)
alignmentToRow(alignment, pairRow, alignType=ALIGN_2D)
pairRow.setValue(xmippLib.MDL_IMAGE_TILTED,
getImageLocation(tImg))
tMic = tMics[micId]
pairRow.setValue(xmippLib.MDL_MICROGRAPH_TILTED,
tMic.getFileName())
(angleY, angleY2, angleTilt) = angles.getAngles()
pairRow.setValue(xmippLib.MDL_ANGLE_Y, float(angleY))
pairRow.setValue(xmippLib.MDL_ANGLE_Y2, float(angleY2))
pairRow.setValue(xmippLib.MDL_ANGLE_TILT, float(angleTilt))
pairRow.writeToMd(classMd, objId)
classMd.write(blockMd, xmippLib.MD_APPEND)
def centerFirstHarmonicStep(self, imagesFn, outputCenter):
dims = xmippLib.MetaDataInfo(str(imagesFn))
md = xmippLib.MetaData()
objId = md.addObject()
md.setValue(xmippLib.MDL_X, float(dims[0] / 2), objId)
md.setValue(xmippLib.MDL_Y, float(dims[1] / 2), objId)
md.write(outputCenter)
return [outputCenter] # this file should exists after the step
def _plotCurveAnisotropy(self, fnmd, title, xTitle, yTitle, mdLabelX,
mdLabelY1, mdLabelY2):
"""
This function is called by _showAnisotropyCurve
It shows the FSO curve in terms of the resolution
The horizontal axis is linear in this plot.
"""
md = xmippLib.MetaData(fnmd)
xplotter = XmippPlotter(figure=None)
xplotter.plot_title_fontsize = 11
a = xplotter.createSubPlot(title, xTitle, yTitle, 1, 1)
xplotter.plotMdFile(md, mdLabelX, mdLabelY1, 'g')
xx, yy = self._prepareDataForPlot(md, mdLabelX, mdLabelY1)
_, yyBingham = self._prepareDataForPlot(md, mdLabelX, mdLabelY2)
from matplotlib.ticker import FuncFormatter
a.axes.xaxis.set_major_formatter(FuncFormatter(self._formatFreq))
a.axes.set_ylim([-0.1, 1.1])
a.axes.plot(xx, yy, 'g')
a.axes.set_xlabel('Resolution (A)')
a.axes.set_ylabel('FSO (a.u)')
hthresholds = [0.1, 0.5, 0.9]
a.axes.hlines(hthresholds,
xx[0],
xx[-1],
colors='k',
linestyles='dashed')
a.axes.grid(True)
textstr = ''
res_01, okToPlot_01 = self.interpolRes(0.1, xx, yy)
res_05, okToPlot_05 = self.interpolRes(0.5, xx, yy)
res_09, okToPlot_09 = self.interpolRes(0.9, xx, yy)
a.axes.plot(xx, yyBingham, 'r--')
if (okToPlot_01 and okToPlot_05 and okToPlot_09):
textstr = str(0.9) + ' --> ' + str(
"{:.2f}".format(res_09)) + 'A\n' + str(0.5) + ' --> ' + str(
"{:.2f}".format(res_05)) + 'A\n' + str(
0.1) + ' --> ' + str("{:.2f}".format(res_01)) + 'A'
a.axes.axvspan(1.0 / res_09,
1.0 / res_01,
alpha=0.3,
color='green')
props = dict(boxstyle='round', facecolor='white')
a.axes.text(0.0,
0.0,
textstr,
fontsize=12,
ha="left",
va="bottom",
bbox=props)
return plt.show()
def test_writeSetOfDefocusGroups(self):
#TODO: FIX THIS test according to the new SetOfDefocusGroup
return
#reference metadata
md = xmippLib.MetaData()
objId = md.addObject()
defocusGroupRow = XmippMdRow()
defocusGroupRow.setValue(xmippLib.MDL_ENABLED, 1)
defocusGroupRow.setValue(xmippLib.MDL_CTF_GROUP, 1)
defocusGroupRow.setValue(xmippLib.MDL_MIN, 2000.)
defocusGroupRow.setValue(xmippLib.MDL_MAX, 2500.)
defocusGroupRow.setValue(xmippLib.MDL_AVG, 2100.)
defocusGroupRow.writeToMd(md, objId)
objId = md.addObject()
defocusGroupRow.setValue(xmippLib.MDL_ENABLED, 1)
defocusGroupRow.setValue(xmippLib.MDL_CTF_GROUP, 2)
defocusGroupRow.setValue(xmippLib.MDL_MIN, 3000.)
defocusGroupRow.setValue(xmippLib.MDL_MAX, 5500.)
defocusGroupRow.setValue(xmippLib.MDL_AVG, 5000.)
defocusGroupRow.writeToMd(md, objId)
#
fnScipion = self.getOutputPath("writeSetOfDefocusGroups.sqlite")
setOfDefocus = SetOfDefocusGroup(filename=fnScipion)
df = DefocusGroup()
df.setDefocusMin(2000.)
df.setDefocusMax(2500.)
df.setDefocusAvg(2100.)
setOfDefocus.append(df)
df.cleanObjId()
df.setDefocusMin(3000)
df.setDefocusMax(5500)
df.setDefocusAvg(5000)
setOfDefocus.append(df)
fnXmipp = self.getOutputPath("writeSetOfDefocusGroups.xmd")
fnScipion = self.getOutputPath("writeSetOfDefocusGroups2.xmd")
writeSetOfDefocusGroups(setOfDefocus, fnXmipp)
mdAux = xmippLib.MetaData(fnXmipp)
md.write(fnScipion)
print "Comparing metadatas: \n%s\n%s" % (fnXmipp, fnScipion)
self.assertEqual(md, mdAux, "test writeSetOfDefocusGroups fails")
def test_micrographsToMd(self):
""" Test the conversion of a SetOfMicrographs to Xmipp metadata. """
micSet = SetOfMicrographs(
filename=self.getOutputPath("micrographs.sqlite"))
n = 3
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
micSet.setAcquisition(acquisition)
micSet.setSamplingRate(1.)
mdXmipp = xmippLib.MetaData()
for i in range(n):
p = Micrograph()
file = self.dataset.getFile("mic%s" % (i + 1))
p.setLocation(file)
ctf = ctfs[i % 2]
p.setCTF(ctf)
micSet.append(p)
id = mdXmipp.addObject()
mdXmipp.setValue(xmippLib.MDL_ENABLED, 1, id)
mdXmipp.setValue(xmippLib.MDL_ITEM_ID, long(i + 1), id)
mdXmipp.setValue(xmippLib.MDL_MICROGRAPH, file, id)
# set CTFModel params
mdXmipp.setValue(xmippLib.MDL_CTF_DEFOCUSU, ctf.getDefocusU(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_DEFOCUSV, ctf.getDefocusV(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_DEFOCUS_ANGLE,
ctf.getDefocusAngle(), id)
# set Acquisition params
mdXmipp.setValue(xmippLib.MDL_CTF_Q0,
acquisition.getAmplitudeContrast(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_CS,
acquisition.getSphericalAberration(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_VOLTAGE,
acquisition.getVoltage(), id)
mdScipion = xmippLib.MetaData()
setOfMicrographsToMd(micSet, mdScipion)
writeSetOfMicrographs(micSet, self.getOutputPath("micrographs.xmd"))
self.assertEqual(mdScipion, mdXmipp, "metadata are not the same")
def test_particlesToMd(self):
""" Test the conversion of a SetOfParticles to Xmipp metadata. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
mdXmipp = xmippLib.MetaData()
imgSet.setAcquisition(acquisition)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
imgSet.append(p)
id = mdXmipp.addObject()
mdXmipp.setValue(xmippLib.MDL_ENABLED, 1, id)
mdXmipp.setValue(xmippLib.MDL_ITEM_ID, long(i + 1), id)
mdXmipp.setValue(xmippLib.MDL_IMAGE, locationToXmipp(i + 1, fn),
id)
# set CTFModel params
mdXmipp.setValue(xmippLib.MDL_CTF_DEFOCUSU, ctf.getDefocusU(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_DEFOCUSV, ctf.getDefocusV(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_DEFOCUS_ANGLE,
ctf.getDefocusAngle(), id)
# set Acquisition params
mdXmipp.setValue(xmippLib.MDL_CTF_Q0,
acquisition.getAmplitudeContrast(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_CS,
acquisition.getSphericalAberration(), id)
mdXmipp.setValue(xmippLib.MDL_CTF_VOLTAGE,
acquisition.getVoltage(), id)
mdScipion = xmippLib.MetaData()
setOfParticlesToMd(imgSet, mdScipion)
self.assertEqual(mdScipion, mdXmipp, "metadata are not the same")
def __init__(self, itemClass):
""" Create new set, base on a Metadata.
itemClass: Class that represent the items.
A method .getFileName should be available to store the md.
Items contained in XmippSet are supposed to inherit from XmippMdRow.
"""
self._itemClass = itemClass
#self._fileName = fileName
self._md = xmippLib.MetaData()
def convertInputStep(self, inputParticlesId):
fnDir=self._getExtraPath()
writeSetOfParticles(self.inputParticles.get(),self.imgsFn)
# Choose the target sampling rate
TsOrig=self.inputParticles.get().getSamplingRate()
TsCurrent=max(TsOrig,self.targetResolution.get()/3)
Xdim=self.inputParticles.get().getDimensions()[0]
newXdim=long(round(Xdim*TsOrig/TsCurrent))
if newXdim<40:
newXdim=long(40)
TsCurrent=Xdim*(TsOrig/newXdim)
print "Preparing images to sampling rate=",TsCurrent
self.writeInfoField(fnDir,"size",xmippLib.MDL_XSIZE,newXdim)
self.writeInfoField(fnDir,"sampling",xmippLib.MDL_SAMPLINGRATE,TsCurrent)
# Prepare particles
fnNewParticles=join(fnDir,"images.stk")
if newXdim!=Xdim:
self.runJob("xmipp_image_resize","-i %s -o %s --fourier %d"%(self.imgsFn,fnNewParticles,newXdim),
numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
else:
self.runJob("xmipp_image_convert","-i %s -o %s --save_metadata_stack %s"%(self.imgsFn,fnNewParticles,join(fnDir,"images.xmd")),
numberOfMpi=1)
R=self.particleRadius.get()
if R<=0:
R=self.inputParticles.get().getDimensions()[0]/2
R=min(round(R*TsOrig/TsCurrent*1.1),newXdim/2)
self.runJob("xmipp_transform_mask","-i %s --mask circular -%d"%(fnNewParticles,R),numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
# Prepare mask
imgHandler=ImageHandler()
if self.nextMask.hasValue():
self.convertInputVolume(imgHandler, self.nextMask.get(), getImageLocation(self.nextMask.get()), join(fnDir,"mask.vol"), TsCurrent, newXdim)
# Prepare references
i=0
for vol in self.inputVolumes.get():
fnVol=join(fnDir,"volume%03d.vol"%i)
self.convertInputVolume(imgHandler, vol, getImageLocation(vol), fnVol, TsCurrent, newXdim)
self.runJob("xmipp_image_operate","-i %s --mult 0 -o %s"%(fnVol,join(fnDir,"volume%03d_speed.vol"%i)),numberOfMpi=1)
i+=1
xmippLib.MetaData().write("best@"+self._getExtraPath("swarm.xmd")) # Empty write to guarantee this block is the first one
xmippLib.MetaData().write("bestByVolume@"+self._getExtraPath("swarm.xmd"),xmippLib.MD_APPEND) # Empty write to guarantee this block is the second one
