def checkSplit(self):
outSet = self._getExtraPath('last_classes.xmd')
self.listNameImgs = []
self.listNumImgs = []
self.listRefImgs = []
metadataItem = md.MetaData(outSet)
for item in metadataItem:
nameImg = metadataItem.getValue(md.MDL_IMAGE, item)
self.listNameImgs.append(nameImg)
numImgs = metadataItem.getValue(md.MDL_CLASS_COUNT, item)
self.listNumImgs.append(numImgs)
refImg = metadataItem.getValue(md.MDL_REF, item)
self.listRefImgs.append(refImg)
i = 0
auxList = sorted(self.listNumImgs, reverse=True)
while i < len(self.listNumImgs):
if len(self.listNumImgs) < self.numClasses.get(
): #inside the while
# just in case we lose some class we want to allow one more
# split
if auxList[i] < (1.75 * self.threshold.get()):
i += 1
continue
maxValue = auxList[i]
maxPos = self.listNumImgs.index(maxValue)
self.listNumImgs[maxPos] = -1
bestRef = self.listRefImgs[maxPos]
outputMd = self._getExtraPath('dataClass%06d.xmd' % bestRef)
self.splitStep(outputMd)
i = 0
outSet = self._getExtraPath('split_last_classes.xmd')
self.listNameImgs = []
self.listNumImgs = []
self.listRefImgs = []
metadataItem = md.MetaData(outSet)
for item in metadataItem:
nameImg = metadataItem.getValue(md.MDL_IMAGE, item)
self.listNameImgs.append(nameImg)
numImgs = metadataItem.getValue(md.MDL_CLASS_COUNT, item)
self.listNumImgs.append(numImgs)
refImg = metadataItem.getValue(md.MDL_REF, item)
self.listRefImgs.append(refImg)
copy(outSet, self._getExtraPath('last_classes.xmd'))
auxList = sorted(self.listNumImgs, reverse=True)
i = 0
else:
break
def createOutputStep(self):
inputMovies = self.inputMovies.get()
particleSet = self._createSetOfMovieParticles()
particleSet.copyInfo(inputMovies)
# Create a folder to store the resulting micrographs
# particleFolder = self._getPath('particles')
# makePath(particleFolder)
mData = md.MetaData()
mdAll = md.MetaData()
self._micNameDict = {}
for movie in inputMovies:
self._micNameDict[movie.getObjId()] = movie.getMicName()
movieName = self._getMovieName(movie)
movieStk = movieName.replace('.mrc', '.stk')
movieMdFile = movieName.replace('.mrc', '.xmd')
# Store particle stack and metadata files in final particles folder
if os.path.exists(movieStk):
mData.read(movieMdFile)
mdAll.unionAll(mData)
particleMd = self._getPath('movie_particles.xmd')
mdAll.addItemId()
mdAll.write(particleMd)
readSetOfMovieParticles(particleMd,
particleSet,
removeDisabled=False,
postprocessImageRow=self._postprocessImageRow)
self._defineOutputs(outputParticles=particleSet)
self._defineSourceRelation(self.inputMovies, particleSet)
def evaluationStep(self):
noOfLevRuns = self._getLevRuns(self._level)
iters = self.numberOfIterations.get()
self._newClass = 0
self._clsDict = {}
outModel = self._getFileName('outputModel', lev=self._level)
outStar = self._getFileName('outputData', lev=self._level)
mdInput = md.MetaData()
outMd = md.MetaData()
for rLev in range(1, noOfLevRuns + 1):
rLevId = self._getRunLevId(self._level, rLev)
self._lastCls = None
mdModel = self._getFileName('model',
iter=iters,
lev=self._level,
rLev=rLev)
print('Filename model star: %s' % mdModel)
self._mergeDataStar(outStar, mdInput, iters, rLev)
self._mergeModelStar(outMd, mdModel, rLev)
self._doneList.append(rLevId)
outMd.write('model_classes@' + outModel)
mdInput.write(outStar)
print('Finishing evaluation step')
def joinSetOfParticlesStep(self, mode):
#Create images.xmd metadata joining from different .stk
fnImages = self._getExtraPath("particles_%s.xmd" % mode)
imgsXmd = MD.MetaData()
for i in range(len(self.inputCoordinatesTrue)):
posFiles = glob(
self._getExtraPath(
self.CONSENSUS_COOR_PATH_TEMPLATE % (mode, i), '*.pos'))
for posFn in posFiles:
xmdFn = self._getExtraPath(
self.CONSENSUS_PARTS_PATH_TEMPLATE % (mode, i),
pwutils.replaceBaseExt(posFn, "xmd"))
if os.path.exists(xmdFn):
mdFn = MD.MetaData(xmdFn)
mdPos = MD.MetaData('particles@%s' % posFn)
mdPos.merge(mdFn)
imgsXmd.unionAll(mdPos)
else:
self.warning(
"The coord file %s wasn't used for extraction! " %
os.path.basename(posFn))
self.warning(
"Maybe you are extracting over a subset of micrographs"
)
imgsXmd.write(fnImages)
def _fillClasses(self, outputClasses):
""" Create the SetOfClasses2D """
inputSet = self.inputClasses.get().getImages()
myRep = md.MetaData('classes@' +
self._getExtraPath('final_classes.xmd'))
for row in md.iterRows(myRep):
fn = row.getValue(md.MDL_IMAGE)
rep = Particle(fn)
repId = row.getObjId()
newClass = Class2D(objId=repId)
newClass.setAlignment2D()
newClass.copyInfo(inputSet)
newClass.setAcquisition(inputSet.getAcquisition())
newClass.setRepresentative(rep)
outputClasses.append(newClass)
i = 1
mdBlocks = md.getBlocksInMetaDataFile(
self._getExtraPath('final_classes.xmd'))
for block in mdBlocks:
if block.startswith('class00'):
mdClass = md.MetaData(block + "@" +
self._getExtraPath('final_classes.xmd'))
imgClassId = i
newClass = outputClasses[imgClassId]
newClass.enableAppend()
for row in md.iterRows(mdClass):
part = rowToParticle(row)
newClass.append(part)
i += 1
newClass.setAlignment2D()
outputClasses.update(newClass)
def splitInCTFGroups(imgStar, defocusRange=1000, numParticles=1):
""" Add a new colunm in the image star to separate the particles into ctf groups """
mdAll = md.MetaData(imgStar)
mdAll.sort(md.RLN_CTF_DEFOCUSU)
focusGroup = 1
counter = 0
oldDefocusU = mdAll.getValue(md.RLN_CTF_DEFOCUSU, mdAll.firstObject())
groupName = '%s_%06d_%05d' % ('ctfgroup', oldDefocusU, focusGroup)
for objId in mdAll:
counter = counter + 1
defocusU = mdAll.getValue(md.RLN_CTF_DEFOCUSU, objId)
if counter < numParticles:
pass
else:
if (defocusU - oldDefocusU) > defocusRange:
focusGroup = focusGroup + 1
oldDefocusU = defocusU
groupName = '%s_%06d_%05d' % ('ctfgroup', oldDefocusU,
focusGroup)
counter = 0
mdAll.setValue(md.RLN_MLMODEL_GROUP_NAME, groupName, objId)
mdAll.write(imgStar)
mdCount = md.MetaData()
mdCount.aggregate(mdAll, md.AGGR_COUNT, md.RLN_MLMODEL_GROUP_NAME,
md.RLN_MLMODEL_GROUP_NAME, md.MDL_COUNT)
print "number of particles per group: ", mdCount
def evaluateSingleMicrograph(self, micFn, micDir):
mdCTFparam = xmipp.MetaData(
self._getFileName('ctfparam', micDir=micDir))
ctfDownFactor = mdCTFparam.getValue(xmipp.MDL_CTF_DOWNSAMPLE_PERFORMED,
mdCTFparam.firstObject())
mdEval = md.MetaData()
id = mdEval.addObject()
# Check what happen with movies
# if md.MDL_TYPE == md.MDL_MICROGRAPH_MOVIE:
# mdEval.setValue(md.MDL_MICROGRAPH_MOVIE, micFn, id)
# mdEval.setValue(md.MDL_MICROGRAPH,('%05d@%s'%(1, micFn)), id)
# else:
mdEval.setValue(md.MDL_MICROGRAPH, micFn, id)
mdEval.setValue(md.MDL_PSD, str(self._getFileName('psd',
micDir=micDir)), id)
mdEval.setValue(md.MDL_PSD_ENHANCED,
str(self._getFileName('enhanced_psd', micDir=micDir)),
id)
mdEval.setValue(md.MDL_CTF_MODEL,
str(self._getFileName('ctfparam', micDir=micDir)), id)
mdEval.setValue(
md.MDL_IMAGE1,
str(self._getFileName('ctfmodel_quadrant', micDir=micDir)), id)
mdEval.setValue(
md.MDL_IMAGE2,
str(self._getFileName('ctfmodel_halfplane', micDir=micDir)), id)
mdEval.setValue(md.MDL_CTF_DOWNSAMPLE_PERFORMED, float(ctfDownFactor),
id)
fnEval = self._getFileName('ctf', micDir=micDir)
mdEval.write(fnEval)
# Evaluate if estimated ctf is good enough
# self.runJob("xmipp_ctf_sort_psds","-i %s --downsampling %f"%(fnEval,ctfDownFactor))
try:
self.runJob("xmipp_ctf_sort_psds", "-i %s" % (fnEval))
except Exception:
pass
# print >> sys.stderr, "xmipp_ctf_sort_psds has been Failed!"
# raise ex
# Check if it is a good micrograph
fnRejected = self._getTmpPath(basename(micFn + "_rejected.xmd"))
self.runJob(
"xmipp_metadata_utilities", '-i %s --query select "%s" -o %s' %
(fnEval, self._criterion, fnRejected))
retval = True
if not isMdEmpty(fnRejected):
mdCTF = md.MetaData(fnEval)
mdCTF.setValue(md.MDL_ENABLED, -1, mdCTF.firstObject())
mdCTF.write(fnEval)
retval = False
cleanPath(fnRejected)
return retval
def _plotSSNR(self, a, fn, label):
mdOut = md.MetaData(fn)
mdSSNR = md.MetaData()
# only cross by 1 is important
mdSSNR.importObjects(mdOut, md.MDValueGT(md.RLN_MLMODEL_DATA_VS_PRIOR_REF, 0.9))
mdSSNR.operate("rlnSsnrMap=log(rlnSsnrMap)")
resolution_inv = [mdSSNR.getValue(md.RLN_RESOLUTION, id) for id in mdSSNR]
frc = [mdSSNR.getValue(md.RLN_MLMODEL_DATA_VS_PRIOR_REF, id) for id in mdSSNR]
a.plot(resolution_inv, frc, label=label)
a.xaxis.set_major_formatter(self._plotFormatter)
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
for i, vol in enumerate(self._iterInputVols()):
volDir = self._getVolDir(i + 1)
volume = vol.clone()
volPrefix = 'vol%03d_' % (i + 1)
m_pruned = md.MetaData()
m_pruned.read(volDir + '/pruned_particles_alignability.xmd')
prunedMd = self._getExtraPath(volPrefix +
'pruned_particles_alignability.xmd')
moveFile(join(volDir, 'pruned_particles_alignability.xmd'),
prunedMd)
m_volScore = md.MetaData()
m_volScore.read(volDir + '/validationAlignability.xmd')
validationMd = self._getExtraPath(volPrefix +
'validation_alignability.xmd')
moveFile(join(volDir, 'validationAlignability.xmd'), validationMd)
imgSet = self.inputParticles.get()
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(
prunedMd, sortByLabel=md.MDL_ITEM_ID))
mdValidatoin = md.getFirstRow(validationMd)
weight = mdValidatoin.getValue(
md.MDL_WEIGHT_PRECISION_ALIGNABILITY)
volume.weightAlignabilityPrecision = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_ACCURACY_ALIGNABILITY)
volume.weightAlignabilityAccuracy = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_MIRROR)
volume.weightMirror = Float(weight)
volume.cleanObjId(
) # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
self.createPlot2D(volPrefix, m_pruned)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
def generateMdForClassification(self, classesOut):
listNameImgs = self.listNameImgs
listNumImgs = self.listNumImgs
count = 1
# Construct the new classes with the renumerated old classes
mdNewClasses = md.MetaData()
for i in range(len(listNumImgs)):
if listNumImgs[i] != -1:
name = listNameImgs[i]
fn = name[name.find('@') + 1:-4] + '.xmd'
numRef = int(name[0:6])
mdClass = md.MetaData("classes@" + fn)
for row in iterRows(mdClass):
if mdClass.getValue(md.MDL_REF, row.getObjId()) == numRef:
row.setValue(md.MDL_REF, count)
row.addToMd(mdNewClasses)
count += 1
# Add the two new classes to the list of renumerated classes
mdClass = md.MetaData("classes@" + classesOut)
rows = iterRows(mdClass)
for row in rows:
row.setValue(md.MDL_REF, count)
row.addToMd(mdNewClasses)
count += 1
mdNewClasses.write(
'classes@' + self._getExtraPath('split_last_classes.xmd'),
MD_APPEND)
# Generate the intermediate images and the blocks of the intermediate
# classes for the unchanged classes
count = 1
for i in range(len(listNumImgs)):
if listNumImgs[i] != -1:
# Read the list of images in this class
mdImgsInClass = md.MetaData(
self._getExtraPath('dataClass%06d.xmd' % (i + 1)))
mdImgsInClass.fillConstant(md.MDL_REF, count)
mdImgsInClass.write(
self._getExtraPath('dataClass%06d.xmd' % count))
count += 1
# Add the two new classes
for newRef in range(0, 2):
mdImgsInClass = md.MetaData('class%06d_images@%s' %
(newRef + 1, classesOut))
mdImgsInClass.fillConstant(md.MDL_REF, count)
mdImgsInClass.write(self._getExtraPath('dataClass%06d.xmd' %
count))
count += 1
def _createOutputStep(self):
ctfSet = self._createSetOfCTF()
inputMics = self.inputMicrographs.get()
ctfSet.setMicrographs(inputMics)
defocusList = []
if self.doAutomaticRejection:
fn = self._getPath("micrographs.xmd")
else:
fn = self._getPath("ctfs_selection.xmd")
mdFn = md.MetaData(fn)
mdAux = md.MetaData()
for _, micDir, mic in self._iterMicrographs():
if exists(self._getFileName('ctfparam', micDir=micDir)):
mdCTF = md.MetaData(
self._getFileName('ctfparam', micDir=micDir))
mdQuality = md.MetaData(self._getFileName('ctf',
micDir=micDir))
mdCTF.setValue(
xmipp.MDL_CTF_CRIT_NONASTIGMATICVALIDITY,
mdQuality.getValue(
xmipp.MDL_CTF_CRIT_NONASTIGMATICVALIDITY,
mdQuality.firstObject()), mdCTF.firstObject())
mdCTF.setValue(
xmipp.MDL_CTF_CRIT_FIRSTMINIMUM_FIRSTZERO_DIFF_RATIO,
mdQuality.getValue(
xmipp.MDL_CTF_CRIT_FIRSTMINIMUM_FIRSTZERO_DIFF_RATIO,
mdQuality.firstObject()), mdCTF.firstObject())
else:
fnError = self._getFileName('ctfErrorParam', micDir=micDir)
if not exists(fnError):
self._createErrorCtfParam(micDir)
mdCTF = md.MetaData(fnError)
mdAux.importObjects(
mdFn, md.MDValueEQ(md.MDL_MICROGRAPH_ID, long(mic.getObjId())))
mdAux.merge(mdCTF)
mic.setSamplingRate(mdCTF.getValue(md.MDL_CTF_SAMPLING_RATE, 1))
ctfModel = mdToCTFModel(mdAux, mic)
self._setPsdFiles(ctfModel, micDir)
ctfSet.append(ctfModel)
# save the values of defocus for each micrograph in a list
defocusList.append(ctfModel.getDefocusU())
defocusList.append(ctfModel.getDefocusV())
self._defineOutputs(outputCTF=ctfSet)
self._defineCtfRelation(inputMics, ctfSet)
self._defocusMaxMin(defocusList)
def createInputMd(self, vols):
fnVols = self._getExtraPath('input_volumes.xmd')
if self.copyAlignment:
alignType = vols.getAlignment()
else:
alignType = ALIGN_NONE
writeSetOfVolumes(vols, fnVols,
postprocessImageRow=self._postprocessVolumeRow,
alignType=alignType)
if not vols.hasAlignment() or not self.copyAlignment:
mdFn = md.MetaData(fnVols)
mdFn.fillConstant(md.MDL_ANGLE_ROT, 0.)
mdFn.fillConstant(md.MDL_ANGLE_TILT, 0.)
mdFn.fillConstant(md.MDL_ANGLE_PSI, 0.)
mdFn.fillConstant(md.MDL_SHIFT_X, 0.)
mdFn.fillConstant(md.MDL_SHIFT_Y, 0.)
mdFn.fillConstant(md.MDL_SHIFT_Z, 0.)
mdFn.write(fnVols, md.MD_OVERWRITE)
# set missing angles
missType = ['wedge_y', 'wedge_x', 'pyramid', 'cone']
missNum = self.missingDataType.get()
missAng = self.missingAng.get()
missDataFn = self._getExtraPath('wedges.xmd')
missAngValues = str(missAng).strip().split()
thetaY0, thetaYF, thetaX0, thetaXF = 0, 0, 0, 0
mdFn = md.MetaData()
if missNum == MISSING_WEDGE_X:
thetaX0, thetaXF = missAngValues
elif missNum == MISSING_WEDGE_Y:
thetaY0, thetaYF = missAngValues
elif missNum == MISSING_PYRAMID:
thetaY0, thetaYF, thetaX0, thetaXF = missAngValues
else: # MISSING_CONE
thetaY0 = missAngValues[0]
for i in range(1, vols.getSize() + 1):
row = md.Row()
row.setValue(md.MDL_MISSINGREGION_NR, missNum + 1)
row.setValue(md.MDL_MISSINGREGION_TYPE, missType[missNum])
row.setValue(md.MDL_MISSINGREGION_THX0, float(thetaX0))
row.setValue(md.MDL_MISSINGREGION_THXF, float(thetaXF))
row.setValue(md.MDL_MISSINGREGION_THY0, float(thetaY0))
row.setValue(md.MDL_MISSINGREGION_THYF, float(thetaYF))
row.addToMd(mdFn)
mdFn.write(missDataFn, md.MD_APPEND)
def analyzeOutOfCores(self, subset):
""" Analyze which images are out of cores """
levelMdFiles = self._getAllLevelMdFiles(subset)
for fn in levelMdFiles:
mdAll = md.MetaData()
blocks = md.getBlocksInMetaDataFile(fn)
fnDir = dirname(fn)
# Gather all images in block
for block in blocks:
if block.startswith('class0'):
mdClass = md.MetaData(block + "@" + fn)
mdAll.unionAll(mdClass)
if mdAll.size() > 0:
# Compute difference to images
fnSubset = join(fnDir, "images%s.xmd" % subset)
mdAll.write(fnSubset)
fnOutOfSubset = join(fnDir, "imagesOut.xmd")
inputMd = self._getFileName('input_particles')
args = "-i %s --set subtraction %s -o %s" % (inputMd, fnSubset,
fnOutOfSubset)
self.runJob("xmipp_metadata_utilities",
args,
numberOfMpi=1,
numberOfThreads=1)
# Remove disabled and intermediate files
mdClass = md.MetaData(fnOutOfSubset)
mdClass.removeDisabled()
fnRejected = "images_rejected@" + fn
mdClass.write(fnRejected, md.MD_APPEND)
cleanPath(fnOutOfSubset)
cleanPath(fnSubset)
# If enough images, make a small summary
if mdClass.size() > 100:
from math import ceil
fnRejectedDir = join(fnDir, "rejected%s" % subset)
makePath(fnRejectedDir)
Nclasses = int(ceil(mdClass.size() / 300))
self.runJob(
"xmipp_classify_CL2D",
"-i %s --nref0 1 --nref %d --iter 5 --distance "
"correlation --classicalMultiref "
"--classifyAllImages --odir %s" %
(fnRejected, Nclasses, fnRejectedDir))
def _showPMax(self, paramName=None):
labels = [md.RLN_MLMODEL_AVE_PMAX, md.RLN_PARTICLE_PMAX]
mdIters = md.MetaData()
iterations = range(self.firstIter, self.lastIter+1)
# range (firstIter, # self._visualizeLastIteration+1):
# #always list all iteration
for it in iterations:
objId = mdIters.addObject()
mdIters.setValue(md.MDL_ITER, it, objId)
for i, prefix in enumerate(self.protocol.PREFIXES):
filename = self.protocol._getFileName(prefix + 'model', iter=it)
fn = 'model_general@' + filename
mdModel = md.RowMetaData(fn)
pmax = mdModel.getValue(md.RLN_MLMODEL_AVE_PMAX)
mdIters.setValue(labels[i], pmax, objId)
fn = self.protocol._getFileName('all_avgPmax_xmipp')
mdIters.write(fn)
colors = ['g', 'b']
xplotter = VolSelPlotter()
xplotter.createSubPlot("Avg PMax per Iterations", "Iterations", "Avg PMax")
for label, color in zip(labels, colors):
xplotter.plotMd(mdIters, md.MDL_ITER, label, color)
if len(self.protocol.PREFIXES) > 1:
xplotter.showLegend(self.protocol.PREFIXES)
return [self.createView(fn), xplotter]
def _mergeDataStar(self, outStar, mdInput, iter, rLev):
imgStar = self._getFileName('data',
iter=iter,
lev=self._level,
rLev=rLev)
mdData = md.MetaData(imgStar)
print("reading %s and begin the loop" % imgStar)
for row in md.iterRows(mdData, sortByLabel=md.RLN_PARTICLE_CLASS):
clsPart = row.getValue(md.RLN_PARTICLE_CLASS)
if clsPart != self._lastCls:
self._newClass += 1
self._clsDict['%s.%s' % (rLev, clsPart)] = self._newClass
self._lastCls = clsPart
# write symlink to new Maps
relionFn = self._getFileName('relionMap',
lev=self._level,
iter=self._getnumberOfIters(),
ref3d=clsPart,
rLev=rLev)
newFn = self._getFileName('map',
lev=self._level,
rLev=self._newClass)
print(('link from %s to %s' % (relionFn, newFn)))
createLink(relionFn, newFn)
row.setValue(md.RLN_PARTICLE_CLASS, self._newClass)
row.addToMd(mdInput)
print("writing %s and ending the loop" % outStar)
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
imgSet = self.inputParticles.get()
for i, vol in enumerate(self._iterInputVols()):
volume = vol.clone()
volDir = self._getVolDir(i + 1)
volPrefix = 'vol%03d_' % (i + 1)
validationMd = self._getExtraPath(volPrefix + 'validation.xmd')
moveFile(join(volDir, 'validation.xmd'), validationMd)
clusterMd = self._getExtraPath(volPrefix +
'clusteringTendency.xmd')
moveFile(join(volDir, 'clusteringTendency.xmd'), clusterMd)
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(
clusterMd, sortByLabel=md.MDL_ITEM_ID))
mdValidatoin = md.MetaData(validationMd)
weight = mdValidatoin.getValue(md.MDL_WEIGHT,
mdValidatoin.firstObject())
volume.weight = Float(weight)
volume.clusterMd = String(clusterMd)
volume.cleanObjId(
) # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
def writeSetOfParticles(imgSet, starFile, outputDir, **kwargs):
""" This function will write a SetOfImages as Relion meta
Params:
imgSet: the SetOfImages instance.
starFile: the filename where to write the meta
filesMapping: this dict will help when there is need to replace images names
"""
filesDict = convertBinaryFiles(imgSet, outputDir)
kwargs['filesDict'] = filesDict
partMd = md.MetaData()
setOfImagesToMd(imgSet, partMd, particleToRow, **kwargs)
if kwargs.get('fillMagnification', False):
pixelSize = imgSet.getSamplingRate()
mag = imgSet.getAcquisition().getMagnification()
detectorPxSize = mag * pixelSize / 10000
partMd.fillConstant(md.RLN_CTF_MAGNIFICATION, mag)
partMd.fillConstant(md.RLN_CTF_DETECTOR_PIXEL_SIZE, detectorPxSize)
else:
# Remove Magnification from metadata to avoid wrong values of pixel size.
# In Relion if Magnification and DetectorPixelSize are in metadata,
# pixel size is ignored in the command line.
partMd.removeLabel(md.RLN_CTF_MAGNIFICATION)
blockName = kwargs.get('blockName', 'Particles')
partMd.write('%s@%s' % (blockName, starFile))
def _createClasses(self, partSet):
self._classesDict = {} # store classes info, indexed by class id
pathDict = {}
self.protocol.info('Loading classes info from: %s' %
self._modelStarFile)
modelMd = md.MetaData('model_classes@' + self._modelStarFile)
for classNumber, objId in enumerate(modelMd):
row = md.Row()
row.readFromMd(modelMd, objId)
index, fn = relionToLocation(row.getValue('rlnReferenceImage'))
if fn in pathDict:
newFn = pathDict.get(fn)
else:
clsPath = findRootFrom(self._modelStarFile, fn)
if clsPath is None:
newFn = fn
else:
newFn = self.protocol._getExtraPath(os.path.basename(fn))
self.copyOrLink(os.path.join(clsPath, fn), newFn)
pathDict[fn] = newFn
self._classesDict[classNumber + 1] = (index, newFn, row)
clsSet = self._classesFunc(partSet)
clsSet.classifyItems(updateClassCallback=self._updateClass)
self.protocol._defineOutputs(outputClasses=clsSet)
self.protocol._defineSourceRelation(partSet, clsSet)
def _writeXmippPosFile(self, movieId, movieName, coordinatesName, shiftX,
shiftY):
""" Create an Xmipp coordinates files to be extracted
from the frames of the movie.
"""
##### import em abre una nueva conexion?
#TODO ROB move this import to the header
from pyworkflow.em import SetOfCoordinates
inputCoords = self.inputCoordinates.get()
coordinates = SetOfCoordinates(filename=inputCoords.getFileName())
#####coordinates = self.inputCoordinates.get()
#####micrograph = coordinates.getMicrographs()[movieId]
####if micrograph is None:
#####raise Exception("Micrograph with id %d was not found in SetOfCoordinates!!!" % movieId)
mData = md.MetaData()
coordRow = XmippMdRow()
#### for coord in coordinates.iterCoordinates(micrograph):
for coord in coordinates.iterCoordinates(movieId):
coord.shiftX(int(round(float(shiftX))))
coord.shiftY(int(round(float(shiftY))))
coordinateToRow(coord, coordRow)
coordRow.writeToMd(mData, mData.addObject())
if mData.isEmpty():
return False
else:
self.info("Writing coordinates metadata: %s, with shifts: %s %s" %
(coordinatesName, shiftX, shiftY))
mData.write('particles@' + coordinatesName)
return True
def readSetOfCoordinates(outputDir, partSet, coordSet):
"""This method is used to store an edition of the coordinates obtained by
localized reconstruction.
Read from Xmipp .pos files.
Params:
outputDir: the directory where the .pos files are.
It is also expected a file named: config.xmd
in this directory where the box size can be read.
micSet: the SetOfMicrographs to associate the .pos, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the config.xmd metadata
configfile = join(outputDir, 'config.xmd')
if exists(configfile):
mdFn = md.MetaData('properties@' + join(outputDir, 'config.xmd'))
boxSize = mdFn.getValue(md.MDL_PICKING_PARTICLE_SIZE,
mdFn.firstObject())
coordSet.setBoxSize(boxSize)
for part in partSet:
baseFn = "%06d_at_%s" % (part.getIndex(),
replaceBaseExt(part.getFileName(), 'pos'))
posFile = join(outputDir, baseFn)
readCoordinates(part, posFile, coordSet, outputDir)
coordSet._xmippMd = String(outputDir)
def createOutputStep(self):
if not self.realignMovieFrames:
imgSet = self._getInputParticles()
vol = Volume()
vol.setFileName(self._getExtraPath('relion_class001.mrc'))
vol.setSamplingRate(imgSet.getSamplingRate())
half1 = self._getFileName("final_half1_volume", ref3d=1)
half2 = self._getFileName("final_half2_volume", ref3d=1)
vol.setHalfMaps([half1, half2])
outImgSet = self._createSetOfParticles()
outImgSet.copyInfo(imgSet)
self._fillDataFromIter(outImgSet, self._lastIter())
self._defineOutputs(outputVolume=vol)
self._defineSourceRelation(self.inputParticles, vol)
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(self.inputParticles, outImgSet)
fsc = FSC(objLabel=self.getRunName())
blockName = 'model_class_%d@' % 1
fn = blockName + self._getExtraPath("relion_model.star")
mData = md.MetaData(fn)
fsc.loadFromMd(mData, md.RLN_RESOLUTION,
md.RLN_MLMODEL_FSC_HALVES_REF)
self._defineOutputs(outputFSC=fsc)
self._defineSourceRelation(vol, fsc)
else:
movieSet = self.inputMovieParticles.get()
if self.movieIncludeRotSearch:
vol = Volume()
vol.setFileName(self._getExtraPath('relion_class001.mrc'))
vol.setSamplingRate(movieSet.getSamplingRate())
half1 = self._getFileName("final_half1_volume", ref3d=1)
half2 = self._getFileName("final_half2_volume", ref3d=1)
vol.setHalfMaps([half1, half2])
self._defineOutputs(outputVolume=vol)
self._defineSourceRelation(self.inputParticles, vol)
self._defineSourceRelation(self.inputMovieParticles, vol)
fnOut = self._getFileName('dataFinal')
outMovieSet = self._createSetOfMovieParticles()
outMovieSet.copyInfo(movieSet)
outMovieSet.setAlignmentProj()
# not using copyItems since input movie particle
# set is missing a lot of metadata (CTF, micName etc.)
# that was created in convertInputStep
readSetOfParticles(fnOut,
outMovieSet,
alignType=ALIGN_PROJ,
extraLabels=MOVIE_EXTRA_LABELS,
postprocessImageRow=self._updateParticle)
self._defineOutputs(outputParticles=outMovieSet)
self._defineTransformRelation(self.inputParticles, outMovieSet)
self._defineTransformRelation(self.inputMovieParticles,
outMovieSet)
def _writeXmippPosFile(self, movie, coordinatesName, shiftX, shiftY):
""" Create an Xmipp coordinates files to be extracted
from the frames of the movie.
"""
coordSet = self.inputCoordinates.get()
# to support multiple access to db
coordToIter = SetOfCoordinates()
coordToIter.copy(coordSet)
coordSet.close()
mData = md.MetaData()
coordRow = XmippMdRow()
for coord in coordToIter.iterCoordinates(movie.getObjId()):
coord.shiftX(int(round(float(shiftX))))
coord.shiftY(int(round(float(shiftY))))
coordinateToRow(coord, coordRow)
coordRow.writeToMd(mData, mData.addObject())
coordToIter.close()
if mData.isEmpty():
return False
else:
self.info("Writing coordinates metadata: %s, with shifts: %s %s" %
(coordinatesName, shiftX, shiftY))
mData.write('particles@' + coordinatesName)
return True
def createOutputStep(self):
if not self.realignMovieFrames:
imgSet = self._getInputParticles()
vol = Volume()
vol.setFileName(self._getExtraPath('relion_class001.mrc'))
vol.setSamplingRate(imgSet.getSamplingRate())
half1 = self._getFileName("final_half1_volume", ref3d=1)
half2 = self._getFileName("final_half2_volume", ref3d=1)
vol.setHalfMaps([half1, half2])
outImgSet = self._createSetOfParticles()
outImgSet.copyInfo(imgSet)
self._fillDataFromIter(outImgSet, self._lastIter())
self._defineOutputs(outputVolume=vol)
self._defineSourceRelation(self.inputParticles, vol)
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(self.inputParticles, outImgSet)
fsc = FSC(objLabel=self.getRunName())
blockName = 'model_class_%d@' % 1
fn = blockName + self._getExtraPath("relion_model.star")
mData = md.MetaData(fn)
fsc.loadFromMd(mData, md.RLN_RESOLUTION,
md.RLN_MLMODEL_FSC_HALVES_REF)
self._defineOutputs(outputFSC=fsc)
self._defineSourceRelation(vol, fsc)
else:
pass
def checkContinueClassification(self, level, iter):
diff=0
i=0
metadata = md.MetaData(self._getExtraPath(join('level%03d' % level,
'general_images_level%03d' % level + '.xmd')))
for item in metadata:
refImg = metadata.getValue(md.MDL_REF, item)
nameImg = metadata.getValue(md.MDL_IMAGE, item)
if iter==0:
self.listContinueClass.append(nameImg)
self.listContinueClass.append(refImg)
else:
if nameImg in self.listContinueClass:
idx = self.listContinueClass.index(nameImg) + 1
if refImg!=self.listContinueClass[idx]:
diff+=1
self.listContinueClass[idx]=refImg
else:
diff += 1
self.listContinueClass.append(nameImg)
self.listContinueClass.append(refImg)
i+=1
num=(diff*100/i)
if num<self.percentStopClassify and iter>0:
return True
else:
return False
def createAngDistributionSqlite(self, sqliteFn, numberOfParticles,
itemDataIterator):
import pyworkflow.em.metadata as md
if not os.path.exists(sqliteFn):
# List of list of 3 elements containing angleTilt, anglePsi, weight
projectionList = []
def getCloseProjection(angleRot, angleTilt):
""" Get an existing projection close to angleRot, angleTilt.
Return None if not found close enough.
"""
for projection in projectionList:
if (abs(projection[0] - angleRot) <= 0.5
and abs(projection[1] - angleTilt) <= 0.5):
return projection
return None
weight = 1. / numberOfParticles
for angleRot, angleTilt in itemDataIterator:
projection = getCloseProjection(angleRot, angleTilt)
if projection is None:
projectionList.append([angleRot, angleTilt, weight])
else:
projection[2] = projection[2] + weight
mdProj = md.MetaData()
for projection in projectionList:
mdRow = md.Row()
mdRow.setValue(md.MDL_ANGLE_ROT, projection[0])
mdRow.setValue(md.MDL_ANGLE_TILT, projection[1])
mdRow.setValue(md.MDL_WEIGHT, projection[2])
mdRow.writeToMd(mdProj, mdProj.addObject())
mdProj.write(sqliteFn)
def performNmaStep(self, atomsFn, modesFn):
sampling = self.inputParticles.get().getSamplingRate()
discreteAngularSampling = self.discreteAngularSampling.get()
trustRegionScale = self.trustRegionScale.get()
odir = self._getTmpPath()
imgFn = self.imgsFn
args = "-i %(imgFn)s --pdb %(atomsFn)s --modes %(modesFn)s --sampling_rate %(sampling)f "
args += "--discrAngStep %(discreteAngularSampling)f --odir %(odir)s --centerPDB "
args += "--trustradius_scale %(trustRegionScale)d --resume "
if self.getInputPdb().getPseudoAtoms():
args += "--fixed_Gaussian "
if self.alignmentMethod == NMA_ALIGNMENT_PROJ:
args += "--projMatch "
self.runJob("xmipp_nma_alignment", args % locals())
cleanPath(self._getPath('nmaTodo.xmd'))
inputSet = self.inputParticles.get()
mdImgs = md.MetaData(self.imgsFn)
for objId in mdImgs:
imgPath = mdImgs.getValue(md.MDL_IMAGE, objId)
index, fn = xmippToLocation(imgPath)
# Conside the index is the id in the input set
particle = inputSet[index]
mdImgs.setValue(md.MDL_IMAGE, getImageLocation(particle), objId)
mdImgs.setValue(md.MDL_ITEM_ID, long(particle.getObjId()), objId)
mdImgs.write(self.imgsFn)
def createPlot(inputStar, classes):
inputMd = md.MetaData(inputStar)
if inputMd.containsLabel(md.RLN_PARTICLE_RANDOM_SUBSET):
splitLabel = md.RLN_PARTICLE_RANDOM_SUBSET
prefix = 'half'
else:
splitLabel = md.RLN_PARTICLE_CLASS
prefix = 'class'
projectionListDict = createAngDistribution(inputMd, splitLabel, classes)
nProj = len(projectionListDict)
nRows = (nProj + 1) / 2
nCols = 2 if nProj > 1 else 1
plotter = RelionPlotter(x=nRows, y=nCols,
windowTitle="Angular Distribution")
for key, projectionList in projectionListDict.iteritems():
rot = []
tilt = []
weight = []
n = len(projectionList)
for r, t, w in projectionList:
rot.append(r)
tilt.append(t)
weight.append(w/n) # normalize weights between 0 and 1
plotter.plotAngularDistribution("%s_%03d" % (prefix, key),
rot, tilt, weight)
plotter.show(block=True)
def checkOutput(self, level):
listAuxString = []
listAuxNum = []
listAuxRefs = []
outSet = self._getExtraPath(join('level%03d' % level,
'intermediate_classes.xmd'))
metadataItem = md.MetaData(outSet)
for item in metadataItem:
nameImg = metadataItem.getValue(md.MDL_IMAGE, item)
listAuxString.append(nameImg)
numImgs = metadataItem.getValue(md.MDL_CLASS_COUNT, item)
listAuxNum.append(numImgs)
refImg = metadataItem.getValue(md.MDL_REF, item)
listAuxRefs.append(refImg)
maxValue = max(listAuxNum)
maxPos = listAuxNum.index(maxValue)
listAuxNum[maxPos] = -1
bestRef = listAuxRefs[maxPos]
self._params = {'input': 'class%06d_images' % (bestRef) + '@' + outSet,
'outputMd': self._getExtraPath(join('level%03d' %
level,'images_level%03d' % level
+ '_major.xmd'))
}
args = ('-i %(input)s -o %(outputMd)s')
self.runJob("xmipp_metadata_utilities", args % self._params,
numberOfMpi=1)
return listAuxNum, listAuxString
def writeSetOfMicrographs(micSet, starFile, **kwargs):
""" If 'outputDir' is in kwargs, the micrographs are\
converted or linked in the outputDir.
"""
micMd = md.MetaData()
setOfImagesToMd(micSet, micMd, micrographToRow, **kwargs)
blockName = kwargs.get('blockName', 'Particles')
micMd.write('%s@%s' % (blockName, starFile))
def _newMd(self):
md0 = md.MetaData()
n = 5
xcoor = range(n)
ycoor = [x * x for x in xcoor]
for i in range(n):
self._addRow(md0, '*****@*****.**' % i, xcoor[i], ycoor[i])
return md0