def createOutputStep(self):
fnImgs = self._getExtraPath('images.stk')
if os.path.exists(fnImgs):
cleanPath(fnImgs)
imgSet = self.inputSet.get()
imgFn = self._getExtraPath("anglesCont.xmd")
self.newAssignmentPerformed = os.path.exists(
self._getExtraPath("angles.xmd"))
self.samplingRate = self.inputSet.get().getSamplingRate()
outputSet = self._createSetOfParticles()
outputSet.copyInfo(imgSet)
if not self.newAssignmentPerformed:
outputSet.setAlignmentProj()
self.iterMd = md.iterRows(imgFn, md.MDL_ITEM_ID)
self.lastRow = next(self.iterMd)
outputSet.copyItems(imgSet, updateItemCallback=self._updateItem)
self._defineOutputs(outputParticles=outputSet)
self._defineSourceRelation(self.inputSet, outputSet)
imgSet = self.inputSet.get()
outputSet2 = self._createSetOfParticles('2')
outputSet2.copyInfo(imgSet)
if not self.newAssignmentPerformed:
outputSet2.setAlignmentProj()
self.iterMd2 = md.iterRows(imgFn, md.MDL_ITEM_ID)
self.lastRow2 = next(self.iterMd2)
outputSet2.copyItems(
imgSet,
updateItemCallback=self._updateItem2,
)
self._defineOutputs(outputProjections=outputSet2)
self._defineSourceRelation(self.inputSet, outputSet2)
def _fillClasses(self, outputClasses):
""" Create the SetOfClasses2D """
inputSet = self.inputClasses.get().getImages()
myRep = md.MetaData('classes@' +
self._getExtraPath('final_classes.xmd'))
for row in md.iterRows(myRep):
fn = row.getValue(md.MDL_IMAGE)
rep = Particle(fn)
repId = row.getObjId()
newClass = Class2D(objId=repId)
newClass.setAlignment2D()
newClass.copyInfo(inputSet)
newClass.setAcquisition(inputSet.getAcquisition())
newClass.setRepresentative(rep)
outputClasses.append(newClass)
i = 1
mdBlocks = md.getBlocksInMetaDataFile(
self._getExtraPath('final_classes.xmd'))
for block in mdBlocks:
if block.startswith('class00'):
mdClass = md.MetaData(block + "@" +
self._getExtraPath('final_classes.xmd'))
imgClassId = i
newClass = outputClasses[imgClassId]
newClass.enableAppend()
for row in md.iterRows(mdClass):
part = rowToParticle(row)
newClass.append(part)
i += 1
newClass.setAlignment2D()
outputClasses.update(newClass)
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = em.Particle()
for mic in micList:
# We need to make this dict because there is no ID in the
# coord.star file
coordDict = {}
for coord in self.coordDict[mic.getObjId()]:
coordDict[self._getPos(coord)] = coord
_, partStarFn = self._getStarFiles(mic)
for row in md.iterRows(self._getPath(partStarFn)):
pos = (row.getValue(md.RLN_IMAGE_COORD_X),
row.getValue(md.RLN_IMAGE_COORD_Y))
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME))
p.setLocation(idx, self._getPath(fn[2:]))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
outputParts.append(p)
# Release the list of coordinates for this micrograph since it
# will not be longer needed
del self.coordDict[mic.getObjId()]
def _mergeDataStar(self, outStar, mdInput, iter, rLev):
imgStar = self._getFileName('data',
iter=iter,
lev=self._level,
rLev=rLev)
mdData = md.MetaData(imgStar)
print("reading %s and begin the loop" % imgStar)
for row in md.iterRows(mdData, sortByLabel=md.RLN_PARTICLE_CLASS):
clsPart = row.getValue(md.RLN_PARTICLE_CLASS)
if clsPart != self._lastCls:
self._newClass += 1
self._clsDict['%s.%s' % (rLev, clsPart)] = self._newClass
self._lastCls = clsPart
# write symlink to new Maps
relionFn = self._getFileName('relionMap',
lev=self._level,
iter=self._getnumberOfIters(),
ref3d=clsPart,
rLev=rLev)
newFn = self._getFileName('map',
lev=self._level,
rLev=self._newClass)
print(('link from %s to %s' % (relionFn, newFn)))
createLink(relionFn, newFn)
row.setValue(md.RLN_PARTICLE_CLASS, self._newClass)
row.addToMd(mdInput)
print("writing %s and ending the loop" % outStar)
def _fillDataFromIter(self, imgSet, iteration):
outImgsFn = self._getFileName('data', iter=iteration)
imgSet.setAlignmentProj()
imgSet.copyItems(self._getInputParticles(),
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(
outImgsFn, sortByLabel=md.RLN_IMAGE_ID))
def _fillClassesFromIter(self, clsSet, iteration):
""" Create the SetOfClasses2D from a given iteration. """
self._loadClassesInfo(iteration)
dataStar = self._getFileName('data', iter=iteration)
clsSet.classifyItems(updateItemCallback=self._updateParticle,
updateClassCallback=self._updateClass,
itemDataIterator=md.iterRows(dataStar, sortByLabel=md.RLN_IMAGE_ID))
def readCoordinates(mic, fileName, coordsSet):
for row in md.iterRows(fileName):
coord = rowToCoordinate(row)
coord.setX(coord.getX())
coord.setY(coord.getY())
coord.setMicrograph(mic)
coordsSet.append(coord)
def createOutputStep(self):
fnImgs = self._getExtraPath('images.stk')
if os.path.exists(fnImgs):
cleanPath(fnImgs)
outputSet = self._createSetOfParticles()
imgSet = self.inputSet.get()
imgFn = self._getExtraPath("anglesCont.xmd")
self.newAssignmentPerformed = os.path.exists(
self._getExtraPath("angles.xmd"))
self.samplingRate = self.inputSet.get().getSamplingRate()
if isinstance(imgSet, SetOfClasses2D):
outputSet = self._createSetOfClasses2D(imgSet)
outputSet.copyInfo(imgSet.getImages())
elif isinstance(imgSet, SetOfAverages):
outputSet = self._createSetOfAverages()
outputSet.copyInfo(imgSet)
else:
outputSet = self._createSetOfParticles()
outputSet.copyInfo(imgSet)
if not self.newAssignmentPerformed:
outputSet.setAlignmentProj()
outputSet.copyItems(imgSet,
updateItemCallback=self._processRow,
itemDataIterator=md.iterRows(
imgFn, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=outputSet)
self._defineSourceRelation(self.inputSet, outputSet)
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
imgSet = self.inputParticles.get()
for i, vol in enumerate(self._iterInputVols()):
volume = vol.clone()
volDir = self._getVolDir(i + 1)
volPrefix = 'vol%03d_' % (i + 1)
validationMd = self._getExtraPath(volPrefix + 'validation.xmd')
moveFile(join(volDir, 'validation.xmd'), validationMd)
clusterMd = self._getExtraPath(volPrefix +
'clusteringTendency.xmd')
moveFile(join(volDir, 'clusteringTendency.xmd'), clusterMd)
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(
clusterMd, sortByLabel=md.MDL_ITEM_ID))
mdValidatoin = md.MetaData(validationMd)
weight = mdValidatoin.getValue(md.MDL_WEIGHT,
mdValidatoin.firstObject())
volume.weight = Float(weight)
volume.clusterMd = String(clusterMd)
volume.cleanObjId(
) # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self):
fnImgs = self._getExtraPath('images.stk')
if os.path.exists(fnImgs):
cleanPath(fnImgs)
outputSet = self._createSetOfParticles()
imgSet = self.inputSet.get()
imgFn = self._getExtraPath("anglesCont.xmd")
self.newAssignmentPerformed = os.path.exists(self._getExtraPath("angles.xmd"))
self.samplingRate = self.inputSet.get().getSamplingRate()
if isinstance(imgSet, SetOfClasses2D):
outputSet = self._createSetOfClasses2D(imgSet)
outputSet.copyInfo(imgSet.getImages())
elif isinstance(imgSet, SetOfAverages):
outputSet = self._createSetOfAverages()
outputSet.copyInfo(imgSet)
else:
outputSet = self._createSetOfParticles()
outputSet.copyInfo(imgSet)
if not self.newAssignmentPerformed:
outputSet.setAlignmentProj()
outputSet.copyItems(imgSet,
updateItemCallback=self._processRow,
itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=outputSet)
self._defineSourceRelation(self.inputSet, outputSet)
def createOutputStep(self):
micSet = self.getInputMicrographs()
outputCoordinatesName = 'outputCoordinates'
outputSuffix = ''
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
outputSuffix = '_subset'
outputCoordinatesName = 'outputCoordinatesSubset'
micSubSet = self._createSetOfMicrographs(suffix=outputSuffix)
micSubSet.copyInfo(micSet)
# Use previously written star file for reading the subset of micrographs,
for row in md.iterRows(self._getPath('input_micrographs.star')):
mic = micSet[row.getValue('rlnImageId')]
micSubSet.append(mic)
self._defineOutputs(outputMicrographsSubset=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [
template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet
]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(**{outputCoordinatesName: coordSet})
self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet)
def _fillDataFromIter(self, imgSet, iteration):
outImgsFn = self._getFileName('data', iter=iteration)
imgSet.setAlignmentProj()
imgSet.copyItems(self._getInputParticles(),
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(outImgsFn,
sortByLabel=md.RLN_IMAGE_ID))
def createOutputStep(self):
particles = self.inputParticles.get()
# Generate the SetOfAlignmet
alignedSet = self._createSetOfParticles()
alignedSet.copyInfo(particles)
inputMd = self._getPath('aligned_particles.xmd')
alignedSet.copyItems(particles,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(inputMd, sortByLabel=md.MDL_ITEM_ID))
# Remove alignment 2D
alignedSet.setAlignment(em.ALIGN_NONE)
# Define the output average
avgFile = self._getExtraPath("average.xmp")
imgh = ImageHandler()
avgImage = imgh.computeAverage(alignedSet)
avgImage.write(avgFile)
avg = em.Particle()
avg.setLocation(1, avgFile)
avg.copyInfo(alignedSet)
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.inputParticles, avg)
self._defineOutputs(outputParticles=alignedSet)
self._defineSourceRelation(self.inputParticles, alignedSet)
def _fillClassesFromIter(self, clsSet, iteration):
""" Create the SetOfClasses3D from a given iteration. """
self._loadClassesInfo(iteration)
dataStar = self._getFileName('data', iter=iteration)
clsSet.classifyItems(updateItemCallback=self._updateParticle,
updateClassCallback=self._updateClass,
itemDataIterator=md.iterRows(dataStar, sortByLabel=md.RLN_IMAGE_ID))
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
imgSet = self.inputParticles.get()
for i, vol in enumerate(self._iterInputVols()):
volume = vol.clone()
volDir = self._getVolDir(i + 1)
volPrefix = "vol%03d_" % (i + 1)
validationMd = self._getExtraPath(volPrefix + "validation.xmd")
moveFile(join(volDir, "validation.xmd"), validationMd)
clusterMd = self._getExtraPath(volPrefix + "clusteringTendency.xmd")
moveFile(join(volDir, "clusteringTendency.xmd"), clusterMd)
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(
imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(clusterMd, sortByLabel=md.MDL_ITEM_ID),
)
mdValidatoin = md.MetaData(validationMd)
weight = mdValidatoin.getValue(md.MDL_WEIGHT, mdValidatoin.firstObject())
volume.weight = Float(weight)
volume.clusterMd = String(clusterMd)
volume.cleanObjId() # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self):
micSet = self.getInputMicrographs()
outputCoordinatesName = 'outputCoordinates'
outputSuffix = ''
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
outputSuffix = '_subset'
outputCoordinatesName = 'outputCoordinatesSubset'
micSubSet = self._createSetOfMicrographs(suffix=outputSuffix)
micSubSet.copyInfo(micSet)
# Use previously written star file for reading the subset of micrographs,
for row in md.iterRows(self._getPath('input_micrographs.star')):
mic = micSet[row.getValue('rlnImageId')]
micSubSet.append(mic)
self._defineOutputs(outputMicrographsSubset=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(**{outputCoordinatesName: coordSet})
self._defineSourceRelation(self.getInputMicrographsPointer(),
coordSet)
def readCoordinates(mic, fileName, coordsSet):
for row in md.iterRows(fileName):
coord = rowToCoordinate(row)
coord.setX(coord.getX())
coord.setY(coord.getY())
coord.setMicrograph(mic)
coordsSet.append(coord)
def createOutputStep(self):
particles = self.inputParticles.get()
# Generate the SetOfAlignmet
alignedSet = self._createSetOfParticles()
alignedSet.copyInfo(particles)
inputMd = self._getPath('aligned_particles.xmd')
alignedSet.copyItems(particles,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(
inputMd, sortByLabel=md.MDL_ITEM_ID))
# Remove alignment 2D
alignedSet.setAlignment(em.ALIGN_NONE)
# Define the output average
avgFile = self._getExtraPath("average.xmp")
imgh = ImageHandler()
avgImage = imgh.computeAverage(alignedSet)
avgImage.write(avgFile)
avg = em.Particle()
avg.setLocation(1, avgFile)
avg.copyInfo(alignedSet)
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.inputParticles, avg)
self._defineOutputs(outputParticles=alignedSet)
self._defineSourceRelation(self.inputParticles, alignedSet)
def _fillClassesFromIter(self, clsSet, iteration):
""" Create the SetOfClasses2D from a given iteration. """
self._loadClassesInfo(self._getIterMdClasses(iteration))
dataXmd = self._getIterMdImages(iteration)
clsSet.classifyItems(updateItemCallback=self._updateParticle,
updateClassCallback=self._updateClass,
itemDataIterator=md.iterRows(dataXmd,
sortByLabel=md.MDL_ITEM_ID))
def _fillClassesFromIter(self, clsSet, iteration):
""" Create the SetOfClassesVol from a given iteration. """
self._loadClassesInfo(iteration)
dataFn = self._getFileName('data_it', iter=iteration)
clsSet.classifyItems(updateItemCallback=self._updateParticle,
updateClassCallback=self._updateClass,
itemDataIterator=md.iterRows(dataFn,
sortByLabel=md.MDL_ITEM_ID))
def _fillClassesFromIter(self, clsSet, iteration):
""" Create the SetOfClasses2D from a given iteration. """
self._loadClassesInfo(self._getIterMdClasses(iteration))
dataXmd = self._getIterMdImages(iteration)
clsSet.classifyItems(updateItemCallback=self._updateParticle,
updateClassCallback=self._updateClass,
itemDataIterator=md.iterRows(
dataXmd, sortByLabel=md.MDL_ITEM_ID))
def createOutputStep(self):
inputSet = self.inputParticles.get()
outputSet = self._createSetOfParticles()
outputSet.copyInfo(inputSet)
outputSet.copyItems(inputSet,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(
self.outputMd, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputSet=outputSet)
def _loadClassesInfo(self, filename):
""" Read some information about the produced 2D classes
from the metadata file.
"""
self._classesInfo = {} # store classes info, indexed by class id
mdClasses = md.MetaData(filename)
for classNumber, row in enumerate(md.iterRows(mdClasses)):
index, fn = xmippToLocation(row.getValue(md.MDL_IMAGE))
self._classesInfo[classNumber + 1] = (index, fn, row.clone())
def readCoordinates(mic, fileName, coordsSet):
if os.path.exists(fileName):
mdCoords = md.MetaData()
mdCoords.readPlain(fileName, "xcoor ycoor xSize ySize")
for row in md.iterRows(mdCoords):
coord = em.Coordinate(x = row.getValue("xcoor") + row.getValue("xSize")/2,
y = row.getValue("ycoor") + row.getValue("ySize")/2)
coord.setMicrograph(mic)
coordsSet.append(coord)
def readCoordinates(mic, fileName, coordsSet):
if os.path.exists(fileName):
mdCoords = md.MetaData()
mdCoords.readPlain(fileName, "xcoor ycoor xSize ySize")
for row in md.iterRows(mdCoords):
coord = em.Coordinate(
x=row.getValue("xcoor") + row.getValue("xSize") / 2,
y=row.getValue("ycoor") + row.getValue("ySize") / 2)
coord.setMicrograph(mic)
coordsSet.append(coord)
def createOutputStep(self):
#Output Volume/SetOfVolumes
particlesSet = self._createSetOfParticles()
particlesSet.copyInfo(self.input.get())
inputMd = self._getExtraPath('Noisy.xmd')
particlesSet.copyItems(self.input.get(),
updateItemCallback=self._updateParticle,
itemDataIterator=md.iterRows(inputMd))
self._defineOutputs(outputParticles=particlesSet)
self._defineSourceRelation(self.input.get(), particlesSet)
def _fillParticlesFromIter(self, partSet, iteration):
import pyworkflow.em.metadata as md
imgSet = self.inputParticles.get()
imgFn = "all_exp_images@" + self._getFileName('docfileInputAnglesIters', iter=iteration, ref=1)
partSet.copyInfo(imgSet)
partSet.setAlignmentProj()
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID))
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
def runCreateOutputStep(self):
import pyworkflow.em.metadata as md
''' Create standard output results_images, result_classes'''
#creating results files
imgSet = self.inputParticles.get()
lastIter = self.numberOfIterations.get()
if self.numberOfReferences != 1:
inDocfile = self._getFileName('docfileInputAnglesIters', iter=lastIter)
ClassFnTemplate = '%(rootDir)s/reconstruction_Ref3D_%(ref)03d.vol'
allExpImagesinDocfile = xmipp.FileName()
all_exp_images="all_exp_images"
allExpImagesinDocfile.compose(all_exp_images, inDocfile)
dataClasses = self._getFileName('sqliteClasses')
createClassesFromImages(imgSet, str(allExpImagesinDocfile), dataClasses,
SetOfClasses3D, xmipp.MDL_REF3D, ClassFnTemplate, lastIter)
classes = self._createSetOfClasses3D(imgSet)
clsSet = SetOfClasses3D(dataClasses)
classes.appendFromClasses(clsSet)
volumes = self._createSetOfVolumes()
volumes.setSamplingRate(imgSet.getSamplingRate())
for refN in self.allRefs():
volFn = self._getFileName('reconstructedFileNamesIters', iter=lastIter, ref=refN)
vol = Volume()
vol.setFileName(volFn)
volumes.append(vol)
self._defineOutputs(outputVolumes=volumes)
self._defineOutputs(outputClasses=classes)
self._defineSourceRelation(self.inputParticles, volumes)
self._defineSourceRelation(self.inputParticles, classes)
else:
volFn = self._getFileName('reconstructedFileNamesIters', iter=lastIter, ref=1)
vol = Volume()
vol.setFileName(volFn)
vol.setSamplingRate(imgSet.getSamplingRate())
self._defineOutputs(outputVolume=vol)
self._defineSourceRelation(self.inputParticles, vol)
#create set of images
imgSetOut = self._createSetOfParticles()
imgFn = "all_exp_images@" + self._getFileName('docfileInputAnglesIters', iter=lastIter, ref=1)
imgSetOut.copyInfo(imgSet)
imgSetOut.setAlignmentProj()
imgSetOut.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(imgFn))
self._defineOutputs(outputParticles=imgSetOut)
self._defineSourceRelation(self.inputParticles, imgSetOut)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
if not self._useSeveralClasses():
newTs = inputParticles.getSamplingRate()
else:
newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE)
self.mdClasses = xmippLib.MetaData(self._getDirectionalClassesFn())
self.mdImages = xmippLib.MetaData(self._getDirectionalImagesFn())
classes2D = self._createSetOfClasses2D(inputParticles)
classes2D.getImages().setSamplingRate(newTs)
self.averageSet = self._createSetOfAverages()
self.averageSet.copyInfo(inputParticles)
self.averageSet.setAlignmentProj()
self.averageSet.setSamplingRate(newTs)
# Let's use a SetMdIterator because it should be less particles
# in the metadata produced than in the input set
iterator = md.SetMdIterator(self.mdImages, sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self._updateParticle,
skipDisabled=True)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
if exists(fnHomogeneous):
homogeneousSet = self._createSetOfParticles()
homogeneousSet.copyInfo(inputParticles)
homogeneousSet.setSamplingRate(newTs)
homogeneousSet.setAlignmentProj()
self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID)
self.lastRow = next(self.iterMd)
homogeneousSet.copyItems(inputParticles,
updateItemCallback=self._updateHomogeneousItem)
self._defineOutputs(outputHomogeneous=homogeneousSet)
self._defineSourceRelation(self.inputParticles, homogeneousSet)
classes2D.classifyItems(updateItemCallback=iterator.updateItem,
updateClassCallback=self._updateClass)
self._defineOutputs(outputClasses=classes2D)
self._defineOutputs(outputAverages=self.averageSet)
self._defineSourceRelation(self.inputParticles, classes2D)
self._defineSourceRelation(self.inputParticles, self.averageSet)
if self.splitVolume and self.directionalClasses.get()>1:
volumesSet = self._createSetOfVolumes()
volumesSet.setSamplingRate(newTs)
for i in range(2):
vol = Volume()
vol.setLocation(1, self._getExtraPath("split_v%d.vol"%(i+1)))
volumesSet.append(vol)
self._defineOutputs(outputVolumes=volumesSet)
self._defineSourceRelation(inputParticles, volumesSet)
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
for i, vol in enumerate(self._iterInputVols()):
volDir = self._getVolDir(i + 1)
volume = vol.clone()
volPrefix = 'vol%03d_' % (i + 1)
m_pruned = md.MetaData()
m_pruned.read(volDir + '/pruned_particles_alignability.xmd')
prunedMd = self._getExtraPath(volPrefix +
'pruned_particles_alignability.xmd')
moveFile(join(volDir, 'pruned_particles_alignability.xmd'),
prunedMd)
m_volScore = md.MetaData()
m_volScore.read(volDir + '/validationAlignability.xmd')
validationMd = self._getExtraPath(volPrefix +
'validation_alignability.xmd')
moveFile(join(volDir, 'validationAlignability.xmd'), validationMd)
imgSet = self.inputParticles.get()
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(
prunedMd, sortByLabel=md.MDL_ITEM_ID))
mdValidatoin = md.getFirstRow(validationMd)
weight = mdValidatoin.getValue(
md.MDL_WEIGHT_PRECISION_ALIGNABILITY)
volume.weightAlignabilityPrecision = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_ACCURACY_ALIGNABILITY)
volume.weightAlignabilityAccuracy = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_MIRROR)
volume.weightMirror = Float(weight)
volume.cleanObjId(
) # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
self.createPlot2D(volPrefix, m_pruned)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self):
imgSet = self.predictSetOfParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._updateParticle,
itemDataIterator=md.iterRows(
self._getPath("particles.xmd"),
sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(
self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
imgFn = self._getPath('corrected_ctf_particles.xmd')
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._updateLocation,
itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
def createOutputStep( self, outCoordsPosDir):
noiseCoords= readSetOfCoordsFromPosFnames(outCoordsPosDir, sqliteOutName=self._getExtraPath('consensus_NOISE.sqlite') ,
setOfInputCoords=self.inputCoordinates.get())
coordSet = self._createSetOfCoordinates(self.inputCoordinates.get().getMicrographs() )
coordSet.copyInfo(noiseCoords)
coordSet.copyItems(noiseCoords,
itemDataIterator=MD.iterRows(self._getPath(noiseCoords.getFileName()),
sortByLabel=MD.MDL_ITEM_ID))
coordSet.setBoxSize(noiseCoords.getBoxSize())
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(self.inputCoordinates.get(), coordSet)
def createOutputStep(self):
imgSet1 = self.inputParticles1.get()
imgSetOut = self._createSetOfParticles()
imgSetOut.copyInfo(imgSet1)
imgSetOut.setAlignmentProj()
self.iterMd = md.iterRows(self._getExtraPath("angles1_common.xmd"),
md.MDL_ITEM_ID)
self.lastRow = next(self.iterMd)
imgSetOut.copyItems(imgSet1, updateItemCallback=self._updateItem)
self._defineOutputs(outputParticles=imgSetOut)
self._defineSourceRelation(self.inputParticles1, imgSetOut)
self._defineSourceRelation(self.inputParticles2, imgSetOut)
def createOutputStep(self):
imgSet = self._getInputParticles()
outImgSet = self._createSetOfParticles()
outImgsFn = self._getFileName('output_star')
outImgSet.copyInfo(imgSet)
outImgSet.setAlignmentProj()
outImgSet.copyItems(imgSet,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(outImgsFn))
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(imgSet, outImgSet)
def createOutputStep(self):
imgSet = self._getInputParticles()
outImgSet = self._createSetOfParticles()
outImgsFn = self._getFileName('output_star')
outImgSet.copyInfo(imgSet)
outImgSet.setAlignmentProj()
outImgSet.copyItems(imgSet,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(outImgsFn))
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(imgSet, outImgSet)
def _fillParticlesFromIter(self, partSet, iteration):
import pyworkflow.em.metadata as md
imgSet = self.inputParticles.get()
imgFn = "all_exp_images@" + self._getFileName(
'docfileInputAnglesIters', iter=iteration, ref=1)
partSet.copyInfo(imgSet)
partSet.setAlignmentProj()
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(
imgFn, sortByLabel=md.MDL_ITEM_ID))
def createOutputStep(self, id=1):
imgSet = self.inputParticles.get()
outImgSet = self._createSetOfParticles()
outImgSet.copyInfo(imgSet)
outImgsFn = self._getExtraPath('shiny.star')
rowIterator = md.iterRows(outImgsFn, sortByLabel=md.RLN_IMAGE_ID)
outImgSet.copyItems(imgSet,
updateItemCallback=self._updateItem,
itemDataIterator=rowIterator)
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(self.inputParticles, outImgSet)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
imgFn = self._getPath('corrected_ctf_particles.xmd')
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._updateLocation,
itemDataIterator=md.iterRows(
imgFn, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
def createOutputStep(self):
inputSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
pdbPointer = self.inputModes.get()._pdbPointer
partSet.copyInfo(inputSet)
partSet.copyItems(inputSet,
updateItemCallback=self._updateParticle,
itemDataIterator=md.iterRows(self.imgsFn, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(pdbPointer, partSet)
self._defineTransformRelation(self.inputParticles, partSet)
def _loadClassesInfo(self, iteration):
""" Read some information about the produced MLtomo 3D classes
from the *ref.xmd file.
"""
self._classesInfo = {} # store classes info, indexed by class id
modelFn = md.MetaData('classes@' +
self._getFileName('ref_it', iter=iteration))
for classNumber, row in enumerate(md.iterRows(modelFn)):
fn = str(row.getValue(md.MDL_IMAGE))
# Store info indexed by id, we need to store the row.clone() since
# the same reference is used for iteration
self._classesInfo[classNumber + 1] = (fn, row.clone())
def _loadClassesInfo(self, iteration):
""" Read some information about the produced Relion 3D classes
from the *model.star file.
"""
self._classesInfo = {} # store classes info, indexed by class id
modelStar = md.MetaData('model_classes@' + self._getFileName('model', iter=iteration))
for classNumber, row in enumerate(md.iterRows(modelStar)):
index, fn = relionToLocation(row.getValue('rlnReferenceImage'))
# Store info indexed by id, we need to store the row.clone() since
# the same reference is used for iteration
self._classesInfo[classNumber+1] = (index, fn, row.clone())
def _loadClassesInfo(self, iteration):
""" Read some information about the produced Relion 2D classes
from the *model.star file.
"""
self._classesInfo = {} # store classes info, indexed by class id
modelStar = md.MetaData('model_classes@' + self._getFileName('model', iter=iteration))
for classNumber, row in enumerate(md.iterRows(modelStar)):
index, fn = relionToLocation(row.getValue('rlnReferenceImage'))
# Store info indexed by id, we need to store the row.clone() since
# the same reference is used for iteration
self._classesInfo[classNumber+1] = (index, fn, row.clone())
def _loadClassesInfo(self, filename):
""" Read some information about the produced 2D classes
from the metadata file.
"""
self._classesInfo = {} # store classes info, indexed by class id
mdClasses = md.MetaData(filename)
for classNumber, row in enumerate(md.iterRows(mdClasses)):
index, fn = xmippToLocation(row.getValue(md.MDL_IMAGE))
# Store info indexed by id, we need to store the row.clone() since
# the same reference is used for iteration
self._classesInfo[classNumber+1] = (index, fn, row.clone())
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
self._initializeZscores()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._updateParticle,
itemDataIterator=md.iterRows(
self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
# Store Zcore summary values.
self._store()
def createOutputStep(self):
outputSet = self._createSetOfParticles()
imgSet = self.inputSet.get()
imgFn = self._getExtraPath("anglesCont.xmd")
if isinstance(imgSet, SetOfClasses2D):
outputSet.copyInfo(imgSet.getImages())
else:
outputSet.copyInfo(imgSet)
outputSet.setAlignmentProj()
outputSet.copyItems(imgSet,
updateItemCallback=self._processRow,
itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=outputSet)
self._defineSourceRelation(self.inputSet, outputSet)
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
for i, vol in enumerate(self._iterInputVols()):
volDir = self._getVolDir(i+1)
volume = vol.clone()
volPrefix = 'vol%03d_' % (i+1)
m_pruned = md.MetaData()
m_pruned.read(volDir+'/pruned_particles_alignability.xmd')
prunedMd = self._getExtraPath(volPrefix + 'pruned_particles_alignability.xmd')
moveFile(join(volDir, 'pruned_particles_alignability.xmd'), prunedMd)
m_volScore = md.MetaData()
m_volScore.read(volDir+'/validationAlignability.xmd')
validationMd = self._getExtraPath(volPrefix + 'validation_alignability.xmd')
moveFile(join(volDir, 'validationAlignability.xmd'), validationMd)
imgSet = self.inputParticles.get()
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(prunedMd, sortByLabel=md.MDL_ITEM_ID))
mdValidatoin = md.getFirstRow(validationMd)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_ALIGNABILITY)
volume.weightAlignabilityPrecision = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_ACCURACY_ALIGNABILITY)
volume.weightAlignabilityAccuracy = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_MIRROR)
volume.weightMirror = Float(weight)
volume.cleanObjId() # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
self.createPlot2D(volPrefix,m_pruned)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self):
inputSet = self.inputParticles.get()
# outputSet could be SetOfParticles, SetOfAverages or any future sub-class of SetOfParticles
className = inputSet.getClassName()
outputSet = self._createSetFromName(className)
outputSet.copyInfo(inputSet)
self._preprocessOutput(outputSet)
outputSet.copyItems(inputSet,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(self.outputMd, sortByLabel=md.MDL_ITEM_ID))
self._postprocessOutput(outputSet)
outputKey = className.replace('SetOf', 'output')
self._defineOutputs(**{outputKey: outputSet})
self._defineTransformRelation(inputSet, outputSet)
def readSetOfParticles(filename, partSet, **kwargs):
"""read from Relion image meta
filename: The metadata filename where the image are.
imgSet: the SetOfParticles that will be populated.
rowToParticle: this function will be used to convert the row to Object
"""
imgMd = md.MetaData(filename)
# By default remove disabled items from metadata
# be careful if you need to preserve the original number of items
if kwargs.get('removeDisabled', True):
imgMd.removeDisabled()
for imgRow in md.iterRows(imgMd):
img = rowToParticle(imgRow, **kwargs)
partSet.append(img)
partSet.setHasCTF(img.hasCTF())
partSet.setAlignment(kwargs['alignType'])
def createOutputStep(self, outImgsFn):
inputSet = self.inputSet.get()
if isinstance(inputSet, SetOfClasses2D):
outputSet = self._createSetOfClasses2D(inputSet.getImages())
outputName = 'outputClasses'
else: # SetOfAverages
inputSet.setAlignment3D()
outputSet = self._createSetOfAverages()
outputName = 'outputAverages'
mdOut = md.MetaData(outImgsFn)
outputSet.copyInfo(inputSet)
outputSet.copyItems(inputSet,
updateItemCallback=self.updateItemMaxCC,
itemDataIterator=md.iterRows(mdOut, sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(**{outputName: outputSet})
self._defineTransformRelation(inputSet, outputSet)
def plotAngularDistributionFromMd(self, mdFile, title, **kwargs):
""" Read the values of rot, tilt and weights from
the medata and plot the angular distribution.
In the metadata:
rot: MDL_ANGLE_ROT
tilt: MDL_ANGLE_TILT
weight: MDL_WEIGHT
"""
angMd = md.MetaData(mdFile)
rot = []
tilt = []
weight = []
for row in md.iterRows(angMd):
rot.append(row.getValue(md.MDL_ANGLE_ROT))
tilt.append(row.getValue(md.MDL_ANGLE_TILT))
weight.append(row.getValue(md.MDL_WEIGHT))
return self.plotAngularDistribution(title, rot, tilt, weight, **kwargs)
def createAngDistribution(inputMd, splitLabel, classes):
# List of list of 3 elements containing angleTilt, anglePsi, weight
projectionListDict = {}
def getCloseProjection(angleRot, angleTilt, projectionList):
""" Get an existing projection close to angleRot, angleTilt.
Return None if not found close enough.
"""
for projection in projectionList:
if (abs(projection[0] - angleRot) <= 0.01 and
abs(projection[1] - angleTilt) <= 0.01):
return projection
return None
weight = 1.
for row in md.iterRows(inputMd):
splitValue = row.getValue(splitLabel)
if classes and splitValue not in classes:
continue
angleRot = row.getValue('rlnAngleRot')
angleTilt = row.getValue('rlnAngleTilt')
if splitValue is None:
raise Exception('Label %s not found' % md.label2Str(splitLabel))
if not splitValue in projectionListDict:
projectionListDict[splitValue] = []
projectionList = projectionListDict[splitValue]
projection = getCloseProjection(angleRot, angleTilt, projectionList)
if projection is None:
projectionList.append([angleRot, angleTilt, weight])
else:
projection[2] = projection[2] + weight
return projectionListDict
def createOutputStep(self):
inputSet = self.inputParticles.get()
if isinstance(inputSet, em.SetOfAverages):
imgSet = self._createSetOfAverages()
else:
imgSet = self._createSetOfParticles()
imgSet.copyInfo(inputSet)
outImgsFn = self._getFileName("preprocess_particles_star")
if self.doScale:
oldSampling = inputSet.getSamplingRate()
scaleFactor = self._getScaleFactor(inputSet)
newSampling = oldSampling * scaleFactor
imgSet.setSamplingRate(newSampling)
imgSet.copyItems(inputSet, updateItemCallback=self._setFileName,
itemDataIterator=md.iterRows(outImgsFn,
sortByLabel=md.RLN_IMAGE_ID))
self._defineOutputs(outputParticles=imgSet)
self._defineTransformRelation(inputSet, imgSet)
def createOutputStep(self):
if not self.realignMovieFrames:
imgSet = self._getInputParticles()
vol = Volume()
vol.setFileName(self._getExtraPath('relion_class001.mrc'))
vol.setSamplingRate(imgSet.getSamplingRate())
outImgSet = self._createSetOfParticles()
outImgsFn = self._getFileName('data', iter=self._lastIter())
outImgSet.copyInfo(imgSet)
outImgSet.setAlignmentProj()
outImgSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(outImgsFn))
self._defineOutputs(outputVolume=vol)
self._defineSourceRelation(imgSet, vol)
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(imgSet, outImgSet)
else:
pass
def createCtfModelStep(self):
inputSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(inputSet)
for particle in inputSet:
coord = particle.getCoordinate()
if coord is None:
continue
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
xNew, yNew = (xCoor * self.scale, yCoor * self.scale)
else:
xNew, yNew = (x * self.scale, y * self.scale)
micBase = pwutils.removeBaseExt(coord.getMicName())
for key in self.matchingMics:
micKey = pwutils.removeBaseExt(key.getFileName())
if micBase in micKey:
# micName from mic and micName from coord may be different
ctfFn = pwutils.join(self._getExtraPath(micKey),
micKey + '_local.star')
if pwutils.exists(ctfFn):
mdFn = md.MetaData(ctfFn)
for row in md.iterRows(mdFn):
coordX = row.getValue(md.RLN_IMAGE_COORD_X)
coordY = row.getValue(md.RLN_IMAGE_COORD_Y)
if (int(xNew), int(yNew)) == (coordX, coordY):
newPart = particle.clone()
rowToCtfModel(row, newPart.getCTF())
partSet.append(newPart)
self._defineOutputs(outputParticles=partSet)
self._defineTransformRelation(inputSet, partSet)
def createOutputStep(self):
""" Store the setOfParticles object
as result of the protocol.
"""
particles = self.inputParticles.get()
# Define the output average
avgFile = self._getExtraPath('level_00', 'class_classes.stk')
avg = em.Particle()
avg.setLocation(1, avgFile)
avg.copyInfo(particles)
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.inputParticles, avg)
# Generate the Set of Particles with alignment
alignedSet = self._createSetOfParticles()
alignedSet.copyInfo(particles)
alignedSet.setRepresentative(avg)
alignedSet.copyItems(particles,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(self.imgsFn,
sortByLabel=md.MDL_ITEM_ID))
alignedSet.setAlignment(em.ALIGN_2D)
self._defineOutputs(outputParticles=alignedSet)
self._defineSourceRelation(self.inputParticles, alignedSet)
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = em.Particle()
# Let's create a dict with the names of the mic in Relion star files
# and also create a set with all star files to iterate them once
starSet = set()
micPathDict = {}
for mic in micList:
starSet.add(self._micCoordStarDict[mic.getObjId()])
micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc')
micFile = os.path.join('extra', micName)
micPathDict[micFile] = mic
prevMicFile = None
mic = None
for partStarFn in starSet:
for row in md.iterRows(self._getPath(partStarFn),
sortByLabel=md.RLN_MICROGRAPH_NAME):
micFile = row.getValue(md.RLN_MICROGRAPH_NAME)
if micFile != prevMicFile: # Load some stuff when new mic
if prevMicFile is not None: # cleanup previous mic
# Release the list of coordinates for this micrograph
# since it will not be longer needed
del self.coordDict[mic.getObjId()]
mic = micPathDict[micFile]
coordDict = {self._getPos(coord): coord
for coord in self.coordDict[mic.getObjId()]}
posSet = set() # Set of (x, y) pairs to avoid duplicates
prevMicFile = micFile
pos = (row.getValue(md.RLN_IMAGE_COORD_X),
row.getValue(md.RLN_IMAGE_COORD_Y))
if pos in posSet:
print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
coord = None
else:
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME))
p.setLocation(idx, self._getPath(fn[2:]))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
outputParts.append(p)
posSet.add(pos)
# Clean up the last mic if necessary
if mic is not None:
del self.coordDict[mic.getObjId()]
def importCoordinates(self, fileName, addCoordinate):
for row in md.iterRows(fileName):
coord = rowToCoordinate(row)
addCoordinate(coord)
