def createOutputStep(self): fnImgs = self._getExtraPath('images.stk') if os.path.exists(fnImgs): cleanPath(fnImgs) imgSet = self.inputSet.get() imgFn = self._getExtraPath("anglesCont.xmd") self.newAssignmentPerformed = os.path.exists( self._getExtraPath("angles.xmd")) self.samplingRate = self.inputSet.get().getSamplingRate() outputSet = self._createSetOfParticles() outputSet.copyInfo(imgSet) if not self.newAssignmentPerformed: outputSet.setAlignmentProj() self.iterMd = md.iterRows(imgFn, md.MDL_ITEM_ID) self.lastRow = next(self.iterMd) outputSet.copyItems(imgSet, updateItemCallback=self._updateItem) self._defineOutputs(outputParticles=outputSet) self._defineSourceRelation(self.inputSet, outputSet) imgSet = self.inputSet.get() outputSet2 = self._createSetOfParticles('2') outputSet2.copyInfo(imgSet) if not self.newAssignmentPerformed: outputSet2.setAlignmentProj() self.iterMd2 = md.iterRows(imgFn, md.MDL_ITEM_ID) self.lastRow2 = next(self.iterMd2) outputSet2.copyItems( imgSet, updateItemCallback=self._updateItem2, ) self._defineOutputs(outputProjections=outputSet2) self._defineSourceRelation(self.inputSet, outputSet2)
def _fillClasses(self, outputClasses): """ Create the SetOfClasses2D """ inputSet = self.inputClasses.get().getImages() myRep = md.MetaData('classes@' + self._getExtraPath('final_classes.xmd')) for row in md.iterRows(myRep): fn = row.getValue(md.MDL_IMAGE) rep = Particle(fn) repId = row.getObjId() newClass = Class2D(objId=repId) newClass.setAlignment2D() newClass.copyInfo(inputSet) newClass.setAcquisition(inputSet.getAcquisition()) newClass.setRepresentative(rep) outputClasses.append(newClass) i = 1 mdBlocks = md.getBlocksInMetaDataFile( self._getExtraPath('final_classes.xmd')) for block in mdBlocks: if block.startswith('class00'): mdClass = md.MetaData(block + "@" + self._getExtraPath('final_classes.xmd')) imgClassId = i newClass = outputClasses[imgClassId] newClass.enableAppend() for row in md.iterRows(mdClass): part = rowToParticle(row) newClass.append(part) i += 1 newClass.setAlignment2D() outputClasses.update(newClass)
def readPartsFromMics(self, micList, outputParts): """ Read the particles extract for the given list of micrographs and update the outputParts set with new items. """ p = em.Particle() for mic in micList: # We need to make this dict because there is no ID in the # coord.star file coordDict = {} for coord in self.coordDict[mic.getObjId()]: coordDict[self._getPos(coord)] = coord _, partStarFn = self._getStarFiles(mic) for row in md.iterRows(self._getPath(partStarFn)): pos = (row.getValue(md.RLN_IMAGE_COORD_X), row.getValue(md.RLN_IMAGE_COORD_Y)) coord = coordDict.get(pos, None) if coord is not None: # scale the coordinates according to particles dimension. coord.scale(self.getBoxScale()) p.copyObjId(coord) idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME)) p.setLocation(idx, self._getPath(fn[2:])) p.setCoordinate(coord) p.setMicId(mic.getObjId()) p.setCTF(mic.getCTF()) outputParts.append(p) # Release the list of coordinates for this micrograph since it # will not be longer needed del self.coordDict[mic.getObjId()]
def _mergeDataStar(self, outStar, mdInput, iter, rLev): imgStar = self._getFileName('data', iter=iter, lev=self._level, rLev=rLev) mdData = md.MetaData(imgStar) print("reading %s and begin the loop" % imgStar) for row in md.iterRows(mdData, sortByLabel=md.RLN_PARTICLE_CLASS): clsPart = row.getValue(md.RLN_PARTICLE_CLASS) if clsPart != self._lastCls: self._newClass += 1 self._clsDict['%s.%s' % (rLev, clsPart)] = self._newClass self._lastCls = clsPart # write symlink to new Maps relionFn = self._getFileName('relionMap', lev=self._level, iter=self._getnumberOfIters(), ref3d=clsPart, rLev=rLev) newFn = self._getFileName('map', lev=self._level, rLev=self._newClass) print(('link from %s to %s' % (relionFn, newFn))) createLink(relionFn, newFn) row.setValue(md.RLN_PARTICLE_CLASS, self._newClass) row.addToMd(mdInput) print("writing %s and ending the loop" % outStar)
def _fillDataFromIter(self, imgSet, iteration): outImgsFn = self._getFileName('data', iter=iteration) imgSet.setAlignmentProj() imgSet.copyItems(self._getInputParticles(), updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows( outImgsFn, sortByLabel=md.RLN_IMAGE_ID))
def _fillClassesFromIter(self, clsSet, iteration): """ Create the SetOfClasses2D from a given iteration. """ self._loadClassesInfo(iteration) dataStar = self._getFileName('data', iter=iteration) clsSet.classifyItems(updateItemCallback=self._updateParticle, updateClassCallback=self._updateClass, itemDataIterator=md.iterRows(dataStar, sortByLabel=md.RLN_IMAGE_ID))
def readCoordinates(mic, fileName, coordsSet): for row in md.iterRows(fileName): coord = rowToCoordinate(row) coord.setX(coord.getX()) coord.setY(coord.getY()) coord.setMicrograph(mic) coordsSet.append(coord)
def createOutputStep(self): fnImgs = self._getExtraPath('images.stk') if os.path.exists(fnImgs): cleanPath(fnImgs) outputSet = self._createSetOfParticles() imgSet = self.inputSet.get() imgFn = self._getExtraPath("anglesCont.xmd") self.newAssignmentPerformed = os.path.exists( self._getExtraPath("angles.xmd")) self.samplingRate = self.inputSet.get().getSamplingRate() if isinstance(imgSet, SetOfClasses2D): outputSet = self._createSetOfClasses2D(imgSet) outputSet.copyInfo(imgSet.getImages()) elif isinstance(imgSet, SetOfAverages): outputSet = self._createSetOfAverages() outputSet.copyInfo(imgSet) else: outputSet = self._createSetOfParticles() outputSet.copyInfo(imgSet) if not self.newAssignmentPerformed: outputSet.setAlignmentProj() outputSet.copyItems(imgSet, updateItemCallback=self._processRow, itemDataIterator=md.iterRows( imgFn, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=outputSet) self._defineSourceRelation(self.inputSet, outputSet)
def createOutputStep(self): outputVols = self._createSetOfVolumes() imgSet = self.inputParticles.get() for i, vol in enumerate(self._iterInputVols()): volume = vol.clone() volDir = self._getVolDir(i + 1) volPrefix = 'vol%03d_' % (i + 1) validationMd = self._getExtraPath(volPrefix + 'validation.xmd') moveFile(join(volDir, 'validation.xmd'), validationMd) clusterMd = self._getExtraPath(volPrefix + 'clusteringTendency.xmd') moveFile(join(volDir, 'clusteringTendency.xmd'), clusterMd) outImgSet = self._createSetOfParticles(volPrefix) outImgSet.copyInfo(imgSet) outImgSet.copyItems(imgSet, updateItemCallback=self._setWeight, itemDataIterator=md.iterRows( clusterMd, sortByLabel=md.MDL_ITEM_ID)) mdValidatoin = md.MetaData(validationMd) weight = mdValidatoin.getValue(md.MDL_WEIGHT, mdValidatoin.firstObject()) volume.weight = Float(weight) volume.clusterMd = String(clusterMd) volume.cleanObjId( ) # clean objects id to assign new ones inside the set outputVols.append(volume) self._defineOutputs(outputParticles=outImgSet) outputVols.setSamplingRate(volume.getSamplingRate()) self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self): fnImgs = self._getExtraPath('images.stk') if os.path.exists(fnImgs): cleanPath(fnImgs) outputSet = self._createSetOfParticles() imgSet = self.inputSet.get() imgFn = self._getExtraPath("anglesCont.xmd") self.newAssignmentPerformed = os.path.exists(self._getExtraPath("angles.xmd")) self.samplingRate = self.inputSet.get().getSamplingRate() if isinstance(imgSet, SetOfClasses2D): outputSet = self._createSetOfClasses2D(imgSet) outputSet.copyInfo(imgSet.getImages()) elif isinstance(imgSet, SetOfAverages): outputSet = self._createSetOfAverages() outputSet.copyInfo(imgSet) else: outputSet = self._createSetOfParticles() outputSet.copyInfo(imgSet) if not self.newAssignmentPerformed: outputSet.setAlignmentProj() outputSet.copyItems(imgSet, updateItemCallback=self._processRow, itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=outputSet) self._defineSourceRelation(self.inputSet, outputSet)
def createOutputStep(self): micSet = self.getInputMicrographs() outputCoordinatesName = 'outputCoordinates' outputSuffix = '' # If in optimization phase, let's create a subset of the micrographs if self.isRunOptimize(): outputSuffix = '_subset' outputCoordinatesName = 'outputCoordinatesSubset' micSubSet = self._createSetOfMicrographs(suffix=outputSuffix) micSubSet.copyInfo(micSet) # Use previously written star file for reading the subset of micrographs, for row in md.iterRows(self._getPath('input_micrographs.star')): mic = micSet[row.getValue('rlnImageId')] micSubSet.append(mic) self._defineOutputs(outputMicrographsSubset=micSubSet) self._defineTransformRelation(self.getInputMicrographsPointer(), micSubSet) micSet = micSubSet coordSet = self._createSetOfCoordinates(micSet) template = self._getExtraPath("%s_autopick.star") starFiles = [ template % pwutils.removeBaseExt(mic.getFileName()) for mic in micSet ] readSetOfCoordinates(coordSet, starFiles, micSet) self._defineOutputs(**{outputCoordinatesName: coordSet}) self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet)
def _fillDataFromIter(self, imgSet, iteration): outImgsFn = self._getFileName('data', iter=iteration) imgSet.setAlignmentProj() imgSet.copyItems(self._getInputParticles(), updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows(outImgsFn, sortByLabel=md.RLN_IMAGE_ID))
def createOutputStep(self): particles = self.inputParticles.get() # Generate the SetOfAlignmet alignedSet = self._createSetOfParticles() alignedSet.copyInfo(particles) inputMd = self._getPath('aligned_particles.xmd') alignedSet.copyItems(particles, updateItemCallback=self._updateItem, itemDataIterator=md.iterRows(inputMd, sortByLabel=md.MDL_ITEM_ID)) # Remove alignment 2D alignedSet.setAlignment(em.ALIGN_NONE) # Define the output average avgFile = self._getExtraPath("average.xmp") imgh = ImageHandler() avgImage = imgh.computeAverage(alignedSet) avgImage.write(avgFile) avg = em.Particle() avg.setLocation(1, avgFile) avg.copyInfo(alignedSet) self._defineOutputs(outputAverage=avg) self._defineSourceRelation(self.inputParticles, avg) self._defineOutputs(outputParticles=alignedSet) self._defineSourceRelation(self.inputParticles, alignedSet)
def _fillClassesFromIter(self, clsSet, iteration): """ Create the SetOfClasses3D from a given iteration. """ self._loadClassesInfo(iteration) dataStar = self._getFileName('data', iter=iteration) clsSet.classifyItems(updateItemCallback=self._updateParticle, updateClassCallback=self._updateClass, itemDataIterator=md.iterRows(dataStar, sortByLabel=md.RLN_IMAGE_ID))
def createOutputStep(self): outputVols = self._createSetOfVolumes() imgSet = self.inputParticles.get() for i, vol in enumerate(self._iterInputVols()): volume = vol.clone() volDir = self._getVolDir(i + 1) volPrefix = "vol%03d_" % (i + 1) validationMd = self._getExtraPath(volPrefix + "validation.xmd") moveFile(join(volDir, "validation.xmd"), validationMd) clusterMd = self._getExtraPath(volPrefix + "clusteringTendency.xmd") moveFile(join(volDir, "clusteringTendency.xmd"), clusterMd) outImgSet = self._createSetOfParticles(volPrefix) outImgSet.copyInfo(imgSet) outImgSet.copyItems( imgSet, updateItemCallback=self._setWeight, itemDataIterator=md.iterRows(clusterMd, sortByLabel=md.MDL_ITEM_ID), ) mdValidatoin = md.MetaData(validationMd) weight = mdValidatoin.getValue(md.MDL_WEIGHT, mdValidatoin.firstObject()) volume.weight = Float(weight) volume.clusterMd = String(clusterMd) volume.cleanObjId() # clean objects id to assign new ones inside the set outputVols.append(volume) self._defineOutputs(outputParticles=outImgSet) outputVols.setSamplingRate(volume.getSamplingRate()) self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self): micSet = self.getInputMicrographs() outputCoordinatesName = 'outputCoordinates' outputSuffix = '' # If in optimization phase, let's create a subset of the micrographs if self.isRunOptimize(): outputSuffix = '_subset' outputCoordinatesName = 'outputCoordinatesSubset' micSubSet = self._createSetOfMicrographs(suffix=outputSuffix) micSubSet.copyInfo(micSet) # Use previously written star file for reading the subset of micrographs, for row in md.iterRows(self._getPath('input_micrographs.star')): mic = micSet[row.getValue('rlnImageId')] micSubSet.append(mic) self._defineOutputs(outputMicrographsSubset=micSubSet) self._defineTransformRelation(self.getInputMicrographsPointer(), micSubSet) micSet = micSubSet coordSet = self._createSetOfCoordinates(micSet) template = self._getExtraPath("%s_autopick.star") starFiles = [template % pwutils.removeBaseExt(mic.getFileName()) for mic in micSet] readSetOfCoordinates(coordSet, starFiles, micSet) self._defineOutputs(**{outputCoordinatesName: coordSet}) self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet)
def createOutputStep(self): particles = self.inputParticles.get() # Generate the SetOfAlignmet alignedSet = self._createSetOfParticles() alignedSet.copyInfo(particles) inputMd = self._getPath('aligned_particles.xmd') alignedSet.copyItems(particles, updateItemCallback=self._updateItem, itemDataIterator=md.iterRows( inputMd, sortByLabel=md.MDL_ITEM_ID)) # Remove alignment 2D alignedSet.setAlignment(em.ALIGN_NONE) # Define the output average avgFile = self._getExtraPath("average.xmp") imgh = ImageHandler() avgImage = imgh.computeAverage(alignedSet) avgImage.write(avgFile) avg = em.Particle() avg.setLocation(1, avgFile) avg.copyInfo(alignedSet) self._defineOutputs(outputAverage=avg) self._defineSourceRelation(self.inputParticles, avg) self._defineOutputs(outputParticles=alignedSet) self._defineSourceRelation(self.inputParticles, alignedSet)
def _fillClassesFromIter(self, clsSet, iteration): """ Create the SetOfClasses2D from a given iteration. """ self._loadClassesInfo(self._getIterMdClasses(iteration)) dataXmd = self._getIterMdImages(iteration) clsSet.classifyItems(updateItemCallback=self._updateParticle, updateClassCallback=self._updateClass, itemDataIterator=md.iterRows(dataXmd, sortByLabel=md.MDL_ITEM_ID))
def _fillClassesFromIter(self, clsSet, iteration): """ Create the SetOfClassesVol from a given iteration. """ self._loadClassesInfo(iteration) dataFn = self._getFileName('data_it', iter=iteration) clsSet.classifyItems(updateItemCallback=self._updateParticle, updateClassCallback=self._updateClass, itemDataIterator=md.iterRows(dataFn, sortByLabel=md.MDL_ITEM_ID))
def _fillClassesFromIter(self, clsSet, iteration): """ Create the SetOfClasses2D from a given iteration. """ self._loadClassesInfo(self._getIterMdClasses(iteration)) dataXmd = self._getIterMdImages(iteration) clsSet.classifyItems(updateItemCallback=self._updateParticle, updateClassCallback=self._updateClass, itemDataIterator=md.iterRows( dataXmd, sortByLabel=md.MDL_ITEM_ID))
def createOutputStep(self): inputSet = self.inputParticles.get() outputSet = self._createSetOfParticles() outputSet.copyInfo(inputSet) outputSet.copyItems(inputSet, updateItemCallback=self._updateItem, itemDataIterator=md.iterRows( self.outputMd, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputSet=outputSet)
def _loadClassesInfo(self, filename): """ Read some information about the produced 2D classes from the metadata file. """ self._classesInfo = {} # store classes info, indexed by class id mdClasses = md.MetaData(filename) for classNumber, row in enumerate(md.iterRows(mdClasses)): index, fn = xmippToLocation(row.getValue(md.MDL_IMAGE)) self._classesInfo[classNumber + 1] = (index, fn, row.clone())
def readCoordinates(mic, fileName, coordsSet): if os.path.exists(fileName): mdCoords = md.MetaData() mdCoords.readPlain(fileName, "xcoor ycoor xSize ySize") for row in md.iterRows(mdCoords): coord = em.Coordinate(x = row.getValue("xcoor") + row.getValue("xSize")/2, y = row.getValue("ycoor") + row.getValue("ySize")/2) coord.setMicrograph(mic) coordsSet.append(coord)
def readCoordinates(mic, fileName, coordsSet): if os.path.exists(fileName): mdCoords = md.MetaData() mdCoords.readPlain(fileName, "xcoor ycoor xSize ySize") for row in md.iterRows(mdCoords): coord = em.Coordinate( x=row.getValue("xcoor") + row.getValue("xSize") / 2, y=row.getValue("ycoor") + row.getValue("ySize") / 2) coord.setMicrograph(mic) coordsSet.append(coord)
def createOutputStep(self): #Output Volume/SetOfVolumes particlesSet = self._createSetOfParticles() particlesSet.copyInfo(self.input.get()) inputMd = self._getExtraPath('Noisy.xmd') particlesSet.copyItems(self.input.get(), updateItemCallback=self._updateParticle, itemDataIterator=md.iterRows(inputMd)) self._defineOutputs(outputParticles=particlesSet) self._defineSourceRelation(self.input.get(), particlesSet)
def _fillParticlesFromIter(self, partSet, iteration): import pyworkflow.em.metadata as md imgSet = self.inputParticles.get() imgFn = "all_exp_images@" + self._getFileName('docfileInputAnglesIters', iter=iteration, ref=1) partSet.copyInfo(imgSet) partSet.setAlignmentProj() partSet.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID))
def createOutputStep(self): imgSet = self.inputParticles.get() partSet = self._createSetOfParticles() partSet.copyInfo(imgSet) partSet.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows(self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(imgSet, partSet)
def runCreateOutputStep(self): import pyworkflow.em.metadata as md ''' Create standard output results_images, result_classes''' #creating results files imgSet = self.inputParticles.get() lastIter = self.numberOfIterations.get() if self.numberOfReferences != 1: inDocfile = self._getFileName('docfileInputAnglesIters', iter=lastIter) ClassFnTemplate = '%(rootDir)s/reconstruction_Ref3D_%(ref)03d.vol' allExpImagesinDocfile = xmipp.FileName() all_exp_images="all_exp_images" allExpImagesinDocfile.compose(all_exp_images, inDocfile) dataClasses = self._getFileName('sqliteClasses') createClassesFromImages(imgSet, str(allExpImagesinDocfile), dataClasses, SetOfClasses3D, xmipp.MDL_REF3D, ClassFnTemplate, lastIter) classes = self._createSetOfClasses3D(imgSet) clsSet = SetOfClasses3D(dataClasses) classes.appendFromClasses(clsSet) volumes = self._createSetOfVolumes() volumes.setSamplingRate(imgSet.getSamplingRate()) for refN in self.allRefs(): volFn = self._getFileName('reconstructedFileNamesIters', iter=lastIter, ref=refN) vol = Volume() vol.setFileName(volFn) volumes.append(vol) self._defineOutputs(outputVolumes=volumes) self._defineOutputs(outputClasses=classes) self._defineSourceRelation(self.inputParticles, volumes) self._defineSourceRelation(self.inputParticles, classes) else: volFn = self._getFileName('reconstructedFileNamesIters', iter=lastIter, ref=1) vol = Volume() vol.setFileName(volFn) vol.setSamplingRate(imgSet.getSamplingRate()) self._defineOutputs(outputVolume=vol) self._defineSourceRelation(self.inputParticles, vol) #create set of images imgSetOut = self._createSetOfParticles() imgFn = "all_exp_images@" + self._getFileName('docfileInputAnglesIters', iter=lastIter, ref=1) imgSetOut.copyInfo(imgSet) imgSetOut.setAlignmentProj() imgSetOut.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows(imgFn)) self._defineOutputs(outputParticles=imgSetOut) self._defineSourceRelation(self.inputParticles, imgSetOut)
def createOutputStep(self): inputParticles = self.inputParticles.get() if not self._useSeveralClasses(): newTs = inputParticles.getSamplingRate() else: newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE) self.mdClasses = xmippLib.MetaData(self._getDirectionalClassesFn()) self.mdImages = xmippLib.MetaData(self._getDirectionalImagesFn()) classes2D = self._createSetOfClasses2D(inputParticles) classes2D.getImages().setSamplingRate(newTs) self.averageSet = self._createSetOfAverages() self.averageSet.copyInfo(inputParticles) self.averageSet.setAlignmentProj() self.averageSet.setSamplingRate(newTs) # Let's use a SetMdIterator because it should be less particles # in the metadata produced than in the input set iterator = md.SetMdIterator(self.mdImages, sortByLabel=md.MDL_ITEM_ID, updateItemCallback=self._updateParticle, skipDisabled=True) fnHomogeneous = self._getExtraPath("images_homogeneous.xmd") if exists(fnHomogeneous): homogeneousSet = self._createSetOfParticles() homogeneousSet.copyInfo(inputParticles) homogeneousSet.setSamplingRate(newTs) homogeneousSet.setAlignmentProj() self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID) self.lastRow = next(self.iterMd) homogeneousSet.copyItems(inputParticles, updateItemCallback=self._updateHomogeneousItem) self._defineOutputs(outputHomogeneous=homogeneousSet) self._defineSourceRelation(self.inputParticles, homogeneousSet) classes2D.classifyItems(updateItemCallback=iterator.updateItem, updateClassCallback=self._updateClass) self._defineOutputs(outputClasses=classes2D) self._defineOutputs(outputAverages=self.averageSet) self._defineSourceRelation(self.inputParticles, classes2D) self._defineSourceRelation(self.inputParticles, self.averageSet) if self.splitVolume and self.directionalClasses.get()>1: volumesSet = self._createSetOfVolumes() volumesSet.setSamplingRate(newTs) for i in range(2): vol = Volume() vol.setLocation(1, self._getExtraPath("split_v%d.vol"%(i+1))) volumesSet.append(vol) self._defineOutputs(outputVolumes=volumesSet) self._defineSourceRelation(inputParticles, volumesSet)
def createOutputStep(self): outputVols = self._createSetOfVolumes() for i, vol in enumerate(self._iterInputVols()): volDir = self._getVolDir(i + 1) volume = vol.clone() volPrefix = 'vol%03d_' % (i + 1) m_pruned = md.MetaData() m_pruned.read(volDir + '/pruned_particles_alignability.xmd') prunedMd = self._getExtraPath(volPrefix + 'pruned_particles_alignability.xmd') moveFile(join(volDir, 'pruned_particles_alignability.xmd'), prunedMd) m_volScore = md.MetaData() m_volScore.read(volDir + '/validationAlignability.xmd') validationMd = self._getExtraPath(volPrefix + 'validation_alignability.xmd') moveFile(join(volDir, 'validationAlignability.xmd'), validationMd) imgSet = self.inputParticles.get() outImgSet = self._createSetOfParticles(volPrefix) outImgSet.copyInfo(imgSet) outImgSet.copyItems(imgSet, updateItemCallback=self._setWeight, itemDataIterator=md.iterRows( prunedMd, sortByLabel=md.MDL_ITEM_ID)) mdValidatoin = md.getFirstRow(validationMd) weight = mdValidatoin.getValue( md.MDL_WEIGHT_PRECISION_ALIGNABILITY) volume.weightAlignabilityPrecision = Float(weight) weight = mdValidatoin.getValue(md.MDL_WEIGHT_ACCURACY_ALIGNABILITY) volume.weightAlignabilityAccuracy = Float(weight) weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_MIRROR) volume.weightMirror = Float(weight) volume.cleanObjId( ) # clean objects id to assign new ones inside the set outputVols.append(volume) self._defineOutputs(outputParticles=outImgSet) self.createPlot2D(volPrefix, m_pruned) outputVols.setSamplingRate(volume.getSamplingRate()) self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self): imgSet = self.predictSetOfParticles.get() partSet = self._createSetOfParticles() partSet.copyInfo(imgSet) partSet.copyItems(imgSet, updateItemCallback=self._updateParticle, itemDataIterator=md.iterRows( self._getPath("particles.xmd"), sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(imgSet, partSet)
def createOutputStep(self): imgSet = self.inputParticles.get() partSet = self._createSetOfParticles() partSet.copyInfo(imgSet) partSet.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows( self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(imgSet, partSet)
def createOutputStep(self): imgSet = self.inputParticles.get() partSet = self._createSetOfParticles() imgFn = self._getPath('corrected_ctf_particles.xmd') partSet.copyInfo(imgSet) partSet.copyItems(imgSet, updateItemCallback=self._updateLocation, itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(imgSet, partSet)
def createOutputStep( self, outCoordsPosDir): noiseCoords= readSetOfCoordsFromPosFnames(outCoordsPosDir, sqliteOutName=self._getExtraPath('consensus_NOISE.sqlite') , setOfInputCoords=self.inputCoordinates.get()) coordSet = self._createSetOfCoordinates(self.inputCoordinates.get().getMicrographs() ) coordSet.copyInfo(noiseCoords) coordSet.copyItems(noiseCoords, itemDataIterator=MD.iterRows(self._getPath(noiseCoords.getFileName()), sortByLabel=MD.MDL_ITEM_ID)) coordSet.setBoxSize(noiseCoords.getBoxSize()) self._defineOutputs(outputCoordinates=coordSet) self._defineSourceRelation(self.inputCoordinates.get(), coordSet)
def createOutputStep(self): imgSet1 = self.inputParticles1.get() imgSetOut = self._createSetOfParticles() imgSetOut.copyInfo(imgSet1) imgSetOut.setAlignmentProj() self.iterMd = md.iterRows(self._getExtraPath("angles1_common.xmd"), md.MDL_ITEM_ID) self.lastRow = next(self.iterMd) imgSetOut.copyItems(imgSet1, updateItemCallback=self._updateItem) self._defineOutputs(outputParticles=imgSetOut) self._defineSourceRelation(self.inputParticles1, imgSetOut) self._defineSourceRelation(self.inputParticles2, imgSetOut)
def createOutputStep(self): imgSet = self._getInputParticles() outImgSet = self._createSetOfParticles() outImgsFn = self._getFileName('output_star') outImgSet.copyInfo(imgSet) outImgSet.setAlignmentProj() outImgSet.copyItems(imgSet, updateItemCallback=self._updateItem, itemDataIterator=md.iterRows(outImgsFn)) self._defineOutputs(outputParticles=outImgSet) self._defineTransformRelation(imgSet, outImgSet)
def _fillParticlesFromIter(self, partSet, iteration): import pyworkflow.em.metadata as md imgSet = self.inputParticles.get() imgFn = "all_exp_images@" + self._getFileName( 'docfileInputAnglesIters', iter=iteration, ref=1) partSet.copyInfo(imgSet) partSet.setAlignmentProj() partSet.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows( imgFn, sortByLabel=md.MDL_ITEM_ID))
def createOutputStep(self, id=1): imgSet = self.inputParticles.get() outImgSet = self._createSetOfParticles() outImgSet.copyInfo(imgSet) outImgsFn = self._getExtraPath('shiny.star') rowIterator = md.iterRows(outImgsFn, sortByLabel=md.RLN_IMAGE_ID) outImgSet.copyItems(imgSet, updateItemCallback=self._updateItem, itemDataIterator=rowIterator) self._defineOutputs(outputParticles=outImgSet) self._defineTransformRelation(self.inputParticles, outImgSet)
def createOutputStep(self): imgSet = self.inputParticles.get() partSet = self._createSetOfParticles() imgFn = self._getPath('corrected_ctf_particles.xmd') partSet.copyInfo(imgSet) partSet.copyItems(imgSet, updateItemCallback=self._updateLocation, itemDataIterator=md.iterRows( imgFn, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(imgSet, partSet)
def createOutputStep(self): inputSet = self.inputParticles.get() partSet = self._createSetOfParticles() pdbPointer = self.inputModes.get()._pdbPointer partSet.copyInfo(inputSet) partSet.copyItems(inputSet, updateItemCallback=self._updateParticle, itemDataIterator=md.iterRows(self.imgsFn, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(pdbPointer, partSet) self._defineTransformRelation(self.inputParticles, partSet)
def _loadClassesInfo(self, iteration): """ Read some information about the produced MLtomo 3D classes from the *ref.xmd file. """ self._classesInfo = {} # store classes info, indexed by class id modelFn = md.MetaData('classes@' + self._getFileName('ref_it', iter=iteration)) for classNumber, row in enumerate(md.iterRows(modelFn)): fn = str(row.getValue(md.MDL_IMAGE)) # Store info indexed by id, we need to store the row.clone() since # the same reference is used for iteration self._classesInfo[classNumber + 1] = (fn, row.clone())
def _loadClassesInfo(self, iteration): """ Read some information about the produced Relion 3D classes from the *model.star file. """ self._classesInfo = {} # store classes info, indexed by class id modelStar = md.MetaData('model_classes@' + self._getFileName('model', iter=iteration)) for classNumber, row in enumerate(md.iterRows(modelStar)): index, fn = relionToLocation(row.getValue('rlnReferenceImage')) # Store info indexed by id, we need to store the row.clone() since # the same reference is used for iteration self._classesInfo[classNumber+1] = (index, fn, row.clone())
def _loadClassesInfo(self, iteration): """ Read some information about the produced Relion 2D classes from the *model.star file. """ self._classesInfo = {} # store classes info, indexed by class id modelStar = md.MetaData('model_classes@' + self._getFileName('model', iter=iteration)) for classNumber, row in enumerate(md.iterRows(modelStar)): index, fn = relionToLocation(row.getValue('rlnReferenceImage')) # Store info indexed by id, we need to store the row.clone() since # the same reference is used for iteration self._classesInfo[classNumber+1] = (index, fn, row.clone())
def _loadClassesInfo(self, filename): """ Read some information about the produced 2D classes from the metadata file. """ self._classesInfo = {} # store classes info, indexed by class id mdClasses = md.MetaData(filename) for classNumber, row in enumerate(md.iterRows(mdClasses)): index, fn = xmippToLocation(row.getValue(md.MDL_IMAGE)) # Store info indexed by id, we need to store the row.clone() since # the same reference is used for iteration self._classesInfo[classNumber+1] = (index, fn, row.clone())
def createOutputStep(self): imgSet = self.inputParticles.get() partSet = self._createSetOfParticles() self._initializeZscores() partSet.copyInfo(imgSet) partSet.copyItems(imgSet, updateItemCallback=self._updateParticle, itemDataIterator=md.iterRows( self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=partSet) self._defineSourceRelation(imgSet, partSet) # Store Zcore summary values. self._store()
def createOutputStep(self): outputSet = self._createSetOfParticles() imgSet = self.inputSet.get() imgFn = self._getExtraPath("anglesCont.xmd") if isinstance(imgSet, SetOfClasses2D): outputSet.copyInfo(imgSet.getImages()) else: outputSet.copyInfo(imgSet) outputSet.setAlignmentProj() outputSet.copyItems(imgSet, updateItemCallback=self._processRow, itemDataIterator=md.iterRows(imgFn, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(outputParticles=outputSet) self._defineSourceRelation(self.inputSet, outputSet)
def createOutputStep(self): outputVols = self._createSetOfVolumes() for i, vol in enumerate(self._iterInputVols()): volDir = self._getVolDir(i+1) volume = vol.clone() volPrefix = 'vol%03d_' % (i+1) m_pruned = md.MetaData() m_pruned.read(volDir+'/pruned_particles_alignability.xmd') prunedMd = self._getExtraPath(volPrefix + 'pruned_particles_alignability.xmd') moveFile(join(volDir, 'pruned_particles_alignability.xmd'), prunedMd) m_volScore = md.MetaData() m_volScore.read(volDir+'/validationAlignability.xmd') validationMd = self._getExtraPath(volPrefix + 'validation_alignability.xmd') moveFile(join(volDir, 'validationAlignability.xmd'), validationMd) imgSet = self.inputParticles.get() outImgSet = self._createSetOfParticles(volPrefix) outImgSet.copyInfo(imgSet) outImgSet.copyItems(imgSet, updateItemCallback=self._setWeight, itemDataIterator=md.iterRows(prunedMd, sortByLabel=md.MDL_ITEM_ID)) mdValidatoin = md.getFirstRow(validationMd) weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_ALIGNABILITY) volume.weightAlignabilityPrecision = Float(weight) weight = mdValidatoin.getValue(md.MDL_WEIGHT_ACCURACY_ALIGNABILITY) volume.weightAlignabilityAccuracy = Float(weight) weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_MIRROR) volume.weightMirror = Float(weight) volume.cleanObjId() # clean objects id to assign new ones inside the set outputVols.append(volume) self._defineOutputs(outputParticles=outImgSet) self.createPlot2D(volPrefix,m_pruned) outputVols.setSamplingRate(volume.getSamplingRate()) self._defineOutputs(outputVolumes=outputVols)
def createOutputStep(self): inputSet = self.inputParticles.get() # outputSet could be SetOfParticles, SetOfAverages or any future sub-class of SetOfParticles className = inputSet.getClassName() outputSet = self._createSetFromName(className) outputSet.copyInfo(inputSet) self._preprocessOutput(outputSet) outputSet.copyItems(inputSet, updateItemCallback=self._updateItem, itemDataIterator=md.iterRows(self.outputMd, sortByLabel=md.MDL_ITEM_ID)) self._postprocessOutput(outputSet) outputKey = className.replace('SetOf', 'output') self._defineOutputs(**{outputKey: outputSet}) self._defineTransformRelation(inputSet, outputSet)
def readSetOfParticles(filename, partSet, **kwargs): """read from Relion image meta filename: The metadata filename where the image are. imgSet: the SetOfParticles that will be populated. rowToParticle: this function will be used to convert the row to Object """ imgMd = md.MetaData(filename) # By default remove disabled items from metadata # be careful if you need to preserve the original number of items if kwargs.get('removeDisabled', True): imgMd.removeDisabled() for imgRow in md.iterRows(imgMd): img = rowToParticle(imgRow, **kwargs) partSet.append(img) partSet.setHasCTF(img.hasCTF()) partSet.setAlignment(kwargs['alignType'])
def createOutputStep(self, outImgsFn): inputSet = self.inputSet.get() if isinstance(inputSet, SetOfClasses2D): outputSet = self._createSetOfClasses2D(inputSet.getImages()) outputName = 'outputClasses' else: # SetOfAverages inputSet.setAlignment3D() outputSet = self._createSetOfAverages() outputName = 'outputAverages' mdOut = md.MetaData(outImgsFn) outputSet.copyInfo(inputSet) outputSet.copyItems(inputSet, updateItemCallback=self.updateItemMaxCC, itemDataIterator=md.iterRows(mdOut, sortByLabel=md.MDL_ITEM_ID)) self._defineOutputs(**{outputName: outputSet}) self._defineTransformRelation(inputSet, outputSet)
def plotAngularDistributionFromMd(self, mdFile, title, **kwargs): """ Read the values of rot, tilt and weights from the medata and plot the angular distribution. In the metadata: rot: MDL_ANGLE_ROT tilt: MDL_ANGLE_TILT weight: MDL_WEIGHT """ angMd = md.MetaData(mdFile) rot = [] tilt = [] weight = [] for row in md.iterRows(angMd): rot.append(row.getValue(md.MDL_ANGLE_ROT)) tilt.append(row.getValue(md.MDL_ANGLE_TILT)) weight.append(row.getValue(md.MDL_WEIGHT)) return self.plotAngularDistribution(title, rot, tilt, weight, **kwargs)
def createAngDistribution(inputMd, splitLabel, classes): # List of list of 3 elements containing angleTilt, anglePsi, weight projectionListDict = {} def getCloseProjection(angleRot, angleTilt, projectionList): """ Get an existing projection close to angleRot, angleTilt. Return None if not found close enough. """ for projection in projectionList: if (abs(projection[0] - angleRot) <= 0.01 and abs(projection[1] - angleTilt) <= 0.01): return projection return None weight = 1. for row in md.iterRows(inputMd): splitValue = row.getValue(splitLabel) if classes and splitValue not in classes: continue angleRot = row.getValue('rlnAngleRot') angleTilt = row.getValue('rlnAngleTilt') if splitValue is None: raise Exception('Label %s not found' % md.label2Str(splitLabel)) if not splitValue in projectionListDict: projectionListDict[splitValue] = [] projectionList = projectionListDict[splitValue] projection = getCloseProjection(angleRot, angleTilt, projectionList) if projection is None: projectionList.append([angleRot, angleTilt, weight]) else: projection[2] = projection[2] + weight return projectionListDict
def createOutputStep(self): inputSet = self.inputParticles.get() if isinstance(inputSet, em.SetOfAverages): imgSet = self._createSetOfAverages() else: imgSet = self._createSetOfParticles() imgSet.copyInfo(inputSet) outImgsFn = self._getFileName("preprocess_particles_star") if self.doScale: oldSampling = inputSet.getSamplingRate() scaleFactor = self._getScaleFactor(inputSet) newSampling = oldSampling * scaleFactor imgSet.setSamplingRate(newSampling) imgSet.copyItems(inputSet, updateItemCallback=self._setFileName, itemDataIterator=md.iterRows(outImgsFn, sortByLabel=md.RLN_IMAGE_ID)) self._defineOutputs(outputParticles=imgSet) self._defineTransformRelation(inputSet, imgSet)
def createOutputStep(self): if not self.realignMovieFrames: imgSet = self._getInputParticles() vol = Volume() vol.setFileName(self._getExtraPath('relion_class001.mrc')) vol.setSamplingRate(imgSet.getSamplingRate()) outImgSet = self._createSetOfParticles() outImgsFn = self._getFileName('data', iter=self._lastIter()) outImgSet.copyInfo(imgSet) outImgSet.setAlignmentProj() outImgSet.copyItems(imgSet, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows(outImgsFn)) self._defineOutputs(outputVolume=vol) self._defineSourceRelation(imgSet, vol) self._defineOutputs(outputParticles=outImgSet) self._defineTransformRelation(imgSet, outImgSet) else: pass
def createCtfModelStep(self): inputSet = self.inputParticles.get() partSet = self._createSetOfParticles() partSet.copyInfo(inputSet) for particle in inputSet: coord = particle.getCoordinate() if coord is None: continue x, y = coord.getPosition() if self.applyShifts: shifts = getShifts(particle.getTransform(), self.alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) xNew, yNew = (xCoor * self.scale, yCoor * self.scale) else: xNew, yNew = (x * self.scale, y * self.scale) micBase = pwutils.removeBaseExt(coord.getMicName()) for key in self.matchingMics: micKey = pwutils.removeBaseExt(key.getFileName()) if micBase in micKey: # micName from mic and micName from coord may be different ctfFn = pwutils.join(self._getExtraPath(micKey), micKey + '_local.star') if pwutils.exists(ctfFn): mdFn = md.MetaData(ctfFn) for row in md.iterRows(mdFn): coordX = row.getValue(md.RLN_IMAGE_COORD_X) coordY = row.getValue(md.RLN_IMAGE_COORD_Y) if (int(xNew), int(yNew)) == (coordX, coordY): newPart = particle.clone() rowToCtfModel(row, newPart.getCTF()) partSet.append(newPart) self._defineOutputs(outputParticles=partSet) self._defineTransformRelation(inputSet, partSet)
def createOutputStep(self): """ Store the setOfParticles object as result of the protocol. """ particles = self.inputParticles.get() # Define the output average avgFile = self._getExtraPath('level_00', 'class_classes.stk') avg = em.Particle() avg.setLocation(1, avgFile) avg.copyInfo(particles) self._defineOutputs(outputAverage=avg) self._defineSourceRelation(self.inputParticles, avg) # Generate the Set of Particles with alignment alignedSet = self._createSetOfParticles() alignedSet.copyInfo(particles) alignedSet.setRepresentative(avg) alignedSet.copyItems(particles, updateItemCallback=self._createItemMatrix, itemDataIterator=md.iterRows(self.imgsFn, sortByLabel=md.MDL_ITEM_ID)) alignedSet.setAlignment(em.ALIGN_2D) self._defineOutputs(outputParticles=alignedSet) self._defineSourceRelation(self.inputParticles, alignedSet)
def readPartsFromMics(self, micList, outputParts): """ Read the particles extract for the given list of micrographs and update the outputParts set with new items. """ p = em.Particle() # Let's create a dict with the names of the mic in Relion star files # and also create a set with all star files to iterate them once starSet = set() micPathDict = {} for mic in micList: starSet.add(self._micCoordStarDict[mic.getObjId()]) micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc') micFile = os.path.join('extra', micName) micPathDict[micFile] = mic prevMicFile = None mic = None for partStarFn in starSet: for row in md.iterRows(self._getPath(partStarFn), sortByLabel=md.RLN_MICROGRAPH_NAME): micFile = row.getValue(md.RLN_MICROGRAPH_NAME) if micFile != prevMicFile: # Load some stuff when new mic if prevMicFile is not None: # cleanup previous mic # Release the list of coordinates for this micrograph # since it will not be longer needed del self.coordDict[mic.getObjId()] mic = micPathDict[micFile] coordDict = {self._getPos(coord): coord for coord in self.coordDict[mic.getObjId()]} posSet = set() # Set of (x, y) pairs to avoid duplicates prevMicFile = micFile pos = (row.getValue(md.RLN_IMAGE_COORD_X), row.getValue(md.RLN_IMAGE_COORD_Y)) if pos in posSet: print("Duplicate coordinate at: %s, IGNORED. " % str(pos)) coord = None else: coord = coordDict.get(pos, None) if coord is not None: # scale the coordinates according to particles dimension. coord.scale(self.getBoxScale()) p.copyObjId(coord) idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME)) p.setLocation(idx, self._getPath(fn[2:])) p.setCoordinate(coord) p.setMicId(mic.getObjId()) p.setCTF(mic.getCTF()) outputParts.append(p) posSet.add(pos) # Clean up the last mic if necessary if mic is not None: del self.coordDict[mic.getObjId()]
def importCoordinates(self, fileName, addCoordinate): for row in md.iterRows(fileName): coord = rowToCoordinate(row) addCoordinate(coord)