def validate(self): errors = [] classSelection = getListFromRangeString(self.SelectedClasses) blocks = getBlocksInMetaDataFile(self.ClassifMd) self.ExtractDir = getWorkingDirFromRunName(self.ExtractionRun) for classNo in classSelection: blockName = "class%06d_images" % classNo if not blockName in blocks: errors.append("%s cannot be found at %s" % (blockName, self.ClassifMd)) if self.CenterMaxShift > 100 or self.CenterMaxShift < 0: errors.append("Maximum shift must be in the range 0-100") if self.LowPassFilter > 0.5 or self.LowPassFilter < 0: errors.append("Low pass filter must be in the range 0-0.5") files = [getImagesFilename(self.ExtractDir, s) for s in ["", "untilted", "tilted"]] fnMicrographs = getProtocolFilename("micrographs", WorkingDir=self.ExtractDir) files.append(fnMicrographs) for f in files: if not exists(f): errors.append("Cannot find file: %s" % f) if exists(fnMicrographs): blocks = getBlocksInMetaDataFile(fnMicrographs) if not "micrographPairs" in blocks: errors.append("Cannot find block micrographPairs@" + fnMicrographs) return errors
def runInitAngularReferenceFileStep(self): '''Create Initial angular file. Either fill it with zeros or copy input''' #NOTE: if using angles, self.selFileName file should contain angles info md = xmipp.MetaData(self.selFileName) # Ensure this labels are always md.addLabel(xmipp.MDL_ANGLE_ROT) md.addLabel(xmipp.MDL_ANGLE_TILT) md.addLabel(xmipp.MDL_ANGLE_PSI) expImages = self._getFileName('inputParticlesDoc') ctfImages = self._getFileName('imageCTFpairs') md.write(self._getExpImagesFileName(expImages)) blocklist = xmipp.getBlocksInMetaDataFile(ctfImages) mdCtf = xmipp.MetaData() mdAux = xmipp.MetaData() readLabels = [xmipp.MDL_ITEM_ID, xmipp.MDL_IMAGE] for block in blocklist: #read ctf block from ctf file mdCtf.read(block + '@' + ctfImages, readLabels) #add ctf columns to images file mdAux.joinNatural(md, mdCtf) # write block in images file with ctf info mdCtf.write(block + '@' + expImages, xmipp.MD_APPEND) return [expImages]
def __init__(self, filename): self._filename = filename blocks = xmipp.getBlocksInMetaDataFile(filename) if not blocks: # If there are no block, add an empty one blocks = [''] ds.DataSet.__init__(self, blocks)
def readSetOfClassesVol(classesVolSet, filename, classesBlock='classes', **args): """read from Xmipp image metadata. fnImages: The metadata filename where the particles properties are. imgSet: the SetOfParticles that will be populated. hasCtf: is True if the ctf information exists. """ blocks = xmipp.getBlocksInMetaDataFile(filename) classesMd = xmipp.MetaData('%s@%s' % (classesBlock, filename)) samplingRate = classesVolSet.getImages().getSamplingRate() for objId in classesMd: classVol = ClassVol() classRow = rowFromMd(classesMd, objId) classVol = rowToClass(classRow, classVol) classesVolSet.append(classVol) ref = classVol.getObjId() b = 'class%06d_images' % ref if b in blocks: classImagesMd = xmipp.MetaData('%s@%s' % (b, filename)) for imgId in classImagesMd: img = rowToParticle(classImagesMd, imgId, hasCtf=False) img.setSamplingRate(samplingRate) classVol.append(img) # Update with new properties of class2D such as _size classesVolSet.update(classVol) # Check if write function is necessary classVol.write()
def _pickMicrograph(self, mic, *args): micPath = mic.getFileName() # Get particle picking boxsize from the previous run boxSize = self.particlePickingRun.outputCoordinates.getBoxSize() modelRoot = self._getExtraPath('model') micName = removeBaseExt(micPath) proceed = True if self.micsToPick == MICS_SAMEASPICKING: basePos = replaceBaseExt(micPath, "pos") fnPos = self.particlePickingRun._getExtraPath(basePos) if exists(fnPos): blocks = xmipp.getBlocksInMetaDataFile(fnPos) copy = True if 'header' in blocks: mdheader = xmipp.MetaData("header@" + fnPos) state = mdheader.getValue( xmipp.MDL_PICKING_MICROGRAPH_STATE, mdheader.firstObject()) if state == "Available": copy = False if copy: # Copy manual .pos file of this micrograph copyFile(fnPos, self._getExtraPath(basename(fnPos))) proceed = False if proceed: args = "-i %s " % micPath args += "--particleSize %d " % boxSize args += "--model %s " % modelRoot args += "--outputRoot %s " % self._getExtraPath(micName) args += "--mode autoselect --thr %d" % self.numberOfThreads self.runJob("xmipp_micrograph_automatic_picking", args)
def getFilePreview(self, objFile): self._imgPreview = None self._imgInfo = None filename = objFile.getPath() ext = getExt(filename) if ext == '.xmd' or ext == '.ctfparam' or ext == '.pos' or ext == '.doc': msg = "*Metadata File* " blocks = xmipp.getBlocksInMetaDataFile(filename) nblocks = len(blocks) if nblocks <= 1: mdStr = self._getMdString(filename) msg += " (single block)\n" if self._imgInfo: msg += "\nFirst item: \n" + self._imgInfo msg += '\n' + mdStr else: mdStr = self._getMdString(filename, blocks[0]) msg += " (%d blocks) " % nblocks if self._imgInfo: msg += "\nFirst item: \n" + self._imgInfo msg += "\nFirst block: \n" + mdStr msg += "\nAll blocks:" + ''.join(["\n - %s" % b for b in blocks]) elif ext == '.star': msg = "*Relion STAR file* \n" msg += self._getMdString(filename) return self._imgPreview, msg
def getFilePreview(self, objFile): self._imgPreview = None self._imgInfo = None filename = objFile.getPath() ext = getExt(filename) if ext == '.xmd' or ext == '.ctfparam' or ext == '.pos' or ext == '.doc': msg = "*Metadata File* " blocks = xmipp.getBlocksInMetaDataFile(filename) nblocks = len(blocks) if nblocks <= 1: mdStr = self._getMdString(filename) msg += " (single block)\n" if self._imgInfo: msg += "\nFirst item: \n" + self._imgInfo msg += '\n' + mdStr else: mdStr = self._getMdString(filename, blocks[0]) msg += " (%d blocks) " % nblocks if self._imgInfo: msg += "\nFirst item: \n" + self._imgInfo msg += "\nFirst block: \n" + mdStr msg += "\nAll blocks:" + ''.join( ["\n - %s" % b for b in blocks]) elif ext == '.star': msg = "*Relion STAR file* \n" msg += self._getMdString(filename) return self._imgPreview, msg
def autopickMicrographStep(self, micPath): # Get particle picking boxsize from the previous run boxSize = self.particlePickingRun.outputCoordinates.getBoxSize() modelRoot = self._getExtraPath("model") micName = removeBaseExt(micPath) proceed = True if self.micsToPick == MICS_SAMEASPICKING: fnPos = self.particlePickingRun._getExtraPath(replaceBaseExt(micPath, "pos")) if exists(fnPos): blocks = xmipp.getBlocksInMetaDataFile(fnPos) copy = True if "header" in blocks: mdheader = xmipp.MetaData("header@" + fnPos) state = mdheader.getValue(xmipp.MDL_PICKING_MICROGRAPH_STATE, mdheader.firstObject()) if state == "Available": copy = False if copy: # Copy manual .pos file of this micrograph copyFile(fnPos, self._getExtraPath(basename(fnPos))) proceed = False if proceed: oroot = self._getExtraPath(micName) args = ( "-i %(micPath)s --particleSize %(boxSize)d --model %(modelRoot)s --outputRoot %(oroot)s --mode autoselect" % locals() ) # TODO: What is this? # if self.Fast: # cmd += " --fast " self.runJob("xmipp_micrograph_automatic_picking", args)
def autopickMicrographStep(self, micPath): # Get particle picking boxsize from the previous run boxSize = self.particlePickingRun.outputCoordinates.getBoxSize() modelRoot = self._getExtraPath('model') micName = removeBaseExt(micPath) proceed = True if self.micsToPick == MICS_SAMEASPICKING: fnPos = self.particlePickingRun._getExtraPath(replaceBaseExt(micPath, "pos")) if exists(fnPos): blocks = xmipp.getBlocksInMetaDataFile(fnPos) copy = True if 'header' in blocks: mdheader = xmipp.MetaData("header@" + fnPos) state = mdheader.getValue(xmipp.MDL_PICKING_MICROGRAPH_STATE, mdheader.firstObject()) if state == "Available": copy = False if copy: # Copy manual .pos file of this micrograph copyFile(fnPos, self._getExtraPath(basename(fnPos))) proceed = False if proceed: oroot = self._getExtraPath(micName) thr = self.numberOfThreads.get() args = "-i %(micPath)s --particleSize %(boxSize)d --model %(modelRoot)s " % locals() args += " --outputRoot %(oroot)s --mode autoselect --thr %(thr)d" % locals() # TODO: What is this? # if self.Fast: # args += " --fast " self.runJob("xmipp_micrograph_automatic_picking", args)
def __init__(self, scriptname, project): XmippProtocol.__init__(self, protDict.screen_classes.name, scriptname, project) self.Import = 'from protocol_screen_classes import *' self.trueClass=False if self.Classes.find('@')==-1: blocks=getBlocksInMetaDataFile(self.Classes) if 'classes' in blocks: self.fnImages='classes@'+self.Classes self.trueClass=True else: self.fnImages=self.Classes else: self.fnImages=self.Classes self.Xdim= MetaDataInfo(self.fnImages)[0]
def readSetOfClasses(classesSet, filename, classesBlock='classes', **kwargs): """ Read a set of classes from an Xmipp metadata with the given convention of a block for classes and another block for each set of images assigned to each class. Params: classesSet: the SetOfClasses object that will be populated. filename: the file path where to read the Xmipp metadata. classesBlock (by default 'classes'): the block name of the classes group in the metadata. """ blocks = xmipp.getBlocksInMetaDataFile(filename) classesMd = xmipp.MetaData('%s@%s' % (classesBlock, filename)) # Provide a hook to be used if something is needed to be # done for special cases before converting row to class preprocessClass = kwargs.get('preprocessClass', None) postprocessClass = kwargs.get('postprocessClass', None) for objId in classesMd: classItem = classesSet.ITEM_TYPE() classItem.setObjId(objId) classRow = rowFromMd(classesMd, objId) classItem = rowToClass(classRow, classItem) # FIXME: the following is only valid for SetOfParticles SetOfParticles.copyInfo(classItem, classesSet.getImages()) #classItem.copyInfo(classesSet.getImages()) if preprocessClass: preprocessClass(classItem, classRow) classesSet.append(classItem) ref = classItem.getObjId() b = 'class%06d_images' % ref if b in blocks: #FIXME: we need to adapt the following line # when we face classes of volumes and not just particles readSetOfParticles('%s@%s' % (b, filename), classItem, **kwargs) if postprocessClass: postprocessClass(classItem, classRow) # Update with new properties of classItem such as _size classesSet.update(classItem)
def __readSetOfClasses(classBaseSet, readSetFunc, classesSet, filename, classesBlock='classes', **kwargs): """ Read a set of classes from an Xmipp metadata with the given convention of a block for classes and another block for each set of images assigned to each class. Params: classesSet: the SetOfClasses object that will be populated. filename: the file path where to read the Xmipp metadata. classesBlock (by default 'classes'): the block name of the classes group in the metadata. """ blocks = xmipp.getBlocksInMetaDataFile(filename) classesMd = xmipp.MetaData('%s@%s' % (classesBlock, filename)) # Provide a hook to be used if something is needed to be # done for special cases before converting row to class preprocessClass = kwargs.get('preprocessClass', None) postprocessClass = kwargs.get('postprocessClass', None) for objId in classesMd: classItem = classesSet.ITEM_TYPE() classRow = rowFromMd(classesMd, objId) # class id should be set in rowToClass function using MDL_REF classItem = rowToClass(classRow, classItem) classBaseSet.copyInfo(classItem, classesSet.getImages()) if preprocessClass: preprocessClass(classItem, classRow) classesSet.append(classItem) ref = classItem.getObjId() b = 'class%06d_images' % ref if b in blocks: readSetFunc('%s@%s' % (b, filename), classItem, **kwargs) if postprocessClass: postprocessClass(classItem, classRow) # Update with new properties of classItem such as _size classesSet.update(classItem)
def analyzeOutOfCores(self, subset): """ Analyze which images are out of cores """ levelMdFiles = self._getLevelMdFiles(subset) for fn in levelMdFiles: mdAll = xmipp.MetaData() blocks = xmipp.getBlocksInMetaDataFile(fn) fnDir = dirname(fn) # Gather all images in block for block in blocks: if block.startswith("class0"): mdClass = xmipp.MetaData(block + "@" + fn) mdAll.unionAll(mdClass) if mdAll.size() > 0: # Compute difference to images fnSubset = join(fnDir, "images%s.xmd" % subset) mdAll.write(fnSubset) fnOutOfSubset = join(fnDir, "imagesOut.xmd") self.runJob( "xmipp_metadata_utilities", "-i %s --set subtraction %s -o %s" % (self.imgsFn, fnSubset, fnOutOfSubset), numberOfMpi=1, numberOfThreads=1, ) # Remove disabled and intermediate files mdClass = xmipp.MetaData(fnOutOfSubset) mdClass.removeDisabled() fnRejected = "images_rejected@" + fn mdClass.write(fnRejected, xmipp.MD_APPEND) cleanPath(fnOutOfSubset) cleanPath(fnSubset) # If enough images, make a small summary if mdClass.size() > 100: from math import ceil fnRejectedDir = join(fnDir, "rejected%s" % subset) makePath(fnRejectedDir) Nclasses = int(ceil(mdClass.size() / 300)) self.runJob( "xmipp_classify_CL2D", "-i %s --nref0 1 --nref %d --iter 5 --distance correlation --classicalMultiref --classifyAllImages --odir %s" % (fnRejected, Nclasses, fnRejectedDir), )
def readSetOfClassesVol(classesVolSet, filename, classesBlock='classes', **kwargs): """read from Xmipp image metadata. fnImages: The metadata filename where the particles properties are. imgSet: the SetOfParticles that will be populated. hasCtf: is True if the ctf information exists. """ __readSetOfClasses(SetOfVolumes, readSetOfVolumes, classesVolSet, filename, classesBlock, **kwargs) return # FIXME: Delete from here blocks = xmipp.getBlocksInMetaDataFile(filename) classesMd = xmipp.MetaData('%s@%s' % (classesBlock, filename)) # Provide a hook to be used if something is needed to be # done for special cases before converting row to class preprocessClass = kwargs.get('preprocessClass', None) postprocessClass = kwargs.get('postprocessClass', None) for objId in classesMd: classVol = classesVolSet.ITEM_TYPE() classRow = rowFromMd(classesMd, objId) classVol = rowToClass(classRow, classVol) # FIXME: the following is only valid for SetOfParticles SetOfParticles.copyInfo(classVol, classesVolSet.getImages()) if preprocessClass: preprocessClass(classVol, classRow) classesVolSet.append(classVol) ref = classVol.getObjId() b = 'class%06d_images' % ref if b in blocks: readSetFunc = _readSetFunc() readSetOfVolumes('%s@%s' % (b, filename), classVol, **kwargs) if postprocessClass: postprocessClass(classVol, classRow) # Update with new properties of classItem such as _size classesVolSet.update(classVol)
def readPosCoordinates(posFile): """ Read the coordinates in .pos file and return corresponding metadata. There are two possible blocks with particles: particles: with manual/supervised particles particles_auto: with automatically picked particles. If posFile doesn't exist, the metadata will be empty """ md = MetaData() if exists(posFile): blocks = getBlocksInMetaDataFile(posFile) for b in ['particles', 'particles_auto']: if b in blocks: mdAux = MetaData('%(b)s@%(posFile)s' % locals()) md.unionAll(mdAux) md.removeDisabled() return md
def readPosCoordinates(posFile): """ Read the coordinates in .pos file and return corresponding metadata. There are two possible blocks with particles: particles: with manual/supervised particles particles_auto: with automatically picked particles. If posFile doesn't exist, the metadata will be empty """ md = xmipp.MetaData() if exists(posFile): blocks = xmipp.getBlocksInMetaDataFile(posFile) for b in ['particles', 'particles_auto']: if b in blocks: mdAux = xmipp.MetaData('%(b)s@%(posFile)s' % locals()) md.unionAll(mdAux) md.removeDisabled() return md
def analyzeOutOfCores(self, subset): """ Analyze which images are out of cores """ levelMdFiles = self._getLevelMdFiles(subset) for fn in levelMdFiles: mdAll = xmipp.MetaData() blocks = xmipp.getBlocksInMetaDataFile(fn) fnDir = dirname(fn) # Gather all images in block for block in blocks: if block.startswith('class0'): mdClass = xmipp.MetaData(block + "@" + fn) mdAll.unionAll(mdClass) if mdAll.size() > 0: # Compute difference to images fnSubset = join(fnDir, "images%s.xmd" % subset) mdAll.write(fnSubset) fnOutOfSubset = join(fnDir, "imagesOut.xmd") self.runJob("xmipp_metadata_utilities", "-i %s --set subtraction %s -o %s" % (self.imgsFn, fnSubset, fnOutOfSubset), numberOfMpi=1, numberOfThreads=1) # Remove disabled and intermediate files mdClass = xmipp.MetaData(fnOutOfSubset) mdClass.removeDisabled() fnRejected = "images_rejected@" + fn mdClass.write(fnRejected, xmipp.MD_APPEND) cleanPath(fnOutOfSubset) cleanPath(fnSubset) # If enough images, make a small summary if mdClass.size() > 100: from math import ceil fnRejectedDir = join(fnDir, "rejected%s" % subset) makePath(fnRejectedDir) Nclasses = int(ceil(mdClass.size() / 300)) self.runJob("xmipp_classify_CL2D", "-i %s --nref0 1 --nref %d --iter 5 --distance correlation --classicalMultiref --classifyAllImages --odir %s"\ %(fnRejected,Nclasses,fnRejectedDir))
def classifyGroupsStep(self): # Create two metadatas, one for classes and another one for images mdClasses = xmipp.MetaData() mdImages = xmipp.MetaData() fnNeighbours = self._getExtraPath("neighbours.xmd") fnGallery = self._getExtraPath("gallery.stk") self.classCount = 0 self.classImages = set() for block in xmipp.getBlocksInMetaDataFile(fnNeighbours): # Figure out the projection number from the block name projNumber = int(block.split("_")[1]) self.classifyOneGroup(projNumber, projMdBlock="%s@%s" % (block, fnNeighbours), projRef="%06d@%s" % (projNumber, fnGallery), mdClasses=mdClasses, mdImages=mdImages) galleryMd = xmipp.MetaData(self._getExtraPath("gallery.doc")) # Increment the reference number to starts from 1 galleryMd.operate("ref=ref+1") mdJoined = xmipp.MetaData() # Add extra information from the gallery metadata mdJoined.join1(mdClasses, galleryMd, xmipp.MDL_REF) # Remove unnecessary columns md.keepColumns(mdJoined, "ref", "ref2", "image", "image1", "classCount", "angleRot", "angleTilt") # Write both classes and images fnDirectional = self._getDirectionalClassesFn() self.info("Writting classes info to: %s" % fnDirectional) mdJoined.write(fnDirectional) fnDirectionalImages = self._getDirectionalImagesFn() self.info("Writing images info to: %s" % fnDirectionalImages) mdImages.write(fnDirectionalImages)
def defineSteps(self): self.insertStep("createDir",verifyfiles=[self.ExtraDir],path=self.ExtraDir) md = MetaData(self.Input['config']) for objId in md: self.particleSizeForAuto = md.getValue(MDL_PICKING_PARTICLE_SIZE, objId) filesToImport = [self.Input[k] for k in self.keysToImport] filesToImport += [self.PrevRun.getFilename(k, model=self.model) for k in ['training', 'pca', 'rotpca', 'svm', 'average', 'config', 'templates']] self.insertImportOfFiles(filesToImport) self.insertCopyFile(self.MicrographsMd, self.getFilename('micrographs')) md = MetaData(self.MicrographsMd) particleSize = self.particleSizeForAuto modelRoot = self.extraPath(self.model) for objId in md: # Get micrograph path and name path = md.getValue(MDL_MICROGRAPH, objId) micrographName = removeBasenameExt(path) proceed = True if not self.anotherSet: fnPos = self.PrevRun.getFilename('pos', micrograph=micrographName) if xmippExists(fnPos): blocks = getBlocksInMetaDataFile(fnPos) copy = True if 'header' in blocks: mdheader = MetaData("header@" + fnPos) state = mdheader.getValue(MDL_PICKING_MICROGRAPH_STATE, mdheader.firstObject()) if state == "Available": copy = False if copy: # Copy manual .pos file of this micrograph self.insertCopyFile(fnPos, self.getFilename('pos', micrograph=micrographName)) proceed = False if proceed: oroot = self.extraPath(micrographName) cmd = "-i %(path)s --particleSize %(particleSize)d --model %(modelRoot)s --outputRoot %(oroot)s --mode autoselect" % locals() if self.Fast: cmd += " --fast " self.insertParallelRunJobStep("xmipp_micrograph_automatic_picking", cmd)