def convertInputStep(self, inputParticles, inputVolume, targetResolution): """ Write the input images as a Xmipp metadata file. particlesId: is only need to detect changes in input particles and cause restart from here. """ imgSet = self._getInputParticles() convXmp.writeSetOfParticles(imgSet, self._getInputXmd()) newXdim =self._getNewDim() ih = ImageHandler() inputVol = self.inputVolume.get() fn = ih.fixXmippVolumeFileName(inputVol) img = ih.read(fn) img.scale(newXdim, newXdim, newXdim) img.write(self._getExtraPath("volume.vol")) args = '-i %(parts)s -o %(scaledStk)s --save_metadata_stack ' args += '%(scaledXmd)s --dim %(xDim)d' params = {"parts" : self._getInputXmd(), "scaledStk": self._getExtraPath('scaled_particles.stk'), "scaledXmd": self._getExtraPath('scaled_particles.xmd'), "xDim": newXdim } self.runJob("xmipp_image_resize", args % params)
def appendMicsFromTomogram(self, output, tomo): self.info("Creating micrographs for %s" % tomo.getFileName()) # Load the tomogram ih = ImageHandler() img = ih.read(tomo.getFileName()) data = img.getData() # For each slice for index in range(0, len(data), self.slicesGap.get()): self.debug("Creating micrograph for slice %s" % index) micName = tomoSliceToMicName(tomo, index) outputMicName = self._getExtraPath(micName) outputMicName = replaceExt(outputMicName, "mrc") slice = data[index] micImg = ImageHandler() micImg._img.setData(slice) micImg.write(micImg._img, outputMicName) # Create the micrograph metadata object newMic = Micrograph() newMic.setFileName(outputMicName) newMic.setMicName(micName) newMic.setSamplingRate(tomo.getSamplingRate()) # Append it output.append(newMic)
def _invertScaleVol(self, fn): xdim = self._getInputParticles().getXDim() outputFn = self._getOutputVolFn(fn) ih = ImageHandler() img = ih.read(fn) img.scale(xdim, xdim, xdim) img.write(outputFn)
def _convertVolumes(self): ih = ImageHandler() makePath(self._getExtraPath('input')) inputVols = self.inputVolumes.get() for vol in inputVols: map = ih.read(vol) outputFn = self._getOutputFn(vol.getObjId()) map.write(outputFn)
def projectTomo(self, tomo): outFile = pwutils.removeBaseExt(tomo.getFileName()) + '_projected.mrc' ih = ImageHandler() outProjection = ih.createImage() tomoData = np.squeeze(ih.read(tomo.getFileName()).getData()) projection = np.sum(tomoData, axis=0) outProjection.setData(projection) ih.write(outProjection, self._getExtraPath(outFile)) return self._getExtraPath(outFile)
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from self.thumbPaths. ===== Params ===== - firstThumbIndex: index from which we start generating thumbnails - micScaleFactor: how much to reduce in size the micrographs. """ ih = ImageHandler() numMics = len(self.thumbPaths[MIC_PATH]) for i in range(firstThumbIndex, numMics): print('Generating images for mic %d' % (i+1)) # mic thumbnails dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) if not exists(dstImgPath): if self.micThumbSymlinks: pwutils.copyFile(self.thumbPaths[MIC_PATH][i], dstImgPath) else: ih.computeThumbnail(self.thumbPaths[MIC_PATH][i], dstImgPath, scaleFactor=micScaleFactor, flipOnY=True) # shift plots if SHIFT_THUMBS in self.thumbPaths: dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i]) if not exists(dstImgPath): pwutils.copyFile(self.thumbPaths[SHIFT_PATH][i], dstImgPath) # Psd thumbnails # If there ARE thumbnail for the PSD (no ctf protocol and # moviealignment hasn't computed it if PSD_THUMBS in self.thumbPaths: if self.ctfProtocol is None: srcImgPath = self.thumbPaths[PSD_PATH][i] dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath) and srcImgPath is not None: if srcImgPath.endswith('psd'): psdImg1 = ih.read(srcImgPath) psdImg1.convertPSD() psdImg1.write(dstImgPath) ih.computeThumbnail(dstImgPath, dstImgPath, scaleFactor=1, flipOnY=True) else: pwutils.copyFile(srcImgPath, dstImgPath) else: dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath): ih.computeThumbnail(self.thumbPaths[PSD_PATH][i], dstImgPath, scaleFactor=1, flipOnY=True) return
def rewriteClassBlockStep(self): firstImage = self.inputParticles.get().getFirstItem() fnClasses = self._params['kclasses'] mdClasses = "classes@%s" % fnClasses fnClassStack = self._params['classes'] fnAverageStack = self._params['averages'] md = emlib.MetaData(mdClasses) image = ImageHandler().createImage() counter = 1 for objId in md: imageName = "%06d@%s" % (counter, fnClassStack) averageName = "%06d@%s" % (counter, fnAverageStack) if md.getValue(emlib.MDL_CLASS_COUNT, objId) > 0: # compute the average of images assigned to this class classPrefix = 'class%06d' % counter classMd = '%s_images@%s' % (classPrefix, fnClasses) classRoot = self._getTmpPath(classPrefix) self.runJob( 'xmipp_image_statistics', '-i %s --save_image_stats %s -v 0' % (classMd, classRoot)) image.read(classRoot + 'average.xmp') else: # Create emtpy image as average image.read(firstImage.getLocation() ) # just to take the right dimensions and datatype image.initConstant(0.) image.write(averageName) md.setValue(emlib.MDL_IMAGE, imageName, objId) md.setValue(emlib.MDL_IMAGE2, averageName, objId) counter += 1 md.write(mdClasses, emlib.MD_APPEND)
def _computeRightPreview(self): """ This function should compute the right preview using the self.lastObj that was selected """ # Copy image to filter to Tmp project folder outputName = os.path.join("Tmp", "filtered_particle") outputPath = outputName + ".spi" cleanPath(outputPath) outputLoc = (1, outputPath) ih = ImageHandler() ih.convert(self.lastObj.getLocation(), outputLoc) outputLocSpiStr = locationToSpider(1, outputName) pars = dict() pars["filterType"] = self.protocolParent.filterType.get() pars["filterMode"] = self.protocolParent.filterMode.get() pars["usePadding"] = self.protocolParent.usePadding.get() pars["op"] = "FQ" if self.protocolParent.filterType <= FILTER_FERMI: pars['filterRadius'] = self.getRadius() else: pars['lowFreq'] = self.getLowFreq() pars['highFreq'] = self.getHighFreq() if self.protocolParent.filterType == FILTER_FERMI: pars['temperature'] = self.getTemperature() filter_spider(outputLocSpiStr, outputLocSpiStr, **pars) # Get output image and update filtered image img = ImageHandler()._img locXmippStr = ImageHandler.locationToXmipp((1, outputPath)) img.read(locXmippStr) self.rightImage = img self.updateFilteredImage()
def _scaleImages(self,indx, img): fn = img.getFileName() index = img.getIndex() newFn = self._getTmpPath('particles_subset.mrcs') xdim = self._getNewDim() ih = ImageHandler() image = ih.read((index, fn)) image.scale(xdim, xdim) image.write((indx, newFn)) img.setFileName(newFn) img.setIndex(indx) img.setSamplingRate(self._getPixeSize())
def _showImagesMatplotlib(self, title, *imgs): ih = ImageHandler() xdim = 2 if len(imgs) > 2 else 1 ydim = 2 plotter = EmPlotter(windowTitle=title, x=xdim, y=ydim, figsize=(8, 6)) positions = [(1, 1), (1, 2), (2, 1), (2, 2)] for i, imgFn in enumerate(imgs): x, y = positions[i] ax = plotter.createSubPlot("", "x", "y", x, y) img = ih.read(imgFn) ax.imshow(img.getData(), cmap='jet') #with mrcfile.open(img.replace(":mrc", "")) as mrc: # im = ax.imshow(mrc.data, cmap='jet') return [plotter]
def projectStep(self, start, end, samplingRate, threadNumber): # Project md = emlib.MetaData(self._getInputParticlesSubsetFn(threadNumber)) ## projection = emlib.Image() projection.setDataType(emlib.DT_DOUBLE) ## for id in md: rot = md.getValue(emlib.MDL_ANGLE_ROT, id) tilt = md.getValue(emlib.MDL_ANGLE_TILT, id) psi = md.getValue(emlib.MDL_ANGLE_PSI, id) ##projection =self.vol.projectVolumeDouble(rot, tilt, psi) self.fourierProjectVol.projectVolume(projection, rot, tilt, psi) ## # Apply CTF if self.projType == self.CORRECT_NONE: pass elif self.projType == self.CORRECT_FULL_CTF: emlib.applyCTF(projection, md, samplingRate, id, False) elif self.projType == self.CORRECT_PHASE_FLIP: emlib.applyCTF(projection, md, samplingRate, id, True) else: raise Exception("ERROR: Unknown projection mode: %d" % self.projType) # Shift image projection.applyGeo(md, id, True, False) #onlyapplyshist, wrap ih = ImageHandler() expProj = ih.read(md.getValue(emlib.MDL_IMAGE, id)) expProj.convert2DataType(emlib.DT_DOUBLE) # Subtract from experimental and write result projection.resetOrigin() if self.normalize: expProj = expProj.adjustAndSubtract(projection) else: expProj.inplaceSubtract(projection) expProj.write(self._getProjGalleryIndexFn(id + start - 1))
def createOutputStep(self): ih = ImageHandler() outputStack = self._getPath('particles.mrcs') outputImg = ih.createImage() inputParticles = self.inputParticles.get() inputCoords = self.inputCoordinates.get() outputSet = self._createSetOfParticles() outputSet.copyInfo(inputParticles) boxSize = self.boxSize.get() b2 = int(round(boxSize / 2)) center = np.zeros((boxSize, boxSize)) ih = ImageHandler() i = 0 outliers = 0 partIdExcluded = [] lastPartId = None progress = ProgressBar(len(inputCoords), fmt=ProgressBar.NOBAR) progress.start() step = max(100, len(inputCoords) // 100) for i, coord in enumerate(inputCoords.iterItems(orderBy=['_subparticle._micId', '_micId', 'id'])): if i % step == 0: progress.update(i+1) # The original particle id is stored in the sub-particle as micId partId = coord._micId.get() # Load the particle if it has changed from the last sub-particle if partId != lastPartId: particle = inputParticles[partId] if particle is None: partIdExcluded.append(partId) self.info("WARNING: Missing particle with id %s from " "input particles set" % partId) else: # Now load the particle image to extract later sub-particles img = ih.read(particle) x, y, _, _ = img.getDimensions() data = img.getData() lastPartId = partId # If particle is not in inputParticles, subparticles will not be # generated. Now, subtract from a subset of original particles is # supported. if partId not in partIdExcluded: xpos = coord.getX() ypos = coord.getY() # Check that the sub-particle will not lay out of the particle if (ypos - b2 < 0 or ypos + b2 > y or xpos - b2 < 0 or xpos + b2 > x): outliers += 1 continue # Crop the sub-particle data from the whole particle image center[:, :] = data[ypos - b2:ypos + b2, xpos - b2:xpos + b2] outputImg.setData(center) i += 1 outputImg.write((i, outputStack)) subpart = coord._subparticle subpart.setLocation( (i, outputStack)) # Change path to new stack subpart.setObjId(i) # Ids will be always the same no mater the number of outliers outputSet.append(subpart) progress.finish() if outliers: self.info("WARNING: Discarded %s particles because laid out of the " "particle (for a box size of %d" % (outliers, boxSize)) self._defineOutputs(outputParticles=outputSet) self._defineSourceRelation(self.inputParticles, outputSet)
class SpiderProtClassifyCluster(SpiderProtClassify): """ Base for Clustering Spider classification protocols. """ def __init__(self, script, classDir, **kwargs): SpiderProtClassify.__init__(self, script, classDir, **kwargs) # --------------------------- STEPS functions ----------------------------- def createOutputStep(self): self.buildDendrogram(True) # --------------------------- UTILS functions ----------------------------- def _fillClassesFromNodes(self, classes2D, nodeList): """ Create the SetOfClasses2D from the images of each node in the dendrogram. """ particles = classes2D.getImages() sampling = classes2D.getSamplingRate() # We need to first create a map between the particles index and # the assigned class number classDict = {} nodeDict = {} classCount = 0 for node in nodeList: if node.path: classCount += 1 node.classId = classCount nodeDict[classCount] = node for i in node.imageList: classDict[int(i)] = classCount def updateItem(p, item): classId = classDict.get(item, None) if classId is None: p._appendItem = False else: p.setClassId(classId) def updateClass(cls): node = nodeDict[cls.getObjId()] rep = cls.getRepresentative() rep.setSamplingRate(sampling) rep.setLocation(node.avgCount, self.dendroAverages) particlesRange = range(1, particles.getSize() + 1) classes2D.classifyItems(updateItemCallback=updateItem, updateClassCallback=updateClass, itemDataIterator=iter(particlesRange)) def _fillParticlesFromNodes(self, inputParts, outputParts, nodeList): """ Create the SetOfClasses2D from the images of each node in the dendrogram. """ allImages = set() for node in nodeList: if node.path: for i in node.imageList: allImages.add(i) def updateItem(item, index): item._appendItem = index in allImages particlesRange = range(1, inputParts.getSize() + 1) outputParts.copyItems(inputParts, updateItemCallback=updateItem, itemDataIterator=iter(particlesRange)) def buildDendrogram(self, writeAverages=False): """ Parse Spider docfile with the information to build the dendrogram. Params: writeAverages: whether to write class averages or not. """ dendroFile = self._getFileName('dendroDoc') # Dendrofile is a docfile with at least 3 data columns (class, height, id) doc = SpiderDocFile(dendroFile) values = [] indexes = [] for _, h, i in doc.iterValues(): indexes.append(i) values.append(h) doc.close() self.dendroValues = values self.dendroIndexes = indexes self.dendroImages = self._getFileName('particles') self.dendroAverages = self._getFileName('averages') self.dendroAverageCount = 0 # Write only the number of needed averages self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels self.ih = ImageHandler() return self._buildDendrogram(0, len(values) - 1, 1, writeAverages) def getImage(self, particleNumber): return self.ih.read((int(particleNumber), self.dendroImages)) def addChildNode(self, node, leftIndex, rightIndex, index, writeAverages, level, searchStop): child = self._buildDendrogram(leftIndex, rightIndex, index, writeAverages, level + 1, searchStop) node.addChild(child) node.extendImageList(child.imageList) if writeAverages: node.addImage(child.image) del child.image # Allow to free child image memory def _buildDendrogram(self, leftIndex, rightIndex, index, writeAverages=False, level=0, searchStop=0): """ This function is recursively called to create the dendrogram graph (binary tree) and also to write the average image files. Params: leftIndex, rightIndex: the indexes within the list where to search. index: the index of the class average. writeAverages: flag to select when to write averages searchStop: this could be 1, means that we will search until the last element (used for right childs of the dendrogram or, can be 0, meaning that the last element was already the max (used for left childs ) From self: self.dendroValues: the list with the heights of each node self.dendroImages: image stack filename to read particles self.dendroAverages: stack name where to write averages It will search for the max in values list (between minIndex and maxIndex). Nodes to the left of the max are left childs and the other right childs. """ if level < self.dendroMaxLevel: avgCount = self.dendroAverageCount + 1 self.dendroAverageCount += 1 if rightIndex == leftIndex: # Just only one element height = self.dendroValues[leftIndex] node = DendroNode(index, height) node.extendImageList([self.dendroIndexes[leftIndex]]) node.addImage(self.getImage(node.imageList[0])) elif rightIndex == leftIndex + 1: # Two elements height = max(self.dendroValues[leftIndex], self.dendroValues[rightIndex]) node = DendroNode(index, height) node.extendImageList([ self.dendroIndexes[leftIndex], self.dendroIndexes[rightIndex] ]) node.addImage(self.getImage(node.imageList[0]), self.getImage(node.imageList[1])) else: # 3 or more elements # Find the max value (or height) of the elements maxValue = self.dendroValues[leftIndex] maxIndex = 0 # searchStop could be 0 (do not consider last element, coming from # left child, or 1 (consider also the last one, coming from right) values = self.dendroValues[leftIndex + 1:rightIndex + searchStop] for i, v in enumerate(values): if v > maxValue: maxValue = v maxIndex = i + 1 m = maxIndex + leftIndex node = DendroNode(index, maxValue) hasRightChild = m < rightIndex if maxValue > 0: nextIndex = 2 * index if hasRightChild else index self.addChildNode(node, leftIndex, m, nextIndex, writeAverages, level, 0) if hasRightChild: self.addChildNode(node, m + 1, rightIndex, 2 * index + 1, writeAverages, level, 1) else: # If the node has a single child, we will remove a node # just to advance in the level of the tree to get more # different class averages if node.getChilds(): child = node.getChilds()[0] child.image = node.image child.parents = [] node = child else: node.extendImageList(self.dendroIndexes[leftIndex:rightIndex + 1]) node.addImage(*[self.getImage(img) for img in node.imageList]) if level < self.dendroMaxLevel: node.avgCount = avgCount node.path = '%d@%s' % (node.avgCount, self.dendroAverages) if writeAverages: # normalize the sum of images depending on the number of particles # assigned to this classes # avgImage = node.image / float(node.getSize()) node.image.inplaceDivide(float(node.getSize())) self.ih.write(node.image, (node.avgCount, self.dendroAverages)) fn = self._getTmpPath('doc_class%03d.stk' % index) doc = SpiderDocFile(fn, 'w+') for i in node.imageList: doc.writeValues(i) doc.close() return node
def convertInputAndSaveToDBStep(self): """ init database to store the last name of the files used and convert 3D maps to MRC '.mrc' format. This step is run once even if the protocol is relunched """ databasePath = self._getExtraPath(OUTPUTDATABASENAMESWITHLABELS) # create database and table # this table will be used to record the last version of any file # save in coot conn = sqlite3.connect(databasePath) # create table # saved = 0, means this file need to be converted to a scipion object # type = 0-> Map, 1 -> atom struct # predefined macros for type: # TYPE_3DMAP = 0 # TYPE_ATOMSTRUCT = 1 sqlCommand = """create table if not exists %s (id integer primary key AUTOINCREMENT, modelId integer, fileName text, labelName text, type int, saved integer default 1 )""" % (DATABASETABLENAME) conn.execute(sqlCommand) # create view to retrieve the id of the last copy # for each model id (coot call imol to this id) sqlCommand = """CREATE VIEW lastid AS SELECT modelId, max(id) as id FROM %s GROUP BY modelId """ % DATABASETABLENAME conn.execute(sqlCommand) inVolumes, norVolumesNames = self._getVolumesList() sqlCommand = """INSERT INTO %s (modelId, fileName, labelName, type, saved) values (%d, '%s', '%s', %d, %d)""" #process main atomic Structure counter = 0 pdbFileToBeRefined = self.pdbFileToBeRefined.get().getFileName() base = os.path.basename(pdbFileToBeRefined) conn.execute(sqlCommand % ( DATABASETABLENAME, counter, pdbFileToBeRefined, os.path.splitext(base)[0], TYPE_ATOMSTRUCT, 1 # saved )) counter += 1 # Process another atom structures for pdb in self.inputPdbFiles: fileName = pdb.get().getFileName() base = os.path.basename(fileName) conn.execute(sqlCommand % ( DATABASETABLENAME, counter, fileName, os.path.splitext(base)[0], TYPE_ATOMSTRUCT, 1 # saved )) counter += 1 # Process 3D maps # normalize them if needed ih = ImageHandler() for inVol, norVolName in zip(inVolumes, norVolumesNames): inVolName = inVol.getFileName() if inVolName.endswith(".mrc"): inVolName += ":mrc" if norVolName.endswith(".mrc"): norVolName += ":mrc" if not ih.existsLocation(norVolName): if True: # self.doNormalize: img = ImageHandler()._img img.read(inVolName) mean, dev, min, max = img.computeStats() img.inplaceMultiply(1. / max) img.write(norVolName) else: ImageHandler().convert(inVolName, norVolName) Ccp4Header(norVolName, readHeader=True).copyCCP4Header( inVol.getOrigin(force=True).getShifts(), inVol.getSamplingRate(), originField=Ccp4Header.START) conn.execute(sqlCommand % ( DATABASETABLENAME, counter, norVolName[:-4], os.path.basename(norVolName)[:-8], TYPE_3DMAP, 0 # saved )) counter += 1 conn.commit()