def composePSD(self, psd1, psd2, outputFn, outputFnUncorrected=None, outputFnCorrected=None): """ Compose a single PSD image: left part from psd1 (uncorrected PSD), right-part from psd2 (corrected PSD) """ ih = ImageHandler() psdImg1 = ih.read(psd1) data1 = psdImg1.getData() if outputFnUncorrected is not None: psdImg1.convertPSD() psdImg1.write(outputFnUncorrected) psdImg2 = ih.read(psd2) data2 = psdImg2.getData() if outputFnCorrected is not None: psdImg2.convertPSD() psdImg2.write(outputFnCorrected) # Compute middle index x, _, _, _ = psdImg1.getDimensions() m = int(round(x / 2.)) data1[:, :m] = data2[:, :m] psdImg1.setData(data1) psdImg1.write(outputFn)
def composePSD(self, psd1, psd2, outputFn): """ Compose a single PSD image: left part from psd1 (corrected PSD), right-part from psd2 (uncorrected PSD) """ ih = ImageHandler() psd = ih.read(psd1) data1 = psd.getData() data2 = ih.read(psd2).getData() # Compute middle index x, _, _, _ = psd.getDimensions() m = int(round(x / 2.)) data1[:, m:] = data2[:, m:] psd.setData(data1) psd.write(outputFn)
def _buildDendrogram(self, leftIndex, rightIndex, index, writeAverages=False, level=0): """ This function is recursively called to create the dendogram graph(binary tree) and also to write the average image files. Params: leftIndex, rightIndex: the indinxes within the list where to search. index: the index of the class average. writeImages: flag to select when to write averages. From self: self.dendroValues: the list with the heights of each node self.dendroImages: image stack filename to read particles self.dendroAverages: stack name where to write averages It will search for the max in values list (between minIndex and maxIndex). Nodes to the left of the max are left childs and the other right childs. """ maxValue = self.dendroValues[leftIndex] maxIndex = 0 for i, v in enumerate(self.dendroValues[leftIndex+1:rightIndex]): if v > maxValue: maxValue = v maxIndex = i+1 m = maxIndex + leftIndex node = DendroNode(index, maxValue) ih = ImageHandler() particleNumber = self.dendroIndexes[m+1] node.imageList = [particleNumber] if writeAverages: node.image = ih.read((particleNumber, self.dendroImages)) def addChildNode(left, right, index): if right > left: child = self._buildDendrogram(left, right, index, writeAverages, level+1) node.addChild(child) node.length += child.length node.imageList += child.imageList if writeAverages: node.image += child.image del child.image # Allow to free child image memory if rightIndex > leftIndex + 1 and level < self.dendroMaxLevel: addChildNode(leftIndex, m, 2*index) addChildNode(m+1, rightIndex, 2*index+1) node.avgCount = self.dendroAverageCount + 1 self.dendroAverageCount += 1 node.path = '%d@%s' % (node.avgCount, self.dendroAverages) if writeAverages: #TODO: node['image'] /= float(node['length']) #node.image.inplaceDivide(float(node.length)) #FIXME: not working, noisy images avgImage = node.image / float(node.length) ih.write(avgImage, (node.avgCount, self.dendroAverages)) fn = self._getTmpPath('doc_class%03d.stk' % index) doc = SpiderDocFile(fn, 'w+') for i in node.imageList: doc.writeValues(i) doc.close() return node
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from self.thumbPaths. ===== Params ===== - firstThumbIndex: index from which we start generating thumbnails - micScaleFactor: how much to reduce in size the micrographs. """ ih = ImageHandler() numMics = len(self.thumbPaths[MIC_PATH]) for i in range(firstThumbIndex, numMics): print('Generating images for mic %d' % (i + 1)) # mic thumbnails dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) if not exists(dstImgPath): if self.micThumbSymlinks: pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i], dstImgPath) else: ih.computeThumbnail(self.thumbPaths[MIC_PATH][i], dstImgPath, scaleFactor=micScaleFactor) # shift plots if SHIFT_THUMBS in self.thumbPaths: dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i]) if not exists(dstImgPath): pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i], dstImgPath) # Psd thumbnails if self.ctfProtocol is None: srcImgPath = self.thumbPaths[PSD_PATH][i] dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath): if srcImgPath.endswith('psd'): psdImg1 = ih.read(srcImgPath) psdImg1.convertPSD() psdImg1.write(dstImgPath) ih.computeThumbnail(dstImgPath, dstImgPath, scaleFactor=1) else: pwutils.createAbsLink(srcImgPath, dstImgPath) else: dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath): ih.computeThumbnail(self.thumbPaths[PSD_PATH][i], dstImgPath, scaleFactor=1) return
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from self.thumbPaths. ===== Params ===== - firstThumbIndex: index from which we start generating thumbnails - micScaleFactor: how much to reduce in size the micrographs. """ ih = ImageHandler() numMics = len(self.thumbPaths[MIC_PATH]) for i in range(firstThumbIndex, numMics): print('Generating images for mic %d' % (i+1)) # mic thumbnails dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) if not exists(dstImgPath): if self.micThumbSymlinks: pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i], dstImgPath) else: ih.computeThumbnail(self.thumbPaths[MIC_PATH][i], dstImgPath, scaleFactor=micScaleFactor, flipOnY=True) # shift plots if SHIFT_THUMBS in self.thumbPaths: dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i]) if not exists(dstImgPath): pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i], dstImgPath) # Psd thumbnails # If there ARE thumbnail for the PSD (no ctf protocol and # moviealignment hasn't computed it if PSD_THUMBS in self.thumbPaths: if self.ctfProtocol is None: srcImgPath = self.thumbPaths[PSD_PATH][i] dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath) and srcImgPath is not None: if srcImgPath.endswith('psd'): psdImg1 = ih.read(srcImgPath) psdImg1.convertPSD() psdImg1.write(dstImgPath) ih.computeThumbnail(dstImgPath, dstImgPath, scaleFactor=1, flipOnY=True) else: pwutils.createAbsLink(srcImgPath, dstImgPath) else: dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath): ih.computeThumbnail(self.thumbPaths[PSD_PATH][i], dstImgPath, scaleFactor=1, flipOnY=True) return
def _computeRightPreview(self): """ This function should compute the right preview using the self.lastObj that was selected """ from pyworkflow.em.packages.xmipp3 import locationToXmipp # Copy image to filter to Tmp project folder outputName = os.path.join("Tmp", "filtered_particle") outputPath = outputName + ".spi" cleanPath(outputPath) outputLoc = (1, outputPath) ih = ImageHandler() ih.convert(self.lastObj.getLocation(), outputLoc) outputLocSpiStr = locationToSpider(1, outputName) pars = {} pars["filterType"] = self.protocolParent.filterType.get() pars["filterMode"] = self.protocolParent.filterMode.get() pars["usePadding"] = self.protocolParent.usePadding.get() pars["op"] = "FQ" if self.protocolParent.filterType <= FILTER_SPACE_REAL: pars['filterRadius'] = self.getRadius() else: pars['lowFreq'] = self.getLowFreq() pars['highFreq'] = self.getHighFreq() if self.protocolParent.filterType == FILTER_FERMI: pars['temperature'] = self.getTemperature() filter_spider(outputLocSpiStr, outputLocSpiStr, **pars) # Get output image and update filtered image img = ImageHandler()._img locXmippStr = locationToXmipp(1, outputPath) img.read(locXmippStr) self.rightImage = img self.updateFilteredImage()
def _computeRightPreview(self): """ This function should compute the right preview using the self.lastObj that was selected """ from pyworkflow.em.packages.xmipp3 import locationToXmipp # Copy image to filter to Tmp project folder outputName = os.path.join("Tmp", "filtered_particle") outputPath = outputName + ".spi" cleanPath(outputPath) outputLoc = (1, outputPath) ih = ImageHandler() ih.convert(self.lastObj.getLocation(), outputLoc) outputLocSpiStr = locationToSpider(1, outputName) pars = {} pars["filterType"] = self.protocolParent.filterType.get() pars["filterMode"] = self.protocolParent.filterMode.get() pars["usePadding"] = self.protocolParent.usePadding.get() pars["op"] = "FQ" if self.protocolParent.filterType <= FILTER_FERMI: pars['filterRadius'] = self.getRadius() else: pars['lowFreq'] = self.getLowFreq() pars['highFreq'] = self.getHighFreq() if self.protocolParent.filterType == FILTER_FERMI: pars['temperature'] = self.getTemperature() filter_spider(outputLocSpiStr, outputLocSpiStr, **pars) # Get output image and update filtered image img = ImageHandler()._img locXmippStr = locationToXmipp(1, outputPath) img.read(locXmippStr) self.rightImage = img self.updateFilteredImage()
def createOutputStep(self): # Really load the input, since in the streaming case we can not # use the self.inputMovies directly allFramesSum = self._getPath('all_frames_sum.mrc') allFramesAvg = self._getPath('all_frames_avg.mrc') self._loadInputList() n = len(self.listOfMovies) ih = ImageHandler() sumImg = ih.read(allFramesSum) sumImg.inplaceDivide(float(n)) sumImg.write(allFramesAvg) outputAvg = Image() outputAvg.setFileName(allFramesAvg) outputAvg.setSamplingRate(self.listOfMovies[0].getSamplingRate()) self._defineOutputs(outputAverage=outputAvg) self._defineSourceRelation(self.inputMovies, outputAvg)
class SpiderProtClassifyCluster(SpiderProtClassify): """ Base for Clustering Spider classification protocols. """ def __init__(self, script, classDir, **kwargs): SpiderProtClassify.__init__(self, script, classDir, **kwargs) #--------------------------- STEPS functions ------------------------------ def createOutputStep(self): self.buildDendrogram(True) #--------------------------- UTILS functions ------------------------------ def _fillClassesFromNodes(self, classes2D, nodeList): """ Create the SetOfClasses2D from the images of each node in the dendrogram. """ particles = classes2D.getImages() sampling = classes2D.getSamplingRate() # We need to first create a map between the particles index and # the assigned class number classDict = {} nodeDict = {} classCount = 0 for node in nodeList: if node.path: classCount += 1 node.classId = classCount nodeDict[classCount] = node for i in node.imageList: classDict[int(i)] = classCount def updateItem(p, i): classId = classDict.get(i, None) if classId is None: p._appendItem = False else: p.setClassId(classId) def updateClass(cls): node = nodeDict[cls.getObjId()] rep = cls.getRepresentative() rep.setSamplingRate(sampling) rep.setLocation(node.avgCount, self.dendroAverages) particlesRange = range(1, particles.getSize()+1) classes2D.classifyItems(updateItemCallback=updateItem, updateClassCallback=updateClass, itemDataIterator=iter(particlesRange)) def _fillParticlesFromNodes(self, inputParts, outputParts, nodeList): """ Create the SetOfClasses2D from the images of each node in the dendrogram. """ allImages = set() for node in nodeList: if node.path: for i in node.imageList: allImages.add(i) def updateItem(item, index): item._appendItem = index in allImages particlesRange = range(1, inputParts.getSize()+1) outputParts.copyItems(inputParts, updateItemCallback=updateItem, itemDataIterator=iter(particlesRange)) def buildDendrogram(self, writeAverages=False): """ Parse Spider docfile with the information to build the dendrogram. Params: writeAverages: whether to write class averages or not. """ dendroFile = self._getFileName('dendroDoc') # Dendrofile is a docfile with at least 3 data colums (class, height, id) doc = SpiderDocFile(dendroFile) values = [] indexes = [] for _, h, i in doc.iterValues(): indexes.append(i) values.append(h) doc.close() self.dendroValues = values self.dendroIndexes = indexes self.dendroImages = self._getFileName('particles') self.dendroAverages = self._getFileName('averages') self.dendroAverageCount = 0 # Write only the number of needed averages self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels self.ih = ImageHandler() return self._buildDendrogram(0, len(values)-1, 1, writeAverages) def getImage(self, particleNumber): return self.ih.read((particleNumber, self.dendroImages)) def addChildNode(self, node, leftIndex, rightIndex, index, writeAverages, level, searchStop): child = self._buildDendrogram(leftIndex, rightIndex, index, writeAverages, level+1, searchStop) node.addChild(child) node.extendImageList(child.imageList) if writeAverages: node.addImage(child.image) del child.image # Allow to free child image memory def _buildDendrogram(self, leftIndex, rightIndex, index, writeAverages=False, level=0, searchStop=0): """ This function is recursively called to create the dendrogram graph (binary tree) and also to write the average image files. Params: leftIndex, rightIndex: the indexes within the list where to search. index: the index of the class average. writeAverages: flag to select when to write averages searchStop: this could be 1, means that we will search until the last element (used for right childs of the dendrogram or, can be 0, meaning that the last element was already the max (used for left childs ) From self: self.dendroValues: the list with the heights of each node self.dendroImages: image stack filename to read particles self.dendroAverages: stack name where to write averages It will search for the max in values list (between minIndex and maxIndex). Nodes to the left of the max are left childs and the other right childs. """ if level < self.dendroMaxLevel: avgCount = self.dendroAverageCount + 1 self.dendroAverageCount += 1 if rightIndex == leftIndex: # Just only one element height = self.dendroValues[leftIndex] node = DendroNode(index, height) node.extendImageList([self.dendroIndexes[leftIndex]]) node.addImage(self.getImage(node.imageList[0])) elif rightIndex == leftIndex + 1: # Two elements height = max(self.dendroValues[leftIndex], self.dendroValues[rightIndex]) node = DendroNode(index, height) node.extendImageList([self.dendroIndexes[leftIndex], self.dendroIndexes[rightIndex]]) node.addImage(self.getImage(node.imageList[0]), self.getImage(node.imageList[1])) else: # 3 or more elements # Find the max value (or height) of the elements maxValue = self.dendroValues[leftIndex] maxIndex = 0 # searchStop could be 0 (do not consider last element, coming from # left child, or 1 (consider also the last one, coming from right) values = self.dendroValues[leftIndex+1:rightIndex+searchStop] for i, v in enumerate(values): if v > maxValue: maxValue = v maxIndex = i+1 m = maxIndex + leftIndex node = DendroNode(index, maxValue) hasRightChild = m < rightIndex if maxValue > 0: nextIndex = 2 * index if hasRightChild else index self.addChildNode(node, leftIndex, m, nextIndex, writeAverages, level, 0) if hasRightChild: self.addChildNode(node, m+1, rightIndex, 2 * index + 1, writeAverages, level, 1) else: # If the node has a single child, we will remove a node # just to advance in the level of the tree to get more # different class averages if node.getChilds(): child = node.getChilds()[0] child.image = node.image child.parents = [] node = child else: node.extendImageList(self.dendroIndexes[leftIndex:rightIndex+1]) node.addImage(*[self.getImage(img) for img in node.imageList]) if level < self.dendroMaxLevel: node.avgCount = avgCount node.path = '%d@%s' % (node.avgCount, self.dendroAverages) if writeAverages: # normalize the sum of images depending on the number of particles # assigned to this classes avgImage = node.image / float(node.getSize()) self.ih.write(avgImage, (node.avgCount, self.dendroAverages)) fn = self._getTmpPath('doc_class%03d.stk' % index) doc = SpiderDocFile(fn, 'w+') for i in node.imageList: doc.writeValues(i) doc.close() return node
class SpiderProtClassifyCluster(SpiderProtClassify): """ Base for Clustering Spider classification protocols. """ def __init__(self, script, classDir, **kwargs): SpiderProtClassify.__init__(self, script, classDir, **kwargs) #--------------------------- STEPS functions -------------------------------------------- def createOutputStep(self): self.buildDendrogram(True) #--------------------------- UTILS functions -------------------------------------------- def _fillClassesFromNodes(self, classes, nodeList): """ Create the SetOfClasses2D from the images of each node in the dendogram. """ img = Particle() sampling = classes.getSamplingRate() for node in nodeList: if node.path: #print "node.path: ", node.path class2D = Class2D() avg = Particle() #avg.copyObjId(class2D) avg.setLocation(node.avgCount, self.dendroAverages) avg.setSamplingRate(sampling) class2D.setRepresentative(avg) class2D.setSamplingRate(sampling) classes.append(class2D) #print "class2D.id: ", class2D.getObjId() for i in node.imageList: #img.setObjId(i) # FIXME: this is wrong if the id is different from index img.cleanObjId() img.setLocation(int(i), self.dendroImages) class2D.append(img) classes.update(class2D) def _fillParticlesFromNodes(self, particles, nodeList): """ Create the SetOfClasses2D from the images of each node in the dendogram. """ img = Particle() for node in nodeList: if node.path: for i in node.imageList: #img.setObjId(i) # FIXME: this is wrong if the id is different from index img.cleanObjId() img.setLocation(int(i), self.dendroImages) particles.append(img) def buildDendrogram(self, writeAverages=False): """ Parse Spider docfile with the information to build the dendogram. Params: dendroFile: docfile with a row per image. Each row contains the image id and the height. """ dendroFile = self._getFileName('dendroDoc') # Dendrofile is a docfile with at least 3 data colums (class, height, id) doc = SpiderDocFile(dendroFile) values = [] indexes = [] for c, h, _ in doc.iterValues(): indexes.append(c) values.append(h) doc.close() self.dendroValues = values self.dendroIndexes = indexes self.dendroImages = self._getFileName('particles') self.dendroAverages = self._getFileName('averages') self.dendroAverageCount = 0 # Write only the number of needed averages self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels self.ih = ImageHandler() return self._buildDendrogram(0, len(values)-1, 1, writeAverages) def getImage(self, particleNumber): return self.ih.read((particleNumber, self.dendroImages)) def addChildNode(self, node, leftIndex, rightIndex, index, writeAverages, level): child = self._buildDendrogram(leftIndex, rightIndex, index, writeAverages, level+1) node.addChild(child) node.length += child.length node.imageList += child.imageList if writeAverages: if node.image is None: node.image = child.image else: node.image += child.image del child.image # Allow to free child image memory def _buildDendrogram(self, leftIndex, rightIndex, index, writeAverages=False, level=0): """ This function is recursively called to create the dendogram graph(binary tree) and also to write the average image files. Params: leftIndex, rightIndex: the indinxes within the list where to search. index: the index of the class average. writeImages: flag to select when to write averages. From self: self.dendroValues: the list with the heights of each node self.dendroImages: image stack filename to read particles self.dendroAverages: stack name where to write averages It will search for the max in values list (between minIndex and maxIndex). Nodes to the left of the max are left childs and the other right childs. """ if level < self.dendroMaxLevel: avgCount = self.dendroAverageCount + 1 self.dendroAverageCount += 1 if rightIndex == leftIndex: # Just only one element height = self.dendroValues[leftIndex] node = DendroNode(index, height) node.imageList = [self.dendroIndexes[leftIndex]] node.image = self.getImage(node.imageList[0]) node.length = 1 elif rightIndex == leftIndex + 1: # Two elements height = max(self.dendroValues[leftIndex], self.dendroValues[rightIndex]) node = DendroNode(index, height) node.imageList = [self.dendroIndexes[leftIndex], self.dendroIndexes[rightIndex]] node.image = self.getImage(node.imageList[0]) + self.getImage(node.imageList[1]) node.length = 2 else: # 3 or more elements # Find the max value (or height) of the elements maxValue = self.dendroValues[leftIndex] maxIndex = 0 for i, v in enumerate(self.dendroValues[leftIndex+1:rightIndex]): if v > maxValue: maxValue = v maxIndex = i+1 m = maxIndex + leftIndex node = DendroNode(index, maxValue) self.addChildNode(node, leftIndex, m, 2*index, writeAverages, level) self.addChildNode(node, m+1, rightIndex, 2*index+1, writeAverages, level) if level < self.dendroMaxLevel: node.avgCount = avgCount node.path = '%d@%s' % (node.avgCount, self.dendroAverages) if writeAverages: # normalize the sum of images depending on the number of particles # assigned to this classes avgImage = node.image / float(node.getSize()) self.ih.write(avgImage, (node.avgCount, self.dendroAverages)) fn = self._getTmpPath('doc_class%03d.stk' % index) doc = SpiderDocFile(fn, 'w+') for i in node.imageList: doc.writeValues(i) doc.close() return node
def createOutputStep(self): ih = ImageHandler() outputStack = self._getPath('particles.mrcs') outputImg = ih.createImage() inputParticles = self.inputParticles.get() inputCoords = self.inputCoordinates.get() outputSet = self._createSetOfParticles() outputSet.copyInfo(inputParticles) boxSize = self.boxSize.get() b2 = int(round(boxSize / 2)) center = np.zeros((boxSize, boxSize)) ih = ImageHandler() i = 0 outliers = 0 partIdExcluded = [] lastPartId = None for coord in inputCoords.iterItems( orderBy=['_subparticle._micId', '_micId', 'id']): # The original particle id is stored in the sub-particle as micId partId = coord._micId.get() # Load the particle if it has changed from the last sub-particle if partId != lastPartId: particle = inputParticles[partId] if particle is None: partIdExcluded.append(partId) self.info("WARNING: Missing particle with id %s from " "input particles set" % partId) else: # Now load the particle image to extract later sub-particles img = ih.read(particle) x, y, _, _ = img.getDimensions() data = img.getData() lastPartId = partId # If particle is not in inputParticles, subparticles will not be # generated. Now, subtract from a subset of original particles is # supported. if not partId in partIdExcluded: xpos = coord.getX() ypos = coord.getY() # Check that the sub-particle will not lay out of the particle if (ypos - b2 < 0 or ypos + b2 > y or xpos - b2 < 0 or xpos + b2 > x): outliers += 1 continue # Crop the sub-particle data from the whole particle image center[:, :] = data[ypos - b2:ypos + b2, xpos - b2:xpos + b2] outputImg.setData(center) i += 1 outputImg.write((i, outputStack)) subpart = coord._subparticle subpart.setLocation( (i, outputStack)) # Change path to new stack subpart.setObjId(None) # Force to insert as a new item outputSet.append(subpart) if outliers: self.info( "WARNING: Discarded %s particles because laid out of the " "particle (for a box size of %d" % (outliers, boxSize)) self._defineOutputs(outputParticles=outputSet) self._defineSourceRelation(self.inputParticles, outputSet)