def composePSD(self, psd1, psd2, outputFn,
outputFnUncorrected=None, outputFnCorrected=None):
""" Compose a single PSD image:
left part from psd1 (uncorrected PSD),
right-part from psd2 (corrected PSD)
"""
ih = ImageHandler()
psdImg1 = ih.read(psd1)
data1 = psdImg1.getData()
if outputFnUncorrected is not None:
psdImg1.convertPSD()
psdImg1.write(outputFnUncorrected)
psdImg2 = ih.read(psd2)
data2 = psdImg2.getData()
if outputFnCorrected is not None:
psdImg2.convertPSD()
psdImg2.write(outputFnCorrected)
# Compute middle index
x, _, _, _ = psdImg1.getDimensions()
m = int(round(x / 2.))
data1[:, :m] = data2[:, :m]
psdImg1.setData(data1)
psdImg1.write(outputFn)
def composePSD(self, psd1, psd2, outputFn):
""" Compose a single PSD image:
left part from psd1 (corrected PSD),
right-part from psd2 (uncorrected PSD)
"""
ih = ImageHandler()
psd = ih.read(psd1)
data1 = psd.getData()
data2 = ih.read(psd2).getData()
# Compute middle index
x, _, _, _ = psd.getDimensions()
m = int(round(x / 2.))
data1[:, m:] = data2[:, m:]
psd.setData(data1)
psd.write(outputFn)
def _buildDendrogram(self, leftIndex, rightIndex, index, writeAverages=False, level=0):
""" This function is recursively called to create the dendogram graph(binary tree)
and also to write the average image files.
Params:
leftIndex, rightIndex: the indinxes within the list where to search.
index: the index of the class average.
writeImages: flag to select when to write averages.
From self:
self.dendroValues: the list with the heights of each node
self.dendroImages: image stack filename to read particles
self.dendroAverages: stack name where to write averages
It will search for the max in values list (between minIndex and maxIndex).
Nodes to the left of the max are left childs and the other right childs.
"""
maxValue = self.dendroValues[leftIndex]
maxIndex = 0
for i, v in enumerate(self.dendroValues[leftIndex+1:rightIndex]):
if v > maxValue:
maxValue = v
maxIndex = i+1
m = maxIndex + leftIndex
node = DendroNode(index, maxValue)
ih = ImageHandler()
particleNumber = self.dendroIndexes[m+1]
node.imageList = [particleNumber]
if writeAverages:
node.image = ih.read((particleNumber, self.dendroImages))
def addChildNode(left, right, index):
if right > left:
child = self._buildDendrogram(left, right, index, writeAverages, level+1)
node.addChild(child)
node.length += child.length
node.imageList += child.imageList
if writeAverages:
node.image += child.image
del child.image # Allow to free child image memory
if rightIndex > leftIndex + 1 and level < self.dendroMaxLevel:
addChildNode(leftIndex, m, 2*index)
addChildNode(m+1, rightIndex, 2*index+1)
node.avgCount = self.dendroAverageCount + 1
self.dendroAverageCount += 1
node.path = '%d@%s' % (node.avgCount, self.dendroAverages)
if writeAverages:
#TODO: node['image'] /= float(node['length'])
#node.image.inplaceDivide(float(node.length)) #FIXME: not working, noisy images
avgImage = node.image / float(node.length)
ih.write(avgImage, (node.avgCount, self.dendroAverages))
fn = self._getTmpPath('doc_class%03d.stk' % index)
doc = SpiderDocFile(fn, 'w+')
for i in node.imageList:
doc.writeValues(i)
doc.close()
return node
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
""" Function to generate thumbnails for the report. Uses data from
self.thumbPaths.
===== Params =====
- firstThumbIndex: index from which we start generating thumbnails
- micScaleFactor: how much to reduce in size the micrographs.
"""
ih = ImageHandler()
numMics = len(self.thumbPaths[MIC_PATH])
for i in range(firstThumbIndex, numMics):
print('Generating images for mic %d' % (i + 1))
# mic thumbnails
dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
if not exists(dstImgPath):
if self.micThumbSymlinks:
pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i],
dstImgPath)
else:
ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
dstImgPath,
scaleFactor=micScaleFactor)
# shift plots
if SHIFT_THUMBS in self.thumbPaths:
dstImgPath = join(self.reportDir,
self.thumbPaths[SHIFT_THUMBS][i])
if not exists(dstImgPath):
pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i],
dstImgPath)
# Psd thumbnails
if self.ctfProtocol is None:
srcImgPath = self.thumbPaths[PSD_PATH][i]
dstImgPath = join(self.reportDir,
self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
if srcImgPath.endswith('psd'):
psdImg1 = ih.read(srcImgPath)
psdImg1.convertPSD()
psdImg1.write(dstImgPath)
ih.computeThumbnail(dstImgPath,
dstImgPath,
scaleFactor=1)
else:
pwutils.createAbsLink(srcImgPath, dstImgPath)
else:
dstImgPath = join(self.reportDir,
self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
dstImgPath,
scaleFactor=1)
return
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
""" Function to generate thumbnails for the report. Uses data from
self.thumbPaths.
===== Params =====
- firstThumbIndex: index from which we start generating thumbnails
- micScaleFactor: how much to reduce in size the micrographs.
"""
ih = ImageHandler()
numMics = len(self.thumbPaths[MIC_PATH])
for i in range(firstThumbIndex, numMics):
print('Generating images for mic %d' % (i+1))
# mic thumbnails
dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
if not exists(dstImgPath):
if self.micThumbSymlinks:
pwutils.createAbsLink(self.thumbPaths[MIC_PATH][i], dstImgPath)
else:
ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
dstImgPath, scaleFactor=micScaleFactor,
flipOnY=True)
# shift plots
if SHIFT_THUMBS in self.thumbPaths:
dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i])
if not exists(dstImgPath):
pwutils.createAbsLink(self.thumbPaths[SHIFT_PATH][i], dstImgPath)
# Psd thumbnails
# If there ARE thumbnail for the PSD (no ctf protocol and
# moviealignment hasn't computed it
if PSD_THUMBS in self.thumbPaths:
if self.ctfProtocol is None:
srcImgPath = self.thumbPaths[PSD_PATH][i]
dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath) and srcImgPath is not None:
if srcImgPath.endswith('psd'):
psdImg1 = ih.read(srcImgPath)
psdImg1.convertPSD()
psdImg1.write(dstImgPath)
ih.computeThumbnail(dstImgPath, dstImgPath,
scaleFactor=1, flipOnY=True)
else:
pwutils.createAbsLink(srcImgPath, dstImgPath)
else:
dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
dstImgPath, scaleFactor=1, flipOnY=True)
return
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
from pyworkflow.em.packages.xmipp3 import locationToXmipp
# Copy image to filter to Tmp project folder
outputName = os.path.join("Tmp", "filtered_particle")
outputPath = outputName + ".spi"
cleanPath(outputPath)
outputLoc = (1, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), outputLoc)
outputLocSpiStr = locationToSpider(1, outputName)
pars = {}
pars["filterType"] = self.protocolParent.filterType.get()
pars["filterMode"] = self.protocolParent.filterMode.get()
pars["usePadding"] = self.protocolParent.usePadding.get()
pars["op"] = "FQ"
if self.protocolParent.filterType <= FILTER_SPACE_REAL:
pars['filterRadius'] = self.getRadius()
else:
pars['lowFreq'] = self.getLowFreq()
pars['highFreq'] = self.getHighFreq()
if self.protocolParent.filterType == FILTER_FERMI:
pars['temperature'] = self.getTemperature()
filter_spider(outputLocSpiStr, outputLocSpiStr, **pars)
# Get output image and update filtered image
img = ImageHandler()._img
locXmippStr = locationToXmipp(1, outputPath)
img.read(locXmippStr)
self.rightImage = img
self.updateFilteredImage()
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
from pyworkflow.em.packages.xmipp3 import locationToXmipp
# Copy image to filter to Tmp project folder
outputName = os.path.join("Tmp", "filtered_particle")
outputPath = outputName + ".spi"
cleanPath(outputPath)
outputLoc = (1, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), outputLoc)
outputLocSpiStr = locationToSpider(1, outputName)
pars = {}
pars["filterType"] = self.protocolParent.filterType.get()
pars["filterMode"] = self.protocolParent.filterMode.get()
pars["usePadding"] = self.protocolParent.usePadding.get()
pars["op"] = "FQ"
if self.protocolParent.filterType <= FILTER_FERMI:
pars['filterRadius'] = self.getRadius()
else:
pars['lowFreq'] = self.getLowFreq()
pars['highFreq'] = self.getHighFreq()
if self.protocolParent.filterType == FILTER_FERMI:
pars['temperature'] = self.getTemperature()
filter_spider(outputLocSpiStr, outputLocSpiStr, **pars)
# Get output image and update filtered image
img = ImageHandler()._img
locXmippStr = locationToXmipp(1, outputPath)
img.read(locXmippStr)
self.rightImage = img
self.updateFilteredImage()
def createOutputStep(self):
# Really load the input, since in the streaming case we can not
# use the self.inputMovies directly
allFramesSum = self._getPath('all_frames_sum.mrc')
allFramesAvg = self._getPath('all_frames_avg.mrc')
self._loadInputList()
n = len(self.listOfMovies)
ih = ImageHandler()
sumImg = ih.read(allFramesSum)
sumImg.inplaceDivide(float(n))
sumImg.write(allFramesAvg)
outputAvg = Image()
outputAvg.setFileName(allFramesAvg)
outputAvg.setSamplingRate(self.listOfMovies[0].getSamplingRate())
self._defineOutputs(outputAverage=outputAvg)
self._defineSourceRelation(self.inputMovies, outputAvg)
class SpiderProtClassifyCluster(SpiderProtClassify):
""" Base for Clustering Spider classification protocols.
"""
def __init__(self, script, classDir, **kwargs):
SpiderProtClassify.__init__(self, script, classDir, **kwargs)
#--------------------------- STEPS functions ------------------------------
def createOutputStep(self):
self.buildDendrogram(True)
#--------------------------- UTILS functions ------------------------------
def _fillClassesFromNodes(self, classes2D, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendrogram.
"""
particles = classes2D.getImages()
sampling = classes2D.getSamplingRate()
# We need to first create a map between the particles index and
# the assigned class number
classDict = {}
nodeDict = {}
classCount = 0
for node in nodeList:
if node.path:
classCount += 1
node.classId = classCount
nodeDict[classCount] = node
for i in node.imageList:
classDict[int(i)] = classCount
def updateItem(p, i):
classId = classDict.get(i, None)
if classId is None:
p._appendItem = False
else:
p.setClassId(classId)
def updateClass(cls):
node = nodeDict[cls.getObjId()]
rep = cls.getRepresentative()
rep.setSamplingRate(sampling)
rep.setLocation(node.avgCount, self.dendroAverages)
particlesRange = range(1, particles.getSize()+1)
classes2D.classifyItems(updateItemCallback=updateItem,
updateClassCallback=updateClass,
itemDataIterator=iter(particlesRange))
def _fillParticlesFromNodes(self, inputParts, outputParts, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendrogram.
"""
allImages = set()
for node in nodeList:
if node.path:
for i in node.imageList:
allImages.add(i)
def updateItem(item, index):
item._appendItem = index in allImages
particlesRange = range(1, inputParts.getSize()+1)
outputParts.copyItems(inputParts,
updateItemCallback=updateItem,
itemDataIterator=iter(particlesRange))
def buildDendrogram(self, writeAverages=False):
""" Parse Spider docfile with the information to build the dendrogram.
Params:
writeAverages: whether to write class averages or not.
"""
dendroFile = self._getFileName('dendroDoc')
# Dendrofile is a docfile with at least 3 data colums (class, height, id)
doc = SpiderDocFile(dendroFile)
values = []
indexes = []
for _, h, i in doc.iterValues():
indexes.append(i)
values.append(h)
doc.close()
self.dendroValues = values
self.dendroIndexes = indexes
self.dendroImages = self._getFileName('particles')
self.dendroAverages = self._getFileName('averages')
self.dendroAverageCount = 0 # Write only the number of needed averages
self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels
self.ih = ImageHandler()
return self._buildDendrogram(0, len(values)-1, 1, writeAverages)
def getImage(self, particleNumber):
return self.ih.read((particleNumber, self.dendroImages))
def addChildNode(self, node, leftIndex, rightIndex, index,
writeAverages, level, searchStop):
child = self._buildDendrogram(leftIndex, rightIndex, index,
writeAverages, level+1, searchStop)
node.addChild(child)
node.extendImageList(child.imageList)
if writeAverages:
node.addImage(child.image)
del child.image # Allow to free child image memory
def _buildDendrogram(self, leftIndex, rightIndex, index,
writeAverages=False, level=0, searchStop=0):
""" This function is recursively called to create the dendrogram
graph (binary tree) and also to write the average image files.
Params:
leftIndex, rightIndex: the indexes within the list where to search.
index: the index of the class average.
writeAverages: flag to select when to write averages
searchStop: this could be 1, means that we will search until the
last element (used for right childs of the dendrogram or,
can be 0, meaning that the last element was already the max
(used for left childs )
From self:
self.dendroValues: the list with the heights of each node
self.dendroImages: image stack filename to read particles
self.dendroAverages: stack name where to write averages
It will search for the max in values list (between minIndex and maxIndex).
Nodes to the left of the max are left childs and the other right childs.
"""
if level < self.dendroMaxLevel:
avgCount = self.dendroAverageCount + 1
self.dendroAverageCount += 1
if rightIndex == leftIndex: # Just only one element
height = self.dendroValues[leftIndex]
node = DendroNode(index, height)
node.extendImageList([self.dendroIndexes[leftIndex]])
node.addImage(self.getImage(node.imageList[0]))
elif rightIndex == leftIndex + 1: # Two elements
height = max(self.dendroValues[leftIndex],
self.dendroValues[rightIndex])
node = DendroNode(index, height)
node.extendImageList([self.dendroIndexes[leftIndex],
self.dendroIndexes[rightIndex]])
node.addImage(self.getImage(node.imageList[0]),
self.getImage(node.imageList[1]))
else: # 3 or more elements
# Find the max value (or height) of the elements
maxValue = self.dendroValues[leftIndex]
maxIndex = 0
# searchStop could be 0 (do not consider last element, coming from
# left child, or 1 (consider also the last one, coming from right)
values = self.dendroValues[leftIndex+1:rightIndex+searchStop]
for i, v in enumerate(values):
if v > maxValue:
maxValue = v
maxIndex = i+1
m = maxIndex + leftIndex
node = DendroNode(index, maxValue)
hasRightChild = m < rightIndex
if maxValue > 0:
nextIndex = 2 * index if hasRightChild else index
self.addChildNode(node, leftIndex, m, nextIndex,
writeAverages, level, 0)
if hasRightChild:
self.addChildNode(node, m+1, rightIndex, 2 * index + 1,
writeAverages, level, 1)
else:
# If the node has a single child, we will remove a node
# just to advance in the level of the tree to get more
# different class averages
if node.getChilds():
child = node.getChilds()[0]
child.image = node.image
child.parents = []
node = child
else:
node.extendImageList(self.dendroIndexes[leftIndex:rightIndex+1])
node.addImage(*[self.getImage(img) for img in node.imageList])
if level < self.dendroMaxLevel:
node.avgCount = avgCount
node.path = '%d@%s' % (node.avgCount, self.dendroAverages)
if writeAverages:
# normalize the sum of images depending on the number of particles
# assigned to this classes
avgImage = node.image / float(node.getSize())
self.ih.write(avgImage, (node.avgCount, self.dendroAverages))
fn = self._getTmpPath('doc_class%03d.stk' % index)
doc = SpiderDocFile(fn, 'w+')
for i in node.imageList:
doc.writeValues(i)
doc.close()
return node
class SpiderProtClassifyCluster(SpiderProtClassify):
""" Base for Clustering Spider classification protocols.
"""
def __init__(self, script, classDir, **kwargs):
SpiderProtClassify.__init__(self, script, classDir, **kwargs)
#--------------------------- STEPS functions --------------------------------------------
def createOutputStep(self):
self.buildDendrogram(True)
#--------------------------- UTILS functions --------------------------------------------
def _fillClassesFromNodes(self, classes, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendogram.
"""
img = Particle()
sampling = classes.getSamplingRate()
for node in nodeList:
if node.path:
#print "node.path: ", node.path
class2D = Class2D()
avg = Particle()
#avg.copyObjId(class2D)
avg.setLocation(node.avgCount, self.dendroAverages)
avg.setSamplingRate(sampling)
class2D.setRepresentative(avg)
class2D.setSamplingRate(sampling)
classes.append(class2D)
#print "class2D.id: ", class2D.getObjId()
for i in node.imageList:
#img.setObjId(i) # FIXME: this is wrong if the id is different from index
img.cleanObjId()
img.setLocation(int(i), self.dendroImages)
class2D.append(img)
classes.update(class2D)
def _fillParticlesFromNodes(self, particles, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendogram.
"""
img = Particle()
for node in nodeList:
if node.path:
for i in node.imageList:
#img.setObjId(i) # FIXME: this is wrong if the id is different from index
img.cleanObjId()
img.setLocation(int(i), self.dendroImages)
particles.append(img)
def buildDendrogram(self, writeAverages=False):
""" Parse Spider docfile with the information to build the dendogram.
Params:
dendroFile: docfile with a row per image.
Each row contains the image id and the height.
"""
dendroFile = self._getFileName('dendroDoc')
# Dendrofile is a docfile with at least 3 data colums (class, height, id)
doc = SpiderDocFile(dendroFile)
values = []
indexes = []
for c, h, _ in doc.iterValues():
indexes.append(c)
values.append(h)
doc.close()
self.dendroValues = values
self.dendroIndexes = indexes
self.dendroImages = self._getFileName('particles')
self.dendroAverages = self._getFileName('averages')
self.dendroAverageCount = 0 # Write only the number of needed averages
self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels
self.ih = ImageHandler()
return self._buildDendrogram(0, len(values)-1, 1, writeAverages)
def getImage(self, particleNumber):
return self.ih.read((particleNumber, self.dendroImages))
def addChildNode(self, node, leftIndex, rightIndex, index, writeAverages, level):
child = self._buildDendrogram(leftIndex, rightIndex, index, writeAverages, level+1)
node.addChild(child)
node.length += child.length
node.imageList += child.imageList
if writeAverages:
if node.image is None:
node.image = child.image
else:
node.image += child.image
del child.image # Allow to free child image memory
def _buildDendrogram(self, leftIndex, rightIndex, index, writeAverages=False, level=0):
""" This function is recursively called to create the dendogram graph(binary tree)
and also to write the average image files.
Params:
leftIndex, rightIndex: the indinxes within the list where to search.
index: the index of the class average.
writeImages: flag to select when to write averages.
From self:
self.dendroValues: the list with the heights of each node
self.dendroImages: image stack filename to read particles
self.dendroAverages: stack name where to write averages
It will search for the max in values list (between minIndex and maxIndex).
Nodes to the left of the max are left childs and the other right childs.
"""
if level < self.dendroMaxLevel:
avgCount = self.dendroAverageCount + 1
self.dendroAverageCount += 1
if rightIndex == leftIndex: # Just only one element
height = self.dendroValues[leftIndex]
node = DendroNode(index, height)
node.imageList = [self.dendroIndexes[leftIndex]]
node.image = self.getImage(node.imageList[0])
node.length = 1
elif rightIndex == leftIndex + 1: # Two elements
height = max(self.dendroValues[leftIndex],
self.dendroValues[rightIndex])
node = DendroNode(index, height)
node.imageList = [self.dendroIndexes[leftIndex],
self.dendroIndexes[rightIndex]]
node.image = self.getImage(node.imageList[0]) + self.getImage(node.imageList[1])
node.length = 2
else: # 3 or more elements
# Find the max value (or height) of the elements
maxValue = self.dendroValues[leftIndex]
maxIndex = 0
for i, v in enumerate(self.dendroValues[leftIndex+1:rightIndex]):
if v > maxValue:
maxValue = v
maxIndex = i+1
m = maxIndex + leftIndex
node = DendroNode(index, maxValue)
self.addChildNode(node, leftIndex, m, 2*index, writeAverages, level)
self.addChildNode(node, m+1, rightIndex, 2*index+1, writeAverages, level)
if level < self.dendroMaxLevel:
node.avgCount = avgCount
node.path = '%d@%s' % (node.avgCount, self.dendroAverages)
if writeAverages:
# normalize the sum of images depending on the number of particles
# assigned to this classes
avgImage = node.image / float(node.getSize())
self.ih.write(avgImage, (node.avgCount, self.dendroAverages))
fn = self._getTmpPath('doc_class%03d.stk' % index)
doc = SpiderDocFile(fn, 'w+')
for i in node.imageList:
doc.writeValues(i)
doc.close()
return node
def createOutputStep(self):
ih = ImageHandler()
outputStack = self._getPath('particles.mrcs')
outputImg = ih.createImage()
inputParticles = self.inputParticles.get()
inputCoords = self.inputCoordinates.get()
outputSet = self._createSetOfParticles()
outputSet.copyInfo(inputParticles)
boxSize = self.boxSize.get()
b2 = int(round(boxSize / 2))
center = np.zeros((boxSize, boxSize))
ih = ImageHandler()
i = 0
outliers = 0
partIdExcluded = []
lastPartId = None
for coord in inputCoords.iterItems(
orderBy=['_subparticle._micId', '_micId', 'id']):
# The original particle id is stored in the sub-particle as micId
partId = coord._micId.get()
# Load the particle if it has changed from the last sub-particle
if partId != lastPartId:
particle = inputParticles[partId]
if particle is None:
partIdExcluded.append(partId)
self.info("WARNING: Missing particle with id %s from "
"input particles set" % partId)
else:
# Now load the particle image to extract later sub-particles
img = ih.read(particle)
x, y, _, _ = img.getDimensions()
data = img.getData()
lastPartId = partId
# If particle is not in inputParticles, subparticles will not be
# generated. Now, subtract from a subset of original particles is
# supported.
if not partId in partIdExcluded:
xpos = coord.getX()
ypos = coord.getY()
# Check that the sub-particle will not lay out of the particle
if (ypos - b2 < 0 or ypos + b2 > y or xpos - b2 < 0
or xpos + b2 > x):
outliers += 1
continue
# Crop the sub-particle data from the whole particle image
center[:, :] = data[ypos - b2:ypos + b2, xpos - b2:xpos + b2]
outputImg.setData(center)
i += 1
outputImg.write((i, outputStack))
subpart = coord._subparticle
subpart.setLocation(
(i, outputStack)) # Change path to new stack
subpart.setObjId(None) # Force to insert as a new item
outputSet.append(subpart)
if outliers:
self.info(
"WARNING: Discarded %s particles because laid out of the "
"particle (for a box size of %d" % (outliers, boxSize))
self._defineOutputs(outputParticles=outputSet)
self._defineSourceRelation(self.inputParticles, outputSet)
