def appendMicsFromTomogram(self, output, tomo):
self.info("Creating micrographs for %s" % tomo.getFileName())
# Load the tomogram
ih = ImageHandler()
img = ih.read(tomo.getFileName())
data = img.getData()
# For each slice
for index in range(0, len(data), self.slicesGap.get()):
self.debug("Creating micrograph for slice %s" % index)
micName = tomoSliceToMicName(tomo, index)
outputMicName = self._getExtraPath(micName)
outputMicName = replaceExt(outputMicName, "mrc")
slice = data[index]
micImg = ImageHandler()
micImg._img.setData(slice)
micImg.write(micImg._img, outputMicName)
# Create the micrograph metadata object
newMic = Micrograph()
newMic.setFileName(outputMicName)
newMic.setMicName(micName)
newMic.setSamplingRate(tomo.getSamplingRate())
# Append it
output.append(newMic)
def projectTomo(self, tomo):
outFile = pwutils.removeBaseExt(tomo.getFileName()) + '_projected.mrc'
ih = ImageHandler()
outProjection = ih.createImage()
tomoData = np.squeeze(ih.read(tomo.getFileName()).getData())
projection = np.sum(tomoData, axis=0)
outProjection.setData(projection)
ih.write(outProjection, self._getExtraPath(outFile))
return self._getExtraPath(outFile)
def rewriteClassBlockStep(self):
firstImage = self.inputParticles.get().getFirstItem()
fnClasses = self._params['kclasses']
mdClasses = "classes@%s" % fnClasses
fnClassStack = self._params['classes']
fnAverageStack = self._params['averages']
md = emlib.MetaData(mdClasses)
image = ImageHandler().createImage()
counter = 1
for objId in md:
imageName = "%06d@%s" % (counter, fnClassStack)
averageName = "%06d@%s" % (counter, fnAverageStack)
if md.getValue(emlib.MDL_CLASS_COUNT, objId) > 0:
# compute the average of images assigned to this class
classPrefix = 'class%06d' % counter
classMd = '%s_images@%s' % (classPrefix, fnClasses)
classRoot = self._getTmpPath(classPrefix)
self.runJob(
'xmipp_image_statistics',
'-i %s --save_image_stats %s -v 0' % (classMd, classRoot))
image.read(classRoot + 'average.xmp')
else:
# Create emtpy image as average
image.read(firstImage.getLocation()
) # just to take the right dimensions and datatype
image.initConstant(0.)
image.write(averageName)
md.setValue(emlib.MDL_IMAGE, imageName, objId)
md.setValue(emlib.MDL_IMAGE2, averageName, objId)
counter += 1
md.write(mdClasses, emlib.MD_APPEND)
class SpiderProtClassifyCluster(SpiderProtClassify):
""" Base for Clustering Spider classification protocols.
"""
def __init__(self, script, classDir, **kwargs):
SpiderProtClassify.__init__(self, script, classDir, **kwargs)
# --------------------------- STEPS functions -----------------------------
def createOutputStep(self):
self.buildDendrogram(True)
# --------------------------- UTILS functions -----------------------------
def _fillClassesFromNodes(self, classes2D, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendrogram.
"""
particles = classes2D.getImages()
sampling = classes2D.getSamplingRate()
# We need to first create a map between the particles index and
# the assigned class number
classDict = {}
nodeDict = {}
classCount = 0
for node in nodeList:
if node.path:
classCount += 1
node.classId = classCount
nodeDict[classCount] = node
for i in node.imageList:
classDict[int(i)] = classCount
def updateItem(p, item):
classId = classDict.get(item, None)
if classId is None:
p._appendItem = False
else:
p.setClassId(classId)
def updateClass(cls):
node = nodeDict[cls.getObjId()]
rep = cls.getRepresentative()
rep.setSamplingRate(sampling)
rep.setLocation(node.avgCount, self.dendroAverages)
particlesRange = range(1, particles.getSize() + 1)
classes2D.classifyItems(updateItemCallback=updateItem,
updateClassCallback=updateClass,
itemDataIterator=iter(particlesRange))
def _fillParticlesFromNodes(self, inputParts, outputParts, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendrogram.
"""
allImages = set()
for node in nodeList:
if node.path:
for i in node.imageList:
allImages.add(i)
def updateItem(item, index):
item._appendItem = index in allImages
particlesRange = range(1, inputParts.getSize() + 1)
outputParts.copyItems(inputParts,
updateItemCallback=updateItem,
itemDataIterator=iter(particlesRange))
def buildDendrogram(self, writeAverages=False):
""" Parse Spider docfile with the information to build the dendrogram.
Params:
writeAverages: whether to write class averages or not.
"""
dendroFile = self._getFileName('dendroDoc')
# Dendrofile is a docfile with at least 3 data columns (class, height, id)
doc = SpiderDocFile(dendroFile)
values = []
indexes = []
for _, h, i in doc.iterValues():
indexes.append(i)
values.append(h)
doc.close()
self.dendroValues = values
self.dendroIndexes = indexes
self.dendroImages = self._getFileName('particles')
self.dendroAverages = self._getFileName('averages')
self.dendroAverageCount = 0 # Write only the number of needed averages
self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels
self.ih = ImageHandler()
return self._buildDendrogram(0, len(values) - 1, 1, writeAverages)
def getImage(self, particleNumber):
return self.ih.read((int(particleNumber), self.dendroImages))
def addChildNode(self, node, leftIndex, rightIndex, index, writeAverages,
level, searchStop):
child = self._buildDendrogram(leftIndex, rightIndex, index,
writeAverages, level + 1, searchStop)
node.addChild(child)
node.extendImageList(child.imageList)
if writeAverages:
node.addImage(child.image)
del child.image # Allow to free child image memory
def _buildDendrogram(self,
leftIndex,
rightIndex,
index,
writeAverages=False,
level=0,
searchStop=0):
""" This function is recursively called to create the dendrogram
graph (binary tree) and also to write the average image files.
Params:
leftIndex, rightIndex: the indexes within the list where to search.
index: the index of the class average.
writeAverages: flag to select when to write averages
searchStop: this could be 1, means that we will search until the
last element (used for right childs of the dendrogram or,
can be 0, meaning that the last element was already the max
(used for left childs )
From self:
self.dendroValues: the list with the heights of each node
self.dendroImages: image stack filename to read particles
self.dendroAverages: stack name where to write averages
It will search for the max in values list (between minIndex and maxIndex).
Nodes to the left of the max are left childs and the other right childs.
"""
if level < self.dendroMaxLevel:
avgCount = self.dendroAverageCount + 1
self.dendroAverageCount += 1
if rightIndex == leftIndex: # Just only one element
height = self.dendroValues[leftIndex]
node = DendroNode(index, height)
node.extendImageList([self.dendroIndexes[leftIndex]])
node.addImage(self.getImage(node.imageList[0]))
elif rightIndex == leftIndex + 1: # Two elements
height = max(self.dendroValues[leftIndex],
self.dendroValues[rightIndex])
node = DendroNode(index, height)
node.extendImageList([
self.dendroIndexes[leftIndex], self.dendroIndexes[rightIndex]
])
node.addImage(self.getImage(node.imageList[0]),
self.getImage(node.imageList[1]))
else: # 3 or more elements
# Find the max value (or height) of the elements
maxValue = self.dendroValues[leftIndex]
maxIndex = 0
# searchStop could be 0 (do not consider last element, coming from
# left child, or 1 (consider also the last one, coming from right)
values = self.dendroValues[leftIndex + 1:rightIndex + searchStop]
for i, v in enumerate(values):
if v > maxValue:
maxValue = v
maxIndex = i + 1
m = maxIndex + leftIndex
node = DendroNode(index, maxValue)
hasRightChild = m < rightIndex
if maxValue > 0:
nextIndex = 2 * index if hasRightChild else index
self.addChildNode(node, leftIndex, m, nextIndex, writeAverages,
level, 0)
if hasRightChild:
self.addChildNode(node, m + 1, rightIndex, 2 * index + 1,
writeAverages, level, 1)
else:
# If the node has a single child, we will remove a node
# just to advance in the level of the tree to get more
# different class averages
if node.getChilds():
child = node.getChilds()[0]
child.image = node.image
child.parents = []
node = child
else:
node.extendImageList(self.dendroIndexes[leftIndex:rightIndex +
1])
node.addImage(*[self.getImage(img) for img in node.imageList])
if level < self.dendroMaxLevel:
node.avgCount = avgCount
node.path = '%d@%s' % (node.avgCount, self.dendroAverages)
if writeAverages:
# normalize the sum of images depending on the number of particles
# assigned to this classes
# avgImage = node.image / float(node.getSize())
node.image.inplaceDivide(float(node.getSize()))
self.ih.write(node.image, (node.avgCount, self.dendroAverages))
fn = self._getTmpPath('doc_class%03d.stk' % index)
doc = SpiderDocFile(fn, 'w+')
for i in node.imageList:
doc.writeValues(i)
doc.close()
return node
def convertInputAndSaveToDBStep(self):
""" init database to store the last name of the files used and
convert 3D maps to MRC '.mrc' format.
This step is run once even if the protocol is relunched
"""
databasePath = self._getExtraPath(OUTPUTDATABASENAMESWITHLABELS)
# create database and table
# this table will be used to record the last version of any file
# save in coot
conn = sqlite3.connect(databasePath)
# create table
# saved = 0, means this file need to be converted to a scipion object
# type = 0-> Map, 1 -> atom struct
# predefined macros for type:
# TYPE_3DMAP = 0
# TYPE_ATOMSTRUCT = 1
sqlCommand = """create table if not exists %s
(id integer primary key AUTOINCREMENT,
modelId integer,
fileName text,
labelName text,
type int,
saved integer default 1
)""" % (DATABASETABLENAME)
conn.execute(sqlCommand)
# create view to retrieve the id of the last copy
# for each model id (coot call imol to this id)
sqlCommand = """CREATE VIEW lastid AS
SELECT modelId, max(id) as id
FROM %s
GROUP BY modelId
""" % DATABASETABLENAME
conn.execute(sqlCommand)
inVolumes, norVolumesNames = self._getVolumesList()
sqlCommand = """INSERT INTO %s (modelId, fileName, labelName, type, saved) values
(%d, '%s', '%s', %d, %d)"""
#process main atomic Structure
counter = 0
pdbFileToBeRefined = self.pdbFileToBeRefined.get().getFileName()
base = os.path.basename(pdbFileToBeRefined)
conn.execute(sqlCommand % (
DATABASETABLENAME,
counter,
pdbFileToBeRefined,
os.path.splitext(base)[0],
TYPE_ATOMSTRUCT,
1 # saved
))
counter += 1
# Process another atom structures
for pdb in self.inputPdbFiles:
fileName = pdb.get().getFileName()
base = os.path.basename(fileName)
conn.execute(sqlCommand % (
DATABASETABLENAME,
counter,
fileName,
os.path.splitext(base)[0],
TYPE_ATOMSTRUCT,
1 # saved
))
counter += 1
# Process 3D maps
# normalize them if needed
ih = ImageHandler()
for inVol, norVolName in zip(inVolumes, norVolumesNames):
inVolName = inVol.getFileName()
if inVolName.endswith(".mrc"):
inVolName += ":mrc"
if norVolName.endswith(".mrc"):
norVolName += ":mrc"
if not ih.existsLocation(norVolName):
if True: # self.doNormalize:
img = ImageHandler()._img
img.read(inVolName)
mean, dev, min, max = img.computeStats()
img.inplaceMultiply(1. / max)
img.write(norVolName)
else:
ImageHandler().convert(inVolName, norVolName)
Ccp4Header(norVolName, readHeader=True).copyCCP4Header(
inVol.getOrigin(force=True).getShifts(),
inVol.getSamplingRate(),
originField=Ccp4Header.START)
conn.execute(sqlCommand % (
DATABASETABLENAME,
counter,
norVolName[:-4],
os.path.basename(norVolName)[:-8],
TYPE_3DMAP,
0 # saved
))
counter += 1
conn.commit()