def convertInputStep(self, inputParticles, inputVolume, targetResolution):
""" Write the input images as a Xmipp metadata file.
particlesId: is only need to detect changes in
input particles and cause restart from here.
"""
imgSet = self._getInputParticles()
convXmp.writeSetOfParticles(imgSet, self._getInputXmd())
newXdim =self._getNewDim()
ih = ImageHandler()
inputVol = self.inputVolume.get()
fn = ih.fixXmippVolumeFileName(inputVol)
img = ih.read(fn)
img.scale(newXdim, newXdim, newXdim)
img.write(self._getExtraPath("volume.vol"))
args = '-i %(parts)s -o %(scaledStk)s --save_metadata_stack '
args += '%(scaledXmd)s --dim %(xDim)d'
params = {"parts" : self._getInputXmd(),
"scaledStk": self._getExtraPath('scaled_particles.stk'),
"scaledXmd": self._getExtraPath('scaled_particles.xmd'),
"xDim": newXdim
}
self.runJob("xmipp_image_resize", args % params)
def appendMicsFromTomogram(self, output, tomo):
self.info("Creating micrographs for %s" % tomo.getFileName())
# Load the tomogram
ih = ImageHandler()
img = ih.read(tomo.getFileName())
data = img.getData()
# For each slice
for index in range(0, len(data), self.slicesGap.get()):
self.debug("Creating micrograph for slice %s" % index)
micName = tomoSliceToMicName(tomo, index)
outputMicName = self._getExtraPath(micName)
outputMicName = replaceExt(outputMicName, "mrc")
slice = data[index]
micImg = ImageHandler()
micImg._img.setData(slice)
micImg.write(micImg._img, outputMicName)
# Create the micrograph metadata object
newMic = Micrograph()
newMic.setFileName(outputMicName)
newMic.setMicName(micName)
newMic.setSamplingRate(tomo.getSamplingRate())
# Append it
output.append(newMic)
def _invertScaleVol(self, fn):
xdim = self._getInputParticles().getXDim()
outputFn = self._getOutputVolFn(fn)
ih = ImageHandler()
img = ih.read(fn)
img.scale(xdim, xdim, xdim)
img.write(outputFn)
def _convertVolumes(self):
ih = ImageHandler()
makePath(self._getExtraPath('input'))
inputVols = self.inputVolumes.get()
for vol in inputVols:
map = ih.read(vol)
outputFn = self._getOutputFn(vol.getObjId())
map.write(outputFn)
def projectTomo(self, tomo):
outFile = pwutils.removeBaseExt(tomo.getFileName()) + '_projected.mrc'
ih = ImageHandler()
outProjection = ih.createImage()
tomoData = np.squeeze(ih.read(tomo.getFileName()).getData())
projection = np.sum(tomoData, axis=0)
outProjection.setData(projection)
ih.write(outProjection, self._getExtraPath(outFile))
return self._getExtraPath(outFile)
def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6):
""" Function to generate thumbnails for the report. Uses data from
self.thumbPaths.
===== Params =====
- firstThumbIndex: index from which we start generating thumbnails
- micScaleFactor: how much to reduce in size the micrographs.
"""
ih = ImageHandler()
numMics = len(self.thumbPaths[MIC_PATH])
for i in range(firstThumbIndex, numMics):
print('Generating images for mic %d' % (i+1))
# mic thumbnails
dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i])
if not exists(dstImgPath):
if self.micThumbSymlinks:
pwutils.copyFile(self.thumbPaths[MIC_PATH][i], dstImgPath)
else:
ih.computeThumbnail(self.thumbPaths[MIC_PATH][i],
dstImgPath, scaleFactor=micScaleFactor,
flipOnY=True)
# shift plots
if SHIFT_THUMBS in self.thumbPaths:
dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i])
if not exists(dstImgPath):
pwutils.copyFile(self.thumbPaths[SHIFT_PATH][i], dstImgPath)
# Psd thumbnails
# If there ARE thumbnail for the PSD (no ctf protocol and
# moviealignment hasn't computed it
if PSD_THUMBS in self.thumbPaths:
if self.ctfProtocol is None:
srcImgPath = self.thumbPaths[PSD_PATH][i]
dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath) and srcImgPath is not None:
if srcImgPath.endswith('psd'):
psdImg1 = ih.read(srcImgPath)
psdImg1.convertPSD()
psdImg1.write(dstImgPath)
ih.computeThumbnail(dstImgPath, dstImgPath,
scaleFactor=1, flipOnY=True)
else:
pwutils.copyFile(srcImgPath, dstImgPath)
else:
dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i])
if not exists(dstImgPath):
ih.computeThumbnail(self.thumbPaths[PSD_PATH][i],
dstImgPath, scaleFactor=1, flipOnY=True)
return
def rewriteClassBlockStep(self):
firstImage = self.inputParticles.get().getFirstItem()
fnClasses = self._params['kclasses']
mdClasses = "classes@%s" % fnClasses
fnClassStack = self._params['classes']
fnAverageStack = self._params['averages']
md = emlib.MetaData(mdClasses)
image = ImageHandler().createImage()
counter = 1
for objId in md:
imageName = "%06d@%s" % (counter, fnClassStack)
averageName = "%06d@%s" % (counter, fnAverageStack)
if md.getValue(emlib.MDL_CLASS_COUNT, objId) > 0:
# compute the average of images assigned to this class
classPrefix = 'class%06d' % counter
classMd = '%s_images@%s' % (classPrefix, fnClasses)
classRoot = self._getTmpPath(classPrefix)
self.runJob(
'xmipp_image_statistics',
'-i %s --save_image_stats %s -v 0' % (classMd, classRoot))
image.read(classRoot + 'average.xmp')
else:
# Create emtpy image as average
image.read(firstImage.getLocation()
) # just to take the right dimensions and datatype
image.initConstant(0.)
image.write(averageName)
md.setValue(emlib.MDL_IMAGE, imageName, objId)
md.setValue(emlib.MDL_IMAGE2, averageName, objId)
counter += 1
md.write(mdClasses, emlib.MD_APPEND)
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
# Copy image to filter to Tmp project folder
outputName = os.path.join("Tmp", "filtered_particle")
outputPath = outputName + ".spi"
cleanPath(outputPath)
outputLoc = (1, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), outputLoc)
outputLocSpiStr = locationToSpider(1, outputName)
pars = dict()
pars["filterType"] = self.protocolParent.filterType.get()
pars["filterMode"] = self.protocolParent.filterMode.get()
pars["usePadding"] = self.protocolParent.usePadding.get()
pars["op"] = "FQ"
if self.protocolParent.filterType <= FILTER_FERMI:
pars['filterRadius'] = self.getRadius()
else:
pars['lowFreq'] = self.getLowFreq()
pars['highFreq'] = self.getHighFreq()
if self.protocolParent.filterType == FILTER_FERMI:
pars['temperature'] = self.getTemperature()
filter_spider(outputLocSpiStr, outputLocSpiStr, **pars)
# Get output image and update filtered image
img = ImageHandler()._img
locXmippStr = ImageHandler.locationToXmipp((1, outputPath))
img.read(locXmippStr)
self.rightImage = img
self.updateFilteredImage()
def _scaleImages(self,indx, img):
fn = img.getFileName()
index = img.getIndex()
newFn = self._getTmpPath('particles_subset.mrcs')
xdim = self._getNewDim()
ih = ImageHandler()
image = ih.read((index, fn))
image.scale(xdim, xdim)
image.write((indx, newFn))
img.setFileName(newFn)
img.setIndex(indx)
img.setSamplingRate(self._getPixeSize())
def _showImagesMatplotlib(self, title, *imgs):
ih = ImageHandler()
xdim = 2 if len(imgs) > 2 else 1
ydim = 2
plotter = EmPlotter(windowTitle=title, x=xdim, y=ydim, figsize=(8, 6))
positions = [(1, 1), (1, 2), (2, 1), (2, 2)]
for i, imgFn in enumerate(imgs):
x, y = positions[i]
ax = plotter.createSubPlot("", "x", "y", x, y)
img = ih.read(imgFn)
ax.imshow(img.getData(), cmap='jet')
#with mrcfile.open(img.replace(":mrc", "")) as mrc:
# im = ax.imshow(mrc.data, cmap='jet')
return [plotter]
def projectStep(self, start, end, samplingRate, threadNumber):
# Project
md = emlib.MetaData(self._getInputParticlesSubsetFn(threadNumber))
##
projection = emlib.Image()
projection.setDataType(emlib.DT_DOUBLE)
##
for id in md:
rot = md.getValue(emlib.MDL_ANGLE_ROT, id)
tilt = md.getValue(emlib.MDL_ANGLE_TILT, id)
psi = md.getValue(emlib.MDL_ANGLE_PSI, id)
##projection =self.vol.projectVolumeDouble(rot, tilt, psi)
self.fourierProjectVol.projectVolume(projection, rot, tilt, psi)
##
# Apply CTF
if self.projType == self.CORRECT_NONE:
pass
elif self.projType == self.CORRECT_FULL_CTF:
emlib.applyCTF(projection, md, samplingRate, id, False)
elif self.projType == self.CORRECT_PHASE_FLIP:
emlib.applyCTF(projection, md, samplingRate, id, True)
else:
raise Exception("ERROR: Unknown projection mode: %d" %
self.projType)
# Shift image
projection.applyGeo(md, id, True, False) #onlyapplyshist, wrap
ih = ImageHandler()
expProj = ih.read(md.getValue(emlib.MDL_IMAGE, id))
expProj.convert2DataType(emlib.DT_DOUBLE)
# Subtract from experimental and write result
projection.resetOrigin()
if self.normalize:
expProj = expProj.adjustAndSubtract(projection)
else:
expProj.inplaceSubtract(projection)
expProj.write(self._getProjGalleryIndexFn(id + start - 1))
def createOutputStep(self):
ih = ImageHandler()
outputStack = self._getPath('particles.mrcs')
outputImg = ih.createImage()
inputParticles = self.inputParticles.get()
inputCoords = self.inputCoordinates.get()
outputSet = self._createSetOfParticles()
outputSet.copyInfo(inputParticles)
boxSize = self.boxSize.get()
b2 = int(round(boxSize / 2))
center = np.zeros((boxSize, boxSize))
ih = ImageHandler()
i = 0
outliers = 0
partIdExcluded = []
lastPartId = None
progress = ProgressBar(len(inputCoords), fmt=ProgressBar.NOBAR)
progress.start()
step = max(100, len(inputCoords) // 100)
for i, coord in enumerate(inputCoords.iterItems(orderBy=['_subparticle._micId',
'_micId', 'id'])):
if i % step == 0:
progress.update(i+1)
# The original particle id is stored in the sub-particle as micId
partId = coord._micId.get()
# Load the particle if it has changed from the last sub-particle
if partId != lastPartId:
particle = inputParticles[partId]
if particle is None:
partIdExcluded.append(partId)
self.info("WARNING: Missing particle with id %s from "
"input particles set" % partId)
else:
# Now load the particle image to extract later sub-particles
img = ih.read(particle)
x, y, _, _ = img.getDimensions()
data = img.getData()
lastPartId = partId
# If particle is not in inputParticles, subparticles will not be
# generated. Now, subtract from a subset of original particles is
# supported.
if partId not in partIdExcluded:
xpos = coord.getX()
ypos = coord.getY()
# Check that the sub-particle will not lay out of the particle
if (ypos - b2 < 0 or ypos + b2 > y or
xpos - b2 < 0 or xpos + b2 > x):
outliers += 1
continue
# Crop the sub-particle data from the whole particle image
center[:, :] = data[ypos - b2:ypos + b2, xpos - b2:xpos + b2]
outputImg.setData(center)
i += 1
outputImg.write((i, outputStack))
subpart = coord._subparticle
subpart.setLocation(
(i, outputStack)) # Change path to new stack
subpart.setObjId(i) # Ids will be always the same no mater the number of outliers
outputSet.append(subpart)
progress.finish()
if outliers:
self.info("WARNING: Discarded %s particles because laid out of the "
"particle (for a box size of %d" % (outliers, boxSize))
self._defineOutputs(outputParticles=outputSet)
self._defineSourceRelation(self.inputParticles, outputSet)
class SpiderProtClassifyCluster(SpiderProtClassify):
""" Base for Clustering Spider classification protocols.
"""
def __init__(self, script, classDir, **kwargs):
SpiderProtClassify.__init__(self, script, classDir, **kwargs)
# --------------------------- STEPS functions -----------------------------
def createOutputStep(self):
self.buildDendrogram(True)
# --------------------------- UTILS functions -----------------------------
def _fillClassesFromNodes(self, classes2D, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendrogram.
"""
particles = classes2D.getImages()
sampling = classes2D.getSamplingRate()
# We need to first create a map between the particles index and
# the assigned class number
classDict = {}
nodeDict = {}
classCount = 0
for node in nodeList:
if node.path:
classCount += 1
node.classId = classCount
nodeDict[classCount] = node
for i in node.imageList:
classDict[int(i)] = classCount
def updateItem(p, item):
classId = classDict.get(item, None)
if classId is None:
p._appendItem = False
else:
p.setClassId(classId)
def updateClass(cls):
node = nodeDict[cls.getObjId()]
rep = cls.getRepresentative()
rep.setSamplingRate(sampling)
rep.setLocation(node.avgCount, self.dendroAverages)
particlesRange = range(1, particles.getSize() + 1)
classes2D.classifyItems(updateItemCallback=updateItem,
updateClassCallback=updateClass,
itemDataIterator=iter(particlesRange))
def _fillParticlesFromNodes(self, inputParts, outputParts, nodeList):
""" Create the SetOfClasses2D from the images of each node
in the dendrogram.
"""
allImages = set()
for node in nodeList:
if node.path:
for i in node.imageList:
allImages.add(i)
def updateItem(item, index):
item._appendItem = index in allImages
particlesRange = range(1, inputParts.getSize() + 1)
outputParts.copyItems(inputParts,
updateItemCallback=updateItem,
itemDataIterator=iter(particlesRange))
def buildDendrogram(self, writeAverages=False):
""" Parse Spider docfile with the information to build the dendrogram.
Params:
writeAverages: whether to write class averages or not.
"""
dendroFile = self._getFileName('dendroDoc')
# Dendrofile is a docfile with at least 3 data columns (class, height, id)
doc = SpiderDocFile(dendroFile)
values = []
indexes = []
for _, h, i in doc.iterValues():
indexes.append(i)
values.append(h)
doc.close()
self.dendroValues = values
self.dendroIndexes = indexes
self.dendroImages = self._getFileName('particles')
self.dendroAverages = self._getFileName('averages')
self.dendroAverageCount = 0 # Write only the number of needed averages
self.dendroMaxLevel = 10 # FIXME: remove hard coding if working the levels
self.ih = ImageHandler()
return self._buildDendrogram(0, len(values) - 1, 1, writeAverages)
def getImage(self, particleNumber):
return self.ih.read((int(particleNumber), self.dendroImages))
def addChildNode(self, node, leftIndex, rightIndex, index, writeAverages,
level, searchStop):
child = self._buildDendrogram(leftIndex, rightIndex, index,
writeAverages, level + 1, searchStop)
node.addChild(child)
node.extendImageList(child.imageList)
if writeAverages:
node.addImage(child.image)
del child.image # Allow to free child image memory
def _buildDendrogram(self,
leftIndex,
rightIndex,
index,
writeAverages=False,
level=0,
searchStop=0):
""" This function is recursively called to create the dendrogram
graph (binary tree) and also to write the average image files.
Params:
leftIndex, rightIndex: the indexes within the list where to search.
index: the index of the class average.
writeAverages: flag to select when to write averages
searchStop: this could be 1, means that we will search until the
last element (used for right childs of the dendrogram or,
can be 0, meaning that the last element was already the max
(used for left childs )
From self:
self.dendroValues: the list with the heights of each node
self.dendroImages: image stack filename to read particles
self.dendroAverages: stack name where to write averages
It will search for the max in values list (between minIndex and maxIndex).
Nodes to the left of the max are left childs and the other right childs.
"""
if level < self.dendroMaxLevel:
avgCount = self.dendroAverageCount + 1
self.dendroAverageCount += 1
if rightIndex == leftIndex: # Just only one element
height = self.dendroValues[leftIndex]
node = DendroNode(index, height)
node.extendImageList([self.dendroIndexes[leftIndex]])
node.addImage(self.getImage(node.imageList[0]))
elif rightIndex == leftIndex + 1: # Two elements
height = max(self.dendroValues[leftIndex],
self.dendroValues[rightIndex])
node = DendroNode(index, height)
node.extendImageList([
self.dendroIndexes[leftIndex], self.dendroIndexes[rightIndex]
])
node.addImage(self.getImage(node.imageList[0]),
self.getImage(node.imageList[1]))
else: # 3 or more elements
# Find the max value (or height) of the elements
maxValue = self.dendroValues[leftIndex]
maxIndex = 0
# searchStop could be 0 (do not consider last element, coming from
# left child, or 1 (consider also the last one, coming from right)
values = self.dendroValues[leftIndex + 1:rightIndex + searchStop]
for i, v in enumerate(values):
if v > maxValue:
maxValue = v
maxIndex = i + 1
m = maxIndex + leftIndex
node = DendroNode(index, maxValue)
hasRightChild = m < rightIndex
if maxValue > 0:
nextIndex = 2 * index if hasRightChild else index
self.addChildNode(node, leftIndex, m, nextIndex, writeAverages,
level, 0)
if hasRightChild:
self.addChildNode(node, m + 1, rightIndex, 2 * index + 1,
writeAverages, level, 1)
else:
# If the node has a single child, we will remove a node
# just to advance in the level of the tree to get more
# different class averages
if node.getChilds():
child = node.getChilds()[0]
child.image = node.image
child.parents = []
node = child
else:
node.extendImageList(self.dendroIndexes[leftIndex:rightIndex +
1])
node.addImage(*[self.getImage(img) for img in node.imageList])
if level < self.dendroMaxLevel:
node.avgCount = avgCount
node.path = '%d@%s' % (node.avgCount, self.dendroAverages)
if writeAverages:
# normalize the sum of images depending on the number of particles
# assigned to this classes
# avgImage = node.image / float(node.getSize())
node.image.inplaceDivide(float(node.getSize()))
self.ih.write(node.image, (node.avgCount, self.dendroAverages))
fn = self._getTmpPath('doc_class%03d.stk' % index)
doc = SpiderDocFile(fn, 'w+')
for i in node.imageList:
doc.writeValues(i)
doc.close()
return node
def convertInputAndSaveToDBStep(self):
""" init database to store the last name of the files used and
convert 3D maps to MRC '.mrc' format.
This step is run once even if the protocol is relunched
"""
databasePath = self._getExtraPath(OUTPUTDATABASENAMESWITHLABELS)
# create database and table
# this table will be used to record the last version of any file
# save in coot
conn = sqlite3.connect(databasePath)
# create table
# saved = 0, means this file need to be converted to a scipion object
# type = 0-> Map, 1 -> atom struct
# predefined macros for type:
# TYPE_3DMAP = 0
# TYPE_ATOMSTRUCT = 1
sqlCommand = """create table if not exists %s
(id integer primary key AUTOINCREMENT,
modelId integer,
fileName text,
labelName text,
type int,
saved integer default 1
)""" % (DATABASETABLENAME)
conn.execute(sqlCommand)
# create view to retrieve the id of the last copy
# for each model id (coot call imol to this id)
sqlCommand = """CREATE VIEW lastid AS
SELECT modelId, max(id) as id
FROM %s
GROUP BY modelId
""" % DATABASETABLENAME
conn.execute(sqlCommand)
inVolumes, norVolumesNames = self._getVolumesList()
sqlCommand = """INSERT INTO %s (modelId, fileName, labelName, type, saved) values
(%d, '%s', '%s', %d, %d)"""
#process main atomic Structure
counter = 0
pdbFileToBeRefined = self.pdbFileToBeRefined.get().getFileName()
base = os.path.basename(pdbFileToBeRefined)
conn.execute(sqlCommand % (
DATABASETABLENAME,
counter,
pdbFileToBeRefined,
os.path.splitext(base)[0],
TYPE_ATOMSTRUCT,
1 # saved
))
counter += 1
# Process another atom structures
for pdb in self.inputPdbFiles:
fileName = pdb.get().getFileName()
base = os.path.basename(fileName)
conn.execute(sqlCommand % (
DATABASETABLENAME,
counter,
fileName,
os.path.splitext(base)[0],
TYPE_ATOMSTRUCT,
1 # saved
))
counter += 1
# Process 3D maps
# normalize them if needed
ih = ImageHandler()
for inVol, norVolName in zip(inVolumes, norVolumesNames):
inVolName = inVol.getFileName()
if inVolName.endswith(".mrc"):
inVolName += ":mrc"
if norVolName.endswith(".mrc"):
norVolName += ":mrc"
if not ih.existsLocation(norVolName):
if True: # self.doNormalize:
img = ImageHandler()._img
img.read(inVolName)
mean, dev, min, max = img.computeStats()
img.inplaceMultiply(1. / max)
img.write(norVolName)
else:
ImageHandler().convert(inVolName, norVolName)
Ccp4Header(norVolName, readHeader=True).copyCCP4Header(
inVol.getOrigin(force=True).getShifts(),
inVol.getSamplingRate(),
originField=Ccp4Header.START)
conn.execute(sqlCommand % (
DATABASETABLENAME,
counter,
norVolName[:-4],
os.path.basename(norVolName)[:-8],
TYPE_3DMAP,
0 # saved
))
counter += 1
conn.commit()