def homogeneizeStep(self):
minClass = self.homogeneize.get()
fnNeighbours = self._getExtraPath("neighbours.xmd")
# Look for the block with the minimum number of images
if minClass==0:
minClass = 1e38
for block in xmippLib.getBlocksInMetaDataFile(fnNeighbours):
projNumber = int(block.split("_")[1])
fnDir=self._getExtraPath("direction_%d"%projNumber,"level_00","class_classes.xmd")
if exists(fnDir):
blockSize = md.getSize("class000001_images@"+fnDir)
if blockSize<minClass:
minClass=blockSize
# Construct the homogeneized metadata
mdAll = xmippLib.MetaData()
mdSubset=xmippLib.MetaData()
mdRandom=xmippLib.MetaData()
for block in xmippLib.getBlocksInMetaDataFile(fnNeighbours):
projNumber = int(block.split("_")[1])
fnDir=self._getExtraPath("direction_%d"%projNumber,"level_00","class_classes.xmd")
if exists(fnDir):
mdDirection = xmippLib.MetaData("class000001_images@"+fnDir)
mdRandom.randomize(mdDirection)
mdSubset.selectPart(mdRandom,0L,min(mdRandom.size(),minClass))
mdAll.unionAll(mdSubset)
mdAll.removeDuplicates(md.MDL_ITEM_ID)
mdAll.sort(md.MDL_ITEM_ID)
mdAll.fillConstant(md.MDL_PARTICLE_ID,1)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
mdAll.write(fnHomogeneous)
self.runJob("xmipp_metadata_utilities",'-i %s --operate modify_values "particleId=itemId"'%fnHomogeneous,numberOfMpi=1)
def sortImagesStep(self):
args = "-i Particles@%s -o %s --addToInput " % (self.fnInputMd,
self.fnOutputMd)
if os.path.exists(self.fnInputOldMd):
args += "-t Particles@%s " % self.fnInputOldMd
if self.autoParRejection == self.REJ_MAXZSCORE:
args += "--zcut " + str(self.maxZscore.get())
elif self.autoParRejection == self.REJ_PERCENTAGE:
args += "--percent " + str(self.percentage.get())
if self.addFeatures:
args += "--addFeatures "
self.runJob("xmipp_image_sort_by_statistics", args)
args = "-i Particles@%s -o %s" % (self.fnInputMd, self.fnOutputMd)
if self.autoParRejectionSSNR == self.REJ_PERCENTAGE_SSNR:
args += " --ssnrpercent " + str(self.percentageSSNR.get())
self.runJob("xmipp_image_ssnr", args)
if self.autoParRejectionVar != self.REJ_NONE:
print('Rejecting by variance:')
if self.outputSize == 0:
varList = []
giniList = []
print(' - Reading metadata')
mdata = xmippLib.MetaData(self.fnInputMd)
for objId in mdata:
varList.append(
mdata.getValue(xmippLib.MDL_SCORE_BY_VAR, objId))
giniList.append(
mdata.getValue(xmippLib.MDL_SCORE_BY_GINI, objId))
if self.autoParRejectionVar == self.REJ_VARIANCE:
valuesList = varList
self.mdLabels = [xmippLib.MDL_SCORE_BY_VAR]
else: # not working pretty well
valuesList = [
var * (1 - gini)
for var, gini in zip(varList, giniList)
]
self.mdLabels = [
xmippLib.MDL_SCORE_BY_VAR, xmippLib.MDL_SCORE_BY_GINI
]
self.varThreshold.set(histThresholding(valuesList))
print(' - Variance threshold: %f' % self.varThreshold)
rejectByVariance(self.fnInputMd, self.fnOutputMd,
self.varThreshold, self.autoParRejectionVar)
# update the processed particles
self.outputSize += getSize(self.fnInputMd)
def _createFinalMetadata(self):
fnClasses = self._getExtraPath('last_classes.xmd')
numClasses = md.getSize(fnClasses)
for i in range(numClasses):
num = i + 1
fn = self._getExtraPath('dataClass%06d.xmd' % num)
if exists(fn):
line = ""
numLine = 0
while not line.startswith(' '):
numLine += 1
line = popen('sed -n %ip %s' % (numLine, fn)).read()
aux = self._getExtraPath('aux.xmd')
system('head -%i %s >> %s' % (numLine - 1, fn, aux))
dataHeader = self._getExtraPath('dataHeader.xmd')
fp = open(aux, 'r')
fout = open(dataHeader, 'w')
for i, line in enumerate(fp):
if i < 2:
continue
if not line.startswith('data_noname'):
fout.write(line)
else:
line = line.replace('noname', 'class%06d_images' % num)
fout.write(line)
fp.close()
fout.close()
cleanPath(aux)
fnOut = self._getExtraPath('dataImages%06d.xmd' % num)
system('tail -n +%i %s >> %s' % (i + 2, fn, aux))
system('cat %s %s >> %s' % (dataHeader, aux, fnOut))
if num == 1:
system('cat %s >> %s' %
(fn, self._getExtraPath('last_images.xmd')))
else:
system('cat %s >> %s' %
(aux, self._getExtraPath('last_images.xmd')))
cleanPath(aux)
cleanPath(dataHeader)
system('cat %s >> %s' %
(fnOut, self._getExtraPath('last_classes.xmd')))
def significantStep(self, iterNumber, alpha, args):
iterDir = self._getTmpPath('iter%03d' % iterNumber)
makePath(iterDir)
args += ' --odir %s' % iterDir
args += ' --alpha0 %f --alphaF %f' % (alpha, alpha)
prevVolFn = self.getIterVolume(iterNumber - 1)
volFn = self.getIterVolume(iterNumber)
if iterNumber == 1:
if self.thereisRefVolume:
args += " --initvolumes " + self._getExtraPath(
'input_volumes.xmd')
else:
args += " --numberOfVolumes 1"
else:
args += " --initvolumes %s" % prevVolFn
t = Timer()
t.tic()
self.runJob("xmipp_reconstruct_significant", args)
t.toc('Significant took: ')
anglesFn = self._getExtraPath('angles_iter%03d.xmd' % iterNumber)
moveFile(os.path.join(iterDir, 'angles_iter001_00.xmd'), anglesFn)
reconsArgs = ' -i %s' % anglesFn
reconsArgs += ' -o %s' % volFn
reconsArgs += ' --weight -v 0 --sym %s ' % self.symmetryGroup
print "Number of images for reconstruction: ", metadata.getSize(
anglesFn)
t.tic()
self.runJob("xmipp_reconstruct_fourier", reconsArgs)
t.toc('Reconstruct fourier took: ')
xdim = self.inputSet.get().getDimensions()[0]
maskArgs = "-i %s --mask circular %d -v 0" % (volFn, -xdim / 2)
self.runJob('xmipp_transform_mask', maskArgs, numberOfMpi=1)
if not self.keepIntermediate:
cleanPath(prevVolFn, iterDir)
def significantStep(self, iterNumber, alpha, args):
iterDir = self._getTmpPath('iter%03d' % iterNumber)
makePath(iterDir)
args += ' --odir %s' % iterDir
args += ' --alpha0 %f --alphaF %f' % (alpha, alpha)
prevVolFn = self.getIterVolume(iterNumber-1)
volFn = self.getIterVolume(iterNumber)
if iterNumber == 1:
if self.thereisRefVolume:
args += " --initvolumes " + self._getExtraPath('input_volumes.xmd')
else:
args += " --numberOfVolumes %d" % self.Nvolumes
else:
args += " --initvolumes %s" % prevVolFn
t = Timer()
t.tic()
self.runJob("xmipp_reconstruct_significant", args)
t.toc('Significant took: ')
anglesFn = self._getExtraPath('angles_iter%03d.xmd' % iterNumber)
moveFile(os.path.join(iterDir, 'angles_iter001_00.xmd'), anglesFn)
reconsArgs = ' -i %s' % anglesFn
reconsArgs += ' -o %s' % volFn
reconsArgs += ' --weight -v 0 --sym %s ' % self.symmetryGroup
print "Number of images for reconstruction: ", metadata.getSize(anglesFn)
t.tic()
self.runJob("xmipp_reconstruct_fourier", reconsArgs)
t.toc('Reconstruct fourier took: ')
xdim = self.inputSet.get().getDim()[0]
maskArgs = "-i %s --mask circular %d -v 0" % (volFn, -xdim/2)
self.runJob('xmipp_transform_mask', maskArgs, numberOfMpi=1)
if not self.keepIntermediate:
cleanPath(prevVolFn, iterDir)
def significantStep(self, iterNumber, alpha):
iterDir = self._getTmpPath('iter%03d' % iterNumber)
makePath(iterDir)
prevVolFn = self.getIterVolume(iterNumber - 1)
volFn = self.getIterVolume(iterNumber)
anglesFn = self._getExtraPath('angles_iter%03d.xmd' % iterNumber)
t = Timer()
t.tic()
if self.useGPU and iterNumber > 1:
# Generate projections
fnGalleryRoot = join(iterDir, "gallery")
args = "-i %s -o %s.stk --sampling_rate %f --sym %s " \
"--compute_neighbors --angular_distance -1 " \
"--experimental_images %s --min_tilt_angle %f " \
"--max_tilt_angle %f -v 0 --perturb %f" % \
(prevVolFn, fnGalleryRoot, self.angularSampling,
self.symmetryGroup, self.imgsFn, self.minTilt, self.maxTilt,
math.sin(self.angularSampling.get()) / 4)
self.runJob("xmipp_angular_project_library ", args)
# Align
# TODO check the alpha values for gpu
if self.trueSymsNo != 0:
alphaApply = (alpha * self.trueSymsNo) / 2
else:
alphaApply = alpha / 2
if self.maximumShift == -1:
maxShift = 10
else:
maxShift = self.maximumShift
args = '-i_ref %s.doc -i_exp %s -o %s --significance %f ' \
'--maxShift %f' % \
(fnGalleryRoot, self.imgsFn, anglesFn, alphaApply,
maxShift)
self.runJob("xmipp_cuda_correlation", args, numberOfMpi=1)
cleanPattern(fnGalleryRoot + "*")
else:
args = self.getSignificantArgs(self.imgsFn)
args += ' --odir %s' % iterDir
args += ' --alpha0 %f --alphaF %f' % (alpha, alpha)
if iterNumber == 1:
if self.thereisRefVolume:
args += " --initvolumes " + \
self._getExtraPath('input_volumes.xmd')
else:
args += " --numberOfVolumes 1"
else:
args += " --initvolumes %s" % prevVolFn
self.runJob("xmipp_reconstruct_significant", args)
moveFile(os.path.join(iterDir, 'angles_iter001_00.xmd'), anglesFn)
t.toc('Significant took: ')
reconsArgs = ' -i %s' % anglesFn
reconsArgs += ' -o %s' % volFn
reconsArgs += ' --weight -v 0 --sym %s ' % self.symmetryGroup
print "Number of images for reconstruction: ", metadata.getSize(
anglesFn)
t.tic()
self.runJob("xmipp_reconstruct_fourier", reconsArgs)
t.toc('Reconstruct fourier took: ')
# Center the volume
fnSym = self._getExtraPath('volumeSym_%03d.vol' % iterNumber)
self.runJob("xmipp_transform_mirror",
"-i %s -o %s --flipX" % (volFn, fnSym),
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipY" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipZ" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_image_operate",
"-i %s --plus %s" % (fnSym, volFn),
numberOfMpi=1)
self.runJob("xmipp_volume_align",
'--i1 %s --i2 %s --local --apply' % (fnSym, volFn),
numberOfMpi=1)
cleanPath(fnSym)
# To mask the volume
xdim = self.inputSet.get().getDim()[0]
maskArgs = "-i %s --mask circular %d -v 0" % (volFn, -xdim / 2)
self.runJob('xmipp_transform_mask', maskArgs, numberOfMpi=1)
# TODO mask the final volume in some smart way...
# To filter the volume
if self.useMaxRes:
self.runJob('xmipp_transform_filter',
'-i %s --fourier low_pass %f --sampling %f' % \
(volFn, self.maxResolution.get(), self.TsCurrent),
numberOfMpi=1)
if not self.keepIntermediate:
cleanPath(prevVolFn, iterDir)
def classifyOneGroup(self, projNumber, projMdBlock, projRef,
mdClasses, mdImages):
""" Classify one of the neighbourhood groups if not empty.
Class information will be stored in output metadata: mdOut
"""
blockSize = md.getSize(projMdBlock)
fnToUse = projMdBlock
if self.maxCLimgs>0 and blockSize>self.maxCLimgs:
fnToUSe = self._getTmpPath("coneImages.xmd")
self.runJob("xmipp_metadata_utilities","-i %s -o %s --operate random_subset %d"\
%(projMdBlock,fnToUSe,self.maxCLimgs))
Nclasses = self.directionalClasses.get()
Nlevels = int(math.ceil(math.log(Nclasses) / math.log(2)))
# Skip projection directions with not enough images to
# create a given number of classes
if blockSize / Nclasses < 10:
return
fnDir = self._getExtraPath("direction_%s" % projNumber)
if not exists(join(fnDir,"level_00")):
makePath(fnDir)
# Run CL2D classification for the images assigned to one direction
args = "-i %s " % fnToUse
args += "--odir %s " % fnDir
args += "--ref0 %s --iter %d --nref %d " % (
projRef, self.cl2dIterations, Nclasses)
args += "--distance correlation --classicalMultiref "
args += "--maxShift %f " % self.maxShift
try:
self.runJob("xmipp_classify_CL2D", args, numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get())
except:
return
# After CL2D the stk and xmd files should be produced
classesXmd = join(fnDir, "level_%02d/class_classes.xmd" % Nlevels)
classesStk = join(fnDir, "level_%02d/class_classes.stk" % Nlevels)
# Let's check that the output was produced
if not exists(classesStk):
return
# Run align of the class average and the projection representative
fnAlignRoot = join(fnDir, "classes")
args = "-i %s " % classesStk
args += "--ref %s " % projRef
args += " --oroot %s --iter 1" % fnAlignRoot
self.runJob("xmipp_image_align", args, numberOfMpi=1)
# Apply alignment
args = "-i %s_alignment.xmd --apply_transform" % fnAlignRoot
self.runJob("xmipp_transform_geometry", args, numberOfMpi=1)
for classNo in range(1, Nclasses + 1):
localImagesMd = xmippLib.MetaData("class%06d_images@%s"
% (classNo, classesXmd))
# New class detected
self.classCount += 1
# Check which images have not been assigned yet to any class
# and assign them to this new class
for objId in localImagesMd:
imgId = localImagesMd.getValue(xmippLib.MDL_ITEM_ID, objId)
# Add images not classify yet and store their class number
if imgId not in self.classImages:
self.classImages.add(imgId)
newObjId = mdImages.addObject()
mdImages.setValue(xmippLib.MDL_ITEM_ID, imgId, newObjId)
mdImages.setValue(xmippLib.MDL_REF2, self.classCount, newObjId)
newClassId = mdClasses.addObject()
mdClasses.setValue(xmippLib.MDL_REF, projNumber, newClassId)
mdClasses.setValue(xmippLib.MDL_REF2, self.classCount, newClassId)
mdClasses.setValue(xmippLib.MDL_IMAGE, "%d@%s" %
(classNo, classesStk), newClassId)
mdClasses.setValue(xmippLib.MDL_IMAGE1, projRef, newClassId)
mdClasses.setValue(xmippLib.MDL_CLASS_COUNT, localImagesMd.size(),
newClassId)
def classifyOneGroup(self, projNumber, projMdBlock, projRef,
mdClasses, mdImages):
""" Classify one of the neighbourhood groups if not empty.
Class information will be stored in output metadata: mdOut
"""
blockSize = md.getSize(projMdBlock)
Nclasses = self.directionalClasses.get()
Nlevels = int(math.ceil(math.log(Nclasses) / math.log(2)))
# Skip projection directions with not enough images to
# create a given number of classes
if blockSize / Nclasses < 10:
return
fnDir = self._getExtraPath("direction_%s" % projNumber)
makePath(fnDir)
# Run CL2D classification for the images assigned to one direction
args = "-i %s " % projMdBlock
args += "--odir %s " % fnDir
args += "--ref0 %s --iter 1 --nref %d " % (projRef, Nclasses)
args += "--distance correlation --classicalMultiref "
args += "--maxShift %f " % self.maxShift
try:
self.runJob("xmipp_classify_CL2D", args)
except:
return
# After CL2D the stk and xmd files should be produced
classesXmd = join(fnDir, "level_%02d/class_classes.xmd" % Nlevels)
classesStk = join(fnDir, "level_%02d/class_classes.stk" % Nlevels)
# Let's check that the output was produced
if not exists(classesStk):
return
# Run align of the class average and the projection representative
fnAlignRoot = join(fnDir, "classes")
args = "-i %s " % classesStk
args += "--ref %s " % projRef
args += " --oroot %s --iter 1" % fnAlignRoot
self.runJob("xmipp_image_align", args, numberOfMpi=1)
# Apply alignment
args = "-i %s_alignment.xmd --apply_transform" % fnAlignRoot
self.runJob("xmipp_transform_geometry", args, numberOfMpi=1)
for classNo in range(1, Nclasses+1):
localImagesMd = xmipp.MetaData("class%06d_images@%s"
% (classNo, classesXmd))
# New class detected
self.classCount += 1
# Check which images have not been assigned yet to any class
# and assign them to this new class
for objId in localImagesMd:
imgId = localImagesMd.getValue(xmipp.MDL_ITEM_ID, objId)
# Add images not classify yet and store their class number
if imgId not in self.classImages:
self.classImages.add(imgId)
newObjId = mdImages.addObject()
mdImages.setValue(xmipp.MDL_ITEM_ID, imgId, newObjId)
mdImages.setValue(xmipp.MDL_REF2, self.classCount, newObjId)
newClassId = mdClasses.addObject()
mdClasses.setValue(xmipp.MDL_REF, projNumber, newClassId)
mdClasses.setValue(xmipp.MDL_REF2, self.classCount, newClassId)
mdClasses.setValue(xmipp.MDL_IMAGE, "%d@%s" %
(classNo, classesStk), newClassId)
mdClasses.setValue(xmipp.MDL_IMAGE1, projRef, newClassId)
mdClasses.setValue(xmipp.MDL_CLASS_COUNT,localImagesMd.size(),newClassId)