def classifyStep(self):
listNumImgs = []
listNameImgs = []
change = False
level=0
while len(listNumImgs) is not self.numberOfClasses.get() \
or change is True:
if self.useReferenceImages and self.numberOfClasses.get() \
== getSize(self.refSet):
if not exists(join(self._getExtraPath(), 'level%03d' % level)):
mkdir(join(self._getExtraPath(), 'level%03d' % level))
copy(self.refSet, self._getExtraPath(join('level%03d' % level,
'intermediate_classes.xmd')))
else:
self.splitStep(level)
#######################################
if not self.useAttraction:
self.attractionSplitStep(level)
#######################################
self.generateMetadata(listNameImgs, listNumImgs, level)
self.ref = self._getExtraPath(join('level%03d' % level,
'intermediate_classes.xmd'))
lengthMd = getSize(self.ref)
if lengthMd==self.numberOfClasses.get():
self.classifyWholeSetStep(level)
#######################################################
if not self.useAttraction:
change, listNumImgs, listNameImgs = \
self.attractionGeneralStep(level)
if change:
level = level + 1
continue
#######################################################
copy(self._getExtraPath(join('level%03d' % level,
'general_level%03d' % level
+ '_classes.xmd')),
self._getExtraPath('last_classes.xmd'), )
if exists(self._getExtraPath(join('level%03d' % level,
'general_images_level%03d'
% level + '.xmd'))):
copy(self._getExtraPath(join('level%03d' % level,
'general_images_level%03d'
% level + '.xmd')),
self._getExtraPath('last_images.xmd'), )
return
listNumImgs, listNameImgs = self.checkOutput(level)
self.cleaningPath(level)
level = level + 1
def split(self, i):
from pyworkflow.em.metadata.utils import getSize
from math import ceil
from random import randint
from time import sleep
sleep(randint(1, 10))
fnDirCurrent = self._getExtraPath("Iter001")
fnAngles = self._getExtraPath("Iter001/angles.xmd")
fnSplit = self._getExtraPath('split%05d.xmd' % i)
Nimages = getSize(fnAngles)
Nsplit = ceil(Nimages * self.splitFraction.get())
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" %
(fnAngles, fnSplit, Nsplit),
numberOfMpi=1)
fnVol = self._getExtraPath('split%05d.vol' % i)
args = "-i %s -o %s --sym %s --weight" % (fnSplit, fnVol,
self.symmetryGroup)
row = getFirstRow(fnAngles)
if row.containsLabel(xmipp.MDL_CTF_DEFOCUSU) or row.containsLabel(
xmipp.MDL_CTF_MODEL):
TsCurrent = self.readInfoField(fnDirCurrent, "sampling",
xmipp.MDL_SAMPLINGRATE)
args += " --useCTF --sampling %f" % TsCurrent
if self.phaseFlipped:
args += " --phaseFlipped"
self.runJob("xmipp_reconstruct_fourier", args, numberOfMpi=1)
self.runJob(
"xmipp_image_operate",
"-i %s --minus %s" % (fnVol, join(fnDirCurrent, "volume.vol")))
def reconstructStep(self, fnRoot):
from pyworkflow.em.metadata.utils import getSize
if os.path.exists(fnRoot+".xmd"):
Nimages=getSize(fnRoot+".xmd")
if Nimages>0:
self.runJob("xmipp_reconstruct_fourier","-i %s.xmd -o %s.vol --sym %s " %(fnRoot,fnRoot,self.symmetryGroup.get()))
self.runJob("xmipp_transform_mask","-i %s.vol --mask circular -%d "%(fnRoot,self.Xdim2/2))
else:
print fnRoot+".xmd is empty. The corresponding volume is not generated."
def _createAngDistChimera(self, it):
fnDir = self.protocol._getExtraPath("Iter%03d"%it)
fnAngles = join(fnDir,"angles.xmd")
view=None
if exists(fnAngles):
fnAnglesSqLite = join(fnDir,"angles.sqlite")
from pyworkflow.em.metadata.utils import getSize
self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
view = ChimeraClientView(join(fnDir,"volumeAvg.mrc"), showProjection=True, angularDistFile=fnAnglesSqLite, spheresDistance=self.spheresScale.get())
return view
def _createAngDistChimera(self, it):
fnDir = self.protocol._getExtraPath("Iter%03d"%it)
fnAngles = join(fnDir,"angles.xmd")
view=None
if exists(fnAngles):
fnAnglesSqLite = join(fnDir,"angles.sqlite")
from pyworkflow.em.metadata.utils import getSize
self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
view = ChimeraClientView(join(fnDir,"volumeAvg.mrc"), showProjection=True, angularDistFile=fnAnglesSqLite, spheresDistance=self.spheresScale.get())
return view
def _createAngDist2D(self, it):
fnDir = self.protocol._getExtraPath("Iter%03d"%it)
fnAngles = join(fnDir,"angles.xmd")
view=None
if exists(fnAngles):
fnAnglesSqLite = join(fnDir,"angles.sqlite")
from pyworkflow.em.metadata.utils import getSize
from pyworkflow.em.plotter import EmPlotter
self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % it, windowTitle="Angular distribution")
view.plotAngularDistributionFromMd(fnAnglesSqLite, 'iter %d' % it)
return view
def splitStep(self, expImgMd):
i = 0
classesOut = None
while i < self.numberOfSplitIterations:
outImgs, classesOut = self.iterationStep(classesOut, expImgMd, i,
True)
i += 1
length = getSize(classesOut)
if length == 1:
i = 0
self.generateMdForClassification(classesOut)
def _createAngDist2D(self, it):
fnDir = self.protocol._getExtraPath("Iter%03d"%it)
fnAngles = join(fnDir,"angles.xmd")
view=None
if exists(fnAngles):
fnAnglesSqLite = join(fnDir,"angles.sqlite")
from pyworkflow.em.plotter import EmPlotter
if not exists(fnAnglesSqLite):
from pyworkflow.em.metadata.utils import getSize
self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % it, windowTitle="Angular distribution")
view.plotAngularDistributionFromMd(fnAnglesSqLite, 'iter %d' % it)
return view
def reconstructStep(self, fnRoot):
from pyworkflow.em.metadata.utils import getSize
if os.path.exists(fnRoot + ".xmd"):
Nimages = getSize(fnRoot + ".xmd")
if Nimages > 0:
self.runJob(
"xmipp_reconstruct_fourier",
"-i %s.xmd -o %s.vol --sym %s " %
(fnRoot, fnRoot, self.symmetryGroup.get()))
self.runJob(
"xmipp_transform_mask", "-i %s.vol --mask circular -%d " %
(fnRoot, self.Xdim2 / 2))
else:
print fnRoot + ".xmd is empty. The corresponding volume is not generated."
def split(self,i):
from pyworkflow.em.metadata.utils import getSize
from math import ceil
from random import randint
from time import sleep
sleep(randint(1,10))
fnDirCurrent=self._getExtraPath("Iter001")
fnAngles=self._getExtraPath("Iter001/angles.xmd")
fnSplit=self._getExtraPath('split%05d.xmd'%i)
Nimages=getSize(fnAngles)
Nsplit=ceil(Nimages*self.splitFraction.get())
self.runJob("xmipp_metadata_utilities","-i %s -o %s --operate random_subset %d"%(fnAngles,fnSplit,Nsplit),numberOfMpi=1)
fnVol=self._getExtraPath('split%05d.vol'%i)
args="-i %s -o %s --sym %s --weight"%(fnSplit,fnVol,self.symmetryGroup)
row=getFirstRow(fnAngles)
if row.containsLabel(xmipp.MDL_CTF_DEFOCUSU) or row.containsLabel(xmipp.MDL_CTF_MODEL):
TsCurrent=self.readInfoField(fnDirCurrent,"sampling",xmipp.MDL_SAMPLINGRATE)
args+=" --useCTF --sampling %f"%TsCurrent
if self.phaseFlipped:
args+=" --phaseFlipped"
self.runJob("xmipp_reconstruct_fourier",args,numberOfMpi=1)
self.runJob("xmipp_image_operate","-i %s --minus %s"%(fnVol,join(fnDirCurrent,"volume.vol")))
def generateMetadata(self, listNameImgs, listNumImgs, level):
# Renumerate unchanged classes
listNewNumImages = [-1] * len(listNumImgs)
count = 1
for i in range(len(listNumImgs)):
if listNumImgs[i] is not -1:
listNewNumImages[i] = count
count += 1
# Construct the new classes with the renumerated old classes
mdNewClasses = md.MetaData()
for i in range(len(listNumImgs)):
if listNumImgs[i] is not -1:
name = listNameImgs[i]
fn = name[name.find('@') + 1:-4] + '.xmd'
numRef = int(name[0:6])
mdClass = md.MetaData("[email protected]" + fn)
for row in iterRows(mdClass):
if mdClass.getValue(md.MDL_REF, row.getObjId()) == numRef:
row.setValue(md.MDL_REF, listNewNumImages[i])
row.addToMd(mdNewClasses)
# Add the two new classes to the list of renumerated classes
outSet = self._getExtraPath(join('level%03d' % level, 'level%03d' %
level + '_classes.xmd'))
mdClass = md.MetaData("[email protected]" + outSet)
rows = iterRows(mdClass)
for row in rows:
row.setValue(md.MDL_REF, count)
row.addToMd(mdNewClasses)
count = count + 1
mdNewClasses.write('[email protected]' + self._getExtraPath(join('level%03d'
% level,'intermediate_classes.xmd')),
MD_APPEND)
# Generate the intermediate images and the blocks of the intermediate
# classes for the unchanged classes
for i in range(len(listNumImgs)):
if listNumImgs[i] is not -1:
name = listNameImgs[i]
fn = name[name.find('@') + 1:-4] + '.xmd'
numRef = int(name[0:6])
# Read the list of images in this class
mdImgsInClass = md.MetaData(
'class%[email protected]%s' % (numRef, fn))
mdImgsInClass.fillConstant(md.MDL_REF, listNewNumImages[i])
mdImgsInClass.write('class%06d' % (listNewNumImages[i])
+ '[email protected]' + self._getExtraPath(join(
'level%03d' % level,
'intermediate_classes.xmd')),
MD_APPEND)
# Add the two new classes
if len(listNumImgs) == 0:
count = 1
else:
count = len(listNumImgs)
for newRef in range(getSize(outSet)):
mdImgsInClass = md.MetaData('class%[email protected]%s' % (newRef + 1,
outSet))
mdImgsInClass.fillConstant(md.MDL_REF, count)
mdImgsInClass.write('class%06d' % (count) + '[email protected]'
+ self._getExtraPath(join('level%03d' %
level,'intermediate_classes.xmd')),
MD_APPEND)
count = count + 1
def iterationStep (self, refSet, imgsExp, iter, useReferenceImages,
level, flag_split, flag_attraction):
if not exists(join(self._getExtraPath(), 'level%03d' % level)):
mkdir(join(self._getExtraPath(), 'level%03d' % level))
if not useReferenceImages and iter==0 and flag_split==True:
# First step: divide the metadata input file to generate
# a couple of references
if level==0:
if not flag_attraction:
outDirName = imgsExp[0:imgsExp.find('extra')+6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'extra')+5:-4]
else:
outDirName = imgsExp[0:imgsExp.find('extra') + 6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'level%03d' % level) + 8:-4]
else:
if not flag_attraction:
outDirName = imgsExp[0:imgsExp.find('extra') + 6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'level%03d' % (level-1)) + 8:-4]
else:
outDirName = imgsExp[0:imgsExp.find('extra') + 6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'level%03d' % level) + 8:-4]
self._params = {'imgsExp': imgsExp,
'outDir': outDirName}
args = ('-i %(imgsExp)s -n 2 --oroot %(outDir)s')
self.runJob("xmipp_metadata_split", args % self._params,
numberOfMpi=1)
# Second step: calculate the means of the previous metadata
expSet1 = outDirName + '000001.xmd'
avg1 = outDirName + '_000001'
expSet2 = outDirName + '000002.xmd'
avg2 = outDirName + '_000002'
self._params = {'imgsSet': expSet1,
'outputAvg': avg1}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics", args % self._params,
numberOfMpi=1)
self._params = {'imgsSet': expSet2,
'outputAvg': avg2}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics", args % self._params,
numberOfMpi=1)
# Third step: generate a single metadata with the two previous
# averages
refSet = self._getExtraPath(join('level%03d' % level,'refSet.xmd'))
self._params = {'avg1': avg1 + 'average.xmp',
'avg2': avg2 + 'average.xmp',
'outputMd': refSet}
args = ('-i %(avg1)s --set union %(avg2)s -o %(outputMd)s')
self.runJob("xmipp_metadata_utilities", args % self._params,
numberOfMpi=1)
# Fourth step: calling program xmipp_cuda_correlation
if flag_split:
filename = 'level%03d' % level+'_classes.xmd'
self._params = {'imgsRef': refSet,
'imgsExp': imgsExp,
'outputFile': 'images_level%03d' % level+'.xmd',
'tmpDir': join(self._getExtraPath(),'level%03d'
% level),
'keepBest': self.keepBest.get(),
'maxshift': self.maximumShift,
'outputClassesFile': filename,
}
else:
filename = 'general_level%03d' % level + '_classes.xmd'
self._params = {'imgsRef': refSet,
'imgsExp': imgsExp,
'outputFile': 'general_images_level%03d' % level
+ '.xmd',
'tmpDir': join(self._getExtraPath(),'level%03d'
% level),
'keepBest': self.keepBest.get(),
'maxshift': self.maximumShift,
'outputClassesFile': filename,
}
Nrefs = getSize(refSet)
if Nrefs>2:
args = '-i_ref %(imgsRef)s -i_exp %(imgsExp)s -o %(outputFile)s '\
'--odir %(tmpDir)s --keep_best %(keepBest)d '\
'--maxShift %(maxshift)d --classify %(outputClassesFile)s '\
'--simplifiedMd'
self.runJob("xmipp_cuda_correlation", args % self._params,
numberOfMpi=1)
else:
flag_error=True
while flag_error:
try:
flag_error=False
args='-i %(imgsExp)s --ref0 %(imgsRef)s --nref %(Nrefs)d '\
'--iter 1 --distance correlation --classicalMultiref '\
'--maxShift %(maxshift)d --odir %(cl2dDir)s'
self._params['Nrefs']=Nrefs
self._params['cl2dDir'] = self._getExtraPath(
join('level%03d' % level))
self.runJob("xmipp_mpi_classify_CL2D",
args % self._params)
except Exception as ex:
flag_error = True
if flag_split:
copy(self._getExtraPath(join('level%03d' % level,
"level_00","class_classes.xmd")),
self._getExtraPath(join('level%03d' % level,
'level%03d' % level
+ '_classes.xmd')))
copy(self._getExtraPath(join('level%03d' % level,"images.xmd")),
self._getExtraPath(join('level%03d' % level,
'images_level%03d' % level +
'.xmd')))
else:
copy(self._getExtraPath(join('level%03d' % level,
"level_00", "class_classes.xmd")),
self._getExtraPath(join('level%03d' % level,
'general_level%03d' % level +
'_classes.xmd')))
copy(self._getExtraPath(join('level%03d' % level,
"images.xmd")),
self._getExtraPath(join('level%03d' % level,
'general_images_level%03d' %
level + '.xmd')))
def iterationStep(self, refSet, imgsExp, iter, flag_split):
if flag_split:
outImgs, classesOut = self._getOutputSplitFn()
else:
outImgs, classesOut = self._getOutputClassFn()
outDirName = imgsExp[:-4]
if iter == 0 and flag_split == True:
# First step: divide the metadata input file to generate
# a couple of references
self._params = {'imgsExp': imgsExp, 'outDir': outDirName}
args = ('-i %(imgsExp)s -n 2 --oroot %(outDir)s')
self.runJob("xmipp_metadata_split",
args % self._params,
numberOfMpi=1)
# Second step: calculate the means of the previous metadata
expSet1 = outDirName + '000001.xmd'
avg1 = outDirName + '_000001'
expSet2 = outDirName + '000002.xmd'
avg2 = outDirName + '_000002'
self._params = {'imgsSet': expSet1, 'outputAvg': avg1}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics",
args % self._params,
numberOfMpi=1)
self._params = {'imgsSet': expSet2, 'outputAvg': avg2}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics",
args % self._params,
numberOfMpi=1)
# Third step: generate a single metadata with the two previous avgs
refSet = self._getExtraPath('refSet.xmd')
self._params = {
'avg1': avg1 + 'average.xmp',
'avg2': avg2 + 'average.xmp',
'outputMd': refSet
}
args = ('-i %(avg1)s --set union %(avg2)s -o %(outputMd)s')
self.runJob("xmipp_metadata_utilities",
args % self._params,
numberOfMpi=1)
# Fourth step: calling program xmipp_cuda_correlation
if flag_split:
self._params = {
'imgsRef': refSet,
'imgsExp': imgsExp,
'maxshift': self.maximumShift,
'Nrefs': getSize(refSet),
'outDir': self._getExtraPath(),
'rootFn': classesOut.split('/')[-1].replace('.xmd', '')
}
args = '-i %(imgsExp)s --ref0 %(imgsRef)s --nref %(Nrefs)d ' \
'--iter 1 --distance correlation --classicalMultiref ' \
'--maxShift %(maxshift)d --odir %(outDir)s --oroot %(' \
'rootFn)s'
self.runJob("mpirun -np 4 -bynode xmipp_mpi_classify_CL2D",
args % self._params)
fileTocopy = classesOut.replace('.xmd', '_classes.xmd')
fileTocopy = fileTocopy.replace('extra/', 'extra/level_00/')
copy(fileTocopy, classesOut)
copy(self._getExtraPath("images.xmd"), outImgs)
else:
self._params = {
'imgsRef': refSet,
'imgsExp': imgsExp,
'outputFile': outImgs,
'keepBest': self.keepBest.get(),
'maxshift': self.maximumShift,
'outputClassesFile': classesOut
}
args = '-i_ref %(imgsRef)s -i_exp %(imgsExp)s -o %(outputFile)s '\
'--keep_best %(keepBest)d --maxShift %(maxshift)d ' \
'--classify %(outputClassesFile)s --simplifiedMd'
self.runJob("xmipp_cuda_correlation",
args % self._params,
numberOfMpi=1)
if exists(outDirName + '000001.xmd'):
cleanPath(expSet1)
cleanPath(expSet2)
cleanPath(avg1 + 'average.xmp')
cleanPath(avg2 + 'average.xmp')
cleanPath(avg1 + 'stddev.xmp')
cleanPath(avg2 + 'stddev.xmp')
return outImgs, classesOut
