def classifyStep(self):
listNumImgs = []
listNameImgs = []
change = False
level=0
while len(listNumImgs) is not self.numberOfClasses.get() \
or change is True:
if self.useReferenceImages and self.numberOfClasses.get() \
== getSize(self.refSet):
if not exists(join(self._getExtraPath(), 'level%03d' % level)):
mkdir(join(self._getExtraPath(), 'level%03d' % level))
copy(self.refSet, self._getExtraPath(join('level%03d' % level,
'intermediate_classes.xmd')))
else:
self.splitStep(level)
#######################################
if not self.useAttraction:
self.attractionSplitStep(level)
#######################################
self.generateMetadata(listNameImgs, listNumImgs, level)
self.ref = self._getExtraPath(join('level%03d' % level,
'intermediate_classes.xmd'))
lengthMd = getSize(self.ref)
if lengthMd==self.numberOfClasses.get():
self.classifyWholeSetStep(level)
#######################################################
if not self.useAttraction:
change, listNumImgs, listNameImgs = \
self.attractionGeneralStep(level)
if change:
level = level + 1
continue
#######################################################
copy(self._getExtraPath(join('level%03d' % level,
'general_level%03d' % level
+ '_classes.xmd')),
self._getExtraPath('last_classes.xmd'), )
if exists(self._getExtraPath(join('level%03d' % level,
'general_images_level%03d'
% level + '.xmd'))):
copy(self._getExtraPath(join('level%03d' % level,
'general_images_level%03d'
% level + '.xmd')),
self._getExtraPath('last_images.xmd'), )
return
listNumImgs, listNameImgs = self.checkOutput(level)
self.cleaningPath(level)
level = level + 1
def _runClassifSteps(self, fnOut, fnBlock, fnDir,
imgNo, fnGallery, callbackMethod=None):
nop = self.noOfParticles.get()
if callbackMethod and not exists(fnOut):
if md.getSize(fnBlock) > nop:
# try:
callbackMethod(fnOut, fnBlock, fnDir, imgNo, fnGallery)
def _evalStop(self):
noOfLevRuns = self._getLevRuns(self._level)
if self.IS_3D and self.useReslog:
slope, y0, err = self._getReslogVars()
for rLev in noOfLevRuns:
iters = self._lastIter(rLev)
modelFn = self._getFileName('model', iter=iters,
lev=self._level, rLev=rLev)
modelMd = md.MetaData('model_classes@' + modelFn)
partMdFn = "particles@" + self._getFileName('input_star',
lev=self._level,
rLev=rLev)
partSize = md.getSize(partMdFn)
clsId = 1
for row in md.iterRows(modelMd):
fn = row.getValue(md.RLN_MLMODEL_REF_IMAGE)
clasDist = row.getValue('rlnClassDistribution')
classSize = int(clasDist * partSize)
if self._getClasDistCond(clasDist):
mapId = self._mapsDict[fn]
ptcStop = self.minPartsToStop.get()
if classSize < int(0.95*partSize):
if self.IS_3D and self.useReslog:
suffixSsnr = 'model_class_%d@' % clsId
ssnrMd = md.MetaData(suffixSsnr + modelFn)
f = self._getFunc(ssnrMd)
ExpcVal = slope*np.math.log10(classSize) + y0
val = f(1) + (2*err)
clsId += 1
print("StopValues: \n"
"Val SSnr=1: %0.4f, parts %d, ExpcVal "
"%0.4f" % (val, classSize, ExpcVal))
evalSlope = val < ExpcVal and self._level > 2
else:
evalSlope = False
else:
evalSlope = True
print("Values to stop the classification: ")
print("Lev: ", self._level, "rLev: ", rLev)
print("partSize: ", partSize)
print("class size: ", classSize)
print("min parts to stop: ", ptcStop)
print("evalSlope: ", evalSlope)
if classSize < ptcStop or evalSlope:
self.stopDict[mapId] = True
if not bool(self._clsIdDict):
self._clsIdDict[mapId] = 1
else:
classId = sorted(self._clsIdDict.values())[-1] + 1
self._clsIdDict[mapId] = classId
else:
self.stopDict[mapId] = False
def splitStep(self, expImgMd):
i = 0
classesOut = None
while i < self.numberOfSplitIterations:
outImgs, classesOut = self.iterationStep(classesOut, expImgMd, i,
True)
i += 1
length = md.getSize(classesOut)
if length == 1:
i = 0
self.generateMdForClassification(classesOut)
def sortImagesStep(self):
args = "-i Particles@%s -o %s --addToInput " % (self.fnInputMd,
self.fnOutputMd)
if os.path.exists(self.fnInputOldMd):
args += "-t Particles@%s " % self.fnInputOldMd
if self.autoParRejection == self.REJ_MAXZSCORE:
args += "--zcut " + str(self.maxZscore.get())
elif self.autoParRejection == self.REJ_PERCENTAGE:
args += "--percent " + str(self.percentage.get())
if self.addFeatures:
args += "--addFeatures "
self.runJob("xmipp_image_sort_by_statistics", args)
args = "-i Particles@%s -o %s" % (self.fnInputMd, self.fnOutputMd)
if self.autoParRejectionSSNR == self.REJ_PERCENTAGE_SSNR:
args += " --ssnrpercent " + str(self.percentageSSNR.get())
self.runJob("xmipp_image_ssnr", args)
if self.autoParRejectionVar != self.REJ_NONE:
print('Rejecting by variance:')
if self.outputSize == 0:
varList = []
giniList = []
print(' - Reading metadata')
mdata = emlib.MetaData(self.fnInputMd)
for objId in mdata:
varList.append(mdata.getValue(emlib.MDL_SCORE_BY_VAR, objId))
giniList.append(mdata.getValue(emlib.MDL_SCORE_BY_GINI, objId))
if self.autoParRejectionVar == self.REJ_VARIANCE:
valuesList = varList
self.mdLabels = [emlib.MDL_SCORE_BY_VAR]
else: # not working pretty well
valuesList = [var*(1-gini) for var, gini in zip(varList, giniList)]
self.mdLabels = [emlib.MDL_SCORE_BY_VAR, emlib.MDL_SCORE_BY_GINI]
self.varThreshold.set(histThresholding(valuesList))
print(' - Variance threshold: %f' % self.varThreshold)
rejectByVariance(self.fnInputMd, self.fnOutputMd, self.varThreshold,
self.autoParRejectionVar)
# update the processed particles
self.outputSize += getSize(self.fnInputMd)
def _createAngDist2D(self, it, heatmap):
fnDir = self.protocol._getExtraPath("Iter%03d"%it)
fnAngles = join(fnDir,"angles.xmd")
view=None
if exists(fnAngles):
fnAnglesSqLite = join(fnDir,"angles.sqlite")
from pwem.viewers import EmPlotter
if not exists(fnAnglesSqLite):
from pwem.emlib.metadata import getSize
self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % it, windowTitle="Angular distribution")
if heatmap:
axis = view.plotAngularDistributionFromMd(fnAnglesSqLite, '', histogram=True)
view.getFigure().colorbar(axis)
else:
view.plotAngularDistributionFromMd(fnAnglesSqLite, '')
return view
def _createFinalMetadata(self):
fnClasses = self._getExtraPath('last_classes.xmd')
numClasses = md.getSize(fnClasses)
for i in range(numClasses):
num = i + 1
fn = self._getExtraPath('dataClass%06d.xmd' % num)
if exists(fn):
line = ""
numLine = 0
while not line.startswith(' '):
numLine += 1
line = popen('sed -n %ip %s' % (numLine, fn)).read()
aux = self._getExtraPath('aux.xmd')
system('head -%i %s >> %s' % (numLine - 1, fn, aux))
dataHeader = self._getExtraPath('dataHeader.xmd')
fp = open(aux, 'r')
fout = open(dataHeader, 'w')
for i, line in enumerate(fp):
if i < 2:
continue
if not line.startswith('data_noname'):
fout.write(line)
else:
line = line.replace('noname', 'class%06d_images' % num)
fout.write(line)
fp.close()
fout.close()
cleanPath(aux)
fnOut = self._getExtraPath('dataImages%06d.xmd' % num)
system('tail -n +%i %s >> %s' % (i + 2, fn, aux))
system('cat %s %s >> %s' % (dataHeader, aux, fnOut))
if num == 1:
system('cat %s >> %s' %
(fn, self._getExtraPath('last_images.xmd')))
else:
system('cat %s >> %s' %
(aux, self._getExtraPath('last_images.xmd')))
cleanPath(aux)
cleanPath(dataHeader)
system('cat %s >> %s' %
(fnOut, self._getExtraPath('last_classes.xmd')))
def reconstructStep(self, fnRoot):
from pwem.emlib.metadata import getSize
if os.path.exists(fnRoot+".xmd"):
Nimages=getSize(fnRoot+".xmd")
if Nimages>0:
if self.useGpu.get():
# protocol will run several reconstructions at once, so execute each reconstruction separately
args = "-i %s.xmd -o %s.vol --sym %s --thr %s --fast"\
% (fnRoot, fnRoot, self.symmetryGroup.get(), self.numberOfThreads.get())
#AJ to make it work with and without queue system
if self.numberOfMpi.get()>1:
N_GPUs = len((self.gpuList.get()).split(','))
args += ' -gpusPerNode %d' % N_GPUs
args += ' -threadsPerGPU %d' % max(self.numberOfThreads.get(),4)
count=0
GpuListCuda=''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda+str(count)+' '
count+=1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda+str(count)+' '
GpuListAux = GpuListAux+str(elem)+','
count+=1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
if self.numberOfMpi.get()==1:
args += " --device %s" %GpuListCuda
if self.numberOfMpi.get()>1:
self.runJob('xmipp_cuda_reconstruct_fourier', args, numberOfMpi=len((self.gpuList.get()).split(','))+1)
else:
self.runJob('xmipp_cuda_reconstruct_fourier', args)
else:
self.runJob("xmipp_reconstruct_fourier_accel","-i %s.xmd -o %s.vol --sym %s "
%(fnRoot,fnRoot,self.symmetryGroup.get()))
self.runJob("xmipp_transform_mask","-i %s.vol --mask circular -%d "%(fnRoot,self.Xdim2/2))
else:
print(fnRoot+".xmd is empty. The corresponding volume is not generated.")
def _createAngDistChimera(self, it):
fnDir = self.protocol._getExtraPath("Iter%03d"%it)
fnAngles = join(fnDir,"angles.xmd")
view=None
if exists(fnAngles):
fnAnglesSqLite = join(fnDir,"angles.sqlite")
from pwem.emlib.metadata import getSize
self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles),
itemDataIterator=self._iterAngles(fnAngles))
vol = os.path.join(fnDir, "volumeAvg.mrc")
extraFilesPath = self.protocol._getExtraPath()
volOrigin = self.protocol.outputVolume.getShiftsFromOrigin()
radius = self.spheresScale.get()
view = ChimeraAngDist(vol, extraFilesPath,
angularDistFile=fnAngles,
spheresDistance=radius,
voxelSize=self.protocol.outputVolume.getSamplingRate(),
volOrigin=volOrigin,
showProjection=True)
return view
def homogeneizeStep(self):
minClass = self.homogeneize.get()
fnNeighbours = self._getExtraPath("neighbours.xmd")
# Look for the block with the minimum number of images
if minClass == 0:
minClass = 1e38
for block in emlib.getBlocksInMetaDataFile(fnNeighbours):
projNumber = int(block.split("_")[1])
fnDir = self._getExtraPath("direction_%d" % projNumber,
"level_00", "class_classes.xmd")
if exists(fnDir):
blockSize = md.getSize("class000001_images@" + fnDir)
if blockSize < minClass:
minClass = blockSize
# Construct the homogeneized metadata
mdAll = emlib.MetaData()
mdSubset = emlib.MetaData()
mdRandom = emlib.MetaData()
for block in emlib.getBlocksInMetaDataFile(fnNeighbours):
projNumber = int(block.split("_")[1])
fnDir = self._getExtraPath("direction_%d" % projNumber, "level_00",
"class_classes.xmd")
if exists(fnDir):
mdDirection = emlib.MetaData("class000001_images@" + fnDir)
mdRandom.randomize(mdDirection)
mdSubset.selectPart(mdRandom, 0, min(mdRandom.size(),
minClass))
mdAll.unionAll(mdSubset)
mdAll.removeDuplicates(md.MDL_ITEM_ID)
mdAll.sort(md.MDL_ITEM_ID)
mdAll.fillConstant(md.MDL_PARTICLE_ID, 1)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
mdAll.write(fnHomogeneous)
self.runJob("xmipp_metadata_utilities",
'-i %s --operate modify_values "particleId=itemId"' %
fnHomogeneous,
numberOfMpi=1)
def significantStep(self, iterNumber, alpha):
iterDir = self._getTmpPath('iter%03d' % iterNumber)
makePath(iterDir)
prevVolFn = self.getIterVolume(iterNumber - 1)
volFn = self.getIterVolume(iterNumber)
anglesFn = self._getExtraPath('angles_iter%03d.xmd' % iterNumber)
t = Timer()
t.tic()
if self.useGpu.get() and iterNumber > 1:
# Generate projections
fnGalleryRoot = join(iterDir, "gallery")
args = "-i %s -o %s.stk --sampling_rate %f --sym %s " \
"--compute_neighbors --angular_distance -1 " \
"--experimental_images %s --min_tilt_angle %f " \
"--max_tilt_angle %f -v 0 --perturb %f " % \
(prevVolFn, fnGalleryRoot, self.angularSampling.get(),
self.symmetryGroup, self.imgsFn, self.minTilt, self.maxTilt,
math.sin(self.angularSampling.get()) / 4)
self.runJob("xmipp_angular_project_library ", args, numberOfMpi=1)
if self.trueSymsNo != 0:
alphaApply = (alpha * self.trueSymsNo) / 2
else:
alphaApply = alpha / 2
from pwem.emlib.metadata import getSize
N = int(getSize(fnGalleryRoot + '.doc') * alphaApply * 2)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
args = '-i %s -r %s.doc -o %s --keepBestN %f --dev %s ' % \
(self.imgsFn, fnGalleryRoot, anglesFn, N, GpuListCuda)
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
cleanPattern(fnGalleryRoot + "*")
else:
args = self.getSignificantArgs(self.imgsFn)
args += ' --odir %s' % iterDir
args += ' --alpha0 %f --alphaF %f' % (alpha, alpha)
args += ' --dontCheckMirrors '
if iterNumber == 1:
if self.thereisRefVolume:
args += " --initvolumes " + \
self._getExtraPath('input_volumes.xmd')
else:
args += " --numberOfVolumes 1"
else:
args += " --initvolumes %s" % prevVolFn
self.runJob("xmipp_reconstruct_significant", args)
moveFile(os.path.join(iterDir, 'angles_iter001_00.xmd'), anglesFn)
t.toc('Significant took: ')
reconsArgs = ' -i %s --fast' % anglesFn
reconsArgs += ' -o %s' % volFn
reconsArgs += ' --weight -v 0 --sym %s ' % self.symmetryGroup
print("Number of images for reconstruction: ",
metadata.getSize(anglesFn))
t.tic()
if self.useGpu.get():
cudaReconsArgs = reconsArgs
#AJ to make it work with and without queue system
if self.numberOfMpi.get() > 1:
N_GPUs = len((self.gpuList.get()).split(','))
cudaReconsArgs += ' -gpusPerNode %d' % N_GPUs
cudaReconsArgs += ' -threadsPerGPU %d' % max(
self.numberOfThreads.get(), 4)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
cudaReconsArgs += ' --thr %s' % self.numberOfThreads.get()
if self.numberOfMpi.get() == 1:
cudaReconsArgs += ' --device %s' % (GpuListCuda)
if self.numberOfMpi.get() > 1:
self.runJob('xmipp_cuda_reconstruct_fourier',
cudaReconsArgs,
numberOfMpi=len(
(self.gpuList.get()).split(',')) + 1)
else:
self.runJob('xmipp_cuda_reconstruct_fourier', cudaReconsArgs)
else:
self.runJob("xmipp_reconstruct_fourier_accel", reconsArgs)
t.toc('Reconstruct fourier took: ')
# Center the volume
fnSym = self._getExtraPath('volumeSym_%03d.vol' % iterNumber)
self.runJob("xmipp_transform_mirror",
"-i %s -o %s --flipX" % (volFn, fnSym),
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipY" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipZ" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_image_operate",
"-i %s --plus %s" % (fnSym, volFn),
numberOfMpi=1)
self.runJob("xmipp_volume_align",
'--i1 %s --i2 %s --local --apply' % (fnSym, volFn),
numberOfMpi=1)
cleanPath(fnSym)
# To mask the volume
xdim = self.inputSet.get().getDimensions()[0]
maskArgs = "-i %s --mask circular %d -v 0" % (volFn, -xdim / 2)
self.runJob('xmipp_transform_mask', maskArgs, numberOfMpi=1)
# TODO mask the final volume in some smart way...
# To filter the volume
if self.useMaxRes:
self.runJob('xmipp_transform_filter',
'-i %s --fourier low_pass %f --sampling %f' % \
(volFn, self.maxResolution.get(), self.TsCurrent),
numberOfMpi=1)
if not self.keepIntermediate:
cleanPath(prevVolFn, iterDir)
if self.thereisRefVolume:
cleanPath(self._getExtraPath('filteredVolume.vol'))
def iterationStep(self, refSet, imgsExp, iter, flag_split):
if flag_split:
outImgs, classesOut = self._getOutputSplitFn()
else:
outImgs, classesOut = self._getOutputClassFn()
outDirName = splitext(imgsExp)[0]
if iter == 0 and flag_split == True:
# First step: divide the metadata input file to generate
# a couple of references
self._params = {'imgsExp': imgsExp, 'outDir': outDirName}
args = ('-i %(imgsExp)s -n 2 --oroot %(outDir)s')
self.runJob("xmipp_metadata_split",
args % self._params,
numberOfMpi=1)
# Second step: calculate the means of the previous metadata
expSet1 = outDirName + '000001.xmd'
avg1 = outDirName + '_000001'
expSet2 = outDirName + '000002.xmd'
avg2 = outDirName + '_000002'
self._params = {'imgsSet': expSet1, 'outputAvg': avg1}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics",
args % self._params,
numberOfMpi=1)
self._params = {'imgsSet': expSet2, 'outputAvg': avg2}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics",
args % self._params,
numberOfMpi=1)
# Third step: generate a single metadata with the two previous avgs
refSet = self._getExtraPath('refSet.xmd')
self._params = {
'avg1': avg1 + 'average.xmp',
'avg2': avg2 + 'average.xmp',
'outputMd': refSet
}
args = ('-i %(avg1)s --set union %(avg2)s -o %(outputMd)s')
self.runJob("xmipp_metadata_utilities",
args % self._params,
numberOfMpi=1)
# Fourth step: calling program xmipp_cuda_align_significant
metadataRef = md.MetaData(refSet)
if metadataRef.containsLabel(md.MDL_REF) is False:
args = ('-i %(outputMd)s --fill ref lineal 1 1 -o %(outputMd)s')
self.runJob("xmipp_metadata_utilities",
args % self._params,
numberOfMpi=1)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
if flag_split:
fileTocopy = classesOut.replace('.xmd', '_classes.xmd')
fileTocopy = fileTocopy.replace('extra/', 'extra/level_00/')
self._params = {
'imgsRef': refSet,
'imgsExp': imgsExp,
'maxshift': self.maximumShift,
'Nrefs': md.getSize(refSet),
'outDir': self._getExtraPath(),
'outImgCuda': self._getExtraPath("images.xmd"),
'rootFn': classesOut.split('/')[-1].replace('.xmd', ''),
'keepBest': self.keepBest.get(),
'outClassesCuda': fileTocopy,
}
args = '-i %(imgsExp)s --ref0 %(imgsRef)s --nref %(Nrefs)d ' \
'--iter 1 --distance correlation --classicalMultiref ' \
'--maxShift %(maxshift)d --odir %(outDir)s --oroot %(' \
'rootFn)s --dontMirrorImages '
self.runJob("xmipp_classify_CL2D",
args % self._params,
numberOfMpi=self.numberOfMpi.get())
copy(fileTocopy, classesOut)
copy(self._getExtraPath("images.xmd"), outImgs)
else:
self._params = {
'imgsRef': refSet,
'imgsExp': imgsExp,
'outputFile': outImgs,
'keepBest': self.keepBest.get(),
'maxshift': self.maximumShift,
'outputClassesFile': classesOut,
'device': GpuListCuda,
'outputClassesFileNoExt': splitext(classesOut)[0],
'auxOut': self._getExtraPath('flipReferences%06d.xmd' % iter),
}
args = '-i %(imgsExp)s -r %(imgsRef)s -o %(outputFile)s ' \
'--keepBestN 1 --oUpdatedRefs %(outputClassesFileNoExt)s --dev %(device)s '
self.runJob(CUDA_ALIGN_SIGNIFICANT,
args % self._params,
numberOfMpi=1)
if exists(outDirName + '000001.xmd'):
cleanPath(expSet1)
cleanPath(expSet2)
cleanPath(avg1 + 'average.xmp')
cleanPath(avg2 + 'average.xmp')
cleanPath(avg1 + 'stddev.xmp')
cleanPath(avg2 + 'stddev.xmp')
return outImgs, classesOut
def generateMetadata(self, listNameImgs, listNumImgs, level):
# Renumerate unchanged classes
listNewNumImages = [-1] * len(listNumImgs)
count = 1
for i in range(len(listNumImgs)):
if listNumImgs[i] != -1:
listNewNumImages[i] = count
count += 1
# Construct the new classes with the renumerated old classes
mdNewClasses = md.MetaData()
for i in range(len(listNumImgs)):
if listNumImgs[i] != -1:
name = listNameImgs[i]
fn = name[name.find('@') + 1:-4] + '.xmd'
numRef = int(name[0:6])
mdClass = md.MetaData("classes@" + fn)
for row in iterRows(mdClass):
if mdClass.getValue(md.MDL_REF, row.getObjId()) == numRef:
row.setValue(md.MDL_REF, listNewNumImages[i])
row.addToMd(mdNewClasses)
# Add the two new classes to the list of renumerated classes
outSet = self._getExtraPath(join('level%03d' % level, 'level%03d' %
level + '_classes.xmd'))
mdClass = md.MetaData("classes@" + outSet)
rows = iterRows(mdClass)
for row in rows:
row.setValue(md.MDL_REF, count)
row.addToMd(mdNewClasses)
count = count + 1
mdNewClasses.write('classes@' + self._getExtraPath(join('level%03d'
% level,'intermediate_classes.xmd')),
MD_APPEND)
# Generate the intermediate images and the blocks of the intermediate
# classes for the unchanged classes
for i in range(len(listNumImgs)):
if listNumImgs[i] != -1:
name = listNameImgs[i]
fn = name[name.find('@') + 1:-4] + '.xmd'
numRef = int(name[0:6])
# Read the list of images in this class
mdImgsInClass = md.MetaData(
'class%06d_images@%s' % (numRef, fn))
mdImgsInClass.fillConstant(md.MDL_REF, listNewNumImages[i])
mdImgsInClass.write('class%06d' % (listNewNumImages[i])
+ '_images@' + self._getExtraPath(join(
'level%03d' % level,
'intermediate_classes.xmd')),
MD_APPEND)
# Add the two new classes
if len(listNumImgs) == 0:
count = 1
else:
count = len(listNumImgs)
for newRef in range(getSize(outSet)):
mdImgsInClass = md.MetaData('class%06d_images@%s' % (newRef + 1, outSet))
mdImgsInClass.fillConstant(md.MDL_REF, count)
mdImgsInClass.write('class%06d' % (count) + '_images@'
+ self._getExtraPath(join('level%03d' %
level,'intermediate_classes.xmd')),
MD_APPEND)
count = count + 1
def iterationStep (self, refSet, imgsExp, iter, useReferenceImages,
level, flag_split, flag_attraction):
if not exists(join(self._getExtraPath(), 'level%03d' % level)):
mkdir(join(self._getExtraPath(), 'level%03d' % level))
if not useReferenceImages and iter==0 and flag_split==True:
# First step: divide the metadata input file to generate
# a couple of references
if level==0:
if not flag_attraction:
outDirName = imgsExp[0:imgsExp.find('extra')+6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'extra')+5:-4]
else:
outDirName = imgsExp[0:imgsExp.find('extra') + 6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'level%03d' % level) + 8:-4]
else:
if not flag_attraction:
outDirName = imgsExp[0:imgsExp.find('extra') + 6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'level%03d' % (level-1)) + 8:-4]
else:
outDirName = imgsExp[0:imgsExp.find('extra') + 6] + \
'level%03d' % level + imgsExp[imgsExp.find(
'level%03d' % level) + 8:-4]
self._params = {'imgsExp': imgsExp,
'outDir': outDirName}
args = ('-i %(imgsExp)s -n 2 --oroot %(outDir)s')
self.runJob("xmipp_metadata_split", args % self._params,
numberOfMpi=1)
# Second step: calculate the means of the previous metadata
expSet1 = outDirName + '000001.xmd'
avg1 = outDirName + '_000001'
expSet2 = outDirName + '000002.xmd'
avg2 = outDirName + '_000002'
self._params = {'imgsSet': expSet1,
'outputAvg': avg1}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics", args % self._params,
numberOfMpi=1)
self._params = {'imgsSet': expSet2,
'outputAvg': avg2}
args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
self.runJob("xmipp_image_statistics", args % self._params,
numberOfMpi=1)
# Third step: generate a single metadata with the two previous
# averages
refSet = self._getExtraPath(join('level%03d' % level,'refSet.xmd'))
self._params = {'avg1': avg1 + 'average.xmp',
'avg2': avg2 + 'average.xmp',
'outputMd': refSet}
args = ('-i %(avg1)s --set union %(avg2)s -o %(outputMd)s')
self.runJob("xmipp_metadata_utilities", args % self._params,
numberOfMpi=1)
# Fourth step: calling program xmipp_cuda_align_significant
metadataRef = md.MetaData(refSet)
if metadataRef.containsLabel(md.MDL_REF) is False:
args = ('-i %s --fill ref lineal 1 1 -o %s'%(refSet, refSet))
self.runJob("xmipp_metadata_utilities", args,
numberOfMpi=1)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
if flag_split:
filename = 'level%03d' % level+'_classes.xmd'
self._params = {'imgsRef': refSet,
'imgsExp': imgsExp,
'outputFile': 'images_level%03d' % level+'.xmd',
'tmpDir': join(self._getExtraPath(),'level%03d'
% level),
'keepBest': self.keepBest.get(),
'maxshift': self.maximumShift,
'outputClassesFile': filename,
'device': GpuListCuda,
'outputClassesFileNoExt': splitext(filename)[0],
}
else:
filename = 'general_level%03d' % level + '_classes.xmd'
self._params = {'imgsRef': refSet,
'imgsExp': imgsExp,
'outputFile': 'general_images_level%03d' % level
+ '.xmd',
'tmpDir': join(self._getExtraPath(),'level%03d'
% level),
'keepBest': self.keepBest.get(),
'maxshift': self.maximumShift,
'outputClassesFile': filename,
'device': GpuListCuda,
'outputClassesFileNoExt': splitext(filename)[0],
}
Nrefs = getSize(refSet)
if Nrefs>2:
args = '-i %(imgsExp)s -r %(imgsRef)s -o %(outputFile)s ' \
'--keepBestN 1 --oUpdatedRefs %(outputClassesFileNoExt)s ' \
'--odir %(tmpDir)s --dev %(device)s'
self.runJob(CUDA_ALIGN_SIGNIFICANT, args % self._params, numberOfMpi=1)
else:
self._params['Nrefs'] = Nrefs
self._params['cl2dDir'] = self._getExtraPath(join('level%03d' % level))
self._params['cl2dDirNew'] = self._getExtraPath(join('level%03d' % level, "level_00"))
args='-i %(imgsExp)s --ref0 %(imgsRef)s --nref %(Nrefs)d '\
'--iter 1 --distance correlation --classicalMultiref '\
'--maxShift %(maxshift)d --odir %(cl2dDir)s --dontMirrorImages '
self.runJob("xmipp_classify_CL2D",
args % self._params, numberOfMpi=self.numberOfMpi.get())
if flag_split:
copy(self._getExtraPath(join('level%03d' % level,
"level_00","class_classes.xmd")),
self._getExtraPath(join('level%03d' % level,
'level%03d' % level
+ '_classes.xmd')))
copy(self._getExtraPath(join('level%03d' % level,"images.xmd")),
self._getExtraPath(join('level%03d' % level,
'images_level%03d' % level +
'.xmd')))
else:
copy(self._getExtraPath(join('level%03d' % level,
"level_00", "class_classes.xmd")),
self._getExtraPath(join('level%03d' % level,
'general_level%03d' % level +
'_classes.xmd')))
copy(self._getExtraPath(join('level%03d' % level,
"images.xmd")),
self._getExtraPath(join('level%03d' % level,
'general_images_level%03d' %
level + '.xmd')))
def classifyOneGroup(self, projNumber, projMdBlock, projRef, mdClasses,
mdImages):
""" Classify one of the neighbourhood groups if not empty.
Class information will be stored in output metadata: mdOut
"""
blockSize = md.getSize(projMdBlock)
Nclasses = self.directionalClasses.get()
Nlevels = int(math.ceil(math.log(Nclasses) / math.log(2)))
# Skip projection directions with not enough images to
# create a given number of classes
if blockSize / Nclasses < 10:
return
fnDir = self._getExtraPath("direction_%s" % projNumber)
makePath(fnDir)
# Run CL2D classification for the images assigned to one direction
args = "-i %s " % projMdBlock
args += "--odir %s " % fnDir
args += "--ref0 %s --iter %d --nref %d " % (
projRef, self.cl2dIterations, Nclasses)
args += "--distance correlation --classicalMultiref "
args += "--maxShift %f " % self.maxShift
try:
self.runJob("xmipp_classify_CL2D", args)
except:
return
# After CL2D the stk and xmd files should be produced
classesXmd = join(fnDir, "level_%02d/class_classes.xmd" % Nlevels)
classesStk = join(fnDir, "level_%02d/class_classes.stk" % Nlevels)
# Let's check that the output was produced
if not exists(classesStk):
return
# Run align of the class average and the projection representative
fnAlignRoot = join(fnDir, "classes")
args = "-i %s " % classesStk
args += "--ref %s " % projRef
args += " --oroot %s --iter 1" % fnAlignRoot
self.runJob("xmipp_image_align", args, numberOfMpi=1)
# Apply alignment
args = "-i %s_alignment.xmd --apply_transform" % fnAlignRoot
self.runJob("xmipp_transform_geometry", args, numberOfMpi=1)
for classNo in range(1, Nclasses + 1):
localImagesMd = emlib.MetaData("class%06d_images@%s" %
(classNo, classesXmd))
# New class detected
self.classCount += 1
# Check which images have not been assigned yet to any class
# and assign them to this new class
for objId in localImagesMd:
imgId = localImagesMd.getValue(emlib.MDL_ITEM_ID, objId)
# Add images not classify yet and store their class number
if imgId not in self.classImages:
self.classImages.add(imgId)
newObjId = mdImages.addObject()
mdImages.setValue(emlib.MDL_ITEM_ID, imgId, newObjId)
mdImages.setValue(emlib.MDL_REF2, self.classCount,
newObjId)
newClassId = mdClasses.addObject()
mdClasses.setValue(emlib.MDL_REF, projNumber, newClassId)
mdClasses.setValue(emlib.MDL_REF2, self.classCount, newClassId)
mdClasses.setValue(emlib.MDL_IMAGE,
"%d@%s" % (classNo, classesStk), newClassId)
mdClasses.setValue(emlib.MDL_IMAGE1, projRef, newClassId)
mdClasses.setValue(emlib.MDL_CLASS_COUNT, localImagesMd.size(),
newClassId)
def classifyOneGroup(self, projNumber, projMdBlock, projRef, mdClasses,
mdImages):
""" Classify one of the neighbourhood groups if not empty.
Class information will be stored in output metadata: mdOut
"""
blockSize = md.getSize(projMdBlock)
fnToUse = projMdBlock
if self.maxCLimgs > 0 and blockSize > self.maxCLimgs:
fnToUSe = self._getTmpPath("coneImages.xmd")
self.runJob("xmipp_metadata_utilities","-i %s -o %s --operate random_subset %d"\
%(projMdBlock,fnToUSe,self.maxCLimgs),numberOfMpi=1)
Nclasses = self.directionalClasses.get()
Nlevels = int(math.ceil(math.log(Nclasses) / math.log(2)))
# Skip projection directions with not enough images to
# create a given number of classes
if blockSize / Nclasses < 10:
return
if self.useGpu.get():
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
fnDir = self._getExtraPath("direction_%s" % projNumber)
if not exists(fnDir):
makePath(fnDir)
for i in range(self.class2dIterations.get()):
mdRefName = join(fnDir, 'reference.xmd')
if i == 0:
mdRef = md.MetaData()
for j in range(2):
row = md.Row()
row.setValue(md.MDL_REF, j + 1)
row.setValue(md.MDL_IMAGE, projRef)
row.addToMd(mdRef)
mdRef.write(mdRefName, emlib.MD_APPEND)
else:
mdRefName = join(fnDir, "level_%02d" % (i - 1),
"class_classes.xmd")
if not exists(join(fnDir, "level_%02d" % i)):
makePath(join(fnDir, "level_%02d" % i))
args = '-i %s -r %s -o images.xmd --odir %s' \
' --keepBestN 1 --oUpdatedRefs %s ' % (fnToUse, mdRefName, join(fnDir,"level_%02d"%i), 'class_classes')
args += ' --dev %s ' % GpuListCuda
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
copy(
join(fnDir, "level_%02d" % (self.class2dIterations.get() - 1),
"images.xmd"), join(fnDir, "images.xmd"))
# After classification the stk and xmd files should be produced
classesXmd = join(
fnDir, "level_%02d/class_classes.xmd" %
(self.class2dIterations.get() - 1))
classesStk = join(
fnDir, "level_%02d/class_classes.stk" %
(self.class2dIterations.get() - 1))
else:
fnDir = self._getExtraPath("direction_%s" % projNumber)
if not exists(join(fnDir, "level_00")):
makePath(fnDir)
# Run CL2D classification for the images assigned to one direction
args = "-i %s " % fnToUse
args += "--odir %s " % fnDir
args += "--ref0 %s --iter %d --nref %d " % \
(projRef, self.class2dIterations, Nclasses)
args += "--distance correlation --classicalMultiref "
args += "--maxShift %f " % self.maxShift
try:
self.runJob("xmipp_classify_CL2D",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
except:
return
# After CL2D the stk and xmd files should be produced
classesXmd = join(fnDir, "level_%02d/class_classes.xmd" % Nlevels)
classesStk = join(fnDir, "level_%02d/class_classes.stk" % Nlevels)
# Let's check that the output was produced
if not exists(classesStk):
return
# Run align of the class average and the projection representative
fnAlignRoot = join(fnDir, "classes")
args = "-i %s " % classesStk
args += "--ref %s " % projRef
args += " --oroot %s --iter 1" % fnAlignRoot
self.runJob("xmipp_image_align", args, numberOfMpi=1)
# Apply alignment
args = "-i %s_alignment.xmd --apply_transform" % fnAlignRoot
self.runJob("xmipp_transform_geometry", args, numberOfMpi=1)
for classNo in range(1, Nclasses + 1):
localImagesMd = emlib.MetaData("class%06d_images@%s" %
(classNo, classesXmd))
# New class detected
self.classCount += 1
# Check which images have not been assigned yet to any class
# and assign them to this new class
for objId in localImagesMd:
imgId = localImagesMd.getValue(emlib.MDL_ITEM_ID, objId)
# Add images not classify yet and store their class number
if imgId not in self.classImages:
self.classImages.add(imgId)
newObjId = mdImages.addObject()
mdImages.setValue(emlib.MDL_ITEM_ID, imgId, newObjId)
mdImages.setValue(emlib.MDL_REF2, self.classCount,
newObjId)
newClassId = mdClasses.addObject()
mdClasses.setValue(emlib.MDL_REF, projNumber, newClassId)
mdClasses.setValue(emlib.MDL_REF2, self.classCount, newClassId)
mdClasses.setValue(emlib.MDL_IMAGE,
"%d@%s" % (classNo, classesStk), newClassId)
mdClasses.setValue(emlib.MDL_IMAGE1, projRef, newClassId)
mdClasses.setValue(emlib.MDL_CLASS_COUNT, localImagesMd.size(),
newClassId)