def classifyStep(self):

        listNumImgs = []
        listNameImgs = []
        change = False

        level=0
        while len(listNumImgs) is not self.numberOfClasses.get() \
                or change is True:

            if self.useReferenceImages and self.numberOfClasses.get() \
                    == getSize(self.refSet):
                if not exists(join(self._getExtraPath(), 'level%03d' % level)):
                    mkdir(join(self._getExtraPath(), 'level%03d' % level))
                copy(self.refSet, self._getExtraPath(join('level%03d' % level,
                                                'intermediate_classes.xmd')))
            else:
                self.splitStep(level)

                #######################################
                if not self.useAttraction:
                    self.attractionSplitStep(level)
                #######################################

                self.generateMetadata(listNameImgs, listNumImgs, level)

            self.ref = self._getExtraPath(join('level%03d' % level,
                                               'intermediate_classes.xmd'))
            lengthMd = getSize(self.ref)
            if lengthMd==self.numberOfClasses.get():
                self.classifyWholeSetStep(level)
                #######################################################
                if not self.useAttraction:
                    change, listNumImgs, listNameImgs = \
                        self.attractionGeneralStep(level)
                    if change:
                        level = level + 1
                        continue
                #######################################################
                copy(self._getExtraPath(join('level%03d' % level,
                                             'general_level%03d' % level
                                             + '_classes.xmd')),
                     self._getExtraPath('last_classes.xmd'), )
                if exists(self._getExtraPath(join('level%03d' % level,
                                                  'general_images_level%03d'
                                                  % level + '.xmd'))):
                    copy(self._getExtraPath(join('level%03d' % level,
                                                 'general_images_level%03d'
                                                 % level + '.xmd')),
                         self._getExtraPath('last_images.xmd'), )
                return

            listNumImgs, listNameImgs = self.checkOutput(level)
            self.cleaningPath(level)
            level = level + 1
Exemple #2
0
    def split(self, i):
        from pyworkflow.em.metadata.utils import getSize
        from math import ceil
        from random import randint
        from time import sleep
        sleep(randint(1, 10))
        fnDirCurrent = self._getExtraPath("Iter001")
        fnAngles = self._getExtraPath("Iter001/angles.xmd")
        fnSplit = self._getExtraPath('split%05d.xmd' % i)
        Nimages = getSize(fnAngles)
        Nsplit = ceil(Nimages * self.splitFraction.get())
        self.runJob("xmipp_metadata_utilities",
                    "-i %s -o %s --operate random_subset %d" %
                    (fnAngles, fnSplit, Nsplit),
                    numberOfMpi=1)

        fnVol = self._getExtraPath('split%05d.vol' % i)
        args = "-i %s -o %s --sym %s --weight" % (fnSplit, fnVol,
                                                  self.symmetryGroup)
        row = getFirstRow(fnAngles)
        if row.containsLabel(xmipp.MDL_CTF_DEFOCUSU) or row.containsLabel(
                xmipp.MDL_CTF_MODEL):
            TsCurrent = self.readInfoField(fnDirCurrent, "sampling",
                                           xmipp.MDL_SAMPLINGRATE)
            args += " --useCTF --sampling %f" % TsCurrent
            if self.phaseFlipped:
                args += " --phaseFlipped"
        self.runJob("xmipp_reconstruct_fourier", args, numberOfMpi=1)
        self.runJob(
            "xmipp_image_operate",
            "-i %s --minus %s" % (fnVol, join(fnDirCurrent, "volume.vol")))
 def reconstructStep(self, fnRoot):
     from pyworkflow.em.metadata.utils import getSize
     if os.path.exists(fnRoot+".xmd"):
         Nimages=getSize(fnRoot+".xmd")
         if Nimages>0:
             self.runJob("xmipp_reconstruct_fourier","-i %s.xmd -o %s.vol --sym %s " %(fnRoot,fnRoot,self.symmetryGroup.get()))
             self.runJob("xmipp_transform_mask","-i %s.vol --mask circular -%d "%(fnRoot,self.Xdim2/2))
     else:
         print fnRoot+".xmd is empty. The corresponding volume is not generated." 
 def _createAngDistChimera(self, it):
     fnDir = self.protocol._getExtraPath("Iter%03d"%it)
     fnAngles = join(fnDir,"angles.xmd")
     view=None
     if exists(fnAngles):
         fnAnglesSqLite = join(fnDir,"angles.sqlite")
         from pyworkflow.em.metadata.utils import getSize
         self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
         view = ChimeraClientView(join(fnDir,"volumeAvg.mrc"), showProjection=True, angularDistFile=fnAnglesSqLite, spheresDistance=self.spheresScale.get())
     return view
 def _createAngDistChimera(self, it):
     fnDir = self.protocol._getExtraPath("Iter%03d"%it)
     fnAngles = join(fnDir,"angles.xmd")
     view=None
     if exists(fnAngles):
         fnAnglesSqLite = join(fnDir,"angles.sqlite")
         from pyworkflow.em.metadata.utils import getSize
         self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
         view = ChimeraClientView(join(fnDir,"volumeAvg.mrc"), showProjection=True, angularDistFile=fnAnglesSqLite, spheresDistance=self.spheresScale.get())
     return view
 def _createAngDist2D(self, it):
     fnDir = self.protocol._getExtraPath("Iter%03d"%it)
     fnAngles = join(fnDir,"angles.xmd")
     view=None
     if exists(fnAngles):
         fnAnglesSqLite = join(fnDir,"angles.sqlite")
         from pyworkflow.em.metadata.utils import getSize
         from pyworkflow.em.plotter import EmPlotter
         self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
         view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % it, windowTitle="Angular distribution")
         view.plotAngularDistributionFromMd(fnAnglesSqLite, 'iter %d' % it)
     return view
Exemple #7
0
    def splitStep(self, expImgMd):
        i = 0
        classesOut = None
        while i < self.numberOfSplitIterations:
            outImgs, classesOut = self.iterationStep(classesOut, expImgMd, i,
                                                     True)
            i += 1
            length = getSize(classesOut)
            if length == 1:
                i = 0

        self.generateMdForClassification(classesOut)
 def _createAngDist2D(self, it):
     fnDir = self.protocol._getExtraPath("Iter%03d"%it)
     fnAngles = join(fnDir,"angles.xmd")
     view=None
     if exists(fnAngles):
         fnAnglesSqLite = join(fnDir,"angles.sqlite")
         from pyworkflow.em.plotter import EmPlotter
         if not exists(fnAnglesSqLite):
             from pyworkflow.em.metadata.utils import getSize
             self.createAngDistributionSqlite(fnAnglesSqLite, getSize(fnAngles), itemDataIterator=self._iterAngles(fnAngles))
         view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % it, windowTitle="Angular distribution")
         view.plotAngularDistributionFromMd(fnAnglesSqLite, 'iter %d' % it)
     return view
Exemple #9
0
 def reconstructStep(self, fnRoot):
     from pyworkflow.em.metadata.utils import getSize
     if os.path.exists(fnRoot + ".xmd"):
         Nimages = getSize(fnRoot + ".xmd")
         if Nimages > 0:
             self.runJob(
                 "xmipp_reconstruct_fourier",
                 "-i %s.xmd -o %s.vol --sym %s " %
                 (fnRoot, fnRoot, self.symmetryGroup.get()))
             self.runJob(
                 "xmipp_transform_mask", "-i %s.vol --mask circular -%d " %
                 (fnRoot, self.Xdim2 / 2))
     else:
         print fnRoot + ".xmd is empty. The corresponding volume is not generated."
 def split(self,i):
     from pyworkflow.em.metadata.utils import getSize
     from math import ceil
     from random import randint
     from time import sleep
     sleep(randint(1,10))
     fnDirCurrent=self._getExtraPath("Iter001")
     fnAngles=self._getExtraPath("Iter001/angles.xmd")
     fnSplit=self._getExtraPath('split%05d.xmd'%i)
     Nimages=getSize(fnAngles)
     Nsplit=ceil(Nimages*self.splitFraction.get())
     self.runJob("xmipp_metadata_utilities","-i %s -o %s --operate random_subset %d"%(fnAngles,fnSplit,Nsplit),numberOfMpi=1)
 
     fnVol=self._getExtraPath('split%05d.vol'%i)
     args="-i %s -o %s --sym %s --weight"%(fnSplit,fnVol,self.symmetryGroup)
     row=getFirstRow(fnAngles)
     if row.containsLabel(xmipp.MDL_CTF_DEFOCUSU) or row.containsLabel(xmipp.MDL_CTF_MODEL):
         TsCurrent=self.readInfoField(fnDirCurrent,"sampling",xmipp.MDL_SAMPLINGRATE)
         args+=" --useCTF --sampling %f"%TsCurrent
         if self.phaseFlipped:
             args+=" --phaseFlipped"
     self.runJob("xmipp_reconstruct_fourier",args,numberOfMpi=1)
     self.runJob("xmipp_image_operate","-i %s --minus %s"%(fnVol,join(fnDirCurrent,"volume.vol")))
    def generateMetadata(self, listNameImgs, listNumImgs, level):

        # Renumerate unchanged classes
        listNewNumImages = [-1] * len(listNumImgs)
        count = 1
        for i in range(len(listNumImgs)):
            if listNumImgs[i] is not -1:
                listNewNumImages[i] = count
                count += 1

        # Construct the new classes with the renumerated old classes
        mdNewClasses = md.MetaData()
        for i in range(len(listNumImgs)):
            if listNumImgs[i] is not -1:
                name = listNameImgs[i]
                fn = name[name.find('@') + 1:-4] + '.xmd'
                numRef = int(name[0:6])

                mdClass = md.MetaData("classes@" + fn)
                for row in iterRows(mdClass):
                    if mdClass.getValue(md.MDL_REF, row.getObjId()) == numRef:
                        row.setValue(md.MDL_REF, listNewNumImages[i])
                        row.addToMd(mdNewClasses)

        # Add the two new classes to the list of renumerated classes
        outSet = self._getExtraPath(join('level%03d' % level, 'level%03d' %
                                         level + '_classes.xmd'))
        mdClass = md.MetaData("classes@" + outSet)
        rows = iterRows(mdClass)
        for row in rows:
            row.setValue(md.MDL_REF, count)
            row.addToMd(mdNewClasses)
            count = count + 1
        mdNewClasses.write('classes@' + self._getExtraPath(join('level%03d'
                                        % level,'intermediate_classes.xmd')),
                           MD_APPEND)

        # Generate the intermediate images and the blocks of the intermediate
        # classes for the unchanged classes
        for i in range(len(listNumImgs)):
            if listNumImgs[i] is not -1:
                name = listNameImgs[i]
                fn = name[name.find('@') + 1:-4] + '.xmd'
                numRef = int(name[0:6])

                # Read the list of images in this class
                mdImgsInClass = md.MetaData(
                    'class%06d_images@%s' % (numRef, fn))
                mdImgsInClass.fillConstant(md.MDL_REF, listNewNumImages[i])
                mdImgsInClass.write('class%06d' % (listNewNumImages[i])
                                    + '_images@' + self._getExtraPath(join(
                                    'level%03d' % level,
                                    'intermediate_classes.xmd')),
                                    MD_APPEND)

        # Add the two new classes
        if len(listNumImgs) == 0:
            count = 1
        else:
            count = len(listNumImgs)
        for newRef in range(getSize(outSet)):
            mdImgsInClass = md.MetaData('class%06d_images@%s' % (newRef + 1,
                                                                 outSet))
            mdImgsInClass.fillConstant(md.MDL_REF, count)
            mdImgsInClass.write('class%06d' % (count) + '_images@'
                                + self._getExtraPath(join('level%03d' %
                                level,'intermediate_classes.xmd')),
                                MD_APPEND)
            count = count + 1
    def iterationStep (self, refSet, imgsExp, iter, useReferenceImages,
                       level, flag_split, flag_attraction):

        if not exists(join(self._getExtraPath(), 'level%03d' % level)):
            mkdir(join(self._getExtraPath(), 'level%03d' % level))

        if not useReferenceImages and iter==0 and flag_split==True:

            # First step: divide the metadata input file to generate
            # a couple of references
            if level==0:
                if not flag_attraction:
                    outDirName = imgsExp[0:imgsExp.find('extra')+6] +  \
                                 'level%03d' % level +  imgsExp[imgsExp.find(
                        'extra')+5:-4]
                else:
                    outDirName = imgsExp[0:imgsExp.find('extra') + 6] +  \
                                 'level%03d' % level +  imgsExp[imgsExp.find(
                        'level%03d' % level) + 8:-4]
            else:
                if not flag_attraction:
                    outDirName = imgsExp[0:imgsExp.find('extra') + 6] +  \
                                 'level%03d' % level +  imgsExp[imgsExp.find(
                        'level%03d' % (level-1)) + 8:-4]
                else:
                    outDirName = imgsExp[0:imgsExp.find('extra') + 6] +  \
                                 'level%03d' % level +  imgsExp[imgsExp.find(
                        'level%03d' % level) + 8:-4]
            self._params = {'imgsExp': imgsExp,
                            'outDir': outDirName}
            args = ('-i %(imgsExp)s -n 2 --oroot %(outDir)s')
            self.runJob("xmipp_metadata_split", args % self._params,
                        numberOfMpi=1)

            # Second step: calculate the means of the previous metadata
            expSet1 = outDirName + '000001.xmd'
            avg1 = outDirName + '_000001'
            expSet2 = outDirName + '000002.xmd'
            avg2 = outDirName + '_000002'
            self._params = {'imgsSet': expSet1,
                            'outputAvg': avg1}
            args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
            self.runJob("xmipp_image_statistics", args % self._params,
                        numberOfMpi=1)

            self._params = {'imgsSet': expSet2,
                            'outputAvg': avg2}
            args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
            self.runJob("xmipp_image_statistics", args % self._params,
                        numberOfMpi=1)

            # Third step: generate a single metadata with the two previous
            # averages
            refSet = self._getExtraPath(join('level%03d' % level,'refSet.xmd'))
            self._params = {'avg1': avg1 + 'average.xmp',
                            'avg2': avg2 + 'average.xmp',
                            'outputMd': refSet}
            args = ('-i %(avg1)s --set union %(avg2)s -o %(outputMd)s')
            self.runJob("xmipp_metadata_utilities", args % self._params,
                        numberOfMpi=1)

        # Fourth step: calling program xmipp_cuda_correlation
        if flag_split:
            filename = 'level%03d' % level+'_classes.xmd'
            self._params = {'imgsRef': refSet,
                            'imgsExp': imgsExp,
                            'outputFile': 'images_level%03d' % level+'.xmd',
                            'tmpDir': join(self._getExtraPath(),'level%03d'
                                           % level),
                            'keepBest': self.keepBest.get(),
                            'maxshift': self.maximumShift,
                            'outputClassesFile': filename,
                            }
        else:
            filename = 'general_level%03d' % level + '_classes.xmd'
            self._params = {'imgsRef': refSet,
                            'imgsExp': imgsExp,
                            'outputFile': 'general_images_level%03d' % level
                                          + '.xmd',
                            'tmpDir': join(self._getExtraPath(),'level%03d'
                                           % level),
                            'keepBest': self.keepBest.get(),
                            'maxshift': self.maximumShift,
                            'outputClassesFile': filename,
                            }
        Nrefs = getSize(refSet)
        if Nrefs>2:
            args = '-i_ref %(imgsRef)s -i_exp %(imgsExp)s -o %(outputFile)s '\
                   '--odir %(tmpDir)s --keep_best %(keepBest)d '\
                   '--maxShift %(maxshift)d --classify %(outputClassesFile)s '\
                   '--simplifiedMd'
            self.runJob("xmipp_cuda_correlation", args % self._params,
                        numberOfMpi=1)
        else:
            flag_error=True
            while flag_error:
                try:
                    flag_error=False
                    args='-i %(imgsExp)s --ref0 %(imgsRef)s --nref %(Nrefs)d '\
                         '--iter 1 --distance correlation --classicalMultiref '\
                         '--maxShift %(maxshift)d --odir %(cl2dDir)s'
                    self._params['Nrefs']=Nrefs
                    self._params['cl2dDir'] = self._getExtraPath(
                                              join('level%03d' % level))
                    self.runJob("xmipp_mpi_classify_CL2D",
                                args % self._params)
                except Exception as ex:
                    flag_error = True

            if flag_split:
                copy(self._getExtraPath(join('level%03d' % level,
                                             "level_00","class_classes.xmd")),
                     self._getExtraPath(join('level%03d' % level,
                                             'level%03d' % level
                                             + '_classes.xmd')))
                copy(self._getExtraPath(join('level%03d' % level,"images.xmd")),
                     self._getExtraPath(join('level%03d' % level,
                                             'images_level%03d' % level +
                                             '.xmd')))
            else:
                copy(self._getExtraPath(join('level%03d' % level,
                                             "level_00", "class_classes.xmd")),
                     self._getExtraPath(join('level%03d' % level,
                                             'general_level%03d' % level +
                                             '_classes.xmd')))
                copy(self._getExtraPath(join('level%03d' % level,
                                             "images.xmd")),
                     self._getExtraPath(join('level%03d' % level,
                                             'general_images_level%03d' %
                                             level + '.xmd')))
Exemple #13
0
    def iterationStep(self, refSet, imgsExp, iter, flag_split):

        if flag_split:
            outImgs, classesOut = self._getOutputSplitFn()
        else:
            outImgs, classesOut = self._getOutputClassFn()

        outDirName = imgsExp[:-4]
        if iter == 0 and flag_split == True:

            # First step: divide the metadata input file to generate
            # a couple of references
            self._params = {'imgsExp': imgsExp, 'outDir': outDirName}
            args = ('-i %(imgsExp)s -n 2 --oroot %(outDir)s')
            self.runJob("xmipp_metadata_split",
                        args % self._params,
                        numberOfMpi=1)

            # Second step: calculate the means of the previous metadata
            expSet1 = outDirName + '000001.xmd'
            avg1 = outDirName + '_000001'
            expSet2 = outDirName + '000002.xmd'
            avg2 = outDirName + '_000002'
            self._params = {'imgsSet': expSet1, 'outputAvg': avg1}
            args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
            self.runJob("xmipp_image_statistics",
                        args % self._params,
                        numberOfMpi=1)

            self._params = {'imgsSet': expSet2, 'outputAvg': avg2}
            args = ('-i %(imgsSet)s --save_image_stats %(outputAvg)s -v 0')
            self.runJob("xmipp_image_statistics",
                        args % self._params,
                        numberOfMpi=1)

            # Third step: generate a single metadata with the two previous avgs
            refSet = self._getExtraPath('refSet.xmd')
            self._params = {
                'avg1': avg1 + 'average.xmp',
                'avg2': avg2 + 'average.xmp',
                'outputMd': refSet
            }
            args = ('-i %(avg1)s --set union %(avg2)s -o %(outputMd)s')
            self.runJob("xmipp_metadata_utilities",
                        args % self._params,
                        numberOfMpi=1)

        # Fourth step: calling program xmipp_cuda_correlation
        if flag_split:
            self._params = {
                'imgsRef': refSet,
                'imgsExp': imgsExp,
                'maxshift': self.maximumShift,
                'Nrefs': getSize(refSet),
                'outDir': self._getExtraPath(),
                'rootFn': classesOut.split('/')[-1].replace('.xmd', '')
            }
            args = '-i %(imgsExp)s --ref0 %(imgsRef)s --nref %(Nrefs)d ' \
                   '--iter 1 --distance correlation --classicalMultiref ' \
                   '--maxShift %(maxshift)d --odir %(outDir)s --oroot %(' \
                   'rootFn)s'

            self.runJob("mpirun -np 4 -bynode xmipp_mpi_classify_CL2D",
                        args % self._params)

            fileTocopy = classesOut.replace('.xmd', '_classes.xmd')
            fileTocopy = fileTocopy.replace('extra/', 'extra/level_00/')
            copy(fileTocopy, classesOut)
            copy(self._getExtraPath("images.xmd"), outImgs)
        else:
            self._params = {
                'imgsRef': refSet,
                'imgsExp': imgsExp,
                'outputFile': outImgs,
                'keepBest': self.keepBest.get(),
                'maxshift': self.maximumShift,
                'outputClassesFile': classesOut
            }
            args = '-i_ref %(imgsRef)s -i_exp %(imgsExp)s -o %(outputFile)s '\
                   '--keep_best %(keepBest)d --maxShift %(maxshift)d ' \
                   '--classify %(outputClassesFile)s --simplifiedMd'
            self.runJob("xmipp_cuda_correlation",
                        args % self._params,
                        numberOfMpi=1)

        if exists(outDirName + '000001.xmd'):
            cleanPath(expSet1)
            cleanPath(expSet2)
            cleanPath(avg1 + 'average.xmp')
            cleanPath(avg2 + 'average.xmp')
            cleanPath(avg1 + 'stddev.xmp')
            cleanPath(avg2 + 'stddev.xmp')

        return outImgs, classesOut