def run_extract(suffix=''):
     args = ('--extract --o subparticles --extract_size %s --coord_files '
             '"%s/particles%s_??????.star"') % (subpart_size, output, suffix)
     run_command(cmd + args, "/dev/null")
     print(" Cleaning up temporary files...")
     run_command("rm subparticles.star")
     if deleteParticles:
         pwutils.cleanPattern('%s/particles%s_??????.mrc' % (output, suffix))
    def _showProjMatchLibrary(self, paramName=None):
        #map stack position with ref number
        list = []
        mdIn = emlib.MetaData()
        mdOut = emlib.MetaData()
        cleanPattern(self.protocol._getExtraPath('references_library*'))

        for ref3d in self._refsList:
            for it in self._iterations:
                convert_refno_to_stack_position = {}
                file_name = self.protocol._getFileName(
                    'projectLibrarySampling', iter=it, ref=ref3d)
                file_nameReferences = 'projectionDirections@' + file_name
                #last reference name
                if exists(file_name):
                    mdReferences = emlib.MetaData(file_nameReferences)
                    mdReferencesSize = mdReferences.size()
                    for id in mdReferences:
                        convert_refno_to_stack_position[mdReferences.getValue(
                            emlib.MDL_NEIGHBOR, id)] = id
                    file_nameAverages = self.protocol._getFileName(
                        'outClassesXmd', iter=it, ref=ref3d)
                    if exists(file_nameAverages):
                        # print("OutClassesXmd %s" % OutClassesXmd)
                        mdIn.read(file_nameAverages)
                        mdOut.clear()
                        for i in mdIn:
                            # id1=mdOut.addObject()
                            # mdOut.setValue(MDL_IMAGE,mdIn.getValue(MDL_IMAGE,i),id1)
                            ref2D = mdIn.getValue(emlib.MDL_REF, i)
                            file_references = self.protocol._getFileName(
                                'projectLibraryStk', iter=it, ref=ref3d)
                            file_reference = emlib.FileName()
                            file_reference.compose(
                                convert_refno_to_stack_position[ref2D],
                                file_references)
                            id2 = mdOut.addObject()
                            mdOut.setValue(emlib.MDL_IMAGE, file_reference,
                                           id2)

                        if mdOut.size() == 0:
                            print("Empty metadata: ", file_nameReferences)
                        else:
                            file_nameReferences = self.protocol._getExtraPath(
                                'references_library.xmd')
                            sfn = createUniqueFileName(file_nameReferences)
                            file_nameReferences = 'projectionDirections@' + sfn
                            mdOut.write(file_nameReferences)
                            list.append(
                                self.createDataView(file_nameReferences))
                else:
                    print("File %s does not exist" % file_name)
        return list
Example #3
0
    def extractParticlesStep(self, micId, baseMicName, fnCTF,
                             micrographToExtract, micOps, doInvert,
                             normalizeArgs, doBorders):
        """ Extract particles from one micrograph """
        outputRoot = str(self._getExtraPath(baseMicName))
        fnPosFile = self._getExtraPath(baseMicName + ".pos")

        # If it has coordinates extract the particles
        particlesMd = 'particles@%s' % fnPosFile
        boxSize = self.boxSize.get()
        boxScale = self.getBoxScale()
        print "boxScale: ", boxScale

        if exists(fnPosFile):
            # Apply first all operations required for the micrograph
            for program, args in micOps:
                self.runJob(program, args)

            args = " -i %s --pos %s" % (micrographToExtract, particlesMd)
            args += " -o %s --Xdim %d" % (outputRoot, boxSize)

            if doInvert:
                args += " --invert"

            if fnCTF:
                args += " --ctfparam " + fnCTF

            if doBorders:
                args += " --fillBorders"

            self.runJob("xmipp_micrograph_scissor", args)

            # Normalize
            if normalizeArgs:
                self.runJob('xmipp_transform_normalize',
                            '-i %s.stk %s' % (outputRoot, normalizeArgs))
        else:
            self.warning("The micrograph %s hasn't coordinate file! " %
                         baseMicName)
            self.warning("Maybe you picked over a subset of micrographs")

        # Let's clean the temporary mrc micrographs
        if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
            pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
Example #4
0
    def _showProjMatchLibrary(self, paramName=None):
        #map stack position with ref number
        list = []
        mdIn  = xmipp.MetaData()
        mdOut = xmipp.MetaData()
        cleanPattern(self.protocol._getTmpPath('references_library*'))
        
        for ref3d in self._refsList:
            for it in self._iterations:
                convert_refno_to_stack_position = {}
                file_name = self.protocol._getFileName('projectLibrarySampling', iter=it, ref=ref3d)
                file_nameReferences = 'projectionDirections@' + file_name
                #last reference name
                if exists(file_name):
                    mdReferences     = xmipp.MetaData(file_nameReferences)
                    mdReferencesSize = mdReferences.size()
                    for id in mdReferences:
                        convert_refno_to_stack_position[mdReferences.getValue(xmipp.MDL_NEIGHBOR,id)]=id
                    file_nameAverages   = self.protocol._getFileName('outClassesXmd', iter=it, ref=ref3d)
                    if exists(file_nameAverages):
                        #print "OutClassesXmd", OutClassesXmd
                        mdIn.read(file_nameAverages)
                        mdOut.clear()
                        for i in mdIn:
                            #id1=mdOut.addObject()
                            #mdOut.setValue(MDL_IMAGE,mdIn.getValue(MDL_IMAGE,i),id1)
                            ref2D = mdIn.getValue(xmipp.MDL_REF,i)
                            file_references = self.protocol._getFileName('projectLibraryStk', iter=it, ref=ref3d)
                            file_reference = xmipp.FileName()
                            file_reference.compose(convert_refno_to_stack_position[ref2D],file_references)
                            id2=mdOut.addObject()
                            mdOut.setValue(xmipp.MDL_IMAGE, file_reference, id2)

                        if mdOut.size() == 0:
                            print "Empty metadata: ", file_nameReferences
                        else:
                            file_nameReferences = self.protocol._getTmpPath('references_library.xmd')
                            sfn = createUniqueFileName(file_nameReferences)
                            file_nameReferences = 'projectionDirections@' + sfn
                            mdOut.write(file_nameReferences)
                            list.append(self.createDataView(file_nameReferences))
                else:
                    print "File %s does not exist" % file_name
        return list
 def extractParticlesStep(self, micId, baseMicName, fnCTF,
                          micrographToExtract, micOps,
                          doInvert, normalizeArgs, doBorders):
     """ Extract particles from one micrograph """
     outputRoot = str(self._getExtraPath(baseMicName))
     fnPosFile = self._getExtraPath(baseMicName + ".pos")
 
     # If it has coordinates extract the particles
     particlesMd = 'particles@%s' % fnPosFile
     boxSize = self.boxSize.get()
     boxScale = self.getBoxScale()
     print "boxScale: ", boxScale
 
     if exists(fnPosFile):
         # Apply first all operations required for the micrograph
         for program, args in micOps:
             self.runJob(program, args)
     
         args = " -i %s --pos %s" % (micrographToExtract, particlesMd)
         args += " -o %s --Xdim %d" % (outputRoot, boxSize)
     
         if doInvert:
             args += " --invert"
     
         if fnCTF:
             args += " --ctfparam " + fnCTF
     
         if doBorders:
             args += " --fillBorders"
     
         self.runJob("xmipp_micrograph_scissor", args)
     
         # Normalize
         if normalizeArgs:
             self.runJob('xmipp_transform_normalize',
                         '-i %s.stk %s' % (outputRoot, normalizeArgs))
     else:
         self.warning("The micrograph %s hasn't coordinate file! "
                      % baseMicName)
         self.warning("Maybe you picked over a subset of micrographs")
 
     # Let's clean the temporary mrc micrographs
     if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
         pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
Example #6
0
    def _restimateCTF(self, ctfId):
        """ Run ctftilt with required parameters """

        ctfModel = self.recalculateSet[ctfId]
        mic = ctfModel.getMicrograph()
        micFn = mic.getFileName()
        micDir = self._getMicrographDir(mic)

        out = self._getCtfOutPath(micDir)
        psdFile = self._getPsdPath(micDir)

        pwutils.cleanPath(out)
        micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
        em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
        pwutils.cleanPath(psdFile)
        try:
            program, args = self._getRecalCommand(ctfModel,
                                                  micFn=micFnMrc,
                                                  ctftiltOut=out,
                                                  ctftiltPSD=psdFile)
            self.runJob(program, args)
        except Exception as ex:
            print >> sys.stderr, "ctftilt has failed with micrograph %s" % micFnMrc
        pwutils.cleanPattern(micFnMrc)
        micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
        em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)

        # Update _params dictionary
        self._prepareRecalCommand(ctfModel)
        self._params['micFn'] = micFnMrc
        self._params['micDir'] = micDir
        self._params['ctffindOut'] = out
        self._params['ctffindPSD'] = psdFile
        
        pwutils.cleanPath(psdFile)
        try:
            self.runJob(self._program, self._args % self._params)
        except Exception, ex:
            print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
        pwutils.cleanPattern(micFnMrc)
    
    def _createNewCtfModel(self, mic):
        micDir = self._getMicrographDir(mic)
        out = self._getCtfOutPath(micDir)
        psdFile = self._getPsdPath(micDir)
        ctfModel2 = em.CTFModel()
        readCtfModel(ctfModel2, out, ctf4=self.useCtffind4.get())
        ctfModel2.setPsdFile(psdFile)
        ctfModel2.setMicrograph(mic)
        return ctfModel2
    
    def _createOutputStep(self):
        ctfSet = self._createSetOfCTF()
        ctfSet.setMicrographs(self.inputMics)
        defocusList = []
    def _extractMicrograph(self, mic, doInvert, normalizeArgs, doBorders):
        """ Extract particles from one micrograph """
        fnLast = mic.getFileName()
        baseMicName = pwutils.removeBaseExt(fnLast)
        outputRoot = str(self._getExtraPath(baseMicName))
        fnPosFile = self._getMicPos(mic)
        boxSize = self.boxSize.get()
        downFactor = self.downFactor.get()
        patchSize = self.patchSize.get() if self.patchSize.get() > 0 \
                    else int(boxSize*1.5*downFactor)

        particlesMd = 'particles@%s' % fnPosFile
        # If it has coordinates extract the particles
        if exists(fnPosFile):
            # Create a list with micrographs operations (programs in xmipp) and
            # the required command line parameters (except input/ouput files)
            micOps = []

            # Compute the variance and Gini coeff. of the part. and mic., resp.
            args = '--pos %s' % fnPosFile
            args += ' --mic %s' % fnLast
            args += ' --patchSize %d' % patchSize
            self.runJob('xmipp_coordinates_noisy_zones_filter', args)

            def getMicTmp(suffix):
                return self._getTmpPath(baseMicName + suffix)

            # Check if it is required to downsample our micrographs
            if self.notOne(downFactor):
                fnDownsampled = getMicTmp("_downsampled.xmp")
                args = "-i %s -o %s --step %f --method fourier"
                self.runJob('xmipp_transform_downsample',
                            args % (fnLast, fnDownsampled, downFactor))
                fnLast = fnDownsampled

            if self.doRemoveDust:
                fnNoDust = getMicTmp("_noDust.xmp")
                args = " -i %s -o %s --bad_pixels outliers %f"
                self.runJob('xmipp_transform_filter',
                            args % (fnLast, fnNoDust, self.thresholdDust))
                fnLast = fnNoDust

            if self._useCTF():
                # We need to write a Xmipp ctfparam file
                # to perform the phase flip on the micrograph
                fnCTF = self._getTmpPath("%s.ctfParam" % baseMicName)
                micrographToCTFParam(mic, fnCTF)
                # Insert step to flip micrograph
                if self.doFlip:
                    fnFlipped = getMicTmp('_flipped.xmp')
                    args = " -i %s -o %s --ctf %s --sampling %f"
                    self.runJob(
                        'xmipp_ctf_phase_flip', args %
                        (fnLast, fnFlipped, fnCTF, self._getNewSampling()))
                    fnLast = fnFlipped
            else:
                fnCTF = None

            args = " -i %s --pos %s" % (fnLast, particlesMd)
            args += " -o %s --Xdim %d" % (outputRoot, boxSize)

            if doInvert:
                args += " --invert"

            if fnCTF:
                args += " --ctfparam " + fnCTF

            if doBorders:
                args += " --fillBorders"

            self.runJob("xmipp_micrograph_scissor", args)

            # Normalize
            if normalizeArgs:
                self.runJob('xmipp_transform_normalize',
                            '-i %s.stk %s' % (outputRoot, normalizeArgs))
        else:
            self.warning("The micrograph %s hasn't coordinate file! " %
                         baseMicName)
            self.warning("Maybe you picked over a subset of micrographs")

        # Let's clean the temporary mrc micrographs
        if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
            pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
    def refineAnglesStep(self):
        fnTmpDir = self._getTmpPath()
        fnDirectional = self._getDirectionalClassesFn()
        inputParticles = self.inputParticles.get()
        newTs = readInfoField(self._getExtraPath(), "sampling",
                              emlib.MDL_SAMPLINGRATE)
        newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE)

        # Generate projections
        fnGallery = join(fnTmpDir, "gallery.stk")
        fnGalleryMd = join(fnTmpDir, "gallery.doc")
        fnVol = self._getInputVolFn()
        args = "-i %s -o %s --sampling_rate %f --sym %s" % \
               (fnVol, fnGallery, 5.0, self.symmetryGroup)
        args += " --compute_neighbors --angular_distance -1 --experimental_images %s" % fnDirectional
        self.runJob("xmipp_angular_project_library",
                    args,
                    numberOfMpi=self.numberOfMpi.get() *
                    self.numberOfThreads.get())

        # Global angular assignment
        maxShift = 0.15 * newXdim
        fnAngles = join(fnTmpDir, "angles_iter001_00.xmd")
        if not self.useGpu.get():
            args = '-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0' % \
                   (fnDirectional, fnGalleryMd, maxShift, fnTmpDir)
            self.runJob('xmipp_reconstruct_significant',
                        args,
                        numberOfMpi=self.numberOfMpi.get() *
                        self.numberOfThreads.get())
        else:
            count = 0
            GpuListCuda = ''
            if self.useQueueForSteps() or self.useQueue():
                GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
                GpuList = GpuList.split(",")
                for elem in GpuList:
                    GpuListCuda = GpuListCuda + str(count) + ' '
                    count += 1
            else:
                GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
                GpuListAux = ''
                for elem in self.getGpuList():
                    GpuListCuda = GpuListCuda + str(count) + ' '
                    GpuListAux = GpuListAux + str(elem) + ','
                    count += 1
                os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux

            args = '-i %s -r %s -o %s --dev %s ' % (fnDirectional, fnGalleryMd,
                                                    fnAngles, GpuListCuda)
            self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)

        self.runJob("xmipp_metadata_utilities",
                    "-i %s --operate drop_column ref" % fnAngles,
                    numberOfMpi=1)
        self.runJob("xmipp_metadata_utilities",
                    "-i %s --set join %s ref2" % (fnAngles, fnDirectional),
                    numberOfMpi=1)

        # Local angular assignment
        fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
        args = "-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 " % \
               (fnAngles, fnAnglesLocalStk, newTs, newXdim / 2, 2, fnVol,
                self.targetResolution)
        args += " --optimizeShift --max_shift %f" % maxShift
        args += " --optimizeAngles --max_angular_change %f" % self.angularDistance
        self.runJob("xmipp_angular_continuous_assign2",
                    args,
                    numberOfMpi=self.numberOfMpi.get() *
                    self.numberOfThreads.get())
        moveFile(self._getPath("directional_local_classes.xmd"),
                 self._getDirectionalClassesFn())

        cleanPattern(self._getExtraPath("direction_*"))