Ejemplo n.º 1
0
    def _restimateCTF(self, ctfId):
        """ Run Gctf with required parameters """
        ctfModel = self.recalculateSet[ctfId]
        mic = ctfModel.getMicrograph()
        micFn = mic.getFileName()
        micDir = self._getMicrographDir(mic)
        micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
        micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')

        out = self._getCtfOutPath(micDir)
        psdFile = self._getPsdPath(micDir)
        ctffitFile = self._getCtfFitOutPath(micDir)

        pwutils.cleanPath(out)

        micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
        em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)

        # Update _params dictionary
        self._prepareRecalCommand(ctfModel)
        self._params['micFn'] = micFnMrc
        self._params['micDir'] = micDir
        self._params['gctfOut'] = out
        pwutils.cleanPath(psdFile)

        try:
            self.runJob(self._getProgram(), self._args % self._params)
        except:
            print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
        pwutils.moveFile(micFnCtf, psdFile)
        pwutils.moveFile(micFnCtfFit, ctffitFile)
        pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
        pwutils.cleanPattern(micFnMrc)
Ejemplo n.º 2
0
 def _renameFiles(self, pattern1, pattern2):
     # find files by pattern1, move and rename them by replacing pattern2
     filesList = sorted(glob(self._getPath(pattern1)))
     for fn in filesList:
         oldFn = os.path.basename(fn)
         newFn = pwutils.join(self._getExtraPath('shiny'), oldFn.replace(pattern2, ''))
         pwutils.moveFile(fn, newFn)
    def refineAnglesStep(self):
        fnTmpDir = self._getTmpPath()
        fnDirectional = self._getDirectionalClassesFn()
        inputParticles = self.inputParticles.get()
        newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE)
        newXdim = self.readInfoField(self._getExtraPath(),"size",xmippLib.MDL_XSIZE)

        # Generate projections
        fnGallery=join(fnTmpDir,"gallery.stk")
        fnGalleryMd=join(fnTmpDir,"gallery.doc")
        fnVol = self._getInputVolFn()
        args="-i %s -o %s --sampling_rate %f --sym %s"%\
             (fnVol,fnGallery,5.0,self.symmetryGroup)
        args+=" --compute_neighbors --angular_distance -1 --experimental_images %s"%fnDirectional
        self.runJob("xmipp_angular_project_library",args,numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())

        # Global angular assignment
        maxShift=0.15*newXdim
        args='-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0'%\
             (fnDirectional,fnGalleryMd,maxShift,fnTmpDir)
        self.runJob('xmipp_reconstruct_significant',args,numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
        fnAngles = join(fnTmpDir,"angles_iter001_00.xmd")
        self.runJob("xmipp_metadata_utilities","-i %s --operate drop_column ref"%fnAngles,numberOfMpi=1)
        self.runJob("xmipp_metadata_utilities","-i %s --set join %s ref2"%(fnAngles,fnDirectional),numberOfMpi=1)

        # Local angular assignment
        fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
        args="-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 --Nsimultaneous %d"%\
           (fnAngles,fnAnglesLocalStk,newTs,newXdim/2,2,fnVol,self.targetResolution,8)
        args+=" --optimizeShift --max_shift %f"%maxShift
        args+=" --optimizeAngles --max_angular_change %f"%self.angularDistance
        self.runJob("xmipp_angular_continuous_assign2",args,numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
        moveFile(self._getPath("directional_local_classes.xmd"),self._getDirectionalClassesFn())

        cleanPattern(self._getExtraPath("direction_*"))
    def convertToPseudoAtomsStep(self, inputFn, fnMask, sampling, prefix=''):
        pseudoatoms = 'pseudoatoms%s' % prefix
        outputFn = self._getPath(pseudoatoms)
        sigma = sampling * self.pseudoAtomRadius.get()
        targetErr = self.pseudoAtomTarget.get()
        #volume-to-pseudoatom conversion was not MPI-parallelized and the number of MPIs was removed from the gui
        #nthreads = self.numberOfThreads.get() * self.numberOfMpi.get()
        nthreads = self.numberOfThreads.get()

        params = "-i %(inputFn)s -o %(outputFn)s --sigma %(sigma)f --thr " \
                 "%(nthreads)d "
        params += "--targetError %(targetErr)f --sampling_rate %(sampling)f " \
                  "-v 2 --intensityColumn Bfactor"
        if fnMask:
            params += " --mask binary_file %(fnMask)s"
        self.runJob("xmipp_volume_to_pseudoatoms", params % locals())
        for suffix in ["_approximation.vol", "_distance.hist"]:
            moveFile(self._getPath(pseudoatoms + suffix),
                     self._getExtraPath(pseudoatoms + suffix))
        self.runJob(
            "xmipp_image_convert",
            "-i %s_approximation.vol -o %s_approximation.mrc -t vol" %
            (self._getExtraPath(pseudoatoms), self._getExtraPath(pseudoatoms)))
        self.runJob(
            "xmipp_image_header",
            "-i %s_approximation.mrc --sampling_rate %f" %
            (self._getExtraPath(pseudoatoms), sampling))
        cleanPattern(self._getPath(pseudoatoms + '_*'))
    def _checkNewOutput(self):
        if getattr(self, 'finished', False):
            return
        self.finished = self.streamClosed and self.checkedMics == self.processedMics
        streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN

        newFiles = getFiles(self._getTmpPath())
        if newFiles or self.finished:  # when finished to close the output set
            outSet = self._loadOutputSet(SetOfCoordinates,
                                         'coordinates.sqlite')

            for fnTmp in newFiles:
                coords = np.loadtxt(fnTmp)
                moveFile(fnTmp, self._getExtraPath())
                if coords.size == 2:  # special case with only one coordinate
                    coords = [coords]
                for coord in coords:
                    newCoord = Coordinate()
                    micrographs = self.getMainInput().getMicrographs()
                    newCoord.setMicrograph(micrographs[self.getMicId(fnTmp)])
                    newCoord.setPosition(coord[0], coord[1])
                    outSet.append(newCoord)

            firstTime = not self.hasAttribute(self.outputName)
            self._updateOutputSet(self.outputName, outSet, streamMode)
            if firstTime:
                self.defineRelations(outSet)
            outSet.close()

        if self.finished:  # Unlock createOutputStep if finished all jobs
            outputStep = self._getFirstJoinStep()
            if outputStep and outputStep.isWaiting():
                outputStep.setStatus(STATUS_NEW)
Ejemplo n.º 6
0
 def _renameFiles(self, pattern1, pattern2):
     # find files by pattern1, move and rename them by replacing pattern2
     filesList = sorted(glob(self._getPath(pattern1)))
     for fn in filesList:
         oldFn = os.path.basename(fn)
         newFn = pwutils.join(self._getExtraPath('shiny'),
                              oldFn.replace(pattern2, ''))
         pwutils.moveFile(fn, newFn)
    def readPartsFromMics(self, micList, outputParts):
        """ Read the particles extract for the given list of micrographs
        and update the outputParts set with new items.
        """
        relionToLocation = relion.convert.relionToLocation
        p = Particle()
        p._rlnOpticsGroup = Integer()
        acq = self.getInputMicrographs().getAcquisition()
        # JMRT: Ideally I would like to disable the whole Acquisition for each
        #       particle row, but the SetOfImages will set it again.
        #       Another option could be to disable in the set, but then in
        #       streaming, other protocols might get the wrong optics info
        pAcq = Acquisition(magnification=acq.getMagnification(),
                           voltage=acq.getVoltage(),
                           amplitudeContrast=acq.getAmplitudeContrast(),
                           sphericalAberration=acq.getSphericalAberration())
        p.setAcquisition(pAcq)

        tmp = self._getTmpPath()
        extra = self._getExtraPath()

        for mic in micList:
            posSet = set()
            coordDict = {self._getPos(c): c
                         for c in self.coordDict[mic.getObjId()]}
            del self.coordDict[mic.getObjId()]

            ogNumber = mic.getAttributeValue('_rlnOpticsGroup', 1)

            partsStar = self.__getMicFile(mic, '_extract.star', folder=tmp)
            partsTable = relion.convert.Table(fileName=partsStar)
            stackFile = self.__getMicFile(mic, '.mrcs', folder=tmp)
            endStackFile = self.__getMicFile(mic, '.mrcs', folder=extra)
            pwutils.moveFile(stackFile, endStackFile)

            for part in partsTable:
                pos = (int(float(part.rlnCoordinateX)),
                       int(float(part.rlnCoordinateY)))

                if pos in posSet:
                    print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
                    coord = None
                else:
                    coord = coordDict.get(pos, None)

                if coord is not None:
                    # scale the coordinates according to particles dimension.
                    coord.scale(self.getBoxScale())
                    p.copyObjId(coord)
                    idx, fn = relionToLocation(part.rlnImageName)
                    p.setLocation(idx, endStackFile)
                    p.setCoordinate(coord)
                    p.setMicId(mic.getObjId())
                    p.setCTF(mic.getCTF())
                    p._rlnOpticsGroup.set(ogNumber)
                    outputParts.append(p)
                    posSet.add(pos)
Ejemplo n.º 8
0
 def filterStep(self, args):
     """ Apply the selected filter to particles. 
     Create the set of particles.
     """
     particlesStk = self._getPath("particles.spi")
     tmpStk = particlesStk.replace(".spi", "_tmp.spi")
     self.runJob("bfilter", args + " %s %s" % (particlesStk, tmpStk))
     pwutils.moveFile(tmpStk, particlesStk.replace(".spi", ".stk"))  # just we prefer stk as stack of spider images
     pwutils.cleanPath(particlesStk)
Ejemplo n.º 9
0
 def filterStep(self, args):
     """ Apply the selected filter to particles. 
     Create the set of particles.
     """
     particlesStk = self._getPath('particles.spi')
     tmpStk = particlesStk.replace('.spi', '_tmp.spi')
     self.runJob('bfilter', args + ' %s %s' % (particlesStk, tmpStk))
     pwutils.moveFile(tmpStk, particlesStk.replace('.spi', '.stk')) # just we prefer stk as stack of spider images
     pwutils.cleanPath(particlesStk)
Ejemplo n.º 10
0
    def createReport(self):
        fnTex = "report.tex"
        fhTex = open(self._getExtraPath(fnTex), "w")
        template = """
\\documentclass[12pt]{article}
\\usepackage{amsmath,amsthm,amssymb,amsfonts} 
\\usepackage{graphicx}
\\usepackage{pdfpages}
\\DeclareGraphicsExtensions{.pdf,.png,.jpg}
\\begin{document}
\\title{User Report}
\\author{Created by Scipion}
\\maketitle
"""
        fhTex.write(template)
        
        fnDir = self.filesPath.get()

        if not os.path.isdir(fnDir):
            fnDir = os.path.basename(fnDir)

        for fileName in sorted(glob.glob(os.path.join(fnDir, "*"))):
            fnDest = os.path.basename(fileName).lower()
            fnDest = fnDest.replace(" ", "_")
            fnDest = fnDest.replace(":", "_")
            fnDest = fnDest.replace(";", "_")
            pwutils.copyFile(fileName, self._getExtraPath(fnDest))

            if fnDest.endswith(".tex") or fnDest.endswith(".txt"):
                fhTex.write("\\input{%s}\n" % fnDest)
                fhTex.write("\n")
            elif fnDest.endswith(".png") or fnDest.endswith(".jpg"):
                fhTex.write("\\begin{center}\n")
                fhTex.write("\\includegraphics[width=14cm]{%s}\n" % fnDest)
                fhTex.write("\\end{center}\n")
                fhTex.write("\n")
            elif fnDest.endswith(".pdf"):
                fhTex.write("\\includepdf[pages=-]{%s}\n" % fnDest)
                fhTex.write("\\clearpage\n")
                fhTex.write("\n")
        
        template = """ 
\\end{document}
"""
        fhTex.write(template)
        fhTex.close()

        args = "-interaction=nonstopmode " + fnTex
        self.runJob(PDFLATEX, args, cwd=self._getExtraPath())

        fnPDF = self._getExtraPath("report.pdf")

        if os.path.exists(fnPDF):
            pwutils.moveFile(fnPDF, self._getPath("report.pdf"))
        else:
            raise Exception("PDF file was not produced.")
Ejemplo n.º 11
0
    def _fixMovie(self, movie):
        if self.doSaveMovie and self.useMotioncor2 and self._isOutStackSupport():
            outputMicFn = self._getExtraPath(self._getOutputMicName(movie))
            outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
            movieFn = outputMicFn.replace('_aligned_mic.mrc',
                                          '_aligned_mic_Stk.mrc')
            pwutils.moveFile(movieFn, outputMovieFn)

        if self.useMotioncor2 and not self.doSaveUnweightedMic:
            fnToDelete = self._getExtraPath(self._getOutputMicName(movie))
            pwutils.cleanPath(fnToDelete)
Ejemplo n.º 12
0
    def _fixMovie(self, movie):
        if self.doSaveMovie and self.useMotioncor2 and self._isOutStackSupport(
        ):
            outputMicFn = self._getExtraPath(self._getOutputMicName(movie))
            outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
            movieFn = outputMicFn.replace('_aligned_mic.mrc',
                                          '_aligned_mic_Stk.mrc')
            pwutils.moveFile(movieFn, outputMovieFn)

        if self.useMotioncor2 and not self.doSaveUnweightedMic:
            fnToDelete = self._getExtraPath(self._getOutputMicName(movie))
            pwutils.cleanPath(fnToDelete)
Ejemplo n.º 13
0
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the micrographs in
        # the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            # create tmp set with all mics from coords set
            coordMics = SetOfMicrographs(filename=':memory:')
            coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())

            for micU, micT in izip(
                    self.inputCoords.getUntilted().getMicrographs(),
                    self.inputCoords.getTilted().getMicrographs()):
                micU.cleanObjId()
                micT.cleanObjId()
                coordMics.append(micU)
                coordMics.append(micT)

            for mic in coordMics:
                micBase = pwutils.removeBaseExt(mic.getFileName())
                micPos = self._getExtraPath(micBase + ".pos")
                micDict[pwutils.removeExt(mic.getMicName())] = micPos

            # now match micDict and inputMics
            if any(
                    pwutils.removeExt(mic.getMicName()) in micDict
                    for mic in self.inputMics):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception(
                    'Could not match input micrographs and coordinates '
                    'by micName.')

            for mic in self.inputMics:  # micrograph from input (other)
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' %
                                      (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)
    def _convertInput(self, imgSet):
        newDim = self._getNewDim()
        bg = int(newDim / 2)

        args = '--operate_on %s --operate_out %s --norm --bg_radius %d'

        params = args % (self._getFileName('input_star'),
                         self._getFileName('preprocess_parts_star'), bg)
        self.runJob(self._getProgram(program='relion_preprocess'), params)

        from pyworkflow.utils import moveFile

        moveFile(self._getFileName('preprocess_parts'),
                 self._getTmpPath('particles_subset.mrcs'))
Ejemplo n.º 15
0
    def convertInputStep(self, particlesId):
        """ Convert all needed inputs before running the refinement script. """
        partSet = self.inputParticles.get()

        self._writeParamsFile(partSet)        
        
        self._writeGroupFiles(partSet)
        
        # Convert the input volume
        volPath = self._getExtraPath('vol01.vol')
        em.ImageHandler().convert(self.input3DReference.get(), volPath)
        pwutils.moveFile(volPath, volPath.replace('.vol', '.stk'))
        
        self._writeRefinementScripts()
Ejemplo n.º 16
0
    def organizeDataStep(self):
        from convert import relionToLocation, locationToRelion
        if getVersion() == V1_3:
            mdColumn = md.RLN_PARTICLE_NAME
        else:
            mdColumn = md.RLN_PARTICLE_ORI_NAME

        shinyStar = self._getFileName('shiny')
        newDir = self._getExtraPath('polished_particles')
        pwutils.makePath(newDir)

        if not isVersion2():
            pwutils.makePath(self._getExtraPath('shiny'))
            shinyOld = "shiny.star"
            inputFit = "movie_particles_shiny.star"
            try:
                pwutils.moveFile(shinyOld, shinyStar)
                pwutils.moveFile(
                    self._getPath(inputFit),
                    self._getExtraPath("shiny/all_movies_input_fit.star"))
                for half in self.PREFIXES:
                    pwutils.moveFile(
                        self._getPath(
                            'movie_particles_shiny_%sclass001_unfil.mrc' %
                            half),
                        self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
                                           half))
                self._renameFiles('movie_particles_shiny_post*',
                                  'movie_particles_')
                self._renameFiles('movie_particles_shiny*',
                                  'movie_particles_shiny_')
            except:
                raise Exception('ERROR: some file(s) were not found!')

        # move polished particles from Tmp to Extra path
        # and restore previous mdColumn
        mdShiny = md.MetaData(shinyStar)
        oldPath = ""

        for objId in mdShiny:
            index, imgPath = relionToLocation(
                mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
            newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
            newLoc = locationToRelion(index, newPath)
            mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
            if oldPath != imgPath and exists(imgPath):
                pwutils.moveFile(imgPath, newPath)
                oldPath = imgPath

            index2, imgPath2 = relionToLocation(
                mdShiny.getValue(mdColumn, objId))
            absPath = os.path.realpath(imgPath2)
            newPath2 = 'Runs' + str(absPath).split('Runs')[1]
            newLoc2 = locationToRelion(index2, newPath2)
            mdShiny.setValue(mdColumn, newLoc2, objId)

        mdShiny.write(shinyStar, md.MD_OVERWRITE)
        pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the
        #  micrographs in the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            # create tmp set with all mics from coords set
            coordMics = SetOfMicrographs(filename=':memory:')
            coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())

            for micU, micT in izip(self.inputCoords.getUntilted().getMicrographs(),
                                   self.inputCoords.getTilted().getMicrographs()):
                micU.cleanObjId()
                micT.cleanObjId()
                coordMics.append(micU)
                coordMics.append(micT)

            for mic in coordMics:
                micBase = pwutils.removeBaseExt(mic.getFileName())
                micPos = self._getExtraPath(micBase + ".pos")
                micDict[pwutils.removeExt(mic.getMicName())] = micPos

            # now match micDict and inputMics
            if any(pwutils.removeExt(mic.getMicName()) in micDict for mic in self.inputMics):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception('Could not match input micrographs and coordinates '
                                'by micName.')

            for mic in self.inputMics:  # micrograph from input (other)
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' % (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)
    def _pickMicrographStep(self, mics, args):
        """ Main func that runs the picking job.
        :param mics: micrograph list
        :param args: programs args
        """
        for mic in mics:
            micName = mic.getFileName()
            outMic = os.path.join(self._getTmpPath(),
                                  pwutils.replaceBaseExt(micName, 'mrc'))
            ctf = self.ctfDict[mic.getMicName()]

            args.update({'micName': outMic,
                         'logFn': self._getLogFn(mic),
                         'outStack': self._getStackFn(mic),
                         'phaseShift': ctf.getPhaseShift() or 0.0,
                         'defocusU': ctf.getDefocusU(),
                         'defocusV': ctf.getDefocusV(),
                         'defocusAngle': ctf.getDefocusAngle()
                         })

            if self.pickType == 1:
                args.update({
                    'refsFn': self._getExtraPath('references.mrc'),
                    'useRadAvg': 'YES' if self.useRadAvg else 'NO',
                    'rotateRef': self.rotateRef.get(),
                })

            argsStr = self._getArgsStr()
            cmdArgs = argsStr % args

            try:
                self.runJob(self._getProgram(), cmdArgs,
                            env=Plugin.getEnviron())

                # Move output coords from tmp to extra
                pltFn = pwutils.replaceExt(self._getStackFn(mic), 'plt')
                pwutils.moveFile(pltFn, self._getPltFn(mic))

                # Clean tmp folder
                pwutils.cleanPath(outMic)
                pwutils.cleanPath(self._getLogFn(mic))
                pwutils.cleanPath(self._getStackFn(mic))
            except Exception as e:
                self.error("ERROR: Picking has failed for %s. %s" % (
                    outMic, self._getErrorFromPickerTxt(mic, e)))
Ejemplo n.º 19
0
    def _processMovie(self, movieId, movieName, movieFolder):
        inputName = movieName
        micName = self._getMicName(movieId)
        logFile = self._getLogFile(movieId)
        gainFile = self.inputMovies.get().getGain()
        gpuId = self.gpuId.get()

        # TODO Check better way to handle gain correction
        #         if gainFile is not None:
        #             # Apply the gain correction to flat the raw movie
        #             correctedName = movieName.replace('.mrc', '_corrected.mrc')
        #
        #             self.runJob('dosefgpu_flat',
        #                         '%(inputName)s %(correctedName)s %(gainFile)s %(gpuId)s' % locals(),
        #                         cwd=movieFolder)
        #
        #            inputName = correctedName

        args = {
            '-crx': self.cropOffsetX.get(),
            '-cry': self.cropOffsetY.get(),
            '-cdx': self.cropDimX.get(),
            '-cdy': self.cropDimY.get(),
            '-bin': self.binFactor.get(),
            '-nst': self.alignFrame0.get(),
            '-ned': self.alignFrameN.get(),
            '-nss': self.sumFrame0.get(),
            '-nes': self.sumFrameN.get(),
            '-gpu': gpuId,
            '-flg': logFile,
        }

        #TODO: check the gain can be handle in dosefgpu_driftcoor program
        #if gainFile is not None:
        #    args['-fgr'] = gainFile

        command = '%(inputName)s -fcs %(micName)s ' % locals()
        command += ' '.join(['%s %s' % (k, v) for k, v in args.iteritems()])
        command += ' ' + self.extraParams.get()

        self.runJob('dosefgpu_driftcorr', command, cwd=movieFolder)
        # Move the micrograph and alignment text file
        # before clean of movie folder
        moveFile(join(movieFolder, micName), self._getExtraPath())
        moveFile(join(movieFolder, logFile), self._getExtraPath())
Ejemplo n.º 20
0
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the
        #  micrographs in the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            for micU, micT in izip(
                    self.inputCoords.getUntilted().getMicrographs(),
                    self.inputCoords.getTilted().getMicrographs()):
                micBaseU = pwutils.removeBaseExt(micU.getFileName())
                micPosU = self._getExtraPath(micBaseU + ".pos")
                micDict[pwutils.removeExt(micU.getMicName())] = micPosU
                micBaseT = pwutils.removeBaseExt(micT.getFileName())
                micPosT = self._getExtraPath(micBaseT + ".pos")
                micDict[pwutils.removeExt(micT.getMicName())] = micPosT

            # now match micDict and other mics (in self.ctfDict)
            if any(
                    pwutils.removeExt(mic.getMicName()) in micDict
                    for mic in self.ctfDict):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception(
                    'Could not match input micrographs and coordinates '
                    'by micName.')

            for mic in self.ctfDict:
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' %
                                      (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)
Ejemplo n.º 21
0
    def _processMovie(self, movieId, movieName, movieFolder):
        inputName = movieName
        micName = self._getMicName(movieId)
        logFile = self._getLogFile(movieId)
        gainFile = self.inputMovies.get().getGain()
        gpuId = self.gpuId.get()

# TODO Check better way to handle gain correction
#         if gainFile is not None:
#             # Apply the gain correction to flat the raw movie
#             correctedName = movieName.replace('.mrc', '_corrected.mrc')
#             
#             self.runJob('dosefgpu_flat', 
#                         '%(inputName)s %(correctedName)s %(gainFile)s %(gpuId)s' % locals(),
#                         cwd=movieFolder)
#            
#            inputName = correctedName
        
        args = {'-crx': self.cropOffsetX.get(),
                '-cry': self.cropOffsetY.get(),
                '-cdx': self.cropDimX.get(),
                '-cdy': self.cropDimY.get(),
                '-bin': self.binFactor.get(),
                '-nst': self.alignFrame0.get(),
                '-ned': self.alignFrameN.get(),
                '-nss': self.sumFrame0.get(),
                '-nes': self.sumFrameN.get(),
                '-gpu': gpuId,
                '-flg': logFile,
                }
        
        #TODO: check the gain can be handle in dosefgpu_driftcoor program
        #if gainFile is not None:
        #    args['-fgr'] = gainFile
        
        command = '%(inputName)s -fcs %(micName)s ' % locals()
        command += ' '.join(['%s %s' % (k, v) for k, v in args.iteritems()])
        command += ' ' + self.extraParams.get()

        self.runJob('dosefgpu_driftcorr', command, cwd=movieFolder)
        # Move the micrograph and alignment text file
        # before clean of movie folder
        moveFile(join(movieFolder, micName), self._getExtraPath())
        moveFile(join(movieFolder, logFile), self._getExtraPath())        
    def _combineTrainDataFiles(pattern, outputFile):
        files = glob.glob(pattern)
        if len(files) == 1:
            moveFile(files[0], outputFile)
        else:
            # Create a dictionary with the data fields contained in each npz file
            dataDict = {}
            with np.load(files[0]) as data:
                for field in data.files:
                    dataDict[field] = []

            # Read and combine the data from all files
            for i, name in enumerate(files):
                with np.load(name) as data:
                    for field in data.files:
                        dataDict[field].append(data[field])

            # Save the combined data into a npz file
            np.savez(outputFile, **dataDict)
Ejemplo n.º 23
0
    def _moveOutput(self, movie):
        """ Move output from tmp to extra folder. """
        def _moveToExtra(fn):
            """ Move file from movies tmp folder to extra """
            pwutils.moveFile(self._getCwdPath(movie, fn),
                             self._getExtraPath(fn))

        _moveToExtra(self._getMovieLogFile(movie))
        _moveToExtra(self._getMicFn(movie))

        if self._doSaveUnweightedMic():
            _moveToExtra(self._getOutputMicName(movie))

        if self.doSaveMovie:
            outputMicFn = self._getCwdPath(movie,
                                           self._getOutputMicName(movie))
            outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
            movieFn = outputMicFn.replace('_aligned_mic.mrc',
                                          '_aligned_mic_Stk.mrc')
            pwutils.moveFile(movieFn, outputMovieFn)
Ejemplo n.º 24
0
 def createMaskStep(self):
     inputObj = self.inputObj.get()
     maskSrc=self.maskFile.get()
     basename = os.path.basename(maskSrc)
     maskDst = self._getPath(basename)
     moveFile(maskSrc, maskDst)
     samplingRate = None
     if(hasattr(inputObj, "getSamplingRate")):
         samplingRate = inputObj.getSamplingRate()
     else:
         for key, attr in inputObj.iterInputAttributes():
             if hasattr(attr.get(), "getSamplingRate"):
                 samplingRate = attr.get().getSamplingRate()
     if  not samplingRate:
         raise Exception("sampling rate required")
     
     mask = Mask()
     mask.setFileName(maskDst)
     mask.setSamplingRate(samplingRate)
     self._defineOutputs(outputMask=mask)
     self._defineSourceRelation(self.inputObj, self.outputMask)
Ejemplo n.º 25
0
    def createMaskStep(self):
        inputObj = self.inputObj.get()
        maskSrc = self.maskFile.get()
        basename = os.path.basename(maskSrc)
        maskDst = self._getPath(basename)
        moveFile(maskSrc, maskDst)
        samplingRate = None
        if hasattr(inputObj, "getSamplingRate"):
            samplingRate = inputObj.getSamplingRate()
        else:
            for key, attr in inputObj.iterInputAttributes():
                if hasattr(attr.get(), "getSamplingRate"):
                    samplingRate = attr.get().getSamplingRate()
        if not samplingRate:
            raise Exception("sampling rate required")

        mask = emobj.Mask()
        mask.setFileName(maskDst)
        mask.setSamplingRate(samplingRate)
        self._defineOutputs(outputMask=mask)
        self._defineSourceRelation(self.inputObj, self.outputMask)
Ejemplo n.º 26
0
    def _estimateCtfList(self, micList, *args, **kwargs):
        """ Estimate several micrographs at once, probably a bit more
        efficient. """
        try:
            micPath = self._getMicrographDir(micList[0])
            if len(micList) > 1:
                micPath += ('-%04d' % micList[-1].getObjId())

            pwutils.makePath(micPath)
            ih = emlib.image.ImageHandler()

            for mic in micList:
                micFn = mic.getFileName()
                # We convert the input micrograph on demand if not in .mrc
                downFactor = self.ctfDownFactor.get()
                micFnMrc = os.path.join(micPath,
                                        pwutils.replaceBaseExt(micFn, 'mrc'))

                if downFactor != 1:
                    # Replace extension by 'mrc' cause there are some formats
                    # that cannot be written (such as dm3)
                    ih.scaleFourier(micFn, micFnMrc, downFactor)
                    sps = self._params['scannedPixelSize'] * downFactor
                    kwargs['scannedPixelSize'] = sps
                else:
                    ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)

            program, args = self._gctfProgram.getCommand(**kwargs)
            args += ' %s/*.mrc' % micPath
            self.runJob(program, args)  # , env=gctf.Plugin.getEnviron())

            def _getFile(micBase, suffix):
                return os.path.join(micPath, micBase + suffix)

            for mic in micList:
                micFn = mic.getFileName()
                micBase = pwutils.removeBaseExt(micFn)
                micFnMrc = _getFile(micBase, '.mrc')
                # Let's clean the temporary mrc micrograph
                pwutils.cleanPath(micFnMrc)

                # move output from tmp to extra
                micFnCtf = _getFile(micBase, self._gctfProgram.getExt())
                micFnCtfLog = _getFile(micBase, '_gctf.log')
                micFnCtfFit = _getFile(micBase, '_EPA.log')

                micFnCtfOut = self._getPsdPath(micFn)
                micFnCtfLogOut = self._getCtfOutPath(micFn)
                micFnCtfFitOut = self._getCtfFitOutPath(micFn)

                pwutils.moveFile(micFnCtf, micFnCtfOut)
                pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
                pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)

            pwutils.cleanPath(micPath)

        except:
            print("ERROR: Gctf has failed on %s/*.mrc" % micPath)
            import traceback
            traceback.print_exc()
Ejemplo n.º 27
0
    def organizeDataStep(self):
        from convert import relionToLocation, locationToRelion
        if getVersion() == V1_3:
            mdColumn = md.RLN_PARTICLE_NAME
        else:
            mdColumn = md.RLN_PARTICLE_ORI_NAME

        shinyStar = self._getFileName('shiny')
        newDir = self._getExtraPath('polished_particles')
        pwutils.makePath(newDir)

        if not isVersion2():
            pwutils.makePath(self._getExtraPath('shiny'))
            shinyOld = "shiny.star"
            inputFit = "movie_particles_shiny.star"
            try:
                pwutils.moveFile(shinyOld, shinyStar)
                pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
                for half in self.PREFIXES:
                    pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
                                     self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
                self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
                self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
            except:
                raise Exception('ERROR: some file(s) were not found!')

        # move polished particles from Tmp to Extra path
        # and restore previous mdColumn
        mdShiny = md.MetaData(shinyStar)
        oldPath = ""

        for objId in mdShiny:
            index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
            newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
            newLoc = locationToRelion(index, newPath)
            mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
            if oldPath != imgPath and exists(imgPath):
                pwutils.moveFile(imgPath, newPath)
                oldPath = imgPath

            index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
            absPath = os.path.realpath(imgPath2)
            newPath2 = 'Runs' + str(absPath).split('Runs')[1]
            newLoc2 = locationToRelion(index2, newPath2)
            mdShiny.setValue(mdColumn, newLoc2, objId)

        mdShiny.write(shinyStar, md.MD_OVERWRITE)
        pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
Ejemplo n.º 28
0
    def _moveFiles(self, movie):
        # It really annoying that Relion default names changes if you use DW or not
        # if use DW, the default name are DW and the others noDW
        if self.doDW:
            pwutils.moveFile(
                self._getMovieOutFn(movie, '.mrc'),
                self._getExtraPath(self._getOutputMicWtName(movie)))
            if self.saveNonDW:
                pwutils.moveFile(
                    self._getMovieOutFn(movie, '_noDW.mrc'),
                    self._getExtraPath(self._getOutputMicName(movie)))
        else:
            pwutils.moveFile(self._getMovieOutFn(movie, '.mrc'),
                             self._getExtraPath(self._getOutputMicName(movie)))

        # Keep some local files of this movie in the extra folder
        for suffix in ['.star', '.log']:
            pwutils.moveFile(self._getMovieOutFn(movie, suffix),
                             self._getMovieExtraFn(movie, suffix))

        suffix = 'corrected_micrographs.star'
        fn = os.path.join(self._getOutputMovieFolder(movie), 'output', suffix)
        pwutils.moveFile(fn, self._getMovieExtraFn(movie, suffix))
Ejemplo n.º 29
0
 def organizeDataStep(self):
     from pyworkflow.utils import moveFile
     import pyworkflow.em.metadata as md
     from convert import relionToLocation, locationToRelion
     
     # moving shiny.star form project base path to the current protocol extra path.
     shinyStar = "shiny.star"
     pathFixedShiny = self._getExtraPath(shinyStar)
     
     if exists(shinyStar):
         moveFile(shinyStar, pathFixedShiny)
     mdShiny = md.MetaData(pathFixedShiny)
     
     oldImgPath = ""
     for objId in mdShiny:
         index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
         newPath = self._getExtraPath(os.path.basename(imgPath))
         newLoc = locationToRelion(index, newPath)
         mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
         if oldImgPath != imgPath and exists(imgPath):
             moveFile(imgPath, newPath)
             oldImgPath = imgPath
     mdShiny.write(pathFixedShiny)
Ejemplo n.º 30
0
 def writePosFilesStep(self):
     """ Write the pos file for each micrograph on metadata format. """
     writeSetOfCoordinates(self._getExtraPath(), self.inputCoords,
                           scale=self.getBoxScale())
     # We need to find the mapping (either by micName (without ext) or micId)
     # between the micrographs in the SetOfCoordinates and
     # the Other micrographs if necessary
     if self._micsOther():
         micDict = {}
         coordMics = self.inputCoords.getMicrographs()
         for mic in coordMics:
             micBase = pwutils.removeBaseExt(mic.getFileName())
             micPos = self._getExtraPath(micBase + ".pos")
             micDict[pwutils.removeBaseExt(mic.getMicName())] = micPos
             micDict[mic.getObjId()] = micPos
             
         if any(pwutils.removeBaseExt(mic.getMicName()) in micDict
                for mic in self.inputMics):
             micKey = lambda mic: pwutils.removeBaseExt(mic.getMicName())
         elif any(mic.getObjId() in micDict for mic in self.inputMics):
             self.warning('Could not match input micrographs and coordinates'
                          ' micrographs by micName, using micId.')
             micKey = lambda mic: mic.getObjId()
         else:
             raise Exception('Could not match input micrographs and '
                             'coordinates neither by micName or micId.')
         
         for mic in self.inputMics: # micrograph from input (other)
             mk = micKey(mic)
             if mk in micDict:
                 micPosCoord = micDict[mk]
                 if exists(micPosCoord):
                     micBase = pwutils.removeBaseExt(mic.getFileName())
                     micPos = self._getExtraPath(micBase + ".pos")
                     if micPos != micPosCoord:
                         self.info('Moving %s -> %s' % (micPosCoord, micPos))
                         pwutils.moveFile(micPosCoord, micPos)
Ejemplo n.º 31
0
    def organizeDataStep(self):
        from pyworkflow.utils import moveFile
        import pyworkflow.em.metadata as md
        from convert import relionToLocation, locationToRelion

        # moving shiny.star form project base path to the current protocol extra path.
        shinyStar = "shiny.star"
        pathFixedShiny = self._getExtraPath(shinyStar)

        if exists(shinyStar):
            moveFile(shinyStar, pathFixedShiny)
        mdShiny = md.MetaData(pathFixedShiny)

        oldImgPath = ""
        for objId in mdShiny:
            index, imgPath = relionToLocation(
                mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
            newPath = self._getExtraPath(os.path.basename(imgPath))
            newLoc = locationToRelion(index, newPath)
            mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
            if oldImgPath != imgPath and exists(imgPath):
                moveFile(imgPath, newPath)
                oldImgPath = imgPath
        mdShiny.write(pathFixedShiny)
    def denoisingStep(self):

        input_mics = self.inputMicrographs.get()
        # Create links to the movies desired to denoise in tmp folder (subset case, janni only accepts directories
        # and work with all the files contained in them)
        for mic in input_mics:
            micName = mic.getFileName()
            createLink(mic.getFileName(), self._getTmpPath(basename(micName)))

        args = "denoise {}/ {}/ {}".format(self._getTmpPath(),
                                           self._getTmpPath(),
                                           self.getInputModel())
        Plugin.runCryolo(self, 'janni_denoise.py', args)
        # Move the resulting denoised files to the extra folder
        [
            moveFile(file, self._getExtraPath(basename(file)))
            for file in glob.glob(self._getTmpPath(join('tmp', '*.mrc')))
        ]
Ejemplo n.º 33
0
class ProtGctf(em.ProtCTFMicrographs):
    """
    Estimates CTF on a set of micrographs
    using GPU-accelerated Gctf program.

    To find more information about Gctf go to:
    http://www.mrc-lmb.cam.ac.uk/kzhang
    """
    _label = 'CTF estimation on GPU'
    _lastUpdateVersion = VERSION_1_1

    def _defineParams(self, form):
        form.addSection(label=Message.LABEL_CTF_ESTI)
        form.addParam('recalculate', params.BooleanParam, default=False,
                      condition='recalculate',
                      label="Do recalculate ctf?")

        form.addParam('continueRun', params.PointerParam, allowsNull=True,
                      condition='recalculate', label="Input previous run",
                      pointerClass=self.getClassName())
        form.addHidden('sqliteFile', params.FileParam, condition='recalculate',
                       allowsNull=True)

        form.addParam('inputMicrographs', params.PointerParam, important=True,
                      condition='not recalculate', label=Message.LABEL_INPUT_MIC,
                      pointerClass='SetOfMicrographs')
        form.addParam('ctfDownFactor', params.FloatParam, default=1.,
                      label='CTF Downsampling factor',
                      condition='not recalculate',
                      help='Set to 1 for no downsampling. Non-integer '
                           'downsample factors are possible. This downsampling '
                           'is only used for estimating the CTF and it does not '
                           'affect any further calculation. Ideally the estimation '
                           'of the CTF is optimal when the Thon rings are not too '
                           'concentrated at the origin (too small to be seen) and '
                           'not occupying the whole power spectrum (since this '
                           'downsampling might entail aliasing).')

        line = form.addLine('Resolution', condition='not recalculate',
                            help='Give a value in digital frequency (i.e. between '
                                 '0.0 and 0.5). These cut-offs prevent the typical '
                                 'peak at the center of the PSD and high-resolution '
                                 'terms where only noise exists, to interfere with '
                                 'CTF estimation. The default lowest value is 0.05 '
                                 'but for micrographs with a very fine sampling this '
                                 'may be lowered towards 0. The default highest '
                                 'value is 0.35, but it should be increased for '
                                 'micrographs with signals extending beyond this '
                                 'value. However, if your micrographs extend further '
                                 'than 0.35, you should consider sampling them at a '
                                 'finer rate.')
        line.addParam('lowRes', params.FloatParam, default=0.05,
                      label='Lowest')
        line.addParam('highRes', params.FloatParam, default=0.35,
                      label='Highest')

        line = form.addLine('Defocus search range (microns)',
                            expertLevel=params.LEVEL_ADVANCED,
                            condition='not recalculate',
                            help='Select _minimum_ and _maximum_ values for '
                                 'defocus search range (in microns). '
                                 'Underfocus is represented by a positive '
                                 'number.')
        line.addParam('minDefocus', params.FloatParam, default=0.25,
                      label='Min')
        line.addParam('maxDefocus', params.FloatParam, default=4.,
                      label='Max')

        form.addParam('astigmatism', params.FloatParam, default=100.0,
                      label='Expected (tolerated) astigmatism',
                      help='Estimated astigmatism in Angstroms',
                      expertLevel=params.LEVEL_ADVANCED)
        form.addParam('windowSize', params.IntParam, default=512,
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Window size', condition='not recalculate',
                      help='The PSD is estimated from small patches of this '
                           'size. Bigger patches allow identifying more '
                           'details. However, since there are fewer windows, '
                           'estimations are noisier.')
        form.addParam('plotResRing', params.BooleanParam, default=True,
                      label='Plot a resolution ring on a PSD file',
                      help='Whether to plot an estimated resolution ring '
                           'on the power spectrum',
                      expertLevel=params.LEVEL_ADVANCED)
        form.addParam('GPUCore', params.IntParam, default=0,
                      expertLevel=params.LEVEL_ADVANCED,
                      label="Choose GPU core",
                      help='GPU may have several cores. Set it to zero if '
                           'you do not know what we are talking about. '
                           'First core index is 0, second 1 and so on.')

        form.addSection(label='Advanced')
        form.addParam('doEPA', params.BooleanParam, default=False,
                      label="Do EPA",
                      help='Do Equiphase average used for output CTF file. '
                           'Only for nice output, will NOT be used for CTF '
                           'determination.')
        form.addParam('EPAsmp', params.IntParam, default=4,
                      expertLevel=params.LEVEL_ADVANCED,
                      condition='not _oldVersion',
                      label="Over-sampling factor for EPA")
        form.addParam('doBasicRotave', params.BooleanParam, default=False,
                      expertLevel=params.LEVEL_ADVANCED,
                      condition='_oldVersion',
                      label="Do rotational average",
                      help='Do rotational average used for output CTF file. '
                           'Only for nice output, will NOT be used for CTF '
                           'determination.')
        form.addParam('bfactor', params.IntParam, default=150,
                      expertLevel=params.LEVEL_ADVANCED,
                      label="B-factor",
                      help='B-factors used to decrease high resolution '
                           'amplitude, A^2; suggested range 50~300 except '
                           'using REBS method')
        form.addParam('overlap', params.FloatParam, default=0.5,
                      expertLevel=params.LEVEL_ADVANCED,
                      label="Overlap factor",
                      help='Overlapping factor for grid boxes sampling, '
                      'for windowsize=512, 0.5 means 256 pixels overlapping.')
        form.addParam('convsize', params.IntParam, default=85,
                      expertLevel=params.LEVEL_ADVANCED,
                      label="Boxsize for smoothing",
                      help='Boxsize to be used for smoothing, '
                           'suggested 1/5 ~ 1/20 of window size in pixel, '
                           'e.g. 99 for 512 window')

        group = form.addGroup('High-res refinement')
        group.addParam('doHighRes', params.BooleanParam, default=False,
                       label="Do high-resolution refinement",
                       help='Whether to do High-resolution refinement or not, '
                            'very useful for selecting high quality micrographs. '
                            'Especially useful when your data has strong '
                            'low-resolution bias')
        group.addParam('HighResL', params.FloatParam, default=15.0,
                       condition='doHighRes',
                       label="Lowest resolution",
                       help='Lowest resolution  to be used for High-resolution '
                            'refinement, in Angstroms')
        group.addParam('HighResH', params.FloatParam, default=4.0,
                       condition='doHighRes',
                       label="Highest resolution",
                       help='Highest resolution  to be used for High-resolution '
                            'refinement, in Angstroms')
        group.addParam('HighResBf', params.IntParam, default=50,
                       condition='doHighRes',
                       label="B-factor",
                       help='B-factor to be used for High-resolution '
                            'refinement, in Angstroms')

        form.addParam('doValidate', params.BooleanParam, default=False,
                      expertLevel=params.LEVEL_ADVANCED,
                      label="Do validation",
                      help='Whether to validate the CTF determination.')

        form.addSection(label='Phase shift')
        form.addParam('doPhShEst', params.BooleanParam, default=False,
                      label="Estimate phase shift?",
                      help='For micrographs collected with phase-plate. '
                           'It is suggested to import such micrographs with '
                           'amplitude contrast = 0. Also, using smaller '
                           '_lowest resolution_ (e.g. 15A) and smaller '
                           '_boxsize for smoothing_ (e.g. 50 for 1024 '
                           'window size) might be better.')

        line = form.addLine('Phase shift range range (deg)',
                            condition='doPhShEst',
                            help='Select _lowest_ and _highest_ phase shift '
                                 '(in degrees).')
        line.addParam('phaseShiftL', params.FloatParam, default=0.0,
                      condition='doPhShEst',
                      label="Min")
        line.addParam('phaseShiftH', params.FloatParam, default=180.0,
                      condition='doPhShEst',
                      label="Max")

        form.addParam('phaseShiftS', params.FloatParam, default=10.0,
                       condition='doPhShEst',
                       label="Step",
                       help='Phase shift search step. Do not worry about '
                            'the accuracy; this is just the search step, '
                            'Gctf will refine the phase shift anyway.')
        form.addParam('phaseShiftT', params.EnumParam, default=CCC,
                      condition='doPhShEst',
                      label='Target',
                      choices=['CCC', 'Resolution limit'],
                      display=params.EnumParam.DISPLAY_HLIST,
                      help='Phase shift target in the search: CCC or '
                           'resolution limit')

    #--------------------------- STEPS functions -------------------------------
    def _estimateCTF(self, micFn, micDir, micName):
        """ Run Gctf with required parameters """
        doneFile = os.path.join(micDir, 'done.txt')

        if self.isContinued() and os.path.exists(doneFile):
            return

        try:
            # Create micrograph dir
            pwutils.makePath(micDir)
            downFactor = self.ctfDownFactor.get()
            micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
            micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
            micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')

            if downFactor != 1:
                # Replace extension by 'mrc' cause there are some formats
                # that cannot be written (such as dm3)
                import pyworkflow.em.packages.xmipp3 as xmipp3
                self.runJob("xmipp_transform_downsample",
                            "-i %s -o %s --step %f --method fourier"
                            % (micFn, micFnMrc, downFactor),
                            env=xmipp3.getEnviron())
                sps = self.inputMicrographs.get().getScannedPixelSize() * downFactor
                self._params['scannedPixelSize'] = sps
            else:
                ih = em.ImageHandler()
                if ih.existsLocation(micFn):
                    ih.convert(micFn, micFnMrc, em.DT_FLOAT)
                else:
                    print >> sys.stderr, "Missing input micrograph %s" % micFn

            # Update _params dictionary
            self._params['micFn'] = micFnMrc
            self._params['micDir'] = micDir
            self._params['gctfOut'] = self._getCtfOutPath(micDir)

        except Exception, ex:
            print >> sys.stderr, "Some error happened: %s" % ex
            import traceback
            traceback.print_exc()

        try:
            self.runJob(self._getProgram(), self._args % self._params,  env=self._getEnviron())
        except:
            print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)

        psdFile = self._getPsdPath(micDir)
        ctffitFile = self._getCtfFitOutPath(micDir)
        pwutils.moveFile(micFnCtf, psdFile)
        pwutils.moveFile(micFnCtfFit, ctffitFile)
        pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))

        # Let's notify that this micrograph has been processed
        # just creating an empty file at the end (after success or failure)
        open(doneFile, 'w')
        # Let's clean the temporary mrc micrographs
        pwutils.cleanPath(micFnMrc)
    def _processMovie(self, movie):
        numberOfFrames = self._getNumberOfFrames(movie)
        #FIXME: Figure out how to properly write shifts for unblur
        #self._writeMovieAlignment(movie, numberOfFrames)
        
        a0, aN = self._getRange(movie, 'align')
        _, lstFrame, _ = movie.getFramesRange()

        movieBaseName = pwutils.removeExt(movie.getFileName())
        aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
        
        if a0 > 1 or aN < lstFrame:
            from pyworkflow.em import ImageHandler
            ih = ImageHandler()
            movieInputFn = movie.getFileName()
            
            if movieInputFn.endswith("mrc"):
                movieInputFn += ":mrcs"
            
            movieConverted = pwutils.removeExt(movieInputFn) + "_tmp.mrcs"
            ih.convertStack(movieInputFn, movieConverted, a0, aN)
            # Here, only temporal movie file (or link) stored in
            # tmp/movie_?????? is removed before move the converted file. It
            #  is necessary 'cause if it is overwritten you may lost your
            # original data.
            os.remove(movie.getFileName())
            pwutils.moveFile(movieConverted, movie.getFileName())
        
        movieSet = self.inputMovies.get()
        self._createLink(movie)
        range = aN - a0 + 1
        self._argsUnblur(movie, range)
        
        try:
            self.runJob(self._program, self._args)

            outMicFn = self._getExtraPath(self._getOutputMicName(movie))
            if not os.path.exists(outMicFn):
                # if only DW mic is saved
                outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))

            if self.doComputePSD:
                # Compute uncorrected avg mic
                roi = [0, 0, 0, 0]
                fakeShiftsFn = self.writeZeroShifts(movie)
                self.averageMovie(movie, fakeShiftsFn, aveMicFn,
                                  binFactor=1,
                                  roi=roi, dark=None,
                                  gain=movieSet.getGain())

                self.computePSDs(movie, aveMicFn, outMicFn,
                                 outputFnCorrected=self._getPsdJpeg(movie))

            self._saveAlignmentPlots(movie)

            if self._doComputeMicThumbnail():
                self.computeThumbnail(outMicFn,
                                      outputFn=self._getOutputMicThumbnail(
                                          movie))
        except:
            print("ERROR: Movie %s failed\n" % movie.getFileName())
            import traceback
            traceback.print_exc()
Ejemplo n.º 35
0
    def refineCtfStep(self):
        self._defineValues()
        self._prepareCommand()

        for mic in self.matchingMics:
            micName = mic.getFileName()
            micBase = pwutils.removeBaseExt(micName)
            micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
            outMic = pwutils.join(micDirTmp,
                                  pwutils.replaceBaseExt(micName, 'mrc'))
            micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
            micFnOut = self._getCtfOutPath(micDirTmp)
            micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
            micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')

            # Update _params dictionary
            self._params['micFn'] = outMic
            self._params['gctfOut'] = micFnOut

            if self.useInputCtf and self.ctfRelations.get():
                # get input CTFs from a mic
                ctfs = self.ctfRelations.get()
                micKey = mic.getMicName() if self.hasMicName else mic.getObjId(
                )

                for ctf in ctfs:
                    ctfMicName = ctf.getMicrograph().getMicName()
                    ctfMicId = ctf.getMicrograph().getObjId()
                    if micKey == ctfMicName or micKey == ctfMicId:
                        # add CTF refine options
                        self._params.update({
                            'refine_input_ctf': 1,
                            'defU_init': ctf.getDefocusU(),
                            'defV_init': ctf.getDefocusV(),
                            'defA_init': ctf.getDefocusAngle(),
                            'B_init': self.bfactor.get()
                        })
                        self._args += "--refine_input_ctf %d " % self._params[
                            'refine_input_ctf']
                        self._args += "--defU_init %f " % self._params[
                            'defU_init']
                        self._args += "--defV_init %f " % self._params[
                            'defV_init']
                        self._args += "--defA_init %f " % self._params[
                            'defA_init']
                        self._args += "--B_init %f " % self._params['B_init']
                        self._args += "--defU_err %f " % self.defUerr.get()
                        self._args += "--defV_err %f " % self.defVerr.get()
                        self._args += "--defA_err %f " % self.defAerr.get()
                        self._args += "--B_err %f " % self.Berr.get()

                        break

            # final args
            self._args += "--do_validation %d " % (1 if self.doValidate else 0)
            self._args += "%(micFn)s "
            self._args += "> %(gctfOut)s"

            try:
                self.runJob(self._getProgram(),
                            self._args % self._params,
                            env=self._getEnviron())
            except:
                print("ERROR: Gctf has failed for micrograph %s" % outMic)

            # move results from tmp to extra folder
            micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
            pwutils.makePath(micDir)
            psdFile = self._getPsdPath(micDir)
            ctfOutFile = self._getCtfOutPath(micDir)
            ctffitFile = self._getCtfFitOutPath(micDir)
            ctflocalFile = self._getCtfLocalPath(micDir, micBase)

            pwutils.moveFile(micFnCtf, psdFile)
            pwutils.moveFile(micFnOut, ctfOutFile)
            pwutils.moveFile(micFnCtfFit, ctffitFile)
            pwutils.moveFile(micFnLocalCtf, ctflocalFile)

            # Let's clean the temporary micrographs
            pwutils.cleanPath(outMic)
            pwutils.cleanPath(micDirTmp)

        pwutils.cleanPath(self.matchingMics.getFileName())
        pwutils.cleanPath(
            self.getProject().getPath('micrographs_all_gctf.star'))
Ejemplo n.º 36
0
 def cryoloModelingStep(self, extraArgs=''):
     self.runCryoloTrain(5, extraArgs=extraArgs)
     pwutils.moveFile(self._getWorkDir(self.MODEL),
                      self.getOutputModelPath())
Ejemplo n.º 37
0
    def refineCtfStep(self, micFn, micKey):
        micPath = self._getTmpPath(pwutils.removeBaseExt(micFn))
        # We convert the input micrograph on demand if not in .mrc

        downFactor = self.ctfDownFactor.get()
        ih = emlib.image.ImageHandler()
        micFnMrc = os.path.join(micPath, pwutils.replaceBaseExt(micFn, 'mrc'))

        if downFactor != 1:
            # Replace extension by 'mrc' cause there are some formats
            # that cannot be written (such as dm3)
            ih.scaleFourier(micFn, micFnMrc, downFactor)
            sps = self.inputMicrographs.get().getScannedPixelSize(
            ) * downFactor
            self._params['scannedPixelSize'] = sps
        else:
            ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)

        # Refine input CTFs, match ctf by micName
        if self.useInputCtf and self.ctfRelations.hasValue():
            ctfs = self._getCtfs()

            for ctf in ctfs:
                ctfMicName = ctf.getMicrograph().getMicName()
                ctfMicId = ctf.getMicrograph().getObjId()
                if micKey == ctfMicName or micKey == ctfMicId:
                    # add CTF refine options
                    self._params.update({
                        'refine_input_ctf': 1,
                        'defU_init': ctf.getDefocusU(),
                        'defV_init': ctf.getDefocusV(),
                        'defA_init': ctf.getDefocusAngle(),
                        'B_init': self.bfactor.get()
                    })
                    self._args += "--refine_input_ctf %d " % self._params[
                        'refine_input_ctf']
                    self._args += "--defU_init %f " % self._params['defU_init']
                    self._args += "--defV_init %f " % self._params['defV_init']
                    self._args += "--defA_init %f " % self._params['defA_init']
                    self._args += "--B_init %f " % self._params['B_init']
                    self._args += "--defU_err %f " % self.defUerr.get()
                    self._args += "--defV_err %f " % self.defVerr.get()
                    self._args += "--defA_err %f " % self.defAerr.get()
                    self._args += "--B_err %f " % self.Berr.get()
                    break

        # Run Gctf refine
        try:
            args = self._args % self._params
            args += ' %s' % micFnMrc
            self.runJob(Plugin.getProgram(), args, env=Plugin.getEnviron())

            # Let's clean the temporary mrc micrograph
            pwutils.cleanPath(micFnMrc)

            # move output from tmp to extra
            micFnCtf = os.path.join(micPath,
                                    pwutils.replaceBaseExt(micFn, 'ctf'))
            micFnCtfLog = os.path.join(
                micPath,
                pwutils.removeBaseExt(micFn) + '_gctf.log')
            micFnCtfFit = os.path.join(
                micPath,
                pwutils.removeBaseExt(micFn) + '_EPA.log')
            micFnCtfLocal = os.path.join(
                micPath,
                pwutils.removeBaseExt(micFn) + '_local.star')

            micFnCtfOut = self._getPsdPath(micFn)
            micFnCtfLogOut = self._getCtfOutPath(micFn)
            micFnCtfFitOut = self._getCtfFitOutPath(micFn)
            micFnCtfLocalOut = self._getCtfLocalOutPath(micFn)

            pwutils.moveFile(micFnCtf, micFnCtfOut)
            pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
            pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
            pwutils.moveFile(micFnCtfLocal, micFnCtfLocalOut)
        except:
            print("ERROR: Gctf has failed on %s" % micFnMrc)
            import traceback
            traceback.print_exc()
Ejemplo n.º 38
0
    def _processMovie(self, movie):
        numberOfFrames = self._getNumberOfFrames(movie)
        #FIXME: Figure out how to properly write shifts for unblur
        #self._writeMovieAlignment(movie, numberOfFrames)
        
        a0, aN = self._getRange(movie, 'align')
        _, lstFrame, _ = movie.getFramesRange()

        movieBaseName = pwutils.removeExt(movie.getFileName())
        aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
        
        if a0 > 1 or aN < lstFrame:
            from pyworkflow.em import ImageHandler
            ih = ImageHandler()
            movieInputFn = movie.getFileName()
            
            if movieInputFn.endswith("mrc"):
                movieInputFn += ":mrcs"
            
            movieConverted = pwutils.removeExt(movieInputFn) + "_tmp.mrcs"
            ih.convertStack(movieInputFn, movieConverted, a0, aN)
            # Here, only temporal movie file (or link) stored in
            # tmp/movie_?????? is removed before move the converted file. It
            #  is necessary 'cause if it is overwritten you may lost your
            # original data.
            os.remove(movie.getFileName())
            pwutils.moveFile(movieConverted, movie.getFileName())
        
        movieSet = self.inputMovies.get()
        self._createLink(movie)
        range = aN - a0 + 1
        self._argsUnblur(movie, range)
        
        try:
            self.runJob(self._program, self._args)

            outMicFn = self._getExtraPath(self._getOutputMicName(movie))
            if not os.path.exists(outMicFn):
                # if only DW mic is saved
                outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))

            if self.doComputePSD:
                # Compute uncorrected avg mic
                roi = [0, 0, 0, 0]
                fakeShiftsFn = self.writeZeroShifts(movie)
                self.averageMovie(movie, fakeShiftsFn, aveMicFn,
                                  binFactor=1,
                                  roi=roi, dark=None,
                                  gain=movieSet.getGain())

                self.computePSDs(movie, aveMicFn, outMicFn,
                                 outputFnCorrected=self._getPsdJpeg(movie))

            self._saveAlignmentPlots(movie)

            if self._doComputeMicThumbnail():
                self.computeThumbnail(outMicFn,
                                      outputFn=self._getOutputMicThumbnail(
                                          movie))
        except:
            print("ERROR: Movie %s failed\n" % movie.getFileName())
Ejemplo n.º 39
0
    def createOutputStep(self):
        inputMovies = self.getInputMovies()
        nFrames = inputMovies.getFirstItem().getNumberOfFrames()

        inputParts = self.getParticles()
        movieParticles = self._createSetOfMovieParticles()
        movieParticles.copyInfo(inputParts)
        movieParticles.setSamplingRate(self._getNewSampling())

        self.lastMicName = None
        self.partList = []

        def _addPartsFromMic():
            # To avoid parsing the Relion star files...we are assuming here
            # the order in which Relion is generating the movie-particles per stack
            # it start part 1, 2, N of frame 1, then 1, 2..N of frame 2 and so on.
            # If this way changes in the future, the following code could break.
            # For the sake of performance, I will take the risk now.
            count = 0
            avgFrames = self.avgFrames.get()

            for frame in range(0, nFrames, avgFrames):
                frameId = min(frame + avgFrames, nFrames)

                for mPart in self.partList:
                    mPart.setObjId(None)  # clear objId to insert a new one
                    mPart.setFrameId(frameId)
                    count += 1
                    mPart.setIndex(count)
                    mPart._rlnAverageNrOfFrames = em.Integer(avgFrames)
                    movieParticles.append(mPart)

            del self.partList  # free unnecessary particle list memory
            self.partList = []

        for part in inputParts.iterItems(orderBy='_micId'):
            micName = part.getCoordinate().getMicName()

            if micName != self.lastMicName:
                if self.lastMicName is not None:
                    _addPartsFromMic()
                self.lastMicName = micName
                movieBase = '%s_movie.mrcs' % pwutils.removeBaseExt(micName)

                def _replaceSuffix(suffix):
                    return movieBase.replace('_movie.mrcs', suffix)

                # Move the resulting stack of movie-particles to extra directly
                movieStack = self._getExtraPath('output', 'extra', movieBase)
                self.newMovieStack = self._getExtraPath(
                    _replaceSuffix('_ptcls.mrcs'))
                pwutils.moveFile(movieStack, self.newMovieStack)

                # Clean up intermediate files (either links or converted)
                # plus generated files not needed anymore
                if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
                    pwutils.cleanPath(
                        self._getExtraPath(movieBase),
                        self._getExtraPath(_replaceSuffix('.mrcs')))

            # Create a movie particles based on that one and
            # store in the list of this movie
            mPart = em.MovieParticle()
            mPart.copy(part)  # copy all information from part
            mPart.setParticleId(part.getObjId())

            mPart.setFileName(self.newMovieStack)
            self.partList.append(mPart)

        pwutils.cleanPath(self._getExtraPath('output'))

        _addPartsFromMic()

        self._defineOutputs(outputParticles=movieParticles)
        self._defineSourceRelation(self.inputMovies, movieParticles)
        self._defineSourceRelation(self.inputParticles, movieParticles)
Ejemplo n.º 40
0
    def refineCtfStep(self):
        self._defineValues()
        self._prepareCommand()

        for mic in self.matchingMics:
            micName = mic.getFileName()
            micBase = pwutils.removeBaseExt(micName)
            micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
            outMic = pwutils.join(micDirTmp, pwutils.replaceBaseExt(micName, 'mrc'))
            micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
            micFnOut = self._getCtfOutPath(micDirTmp)
            micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
            micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')

            # Update _params dictionary
            self._params['micFn'] = outMic
            self._params['gctfOut'] = micFnOut

            if self.useInputCtf and self.ctfRelations.get():
                # get input CTFs from a mic
                ctfs = self.ctfRelations.get()
                micKey = mic.getMicName() if self.hasMicName else mic.getObjId()

                for ctf in ctfs:
                    ctfMicName = ctf.getMicrograph().getMicName()
                    ctfMicId = ctf.getMicrograph().getObjId()
                    if micKey == ctfMicName or micKey == ctfMicId:
                        # add CTF refine options
                        self._params.update({'refine_input_ctf': 1,
                                             'defU_init': ctf.getDefocusU(),
                                             'defV_init': ctf.getDefocusV(),
                                             'defA_init': ctf.getDefocusAngle(),
                                             'B_init': self.bfactor.get()
                                             })
                        self._args += "--refine_input_ctf %d " % self._params['refine_input_ctf']
                        self._args += "--defU_init %f " % self._params['defU_init']
                        self._args += "--defV_init %f " % self._params['defV_init']
                        self._args += "--defA_init %f " % self._params['defA_init']
                        self._args += "--B_init %f " % self._params['B_init']
                        self._args += "--defU_err %f " % self.defUerr.get()
                        self._args += "--defV_err %f " % self.defVerr.get()
                        self._args += "--defA_err %f " % self.defAerr.get()
                        self._args += "--B_err %f " % self.Berr.get()

                        break

            # final args
            self._args += "--do_validation %d " % (1 if self.doValidate else 0)
            self._args += "%(micFn)s "
            self._args += "> %(gctfOut)s"

            try:
                self.runJob(self._getProgram(), self._args % self._params,
                            env=self._getEnviron())
            except:
                print("ERROR: Gctf has failed for micrograph %s" % outMic)

            # move results from tmp to extra folder
            micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
            pwutils.makePath(micDir)
            psdFile = self._getPsdPath(micDir)
            ctfOutFile = self._getCtfOutPath(micDir)
            ctffitFile = self._getCtfFitOutPath(micDir)
            ctflocalFile = self._getCtfLocalPath(micDir, micBase)

            pwutils.moveFile(micFnCtf, psdFile)
            pwutils.moveFile(micFnOut, ctfOutFile)
            pwutils.moveFile(micFnCtfFit, ctffitFile)
            pwutils.moveFile(micFnLocalCtf, ctflocalFile)

            # Let's clean the temporary micrographs
            pwutils.cleanPath(outMic)
            pwutils.cleanPath(micDirTmp)

        pwutils.cleanPath(self.matchingMics.getFileName())
        pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))