Exemplo n.º 1
0
    def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair):
        """ Create a subset of Micrographs Tilt Pair. """
        output = MicrographsTiltPair(filename=self._getPath('micrographs_pairs.sqlite'))
        print "self._dbName=%s" % self._dbName
        modifiedSet = MicrographsTiltPair(filename=self._dbName, prefix=self._dbPrefix)
        inputU = micrographsTiltPair.getUntilted()
        inputT = micrographsTiltPair.getTilted()
        outputU = SetOfMicrographs(filename=self._getPath('mics_untilted.sqlite'))
        outputT = SetOfParticles(filename=self._getPath('mics_tilted.sqlite'))
        outputU.copyInfo(inputU)
        outputT.copyInfo(inputT)
        for micPairI in modifiedSet:
            untilted = micPairI.getUntilted()
            tilted = micPairI.getTilted()
            if micPairI.isEnabled():

                micPairO = TiltPair()
                micPairO.setUntilted(untilted)
                micPairO.setTilted(tilted)
                output.append(micPairO)
                outputU.append(untilted)
                outputT.append(tilted)
        output.setUntilted(outputU)
        output.setTilted(outputT)
        # Register outputs
        outputDict = {'outputMicrographsTiltPair': output}
        self._defineOutputs(**outputDict)
        self._defineTransformRelation(micrographsTiltPair, output)
        return output
Exemplo n.º 2
0
    def importCTFStep(self, importFrom):

        """ Copy ctfs matching the filename pattern.
        """
        ci = self.getImportClass()

        inputMics = self.inputMicrographs.get()
        ctfSet = self._createSetOfCTF()
        ctfSet.setMicrographs(inputMics)

        outputMics = self._createSetOfMicrographs()
        SetOfMicrographs.copyInfo(outputMics, inputMics)

        createOutputMics = False

        for mic in inputMics:
            for i, (fileName, fileId) in enumerate(self.iterFiles()):
                #TODO: Define how to match mics and ctf files (temporary solution is to expect micname inside ctf file name)
                if removeBaseExt(mic.getFileName()) in removeBaseExt(fileName):
                    ctf = ci.importCTF(mic, fileName)
                    ctfSet.append(ctf)
                    outputMics.append(mic)
                    break
            else:
                # If not CTF is found for a micrograph remove it from output mics
                self.warning("CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId())
                createOutputMics = True

        self._defineOutputs(outputCTF=ctfSet)
        # If some of the micrographs had not ctf a subset of micrographs have been produced
        if createOutputMics:
            self._defineOutputs(outputMicrographs=outputMics)
            self._defineCtfRelation(outputMics, ctfSet)
        else:
            self._defineCtfRelation(inputMics, ctfSet)
Exemplo n.º 3
0
    def _checkProcessedData(self):
        if self.setof == SET_OF_MOVIES:
            objSet = SetOfMovies(filename=self._getPath('movies.sqlite'))
        elif self.setof == SET_OF_MICROGRAPHS:
            objSet = \
                SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
        elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
            objSet = \
                SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
        elif self.setof == SET_OF_PARTICLES:
            objSet = SetOfParticles(filename=self._getPath('particles.sqlite'))
        else:
            raise Exception('Unknown data type')

        newObjSet, newObj = self._checkNewItems(objSet)

        if self.setof == SET_OF_MOVIES:
            newObjSet.setFramesRange(self.inputMovies.get().getFramesRange())

        # check if end ....
        if self.setof != SET_OF_PARTICLES:
            self.nDims = self.nDim.get()
        endObjs = newObjSet.getSize() == self.nDims

        if newObj:
            if endObjs:
                newObjSet.setStreamState(newObjSet.STREAM_CLOSED)
            self._updateOutput(newObjSet)
        newObjSet.close()
Exemplo n.º 4
0
    def writeCtfStarStep(self):
        inputCTF = self.inputCTF.get()

        if self.micrographSource == 0: # same as CTF estimation
            ctfMicSet = inputCTF.getMicrographs()
        else:
            ctfMicSet = self.inputMicrographs.get()

        micSet = SetOfMicrographs(filename=':memory:')

        psd = inputCTF.getFirstItem().getPsdFile()
        hasPsd = psd and os.path.exists(psd)

        if hasPsd:
            psdPath = self._getPath('PSD')
            pwutils.makePath(psdPath)
            print "Writing PSD files to %s" % psdPath

        for ctf in inputCTF:
            # Get the corresponding micrograph
            mic = ctfMicSet[ctf.getObjId()]
            if mic is None:
                print("Skipping CTF id: %s, it is missing from input "
                      "micrographs. " % ctf.getObjId())
                continue

            micFn = mic.getFileName()
            if not os.path.exists(micFn):
                print "Skipping micrograph %s, it does not exists. " % micFn
                continue

            mic2 = mic.clone()
            mic2.setCTF(ctf)
            if hasPsd:
                psdFile = ctf.getPsdFile()
                newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
                if not os.path.exists(psdFile):
                    print "PSD file %s does not exits" % psdFile
                    print "Skipping micrograph %s" % micFn
                    continue
                pwutils.copyFile(psdFile, newPsdFile)
                ctf.setPsdFile(newPsdFile)
            micSet.append(mic2)

        starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
        print "Writing set: %s" % inputCTF
        print " to: %s" % starFile

        acq = ctfMicSet.getAcquisition()
        self.samplingRate = ctfMicSet.getSamplingRate()
        mag = acq.getMagnification()
        self.detectorPixelSize = 1e-4 * self.samplingRate * mag

        writeSetOfMicrographs(micSet, starFile,
                              preprocessImageRow=self.preprocessMicrograph)

        # Let's create a link from the project roots to facilitate the import
        # of the star file into a Relion project
        pwutils.createLink(starFile, os.path.basename(starFile))
Exemplo n.º 5
0
    def writeCtfStarStep(self):
        inputCTF = self.inputCTF.get()

        if self.micrographSource == 0:  # same as CTF estimation
            ctfMicSet = inputCTF.getMicrographs()
        else:
            ctfMicSet = self.inputMicrographs.get()

        micSet = SetOfMicrographs(filename=':memory:')

        psd = inputCTF.getFirstItem().getPsdFile()
        hasPsd = psd and os.path.exists(psd)

        if hasPsd:
            psdPath = self._getPath('PSD')
            pwutils.makePath(psdPath)
            print "Writing PSD files to %s" % psdPath

        for ctf in inputCTF:
            # Get the corresponding micrograph
            mic = ctfMicSet[ctf.getObjId()]
            if mic is None:
                print(
                    "Skipping CTF id: %s, it is missing from input "
                    "micrographs. " % ctf.getObjId())
                continue

            micFn = mic.getFileName()
            if not os.path.exists(micFn):
                print "Skipping micrograph %s, it does not exists. " % micFn
                continue

            mic2 = mic.clone()
            mic2.setCTF(ctf)
            if hasPsd:
                psdFile = ctf.getPsdFile()
                newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
                if not os.path.exists(psdFile):
                    print "PSD file %s does not exits" % psdFile
                    print "Skipping micrograph %s" % micFn
                    continue
                pwutils.copyFile(psdFile, newPsdFile)
                ctf.setPsdFile(newPsdFile)
            micSet.append(mic2)

        starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
        print "Writing set: %s" % inputCTF
        print " to: %s" % starFile

        writeSetOfMicrographs(micSet,
                              starFile,
                              preprocessImageRow=self.preprocessMicrograph)

        # Let's create a link from the project roots to facilitate the import
        # of the star file into a Relion project
        pwutils.createLink(starFile, os.path.basename(starFile))
Exemplo n.º 6
0
    def importCTFStep(self, importFrom):
        """ Copy ctfs matching the filename pattern. """
        ci = self.getImportClass()

        inputMics = self.inputMicrographs.get()
        ctfSet = self._createSetOfCTF()
        ctfSet.setMicrographs(inputMics)

        outputMics = self._createSetOfMicrographs()
        SetOfMicrographs.copyInfo(outputMics, inputMics)

        createOutputMics = False

        files = [f for f, _ in self.iterFiles()]
        n = len(files)
        if n == 0:
            raise Exception("No files where found in path: '%s'\n"
                            "matching the pattern: '%s'" %
                            (self.filesPath, self.filesPattern))
        print "Matching files: ", len(files)

        def _getMicCTF(mic):
            micName = mic.getMicName()
            micBase = removeBaseExt(mic.getFileName())

            for fileName, fileId in self.iterFiles():
                if (fileId == mic.getObjId() or micBase in fileName
                        or micName in fileName):
                    return ci.importCTF(mic, fileName)

            return None

        # Check if the CTF import class has a method to retrieve the CTF
        # from a given micrograph. If not, we will try to associated based
        # on matching the filename or id
        getMicCTF = getattr(ci, 'getMicCTF', None) or _getMicCTF

        for mic in inputMics:
            ctf = getMicCTF(mic)
            if ctf is not None:
                ctfSet.append(ctf)
                outputMics.append(mic)
            else:
                # If CTF is not found for a micrograph remove it from output mics
                self.warning("CTF for micrograph id %d was not found. Removed "
                             "from set of micrographs." % mic.getObjId())
                createOutputMics = True

        self._defineOutputs(outputCTF=ctfSet)
        # If some of the micrographs had not ctf a subset of micrographs
        # have been produced
        if createOutputMics:
            self._defineOutputs(outputMicrographs=outputMics)
            self._defineCtfRelation(outputMics, ctfSet)
        else:
            self._defineCtfRelation(inputMics, ctfSet)
Exemplo n.º 7
0
    def test_fromScipion(self):
        """ Import an EMX file with micrographs and defocus
        """
        micsSqlite = self.dsXmipp.getFile('micrographs/micrographs.sqlite')
        print "Importing from sqlite: ", micsSqlite
        
        micSet = SetOfMicrographs(filename=micsSqlite)
        # Gold values
        #_samplingRate -> 1.237
        #_acquisition._voltage -> 300.0
        #_acquisition._magnification -> 50000.0

        for k in ['_samplingRate','_acquisition._voltage','_acquisition._magnification']:
            print k, "->", micSet.getProperty(k)
        prot = self.newProtocol(ProtImportMicrographs,
                                 objLabel='from scipion',
                                 importFrom=ProtImportMicrographs.IMPORT_FROM_SCIPION,
                                 sqliteFile=micsSqlite,
                                 samplingRate=float(micSet.getProperty('_samplingRate')),
                                 voltage=float(micSet.getProperty('_acquisition._voltage')),
                                 magnification=int(float(micSet.getProperty('_acquisition._magnification')))
                                )
         
        self.launchProtocol(prot)
        
        micSet = getattr(prot, 'outputMicrographs', None)
        self.assertIsNotNone(micSet)
        
        self.assertAlmostEqual(1.237, micSet.getSamplingRate())
        
        acq = micSet.getAcquisition()
        self.assertAlmostEqual(300., acq.getVoltage())
        self.assertAlmostEqual(50000., acq.getMagnification())
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the
        #  micrographs in the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            # create tmp set with all mics from coords set
            coordMics = SetOfMicrographs(filename=':memory:')
            coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())

            for micU, micT in izip(self.inputCoords.getUntilted().getMicrographs(),
                                   self.inputCoords.getTilted().getMicrographs()):
                micU.cleanObjId()
                micT.cleanObjId()
                coordMics.append(micU)
                coordMics.append(micT)

            for mic in coordMics:
                micBase = pwutils.removeBaseExt(mic.getFileName())
                micPos = self._getExtraPath(micBase + ".pos")
                micDict[pwutils.removeExt(mic.getMicName())] = micPos

            # now match micDict and inputMics
            if any(pwutils.removeExt(mic.getMicName()) in micDict for mic in self.inputMics):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception('Could not match input micrographs and coordinates '
                                'by micName.')

            for mic in self.inputMics:  # micrograph from input (other)
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' % (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)
Exemplo n.º 9
0
    def importCTFStep(self, importFrom):
        """ Copy ctfs matching the filename pattern. """
        ci = self.getImportClass()

        inputMics = self.inputMicrographs.get()
        ctfSet = self._createSetOfCTF()
        ctfSet.setMicrographs(inputMics)

        outputMics = self._createSetOfMicrographs()
        SetOfMicrographs.copyInfo(outputMics, inputMics)

        createOutputMics = False

        files = [f for f, _ in self.iterFiles()]
        n = len(files)
        if n == 0:
            raise Exception("No files where found in path: '%s'\n"
                            "matching the pattern: '%s'" %
                            (self.filesPath, self.filesPattern))
        print "Matching files: ", len(files)

        for mic in inputMics:
            micName = mic.getMicName()
            micBase = removeBaseExt(mic.getFileName())
            for fileName, fileId in self.iterFiles():
                if (fileId == mic.getObjId() or micBase in fileName
                        or micName in fileName):
                    ctf = ci.importCTF(mic, fileName)
                    ctfSet.append(ctf)
                    outputMics.append(mic)
                    break
            else:
                # If not CTF is found for a micrograph remove it from output mics
                self.warning(
                    "CTF for micrograph id %d was not found. Removed from set of micrographs."
                    % mic.getObjId())
                createOutputMics = True

        self._defineOutputs(outputCTF=ctfSet)
        # If some of the micrographs had not ctf a subset of micrographs have been produced
        if createOutputMics:
            self._defineOutputs(outputMicrographs=outputMics)
            self._defineCtfRelation(outputMics, ctfSet)
        else:
            self._defineCtfRelation(inputMics, ctfSet)
Exemplo n.º 10
0
    def loadInputs(self):
        micsFn = self.getInputMicrographs().getFileName()
        micsSet = SetOfMicrographs(filename=micsFn)
        micsSet.loadAllProperties()

        availableMics = []
        for mic in micsSet:
            availableMics.append(mic.getObjId())

        micsSetClosed = micsSet.isStreamClosed()
        micsSet.close()

        partsFn = self.getInputParticles().getFileName()
        partsSet = SetOfParticles(filename=partsFn)
        partsSet.loadAllProperties()

        newParts = []
        newMics = []
        for item in partsSet:
            micKey = item.getCoordinate().getMicId()
            if micKey not in self.micsDone and micKey in availableMics:
                newParts.append(item.getObjId())
                if not micKey in self.micsDone:
                    newMics.append(micKey)
        self.micsDone += newMics
        self.inputSize = partsSet.getSize()
        partSetClosed = partsSet.isStreamClosed()
        partsSet.close()

        return newParts, micsSetClosed and partSetClosed
Exemplo n.º 11
0
    def _setupBasicProperties(self):
        # Get sampling rate and inputMics according to micsSource type
        self.inputCoords = self.getCoords()
        self.uMics, self.tMics = self.getInputMicrographs()
        self.samplingInput = self.inputCoords.getUntilted().getMicrographs(
        ).getSamplingRate()
        self.samplingMics = self.uMics.getSamplingRate()
        self.samplingFactor = float(self.samplingMics / self.samplingInput)

        # create tmp set with all mic pairs
        self.inputMics = SetOfMicrographs(filename=':memory:')
        self.inputMics.copyInfo(self.uMics)
        self.inputMics.setStore(False)

        for micU, micT in izip(self.uMics, self.tMics):
            micU.cleanObjId()
            micT.cleanObjId()
            self.inputMics.append(micU)
            self.inputMics.append(micT)
Exemplo n.º 12
0
    def importCTFStep(self, importFrom):
        """ Copy ctfs matching the filename pattern. """
        ci = self.getImportClass()

        inputMics = self.inputMicrographs.get()
        ctfSet = self._createSetOfCTF()
        ctfSet.setMicrographs(inputMics)

        outputMics = self._createSetOfMicrographs()
        SetOfMicrographs.copyInfo(outputMics, inputMics)

        createOutputMics = False
        
        files = [f for f, _ in self.iterFiles()]
        n = len(files)
        if n == 0:
            raise Exception("No files where found in path: '%s'\n"
                            "matching the pattern: '%s'" % (self.filesPath, self.filesPattern))
        print "Matching files: ", len(files)

        for mic in inputMics:
            micName = mic.getMicName()
            micBase = removeBaseExt(mic.getFileName())
            for fileName, fileId in self.iterFiles():
                if (fileId == mic.getObjId() or
                    micBase in fileName or micName in fileName):
                    ctf = ci.importCTF(mic, fileName)
                    ctfSet.append(ctf)
                    outputMics.append(mic)
                    break
            else:
                # If not CTF is found for a micrograph remove it from output mics
                self.warning("CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId())
                createOutputMics = True

        self._defineOutputs(outputCTF=ctfSet)
        # If some of the micrographs had not ctf a subset of micrographs have been produced
        if createOutputMics:
            self._defineOutputs(outputMicrographs=outputMics)
            self._defineCtfRelation(outputMics, ctfSet)
        else:
            self._defineCtfRelation(inputMics, ctfSet)
Exemplo n.º 13
0
    def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair):
        """ Create a subset of Micrographs Tilt Pair. """
        output = MicrographsTiltPair(
            filename=self._getPath('micrographs_pairs.sqlite'))
        modifiedSet = MicrographsTiltPair(filename=self._dbName,
                                          prefix=self._dbPrefix)
        inputU = micrographsTiltPair.getUntilted()
        inputT = micrographsTiltPair.getTilted()
        outputU = SetOfMicrographs(
            filename=self._getPath('mics_untilted.sqlite'))
        outputT = SetOfParticles(filename=self._getPath('mics_tilted.sqlite'))
        outputU.copyInfo(inputU)
        outputT.copyInfo(inputT)

        for micPairI in modifiedSet:
            untilted = micPairI.getUntilted()
            tilted = micPairI.getTilted()

            if micPairI.isEnabled():
                micPairO = TiltPair()
                micPairO.setUntilted(untilted)
                micPairO.setTilted(tilted)
                output.append(micPairO)
                outputU.append(untilted)
                outputT.append(tilted)
        output.setUntilted(outputU)
        output.setTilted(outputT)
        # Register outputs
        outputDict = {'outputMicrographsTiltPair': output}
        self._defineOutputs(**outputDict)
        self._defineTransformRelation(micrographsTiltPair, output)
        return output
Exemplo n.º 14
0
    def checkMicSet(self, micSet, goldFn):
        """ Compare micrographs of micSet with the 
        ones in the goldFn. Maybe except the full path.
        """
        goldSet  = SetOfMicrographs(filename = goldFn)

        for mic1, mic2 in izip(goldSet, micSet):
            # Remove the absolute path in the micrographs to
            # really check that the attributes should be equal
            mic1.setFileName(os.path.basename(mic1.getFileName()))
            mic2.setFileName(os.path.basename(mic2.getFileName()))

            self.assertTrue(mic1.equalAttributes(mic2, verbose=True))        
Exemplo n.º 15
0
    def _stepsCheck(self):
        if self.setof == SET_OF_MOVIES:
            objSet = SetOfMovies(filename=self._getPath('movies.sqlite'))
        elif self.setof == SET_OF_MICROGRAPHS:
            objSet = \
                SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
        elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
            objSet = \
                SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
        else:
            raise Exception('Unknown data type')

        newObjSet, newObj = self._checkNewItems(objSet)

        # check if end ....
        endObjs = newObjSet.getSize() == self.nDim.get()

        if newObj:
            if endObjs:
                newObjSet.setStreamState(newObjSet.STREAM_CLOSED)
            self._updateOutput(newObjSet)
        newObjSet.close()
    def _stepsCheck(self):
        setOfMicFn = self._getPath('micrographs.sqlite')
        micSet = SetOfMicrographs(filename=setOfMicFn)
        micSet, newMic = self._checkNewMics(micSet)

        #check if end ....
        endMics = micSet.getSize() == self.nDim.get()

        if newMic:
            if endMics:
                micSet.setStreamState(micSet.STREAM_CLOSED)
            self._updateOutput(micSet)
        micSet.close()
Exemplo n.º 17
0
    def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair):
        """ Create a subset of Micrographs Tilt Pair. """
        output = MicrographsTiltPair(filename=self._getPath('micrographs_pairs.sqlite'))
        modifiedSet = MicrographsTiltPair(filename=self._dbName,
                                          prefix=self._dbPrefix)
        inputU = micrographsTiltPair.getUntilted()
        inputT = micrographsTiltPair.getTilted()
        outputU = SetOfMicrographs(filename=self._getPath('mics_untilted.sqlite'))
        outputT = SetOfMicrographs(filename=self._getPath('mics_tilted.sqlite'))
        outputU.copyInfo(inputU)
        outputT.copyInfo(inputT)

        for micPair, u, t in izip(modifiedSet, inputU, inputT):
            if micPair.isEnabled():
                output.append(micPair)
                outputU.append(u)
                outputT.append(t)
        output.setUntilted(outputU)
        output.setTilted(outputT)
        # Register outputs
        outputDict = {'outputMicrographsTiltPair': output}
        self._defineOutputs(**outputDict)
        self._defineTransformRelation(micrographsTiltPair, output)
        return output
 def _checkNewInput(self):
     # Check if there are new micrographs to process from the input set
     micsFile = self.inputMicrographs.get().getFileName()
     micsSet = SetOfMicrographs(filename=micsFile)
     micsSet.loadAllProperties()
     self.SetOfMicrographs = [m.clone() for m in micsSet]
     self.streamClosed = micsSet.isStreamClosed()
     micsSet.close()
     newMics = any(m.getObjId() not in self.insertedDict
                   for m in self.inputMics)
     outputStep = self._getFirstJoinStep()
     if newMics:
         fDeps = self._insertNewMicsSteps(self.insertedDict, self.inputMics)
         if outputStep is not None:
             outputStep.addPrerequisites(*fDeps)
         self.updateSteps()
Exemplo n.º 19
0
    def importMicrographs(self):
        """ Import a SetOfMicrographs from a given sqlite file. """
        inputSet = SetOfMicrographs(filename=self._sqliteFile)
        self._findImagesPath(inputSet)

        micSet = self.protocol._createSetOfMicrographs()
        micSet.setObjComment('Micrographs imported from sqlite file:\n%s' %
                             self._sqliteFile)

        # Update both samplingRate and acquisition with parameters
        # selected in the protocol form
        self.protocol.setSamplingRate(micSet)
        micSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get())
        self.protocol.fillAcquisition(micSet.getAcquisition())
        # Read the micrographs from the 'self._sqliteFile' metadata
        # but fixing the filenames with new ones (linked or copy to extraDir)
        micSet.copyItems(inputSet, updateItemCallback=self._updateParticle)
        self.protocol._defineOutputs(outputMicrographs=micSet)
    def _setupBasicProperties(self):
        # Get sampling rate and inputMics according to micsSource type
        self.inputCoords = self.getCoords()
        self.uMics, self.tMics = self.getInputMicrographs()
        self.samplingInput = self.inputCoords.getUntilted().getMicrographs().getSamplingRate()
        self.samplingMics = self.uMics.getSamplingRate()
        self.samplingFactor = float(self.samplingMics / self.samplingInput)

        # create tmp set with all mic pairs
        self.inputMics = SetOfMicrographs(filename=':memory:')
        self.inputMics.copyInfo(self.uMics)
        self.inputMics.setStore(False)

        for micU, micT in izip(self.uMics, self.tMics):
            micU.cleanObjId()
            micT.cleanObjId()
            self.inputMics.append(micU)
            self.inputMics.append(micT)
Exemplo n.º 21
0
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the micrographs in
        # the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            # create tmp set with all mics from coords set
            coordMics = SetOfMicrographs(filename=':memory:')
            coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())

            for micU, micT in izip(
                    self.inputCoords.getUntilted().getMicrographs(),
                    self.inputCoords.getTilted().getMicrographs()):
                micU.cleanObjId()
                micT.cleanObjId()
                coordMics.append(micU)
                coordMics.append(micT)

            for mic in coordMics:
                micBase = pwutils.removeBaseExt(mic.getFileName())
                micPos = self._getExtraPath(micBase + ".pos")
                micDict[pwutils.removeExt(mic.getMicName())] = micPos

            # now match micDict and inputMics
            if any(
                    pwutils.removeExt(mic.getMicName()) in micDict
                    for mic in self.inputMics):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception(
                    'Could not match input micrographs and coordinates '
                    'by micName.')

            for mic in self.inputMics:  # micrograph from input (other)
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' %
                                      (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)
Exemplo n.º 22
0
    def test_micrographImport(self):
        """ Import an EMX file with micrographs and defocus
        """
        emxFn = self.dataset.getFile('emxMicrographCtf1')
        protEmxImport = self.newProtocol(
            ProtImportMicrographs,
            objLabel='emx - import mics',
            importFrom=ProtImportMicrographs.IMPORT_FROM_EMX,
            emxFile=emxFn,
            magnification=10000,
            samplingRate=2.46,
        )
        self.launchProtocol(protEmxImport)
        micFn = self.dataset.getFile('emxMicrographCtf1Gold')
        mics = SetOfMicrographs(filename=micFn)

        for mic1, mic2 in izip(mics, protEmxImport.outputMicrographs):
            # Remove the absolute path in the micrographs to
            # really check that the attributes should be equal
            mic1.setFileName(os.path.basename(mic1.getFileName()))
            mic2.setFileName(os.path.basename(mic2.getFileName()))
            self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
Exemplo n.º 23
0
    def _stepsCheck(self):
        # For now the streaming is not allowed for recalculate CTF
        if self.recalculate:
            return

        # Check if there are new micrographs to process
        micFn = self.inputMicrographs.get().getFileName()
        micSet = SetOfMicrographs(filename=micFn)
        micSet.loadAllProperties()
        streamClosed = micSet.isStreamClosed()

        outputStep = self._getFirstJoinStep()
        self._checkNewMicrographs(micSet, outputStep)
        ctfSet, newCTFs = self._checkNewCTFs(micSet)

        if ctfSet is None:
            return

        endCTFs = streamClosed and micSet.getSize() == ctfSet.getSize()
        if newCTFs:
            # Check if it is the first time we are registering CTF to
            # create the CTF_RELATION only once
            firstTime = not self.hasAttribute('outputCTF')
            ctfSet.setMicrographs(self.inputMics)
            self._computeDefocusRange(ctfSet)
            streamMode = ctfSet.STREAM_CLOSED if endCTFs else ctfSet.STREAM_OPEN
            self._updateOutputSet('outputCTF', ctfSet, streamMode)
            if firstTime:  # define relation just once
                self._defineCtfRelation(self.inputMics, ctfSet)
        else:
            ctfSet.close()

        if outputStep and outputStep.isWaiting() and endCTFs:
            outputStep.setStatus(STATUS_NEW)

        micSet.close()
Exemplo n.º 24
0
    def test_particleImportDefocus(self):
        """ Import an EMX file with a stack of particles
        that has defocus
        """
        emxFn = self.dataset.getFile('defocusParticleT2')
        protEmxImport = self.newProtocol(
            ProtImportParticles,
            objLabel='emx - import ctf',
            importFrom=ProtImportParticles.IMPORT_FROM_EMX,
            emxFile=emxFn,
            alignType=3,
            magnification=10000,
            samplingRate=2.8)
        self.launchProtocol(protEmxImport)
        micFn = self.dataset.getFile('micrographsGoldT2')
        mics = SetOfMicrographs(filename=micFn)

        for mic1, mic2 in izip(mics, protEmxImport.outputMicrographs):
            # Remove the absolute path in the micrographs to
            # really check that the attributes should be equal
            mic1.setFileName(os.path.basename(mic1.getFileName()))
            mic2.setFileName(os.path.basename(mic2.getFileName()))
            self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
Exemplo n.º 25
0
    def test_pattern(self):
        """ Import several Particles from a given pattern.
        """
        kwargs = {'xDim': 1024,
                  'yDim': 1024,
                  'nDim': MICS,
                  'samplingRate': 1.25,
                  'creationInterval': 5,
                  'delay': 0,
                  'setof': SET_OF_RANDOM_MICROGRAPHS  # SetOfMicrographs
                  }

        # create input micrographs
        protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
        protStream.setObjLabel('create Stream Mic')
        self.proj.launchProtocol(protStream, wait=False)

        counter = 1
        while not protStream.hasAttribute('outputMicrographs'):
            time.sleep(2)
            protStream = self._updateProtocol(protStream)
            if counter > 100:
                break
            counter += 1

        # then introduce monitor, checking all the time ctf and
        # saving to database
        protCTF = ProtCTFFind(useCftfind4=True)
        protCTF.inputMicrographs.set(protStream.outputMicrographs)
        protCTF.ctfDownFactor.set(2)
        protCTF.highRes.set(0.4)
        protCTF.lowRes.set(0.05)
        protCTF.numberOfThreads.set(4)
        self.proj.launchProtocol(protCTF, wait=False)

        counter = 1

        while not protCTF.hasAttribute('outputCTF'):

            time.sleep(2)
            protCTF = self._updateProtocol(protCTF)
            if counter > 100:
                break
            counter += 1

        kwargs = {
            'maxDefocus': 28000,
            'minDefocus': 1000,
            'astigmatism': 1000,
            'resolution': 7
        }

        protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
        protCTFSel.inputCTF.set(protCTF.outputCTF)
        self.proj.launchProtocol(protCTFSel,  wait=False)

        counter = 1

        while not protCTFSel.hasAttribute('outputCTF'):

            time.sleep(2)
            protCTFSel = self._updateProtocol(protCTFSel)
            if counter > 100:
                break
            counter += 1

        kwargs = {
            'maxDefocus': 40000,
            'minDefocus': 1000,
            'astigmatism': 1000,
            'resolution': 3.7,
        }

        protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
        protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
        self.proj.launchProtocol(protCTFSel2)

        counter = 1
        while not (protCTFSel2.hasAttribute('outputCTF') and
                   protCTFSel2.hasAttribute('outputMicrographs')):

            time.sleep(2)
            protCTFSel2 = self._updateProtocol(protCTFSel2)
            if counter > 100:
                self.assertTrue(False)
            counter += 1

        # AJ the number of micrographs discarded and selected in the first CTF
        # selection protocol must be equal to the number of mics in the
        # CTF estimation protocol
        micSetDiscarded1 = SetOfMicrographs(
            filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
        micSet1 = SetOfMicrographs(
            filename=protCTFSel._getPath(MIC_SQLITE))
        counter = 1
        while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
            time.sleep(2)
            micSetDiscarded1 = SetOfMicrographs(
                filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
            micSet1 = SetOfMicrographs(
                filename=protCTFSel._getPath(MIC_SQLITE))
            if counter > 100:
                self.assertTrue(False)
            counter += 1

        # AJ the number of micrographs discarded and selected in the second CTF
        # selection protocol must be equal to the number of ctfs in the first
        # CTF selection protocol
        micSetDiscarded2 = SetOfMicrographs(
            filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
        micSet2 = SetOfMicrographs(
            filename=protCTFSel2._getPath(MIC_SQLITE))
        ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
        counter = 1
        while not (ctfSet1.getSize() == (micSetDiscarded2.getSize() +
                                         micSet2.getSize())):
            time.sleep(2)
            micSetDiscarded2 = SetOfMicrographs(
                filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
            micSet2 = SetOfMicrographs(
                filename=protCTFSel2._getPath(MIC_SQLITE))
            ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
            if counter > 100:
                self.assertTrue(False)
            counter += 1

        ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
        for ctf in ctfSet:
            defocusU = ctf.getDefocusU()
            defocusV = ctf.getDefocusV()
            astigm = defocusU - defocusV
            resol = ctf.getResolution()  # TODO
            if defocusU < 1000 \
                    or defocusU > 28000 \
                    or defocusV < 1000 \
                    or defocusV > 28000 \
                    or astigm > 1000 or resol > 3.7:
                self.assertTrue(False, "A CTF without the correct parameters"
                                       " is included in the output set")
    def test_pattern(self):
        """ Import several Particles from a given pattern.
        """
        kwargs = {
            'xDim': 1024,
            'yDim': 1024,
            'nDim': MICS,
            'samplingRate': 1.25,
            'creationInterval': 5,
            'delay': 0,
            'setof': SET_OF_RANDOM_MICROGRAPHS  # SetOfMicrographs
        }

        # create input micrographs
        protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
        protStream.setObjLabel('create Stream Mic')
        self.proj.launchProtocol(protStream, wait=False)

        counter = 1
        while not protStream.hasAttribute('outputMicrographs'):
            time.sleep(2)
            protStream = self._updateProtocol(protStream)
            if counter > 100:
                break
            counter += 1

        # then introduce monitor, checking all the time ctf and
        # saving to database
        protCTF = ProtCTFFind(useCftfind4=True)
        protCTF.inputMicrographs.set(protStream.outputMicrographs)
        protCTF.ctfDownFactor.set(2)
        protCTF.highRes.set(0.4)
        protCTF.lowRes.set(0.05)
        protCTF.numberOfThreads.set(4)
        self.proj.launchProtocol(protCTF, wait=False)

        counter = 1

        while not protCTF.hasAttribute('outputCTF'):

            time.sleep(2)
            protCTF = self._updateProtocol(protCTF)
            if counter > 100:
                break
            counter += 1

        kwargs = {
            'maxDefocus': 28000,
            'minDefocus': 1000,
            'astigmatism': 1000,
            'resolution': 7
        }

        protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
        protCTFSel.inputCTF.set(protCTF.outputCTF)
        self.proj.launchProtocol(protCTFSel, wait=False)

        counter = 1

        while not protCTFSel.hasAttribute('outputCTF'):

            time.sleep(2)
            protCTFSel = self._updateProtocol(protCTFSel)
            if counter > 100:
                break
            counter += 1

        kwargs = {
            'maxDefocus': 40000,
            'minDefocus': 1000,
            'astigmatism': 1000,
            'resolution': 3.7,
        }

        protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
        protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
        self.proj.launchProtocol(protCTFSel2)

        counter = 1
        while not (protCTFSel2.hasAttribute('outputCTF')
                   and protCTFSel2.hasAttribute('outputMicrographs')):

            time.sleep(2)
            protCTFSel2 = self._updateProtocol(protCTFSel2)
            if counter > 100:
                self.assertTrue(False)
            counter += 1

        # AJ the number of micrographs discarded and selected in the first CTF
        # selection protocol must be equal to the number of mics in the
        # CTF estimation protocol
        micSetDiscarded1 = SetOfMicrographs(
            filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
        micSet1 = SetOfMicrographs(filename=protCTFSel._getPath(MIC_SQLITE))
        counter = 1
        while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
            time.sleep(2)
            micSetDiscarded1 = SetOfMicrographs(
                filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
            micSet1 = SetOfMicrographs(
                filename=protCTFSel._getPath(MIC_SQLITE))
            if counter > 100:
                self.assertTrue(False)
            counter += 1

        # AJ the number of micrographs discarded and selected in the second CTF
        # selection protocol must be equal to the number of ctfs in the first
        # CTF selection protocol
        micSetDiscarded2 = SetOfMicrographs(
            filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
        micSet2 = SetOfMicrographs(filename=protCTFSel2._getPath(MIC_SQLITE))
        ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
        counter = 1
        while not (ctfSet1.getSize()
                   == (micSetDiscarded2.getSize() + micSet2.getSize())):
            time.sleep(2)
            micSetDiscarded2 = SetOfMicrographs(
                filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
            micSet2 = SetOfMicrographs(
                filename=protCTFSel2._getPath(MIC_SQLITE))
            ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
            if counter > 100:
                self.assertTrue(False)
            counter += 1

        ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
        for ctf in ctfSet:
            defocusU = ctf.getDefocusU()
            defocusV = ctf.getDefocusV()
            astigm = defocusU - defocusV
            resol = ctf.getResolution()  # TODO
            if defocusU < 1000 \
                    or defocusU > 28000 \
                    or defocusV < 1000 \
                    or defocusV > 28000 \
                    or astigm > 1000 or resol > 3.7:
                self.assertTrue(
                    False, "A CTF without the correct parameters"
                    " is included in the output set")
class XmippProtExtractParticlesPairs(ProtExtractParticlesPair, XmippProtocol):
    """Protocol to extract particles from a set of tilted pairs coordinates"""
    _label = 'extract particle pairs'

    def __init__(self, **kwargs):
        ProtExtractParticlesPair.__init__(self, **kwargs)
        self.stepsExecutionMode = STEPS_PARALLEL

    # --------------------------- DEFINE param functions -----------------------
    def _defineParams(self, form):
        form.addSection(label='Input')
        form.addParam('inputCoordinatesTiltedPairs', PointerParam,
                      important=True, label="Coordinates tilted pairs",
                      pointerClass='CoordinatesTiltPair',
                      help='Select the CoordinatesTiltPairs')
        form.addParam('downsampleType', EnumParam,
                      choices=['same as picking', 'other'],
                      default=0, important=True, label='Micrographs source',
                      display=EnumParam.DISPLAY_HLIST,
                      help='By default the particles will be extracted '
                           'from the micrographs used in the picking '
                           'step ( _same as picking_ option ).\n'
                           'If you select _other_ option, you must provide '
                           'a different set of micrographs to extract from.\n'
                           '*Note*: In the _other_ case, ensure that provided '
                           'micrographs and coordinates are related '
                           'by micName or by micId. Difference in pixel size '
                           'will be handled automatically.')
        form.addParam('inputMicrographsTiltedPair', PointerParam,
                      pointerClass='MicrographsTiltPair',
                      condition='downsampleType != 0',
                      important=True, label='Input tilt pair micrographs',
                      help='Select the tilt pair micrographs from which to '
                           'extract.')

        # downFactor should always be 1.0 or greater
        geOne = GE(1.0, error='Value should be greater or equal than 1.0')

        form.addParam('downFactor', FloatParam, default=1.0,
                      validators=[geOne],
                      label='Downsampling factor',
                      help='Select a value greater than 1.0 to reduce the size '
                           'of micrographs before extracting the particles. '
                           'If 1.0 is used, no downsample is applied. '
                           'Non-integer downsample factors are possible. ')
        form.addParam('boxSize', IntParam, default=0,
                      label='Particle box size', validators=[Positive],
                      help='In pixels. The box size is the size of the boxed '
                           'particles, actual particles may be smaller than '
                           'this. If you do downsampling after extraction, '
                           'provide final box size here.')
        form.addParam('doBorders', BooleanParam, default=False,
                      label='Fill pixels outside borders',
                      help='Xmipp by default skips particles whose boxes fall '
                           'outside of the micrograph borders. Set this '
                           'option to True if you want those pixels outside '
                           'the borders to be filled with the closest pixel '
                           'value available')

        form.addSection(label='Preprocess')
        form.addParam('doRemoveDust', BooleanParam, default=True,
                      important=True, label='Dust removal (Recommended)',
                      help='Sets pixels with unusually large values to random '
                           'values from a Gaussian with zero-mean and '
                           'unity-standard deviation.')
        form.addParam('thresholdDust', FloatParam, default=3.5,
                      condition='doRemoveDust', expertLevel=LEVEL_ADVANCED,
                      label='Threshold for dust removal',
                      help='Pixels with a signal higher or lower than this '
                           'value times the standard deviation of the image '
                           'will be affected. For cryo, 3.5 is a good value. '
                           'For high-contrast negative stain, the signal '
                           'itself may be affected so that a higher value may '
                           'be preferable.')
        form.addParam('doInvert', BooleanParam, default=None,
                      label='Invert contrast',
                      help='Invert the contrast if your particles are black '
                           'over a white background. Xmipp, Spider, Relion '
                           'and Eman require white particles over a black '
                           'background, Frealign (up to v9.07) requires black '
                           'particles over a white background')
        form.addParam('doNormalize', BooleanParam, default=True,
                      label='Normalize (Recommended)',
                      help='It subtract a ramp in the gray values and '
                           'normalizes so that in the background there is 0 '
                           'mean and standard deviation 1.')
        form.addParam('normType', EnumParam,
                      choices=['OldXmipp', 'NewXmipp', 'Ramp'], default=2,
                      condition='doNormalize', expertLevel=LEVEL_ADVANCED,
                      display=EnumParam.DISPLAY_COMBO,
                      label='Normalization type',
                      help='OldXmipp (mean(Image)=0, stddev(Image)=1).\n'
                           'NewXmipp (mean(background)=0, '
                           'stddev(background)=1)\n'
                           'Ramp (subtract background+NewXmipp).')
        form.addParam('backRadius', IntParam, default=-1,
                      condition='doNormalize',
                      label='Background radius', expertLevel=LEVEL_ADVANCED,
                      help='Pixels outside this circle are assumed to be '
                           'noise and their stddev is set to 1. Radius for '
                           'background circle definition (in pix.). If this '
                           'value is 0, then half the box size is used.')
        form.addParallelSection(threads=4, mpi=1)

    #--------------------------- INSERT steps functions ------------------------
    def _insertAllSteps(self):
        self._setupBasicProperties()
        # Write pos files for each micrograph
        firstStepId = self._insertFunctionStep('writePosFilesStep')

        # For each micrograph insert the steps, run in parallel
        deps = []
        for mic in self.inputMics:
            localDeps = [firstStepId]
            fnLast = mic.getFileName()
            micName = pwutils.removeBaseExt(mic.getFileName())

            def getMicTmp(suffix):
                return self._getTmpPath(micName + suffix)

            # Create a list with micrographs operations (programs in xmipp) and
            # the required command line parameters (except input/ouput files)
            micOps = []

            # Check if it is required to downsample your micrographs
            downFactor = self.downFactor.get()

            if self.notOne(downFactor):
                fnDownsampled = getMicTmp("_downsampled.xmp")
                args = "-i %s -o %s --step %f --method fourier"
                micOps.append(('xmipp_transform_downsample',
                               args % (fnLast, fnDownsampled, downFactor)))
                fnLast = fnDownsampled

            if self.doRemoveDust:
                fnNoDust = getMicTmp("_noDust.xmp")
                args = " -i %s -o %s --bad_pixels outliers %f"
                micOps.append(('xmipp_transform_filter',
                               args % (fnLast, fnNoDust, self.thresholdDust)))
                fnLast = fnNoDust

            # TODO: implement CTF
            # Actually extract
            deps.append(self._insertFunctionStep('extractParticlesStep',
                                                 mic.getObjId(), micName,
                                                 None, fnLast, micOps,
                                                 self.doInvert.get(),
                                                 self._getNormalizeArgs(),
                                                 self.doBorders.get(),
                                                 prerequisites=localDeps))

        metaDeps = self._insertFunctionStep('createMetadataImageStep',
                                            prerequisites=deps)

        # Insert step to create output objects
        self._insertFunctionStep('createOutputStep',
                                 prerequisites=[metaDeps], wait=False)

    # --------------------------- STEPS functions ------------------------------
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the
        #  micrographs in the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            # create tmp set with all mics from coords set
            coordMics = SetOfMicrographs(filename=':memory:')
            coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())

            for micU, micT in izip(self.inputCoords.getUntilted().getMicrographs(),
                                   self.inputCoords.getTilted().getMicrographs()):
                micU.cleanObjId()
                micT.cleanObjId()
                coordMics.append(micU)
                coordMics.append(micT)

            for mic in coordMics:
                micBase = pwutils.removeBaseExt(mic.getFileName())
                micPos = self._getExtraPath(micBase + ".pos")
                micDict[pwutils.removeExt(mic.getMicName())] = micPos

            # now match micDict and inputMics
            if any(pwutils.removeExt(mic.getMicName()) in micDict for mic in self.inputMics):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception('Could not match input micrographs and coordinates '
                                'by micName.')

            for mic in self.inputMics:  # micrograph from input (other)
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' % (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)

    def extractParticlesStep(self, micId, baseMicName, fnCTF,
                             micrographToExtract, micOps,
                             doInvert, normalizeArgs, doBorders):
        """ Extract particles from one micrograph """
        outputRoot = str(self._getExtraPath(baseMicName))
        fnPosFile = self._getExtraPath(baseMicName + ".pos")
    
        # If it has coordinates extract the particles
        particlesMd = 'particles@%s' % fnPosFile
        boxSize = self.boxSize.get()
        boxScale = self.getBoxScale()
        print "boxScale: ", boxScale
    
        if exists(fnPosFile):
            # Apply first all operations required for the micrograph
            for program, args in micOps:
                self.runJob(program, args)
        
            args = " -i %s --pos %s" % (micrographToExtract, particlesMd)
            args += " -o %s --Xdim %d" % (outputRoot, boxSize)
        
            if doInvert:
                args += " --invert"
        
            if fnCTF:
                args += " --ctfparam " + fnCTF
        
            if doBorders:
                args += " --fillBorders"
        
            self.runJob("xmipp_micrograph_scissor", args)
        
            # Normalize
            if normalizeArgs:
                self.runJob('xmipp_transform_normalize',
                            '-i %s.stk %s' % (outputRoot, normalizeArgs))
        else:
            self.warning("The micrograph %s hasn't coordinate file! "
                         % baseMicName)
            self.warning("Maybe you picked over a subset of micrographs")
    
        # Let's clean the temporary mrc micrographs
        if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
            pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')

    def createMetadataImageStep(self):
        mdUntilted = md.MetaData()
        mdTilted = md.MetaData()
        # for objId in mdPairs:
        for uMic, tMic in izip(self.uMics, self.tMics):
            umicName = pwutils.removeBaseExt(uMic.getFileName())
            fnMicU = self._getExtraPath(umicName + ".xmd")
            fnPosU = self._getExtraPath(umicName + ".pos")
            # Check if there are picked particles in these micrographs
            if pwutils.exists(fnMicU):
                mdMicU = md.MetaData(fnMicU)
                mdPosU = md.MetaData('particles@%s' % fnPosU)
                mdPosU.merge(mdMicU)
                mdUntilted.unionAll(mdPosU)
                tmicName = pwutils.removeBaseExt(tMic.getFileName())
                fnMicT = self._getExtraPath(tmicName + ".xmd")
                fnPosT = self._getExtraPath(tmicName + ".pos")
                mdMicT = md.MetaData(fnMicT)
                mdPosT = md.MetaData('particles@%s' % fnPosT)
                mdPosT.merge(mdMicT)
                mdTilted.unionAll(mdPosT)

        # Write image metadata (check if it is really necessary)
        fnTilted = self._getExtraPath("images_tilted.xmd")
        fnUntilted = self._getExtraPath("images_untilted.xmd")
        mdUntilted.write(fnUntilted)
        mdTilted.write(fnTilted)

    def createOutputStep(self):
        fnTilted = self._getExtraPath("images_tilted.xmd")
        fnUntilted = self._getExtraPath("images_untilted.xmd")

        # Create outputs SetOfParticles both for tilted and untilted
        imgSetU = self._createSetOfParticles(suffix="Untilted")
        imgSetU.copyInfo(self.uMics)
        imgSetT = self._createSetOfParticles(suffix="Tilted")
        imgSetT.copyInfo(self.tMics)

        sampling = self.samplingMics if self._micsOther() else self.samplingInput
        if self._doDownsample():
            sampling *= self.downFactor.get()
        imgSetU.setSamplingRate(sampling)
        imgSetT.setSamplingRate(sampling)

        # set coords from the input, will update later if needed
        imgSetU.setCoordinates(self.inputCoordinatesTiltedPairs.get().getUntilted())
        imgSetT.setCoordinates(self.inputCoordinatesTiltedPairs.get().getTilted())

        # Read untilted and tilted particles on a temporary object (also disabled particles)
        imgSetAuxU = SetOfParticles(filename=':memory:')
        imgSetAuxU.copyInfo(imgSetU)
        readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False)

        imgSetAuxT = SetOfParticles(filename=':memory:')
        imgSetAuxT.copyInfo(imgSetT)
        readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False)

        # calculate factor for coords scaling
        factor = 1 / self.samplingFactor
        if self._doDownsample():
            factor /= self.downFactor.get()

        coordsT = self.inputCoordinatesTiltedPairs.get().getTilted()
        # For each untilted particle retrieve micId from SetOfCoordinates untilted
        for imgU, coordU in izip(imgSetAuxU, self.inputCoordinatesTiltedPairs.get().getUntilted()):
            # FIXME: Remove this check when sure that objIds are equal
            id = imgU.getObjId()
            if id != coordU.getObjId():
                raise Exception('ObjIds in untilted img and coord are not the same!!!!')
            imgT = imgSetAuxT[id]
            coordT = coordsT[id]

            # If both particles are enabled append them
            if imgU.isEnabled() and imgT.isEnabled():
                if self._micsOther() or self._doDownsample():
                    coordU.scale(factor)
                    coordT.scale(factor)
                imgU.setCoordinate(coordU)
                imgSetU.append(imgU)
                imgT.setCoordinate(coordT)
                imgSetT.append(imgT)

        imgSetU.write()
        imgSetT.write()

        # Define output ParticlesTiltPair
        outputset = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite'))
        outputset.setTilted(imgSetT)
        outputset.setUntilted(imgSetU)
        for imgU, imgT in izip(imgSetU, imgSetT):
            outputset.append(TiltPair(imgU, imgT))

        outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get())
        self._defineOutputs(outputParticlesTiltPair=outputset)
        self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)

    # --------------------------- INFO functions --------------------------------------------
    def _validate(self):
        errors = []

        if self.doNormalize:
            if self.backRadius > int(self.boxSize.get() / 2):
                errors.append("Background radius for normalization should be "
                              "equal or less than half of the box size.")
        return errors

    def _citations(self):
        return ['delaRosaTrevin2013']

    def _summary(self):
        summary = []
        summary.append("Micrographs source: %s"
                       % self.getEnumText('downsampleType'))
        summary.append("Particle box size: %d" % self.boxSize)

        if not hasattr(self, 'outputParticlesTiltPair'):
            summary.append("Output images not ready yet.")
        else:
            summary.append("Particle pairs extracted: %d" %
                           self.outputParticlesTiltPair.getSize())

        return summary

    def _methods(self):
        methodsMsgs = []

        if self.getStatus() == STATUS_FINISHED:
            msg = "A total of %d particle pairs of size %d were extracted" % (self.getOutput().getSize(),
                                                                              self.boxSize)
            if self._micsOther():
                msg += " from another set of micrographs: %s" % self.getObjectTag('inputMicrographsTiltedPair')

            msg += " using coordinates %s" % self.getObjectTag('inputCoordinatesTiltedPairs')
            msg += self.methodsVar.get('')
            methodsMsgs.append(msg)

            if self.doRemoveDust:
                methodsMsgs.append("Removed dust over a threshold of %s." % (self.thresholdDust))
            if self.doInvert:
                methodsMsgs.append("Inverted contrast on images.")
            if self._doDownsample():
                methodsMsgs.append("Particles downsampled by a factor of %0.2f." % self.downFactor)
            if self.doNormalize:
                methodsMsgs.append("Normalization performed of type %s." % (self.getEnumText('normType')))

        return methodsMsgs

    # --------------------------- UTILS functions ------------------------------
    def _setupBasicProperties(self):
        # Get sampling rate and inputMics according to micsSource type
        self.inputCoords = self.getCoords()
        self.uMics, self.tMics = self.getInputMicrographs()
        self.samplingInput = self.inputCoords.getUntilted().getMicrographs().getSamplingRate()
        self.samplingMics = self.uMics.getSamplingRate()
        self.samplingFactor = float(self.samplingMics / self.samplingInput)

        # create tmp set with all mic pairs
        self.inputMics = SetOfMicrographs(filename=':memory:')
        self.inputMics.copyInfo(self.uMics)
        self.inputMics.setStore(False)

        for micU, micT in izip(self.uMics, self.tMics):
            micU.cleanObjId()
            micT.cleanObjId()
            self.inputMics.append(micU)
            self.inputMics.append(micT)

    def getInputMicrographs(self):
        """ Return pairs of micrographs associated to the SetOfCoordinates or
        Other micrographs. """
        if not self._micsOther():
            
            return self.inputCoordinatesTiltedPairs.get().getUntilted().getMicrographs(), \
                   self.inputCoordinatesTiltedPairs.get().getTilted().getMicrographs()
        else:
            return self.inputMicrographsTiltedPair.get().getUntilted(), \
                   self.inputMicrographsTiltedPair.get().getTilted()

    def getCoords(self):
        return self.inputCoordinatesTiltedPairs.get()

    def getOutput(self):
        if self.hasAttribute('outputParticlesTiltPair') and self.outputParticlesTiltPair.hasValue():
            return self.outputParticlesTiltPair
        else:
            return None

    def getCoordSampling(self):
        return self.getCoords().getUntilted().getMicrographs().getSamplingRate()

    def getMicSampling(self):
        return self.getInputMicrographs()[0].getSamplingRate()

    def notOne(self, value):
        return abs(value - 1) > 0.0001

    def _doDownsample(self):
        return self.downFactor > 1.0

    def _getNormalizeArgs(self):
        if not self.doNormalize:
            return ''

        normType = self.getEnumText("normType")
        args = "--method %s " % normType

        if normType != "OldXmipp":
            bgRadius = self.backRadius.get()
            if bgRadius <= 0:
                bgRadius = int(self.boxSize.get() / 2)
            args += " --background circle %d" % bgRadius

        return args

    def getBoxScale(self):
        """ Computing the sampling factor between input and output.
        We should take into account the differences in sampling rate between
        micrographs used for picking and the ones used for extraction.
        The downsampling factor could also affect the resulting scale.
        """
        samplingPicking = self.getCoordSampling()
        samplingExtract = self.getMicSampling()
        f = samplingPicking / samplingExtract
        return f / self.downFactor.get() if self._doDownsample() else f

    def _micsOther(self):
        """ Return True if other micrographs are used for extract. """
        return self.downsampleType == OTHER
    def testCtfdiscrepancyWorkflow(self):
        # create one micrograph set
        fnMicSet = self.proj.getTmpPath("mics.sqlite")
        fnMic = self.proj.getTmpPath("mic.mrc")
        mic = Micrograph()
        mic.setFileName(fnMic)
        micSet = SetOfMicrographs(filename=fnMicSet)

        # create two CTFsets
        fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
        fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
        ctfSet1 = SetOfCTF(filename=fnCTF1)
        ctfSet2 = SetOfCTF(filename=fnCTF2)

        # create one fake micrographs image
        projSize = 32
        img = xmipp.Image()
        img.setDataType(xmipp.DT_FLOAT)
        img.resize(projSize, projSize)
        img.write(fnMic)

        # fill the sets
        for i in range(1, 4):
            mic = Micrograph()
            mic.setFileName(fnMic)
            micSet.append(mic)

            defocusU = 1000+10*i
            defocusV = 1000+i
            defocusAngle = i*10
            psdFile = "psd_1%04d" % i
            ctf = self._getCTFModel(defocusU,
                                    defocusV,
                                    defocusAngle,
                                    psdFile)
            ctf.setMicrograph(mic)
            ctfSet1.append(ctf)

            defocusU = 1000+20*i
            defocusV = 1000+i
            defocusAngle = i*20
            psdFile = "psd_2%04d" % i
            ctf = self._getCTFModel(defocusU,
                                    defocusV,
                                    defocusAngle,
                                    psdFile)
            ctf.setMicrograph(mic)
            ctfSet2.append(ctf)
        ctfSet1.write()
        ctfSet2.write()
        micSet.write()

        # import micrograph set
        args = {'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
                'sqliteFile': fnMicSet,
                'amplitudConstrast': 0.1,
                'sphericalAberration': 2.,
                'voltage': 100,
                'samplingRate': 2.1
                }

        protMicImport = self.newProtocol(ProtImportMicrographs, **args)
        protMicImport.setObjLabel('import micrographs from sqlite ')
        self.launchProtocol(protMicImport)

        # import ctfsets
        protCTF1 = \
            self.newProtocol(ProtImportCTF,
                             importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
                             filesPath=fnCTF1)
        protCTF2 = \
            self.newProtocol(ProtImportCTF,
                             importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
                             filesPath=fnCTF2)
        protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
        protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
        protCTF1.setObjLabel('import ctfs from scipion_1 ')
        protCTF2.setObjLabel('import ctfs from scipion_2 ')
        self.launchProtocol(protCTF1)
        self.launchProtocol(protCTF2)

        # launch CTF discrepancy protocol
        protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
        protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
        protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
        protCtfDiscrepancy.setObjLabel('ctf discrepancy')
        self.launchProtocol(protCtfDiscrepancy)
        ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
        resolution = int(ctf0.getResolution())
        defocusU = int(ctf0.getDefocusU())
        self.assertEqual(resolution, 2)
        self.assertEqual(defocusU, 1010)
Exemplo n.º 29
0
    def testCtfdiscrepancyWorkflow(self):
        # create one micrograph set
        fnMicSet = self.proj.getTmpPath("mics.sqlite")
        fnMic = self.proj.getTmpPath("mic.mrc")
        mic = Micrograph()
        mic.setFileName(fnMic)
        micSet = SetOfMicrographs(filename=fnMicSet)

        # create two CTFsets
        fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
        fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
        ctfSet1 = SetOfCTF(filename=fnCTF1)
        ctfSet2 = SetOfCTF(filename=fnCTF2)

        # create one fake micrographs image
        projSize = 32
        img = xmipp.Image()
        img.setDataType(xmipp.DT_FLOAT)
        img.resize(projSize, projSize)
        img.write(fnMic)

        # fill the sets
        for i in range(1, 4):
            mic = Micrograph()
            mic.setFileName(fnMic)
            micSet.append(mic)

            defocusU = 1000 + 10 * i
            defocusV = 1000 + i
            defocusAngle = i * 10
            psdFile = "psd_1%04d" % i
            ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
            ctf.setMicrograph(mic)
            ctfSet1.append(ctf)

            defocusU = 1000 + 20 * i
            defocusV = 1000 + i
            defocusAngle = i * 20
            psdFile = "psd_2%04d" % i
            ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
            ctf.setMicrograph(mic)
            ctfSet2.append(ctf)
        ctfSet1.write()
        ctfSet2.write()
        micSet.write()

        # import micrograph set
        args = {
            'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
            'sqliteFile': fnMicSet,
            'amplitudConstrast': 0.1,
            'sphericalAberration': 2.,
            'voltage': 100,
            'samplingRate': 2.1
        }

        protMicImport = self.newProtocol(ProtImportMicrographs, **args)
        protMicImport.setObjLabel('import micrographs from sqlite ')
        self.launchProtocol(protMicImport)

        # import ctfsets
        protCTF1 = \
            self.newProtocol(ProtImportCTF,
                             importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
                             filesPath=fnCTF1)
        protCTF2 = \
            self.newProtocol(ProtImportCTF,
                             importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
                             filesPath=fnCTF2)
        protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
        protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
        protCTF1.setObjLabel('import ctfs from scipion_1 ')
        protCTF2.setObjLabel('import ctfs from scipion_2 ')
        self.launchProtocol(protCTF1)
        self.launchProtocol(protCTF2)

        # launch CTF discrepancy protocol
        protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
        protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
        protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
        protCtfDiscrepancy.setObjLabel('ctf discrepancy')
        self.launchProtocol(protCtfDiscrepancy)
        ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
        resolution = int(ctf0.getResolution())
        defocusU = int(ctf0.getDefocusU())
        self.assertEqual(resolution, 2)
        self.assertEqual(defocusU, 1010)
Exemplo n.º 30
0
    def importCTFStep(self, importFrom):
        """ Copy ctfs matching the filename pattern. """
        ci = self.getImportClass()

        inputMics = self.inputMicrographs.get()
        ctfSet = self._createSetOfCTF()
        ctfSet.setMicrographs(inputMics)

        outputMics = self._createSetOfMicrographs()
        SetOfMicrographs.copyInfo(outputMics, inputMics)

        createOutputMics = False
        
        files = [f for f, _ in self.iterFiles()]
        n = len(files)
        if n == 0:
            raise Exception("No files where found in path: '%s'\n"
                            "matching the pattern: '%s'" %
                            (self.filesPath, self.filesPattern))
        print("Matching files: %s" % n)
        inputMicBases = [removeBaseExt(m.getFileName()) for m in inputMics]

        if len(files) > len(inputMicBases):
            self.warning("WARNING: The number of files matched by your pattern (%d) is larger than "
                         "the number of available micrographs (%d). It is advised to carefully "
                         "review the output of this run or to re-run with a more restrictive pattern."
                         % (n, len(inputMicBases)))

        def _getMicCTF(mic):
            micName = mic.getMicName()
            micBase = removeBaseExt(mic.getFileName())
            # see if the base name of this mic is contained in other base names
            micConflicts = [mc for mc in inputMicBases if micBase in mc and micBase != mc]
            if micConflicts:
                self.warning('WARNING: Micrograph base name "%s" conflicts with micrograph(s) "%s". '
                             'Will try to find a unique match...' % (micBase, '", "'.join(micConflicts)))
                # check which matching file only matches with this mic and not its conflicts
                goodFnMatches = [f for f in files if micBase in f and not any(c in f for c in micConflicts)]
                for goodFnMatch in goodFnMatches:
                    try:
                        micCtf = ci.importCTF(mic, goodFnMatch)
                        self.warning("WARNING: Assigned file %s to micrograph %s." % (goodFnMatch, micBase))
                        return micCtf
                    except Exception as ex:
                        self.warning("WARNING: Can't import ctf for micrograph %s from file %s"
                                     % (micBase, goodFnMatch))
                        continue
                else:
                    return None
            else:
                for fileName, fileId in self.iterFiles():
                    if (fileId == mic.getObjId() or
                                micBase in fileName or micName in fileName):
                        return ci.importCTF(mic, fileName)

            return None
        # Check if the CTF import class has a method to retrieve the CTF
        # from a given micrograph. If not, we will try to associated based
        # on matching the filename or id
        getMicCTF = getattr(ci, 'getMicCTF', None) or _getMicCTF

        for mic in inputMics:
            ctf = getMicCTF(mic)
            if ctf is not None:
                ctfSet.append(ctf)
                outputMics.append(mic)
            else:
                # If CTF is not found for a micrograph remove it from output mics
                self.warning("CTF for micrograph id %d was not found. Removed "
                             "from set of micrographs." % mic.getObjId())
                createOutputMics = True

        self._defineOutputs(outputCTF=ctfSet)
        # If some of the micrographs had not ctf a subset of micrographs
        # have been produced
        if createOutputMics:
            self._defineOutputs(outputMicrographs=outputMics)
            self._defineCtfRelation(outputMics, ctfSet)
        else:
            self._defineCtfRelation(inputMics, ctfSet)
Exemplo n.º 31
0
class XmippProtExtractParticlesPairs(XmippProtExtractParticles):
    """Protocol to extract particles from a set of tilted pairs coordinates"""
    _label = 'extract particle pairs'

    def __init__(self, **args):
        XmippProtExtractParticles.__init__(self, **args)
        self.stepsExecutionMode = STEPS_PARALLEL

    # --------------------------- DEFINE param functions --------------------------------------------

    def _defineParams(self, form):
        form.addSection(label='Input')
        form.addParam('inputCoordinatesTiltedPairs',
                      PointerParam,
                      important=True,
                      label="Coordinates tilted pairs",
                      pointerClass='CoordinatesTiltPair',
                      help='Select the CoordinatesTiltPairs')
        form.addParam(
            'downsampleType',
            EnumParam,
            choices=['same as picking', 'other'],
            default=0,
            important=True,
            label='Micrographs source',
            display=EnumParam.DISPLAY_HLIST,
            help='By default the particles will be extracted \n'
            'from the micrographs used in the picking \n'
            'step ( _same as picking_ option ). \n'
            'If you select _other_ option, you must provide \n'
            'a different set of micrographs to extract from.\n'
            '*Note*: In the _other_ case, ensure that provided \n'
            'micrographs and coordinates are related \n'
            'by micName or by micId. Difference in pixel size will \n'
            'be handled automatically.')
        form.addParam(
            'inputMicrographsTiltedPair',
            PointerParam,
            pointerClass='MicrographsTiltPair',
            condition='downsampleType != 0',
            important=True,
            label='Input tilt pair micrographs',
            help='Select the tilt pair micrographs from which to extract.')

        # downFactor should always be 1.0 or greater
        geOne = GE(1.0, error='Value should be greater or equal than 1.0')

        form.addParam(
            'downFactor',
            FloatParam,
            default=1.0,
            validators=[geOne],
            label='Downsampling factor',
            help='Select a value greater than 1.0 to reduce the size '
            'of micrographs before extracting the particles. '
            'If 1.0 is used, no downsample is applied. '
            'Non-integer downsample factors are possible. ')

        form.addParam(
            'boxSize',
            IntParam,
            default=0,
            label='Particle box size',
            validators=[Positive],
            help='In pixels. The box size is the size of the boxed particles, '
            'actual particles may be smaller than this. If you do downsampling '
            'after extraction, provide final box size here.')

        form.addParam(
            'doBorders',
            BooleanParam,
            default=False,
            label='Fill pixels outside borders',
            help=
            'Xmipp by default skips particles whose boxes fall outside of the micrograph borders.'
            'Set this option to True if you want those pixels outside the borders to be filled with the closest pixel value available'
        )

        form.addSection(label='Preprocess')
        form.addParam(
            'doRemoveDust',
            BooleanParam,
            default=True,
            important=True,
            label='Dust removal (Recommended)',
            help=
            'Sets pixels with unusually large values to random values from a Gaussian '
            'with zero-mean and unity-standard deviation.')
        form.addParam(
            'thresholdDust',
            FloatParam,
            default=3.5,
            condition='doRemoveDust',
            expertLevel=LEVEL_ADVANCED,
            label='Threshold for dust removal',
            help=
            'Pixels with a signal higher or lower than this value times the standard '
            'deviation of the image will be affected. For cryo, 3.5 is a good value.'
            'For high-contrast negative stain, the signal itself may be affected so '
            'that a higher value may be preferable.')
        form.addParam(
            'doInvert',
            BooleanParam,
            default=None,
            label='Invert contrast',
            help=
            'Invert the contrast if your particles are black over a white background.\n'
            'Xmipp, Spider, Relion and Eman require white particles over a black background\n'
            'Frealign (up to v9.07) requires black particles over a white background'
        )
        form.addParam(
            'doNormalize',
            BooleanParam,
            default=True,
            label='Normalize (Recommended)',
            help=
            'It subtract a ramp in the gray values and normalizes so that in the '
            'background there is 0 mean and standard deviation 1.')
        form.addParam(
            'normType',
            EnumParam,
            choices=['OldXmipp', 'NewXmipp', 'Ramp'],
            default=2,
            condition='doNormalize',
            expertLevel=LEVEL_ADVANCED,
            display=EnumParam.DISPLAY_COMBO,
            label='Normalization type',
            help='OldXmipp (mean(Image)=0, stddev(Image)=1).  \n  '
            'NewXmipp (mean(background)=0, stddev(background)=1)  \n  '
            'Ramp (subtract background+NewXmipp).  \n  ')
        form.addParam(
            'backRadius',
            IntParam,
            default=-1,
            condition='doNormalize',
            label='Background radius',
            expertLevel=LEVEL_ADVANCED,
            help=
            'Pixels outside this circle are assumed to be noise and their stddev '
            'is set to 1. Radius for background circle definition (in pix.). '
            'If this value is 0, then half the box size is used.')

        form.addParallelSection(threads=4, mpi=1)

    def _insertAllSteps(self):
        self._setupBasicProperties()
        # Write pos files for each micrograph
        firstStepId = self._insertFunctionStep('writePosFilesStep')

        # For each micrograph insert the steps, run in parallel
        deps = []
        for mic in self.inputMics:
            localDeps = [firstStepId]
            fnLast = mic.getFileName()
            micName = pwutils.removeBaseExt(mic.getFileName())

            def getMicTmp(suffix):
                return self._getTmpPath(micName + suffix)

            # Create a list with micrographs operations (programs in xmipp) and
            # the required command line parameters (except input/ouput files)
            micOps = []

            # Check if it is required to downsample your micrographs
            downFactor = self.downFactor.get()

            if self.notOne(downFactor):
                fnDownsampled = getMicTmp("_downsampled.xmp")
                args = "-i %s -o %s --step %f --method fourier"
                micOps.append(('xmipp_transform_downsample',
                               args % (fnLast, fnDownsampled, downFactor)))
                fnLast = fnDownsampled

            if self.doRemoveDust:
                fnNoDust = getMicTmp("_noDust.xmp")
                args = " -i %s -o %s --bad_pixels outliers %f"
                micOps.append(('xmipp_transform_filter',
                               args % (fnLast, fnNoDust, self.thresholdDust)))
                fnLast = fnNoDust

            # TODO: implement CTF
            # Actually extract
            deps.append(
                self._insertFunctionStep('extractParticlesStep',
                                         mic.getObjId(),
                                         micName,
                                         None,
                                         fnLast,
                                         micOps,
                                         self.doInvert.get(),
                                         self._getNormalizeArgs(),
                                         self.doBorders.get(),
                                         prerequisites=localDeps))

        metaDeps = self._insertFunctionStep('createMetadataImageStep',
                                            prerequisites=deps)

        # Insert step to create output objects
        self._insertFunctionStep('createOutputStep', prerequisites=[metaDeps])

    # --------------------------- STEPS functions --------------------------------------------
    def writePosFilesStep(self):
        """ Write the pos file for each micrograph in metadata format
        (both untilted and tilted). """
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getUntilted(),
                              scale=self.getBoxScale())
        writeSetOfCoordinates(self._getExtraPath(),
                              self.inputCoords.getTilted(),
                              scale=self.getBoxScale())

        # We need to find the mapping by micName (without ext) between the micrographs in
        # the SetOfCoordinates and the Other micrographs
        if self._micsOther():
            micDict = {}
            # create tmp set with all mics from coords set
            coordMics = SetOfMicrographs(filename=':memory:')
            coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())

            for micU, micT in izip(
                    self.inputCoords.getUntilted().getMicrographs(),
                    self.inputCoords.getTilted().getMicrographs()):
                micU.cleanObjId()
                micT.cleanObjId()
                coordMics.append(micU)
                coordMics.append(micT)

            for mic in coordMics:
                micBase = pwutils.removeBaseExt(mic.getFileName())
                micPos = self._getExtraPath(micBase + ".pos")
                micDict[pwutils.removeExt(mic.getMicName())] = micPos

            # now match micDict and inputMics
            if any(
                    pwutils.removeExt(mic.getMicName()) in micDict
                    for mic in self.inputMics):
                micKey = lambda mic: pwutils.removeExt(mic.getMicName())
            else:
                raise Exception(
                    'Could not match input micrographs and coordinates '
                    'by micName.')

            for mic in self.inputMics:  # micrograph from input (other)
                mk = micKey(mic)
                if mk in micDict:
                    micPosCoord = micDict[mk]
                    if exists(micPosCoord):
                        micBase = pwutils.removeBaseExt(mic.getFileName())
                        micPos = self._getExtraPath(micBase + ".pos")
                        if micPos != micPosCoord:
                            self.info('Moving %s -> %s' %
                                      (micPosCoord, micPos))
                            pwutils.moveFile(micPosCoord, micPos)

    def createMetadataImageStep(self):
        mdUntilted = md.MetaData()
        mdTilted = md.MetaData()
        # for objId in mdPairs:
        for uMic, tMic in izip(self.uMics, self.tMics):
            umicName = pwutils.removeBaseExt(uMic.getFileName())
            fnMicU = self._getExtraPath(umicName + ".xmd")
            fnPosU = self._getExtraPath(umicName + ".pos")
            # Check if there are picked particles in these micrographs
            if pwutils.exists(fnMicU):
                mdMicU = md.MetaData(fnMicU)
                mdPosU = md.MetaData('particles@%s' % fnPosU)
                mdPosU.merge(mdMicU)
                mdUntilted.unionAll(mdPosU)
                tmicName = pwutils.removeBaseExt(tMic.getFileName())
                fnMicT = self._getExtraPath(tmicName + ".xmd")
                fnPosT = self._getExtraPath(tmicName + ".pos")
                mdMicT = md.MetaData(fnMicT)
                mdPosT = md.MetaData('particles@%s' % fnPosT)
                mdPosT.merge(mdMicT)
                mdTilted.unionAll(mdPosT)

        # Write image metadata (check if it is really necessary)
        fnTilted = self._getExtraPath("images_tilted.xmd")
        fnUntilted = self._getExtraPath("images_untilted.xmd")
        mdUntilted.write(fnUntilted)
        mdTilted.write(fnTilted)

    def createOutputStep(self):
        fnTilted = self._getExtraPath("images_tilted.xmd")
        fnUntilted = self._getExtraPath("images_untilted.xmd")

        # Create outputs SetOfParticles both for tilted and untilted
        imgSetU = self._createSetOfParticles(suffix="Untilted")
        imgSetU.copyInfo(self.uMics)
        imgSetT = self._createSetOfParticles(suffix="Tilted")
        imgSetT.copyInfo(self.tMics)

        sampling = self.samplingMics if self._micsOther(
        ) else self.samplingInput
        if self._doDownsample():
            sampling *= self.downFactor.get()
        imgSetU.setSamplingRate(sampling)
        imgSetT.setSamplingRate(sampling)

        # set coords from the input, will update later if needed
        imgSetU.setCoordinates(
            self.inputCoordinatesTiltedPairs.get().getUntilted())
        imgSetT.setCoordinates(
            self.inputCoordinatesTiltedPairs.get().getTilted())

        # Read untilted and tilted particles on a temporary object (also disabled particles)
        imgSetAuxU = SetOfParticles(filename=':memory:')
        imgSetAuxU.copyInfo(imgSetU)
        readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False)

        imgSetAuxT = SetOfParticles(filename=':memory:')
        imgSetAuxT.copyInfo(imgSetT)
        readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False)

        # calculate factor for coords scaling
        factor = 1 / self.samplingFactor
        if self._doDownsample():
            factor /= self.downFactor.get()

        coordsT = self.inputCoordinatesTiltedPairs.get().getTilted()
        # For each untilted particle retrieve micId from SetOfCoordinates untilted
        for imgU, coordU in izip(
                imgSetAuxU,
                self.inputCoordinatesTiltedPairs.get().getUntilted()):
            # FIXME: Remove this check when sure that objIds are equal
            id = imgU.getObjId()
            if id != coordU.getObjId():
                raise Exception(
                    'ObjIds in untilted img and coord are not the same!!!!')
            imgT = imgSetAuxT[id]
            coordT = coordsT[id]

            # If both particles are enabled append them
            if imgU.isEnabled() and imgT.isEnabled():
                if self._micsOther() or self._doDownsample():
                    coordU.scale(factor)
                    coordT.scale(factor)
                imgU.setCoordinate(coordU)
                imgSetU.append(imgU)
                imgT.setCoordinate(coordT)
                imgSetT.append(imgT)

        imgSetU.write()
        imgSetT.write()

        # Define output ParticlesTiltPair
        outputset = ParticlesTiltPair(
            filename=self._getPath('particles_pairs.sqlite'))
        outputset.setTilted(imgSetT)
        outputset.setUntilted(imgSetU)
        for imgU, imgT in izip(imgSetU, imgSetT):
            outputset.append(TiltPair(imgU, imgT))

        outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get())
        self._defineOutputs(outputParticlesTiltPair=outputset)
        self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)

    # --------------------------- INFO functions --------------------------------------------
    def _validate(self):
        errors = []

        if self.doNormalize:
            if self.backRadius > int(self.boxSize.get() / 2):
                errors.append("Background radius for normalization should be "
                              "equal or less than half of the box size.")
        return errors

    def _citations(self):
        return ['delaRosaTrevin2013']

    def _summary(self):
        summary = []
        summary.append("Micrographs source: %s" %
                       self.getEnumText('downsampleType'))
        summary.append("Particle box size: %d" % self.boxSize)

        if not hasattr(self, 'outputParticlesTiltPair'):
            summary.append("Output images not ready yet.")
        else:
            summary.append("Particle pairs extracted: %d" %
                           self.outputParticlesTiltPair.getSize())

        return summary

    def _methods(self):
        methodsMsgs = []

        if self.getStatus() == STATUS_FINISHED:
            msg = "A total of %d particle pairs of size %d were extracted" % (
                self.getOutput().getSize(), self.boxSize)
            if self._micsOther():
                msg += " from another set of micrographs: %s" % self.getObjectTag(
                    'inputMicrographsTiltedPair')

            msg += " using coordinates %s" % self.getObjectTag(
                'inputCoordinatesTiltedPairs')
            msg += self.methodsVar.get('')
            methodsMsgs.append(msg)

            if self.doRemoveDust:
                methodsMsgs.append("Removed dust over a threshold of %s." %
                                   (self.thresholdDust))
            if self.doInvert:
                methodsMsgs.append("Inverted contrast on images.")
            if self._doDownsample():
                methodsMsgs.append(
                    "Particles downsampled by a factor of %0.2f." %
                    self.downFactor)
            if self.doNormalize:
                methodsMsgs.append("Normalization performed of type %s." %
                                   (self.getEnumText('normType')))

        return methodsMsgs

    # --------------------------- UTILS functions --------------------------------------------
    def _setupBasicProperties(self):
        # Get sampling rate and inputMics according to micsSource type
        self.inputCoords = self.getCoords()
        self.uMics, self.tMics = self.getInputMicrographs()
        self.samplingInput = self.inputCoords.getUntilted().getMicrographs(
        ).getSamplingRate()
        self.samplingMics = self.uMics.getSamplingRate()
        self.samplingFactor = float(self.samplingMics / self.samplingInput)

        # create tmp set with all mic pairs
        self.inputMics = SetOfMicrographs(filename=':memory:')
        self.inputMics.copyInfo(self.uMics)
        self.inputMics.setStore(False)

        for micU, micT in izip(self.uMics, self.tMics):
            micU.cleanObjId()
            micT.cleanObjId()
            self.inputMics.append(micU)
            self.inputMics.append(micT)

    def getInputMicrographs(self):
        """ Return pairs of micrographs associated to the SetOfCoordinates or
        Other micrographs. """
        if not self._micsOther():
            return self.inputCoordinatesTiltedPairs.get().getUntilted().getMicrographs(), \
                   self.inputCoordinatesTiltedPairs.get().getTilted().getMicrographs()
        else:
            return self.inputMicrographsTiltedPair.get().getUntilted(), \
                   self.inputMicrographsTiltedPair.get().getTilted()

    def getCoords(self):
        return self.inputCoordinatesTiltedPairs.get()

    def getOutput(self):
        if self.hasAttribute('outputParticlesTiltPair'
                             ) and self.outputParticlesTiltPair.hasValue():
            return self.outputParticlesTiltPair
        else:
            return None

    def getCoordSampling(self):
        return self.getCoords().getUntilted().getMicrographs().getSamplingRate(
        )

    def getMicSampling(self):
        return self.getInputMicrographs()[0].getSamplingRate()