Exemple #1
0
 def createOutputStep(self):
     micTiltPairs = self.getInputMicrographs()
     # Get the converted input micrographs in Xmipp format
     writeSetOfMicrographsPairs = Domain.importFromPlugin(
         'xmipp3.convert', 'writeSetOfMicrographsPairs')
     writeSetOfMicrographsPairs(micTiltPairs.getUntilted(),
                                micTiltPairs.getTilted(), self.micsFn)
     uCoordSet, tCoordSet, anglesSet = self.readCoordinates()
     # Create CoordinatesTiltPair object
     coordsSet = self._createCoordinatesTiltPair(micTiltPairs,
                                                 uCoordSet,
                                                 tCoordSet,
                                                 anglesSet,
                                                 suffix='')
     self._defineOutputs(outputCoordinatesTiltPair=coordsSet)
     self._defineSourceRelation(self.inputMicrographsTiltedPair, coordsSet)
    def _visualize(self, obj, **args):
        fnCmd = self.protocol._getExtraPath("chimera_output.cxc")

        self._getVols()
        self._getPdbs()
        dim = float()
        sampling = float()
        if len(self.vols) > 0:
            if self.vols[0] is not None:
                dim, sampling = self._getDimSamplingFromVol(self.vols[0])
            elif self.vols[1] is not None:
                dim, sampling = self._getDimSamplingFromVol(self.vols[1])
        else:
            # To show pdbs only
            dim = 150.
            sampling = 1.

        bildFileName = self.protocol._getExtraPath("axis_output.bild")
        Chimera.createCoordinateAxisFile(dim,
                                         bildFileName=bildFileName,
                                         sampling=sampling)

        with open(fnCmd, 'w') as f:
            # change to workingDir
            # If we do not use cd and the project name has an space
            # the protocol fails even if we pass absolute paths
            f.write('cd %s\n' % os.getcwd())
            f.write("open %s\n" % bildFileName)
            f.write("cofr 0,0,0\n")  # set center of coordinates
            if len(self.vols) > 0:
                for vol in self.vols:
                    sampling, volFileName, x, y, z = self._getXYZFromVol(vol)
                    f.write("open %s\n" % volFileName)
                    f.write("volume #2 style surface voxelSize %f\n"
                            "volume #2 origin %0.2f,%0.2f,%0.2f\n" %
                            (sampling, x, y, z))
            for filename in self.pdbList:
                f.write("open %s\n" % filename)

        # run in the background
        chimeraPlugin = Domain.importFromPlugin('chimera',
                                                'Plugin',
                                                doRaise=True)
        chimeraPlugin.runChimeraProgram(chimeraPlugin.getProgram(),
                                        fnCmd + "&",
                                        cwd=os.getcwd())
        return []
Exemple #3
0
    def _validate(self):
        validateMsgs = []

        if self.mtfFile.hasValue() and not os.path.exists(self.mtfFile.get()):
            validateMsgs.append("MTF file %s does not exist!" % self.mtfFile.get())

        if self.defectFile.hasValue() and not os.path.exists(self.defectFile.get()):
            validateMsgs.append("Defect file %s does not exist!" % self.defectFile.get())

        if self.gainRot or self.gainFlip:
            try:
                from pwem import Domain
                eman2 = Domain.importFromPlugin('eman2', doRaise=True)
            except:
                validateMsgs.append("EMAN2 plugin not found!\nTransforming gain image "
                                    "requires EMAN2 plugin and binaries installed.")

        return validateMsgs
Exemple #4
0
    def homoNoGoldStandardAlign(self, relion1, relion2):
        """test without Goldstandard and alignment"""
        XmippProtVolumeHomogenizer = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtEnrich', doRaise=True)
        protVolumeHomogenizer = self.newProtocol(
            XmippProtVolumeHomogenizer, objLabel='volume homogenizer2')

        protVolumeHomogenizer.referenceVolume.set(relion1.outputVolume)
        protVolumeHomogenizer.inputVolume.set(relion2.outputVolume)
        protVolumeHomogenizer.inputParticles.set(relion2.outputParticles)
        protVolumeHomogenizer.doAlignment.set(True)
        self.launchProtocol(protVolumeHomogenizer)

        self.assertIsNotNone(
            protVolumeHomogenizer.outputParticles.getFileName(),
            "There was a problem with the homoNoGoldStandardAlign")

        return protVolumeHomogenizer
    def getDimensions(self, locationObj):
        """ It will return a tuple with the images dimensions.
        The tuple will contains:
            (x, y, z, n) where x, y, z are image dimensions (z=1 for 2D) and 
            n is the number of elements if stack.
        """
        if self.existsLocation(locationObj):
            location = self._convertToLocation(locationObj)
            fn = location[1]
            ext = pwutils.getExt(fn).lower()

            # Local import to avoid import loop between ImageHandler and Ccp4Header.
            from pwem.convert import headers

            if ext == '.png' or ext == '.jpg':
                im = Image.open(fn)
                x, y = im.size  # (width,height) tuple
                return x, y, 1, 1

            elif headers.getFileFormat(fn) == headers.MRC:
                header = headers.Ccp4Header(fn, readHeader=True)
                return header.getXYZN()

            elif ext == '.img':
                # FIXME Since now we can not read dm4 format in Scipion natively
                # or recent .img format
                # we are opening an Eman2 process to read the dm4 file
                from pwem import Domain
                getImageDimensions = Domain.importFromPlugin(
                    'eman2.convert', 'getImageDimensions', doRaise=True)
                return getImageDimensions(fn)  # we are ignoring index here
            elif ext in ['.eer', '.gain']:
                tif = TiffFile(fn)
                frames = len(tif.pages)  # number of pages in the file
                page = tif.pages[
                    0]  # get shape and dtype of the image in the first page
                x, y = page.shape
                return x, y, frames, 1
            else:
                self._img.read(location, lib.HEADER)
                return self._img.getDimensions()
        else:
            return None, None, None, None
    def test_volume1_AtomStructCIF(self):
        from pwem import Domain
        XmippProtResolution3D = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtResolution3D', doRaise=True)

        prot = self.newProtocol(XmippProtResolution3D)
        prot.inputVolume.set(self.protImportHalf1.outputVolume)
        prot.referenceVolume.set(self.protImportHalf2.outputVolume)
        self.launchProtocol(prot)

        protExp = self.newProtocol(emprot.ProtExportDataBases)
        protExp.setObjLabel("export to DB\ndir1")
        protExp.exportVolume.set(self.protImportHalf1.outputVolume)
        protExp.exportFSC.set(prot.outputFSC)
        protExp.exportAtomStruct.set(self._importAtomStructCIF())

        protExp.filesPath.set(os.getcwd() + "/dir1")
        self.launchProtocol(protExp)

        # Check the files were generated properly.
        # protExp._createFileNamesTemplates()
        dirName = protExp.filesPath.get()
        nameVolume = os.path.join(dirName, protExp.VOLUMENAME)
        nameFsc = os.path.join(dirName, "fsc_%02d.xml" % 1)
        nameAtomStruct = os.path.join(dirName, protExp.COORDINATEFILENAME)
        self.assertTrue(os.path.exists(nameVolume))
        self.assertTrue(os.path.exists(nameFsc))
        self.assertTrue(os.path.exists(nameAtomStruct))

        # Check if the files have the correct data
        orig_list_x, orig_list_y = protExp.exportFSC.get().getData()
        fo = open(nameFsc, "rU")
        saved_x = []
        orig_x = []
        count = 0
        for line in fo:
            if line[0:3] == '<x>':
                saved_x.append(int(float(line[3:-5]) * 1000))
                orig_x.append(int(orig_list_x[count] * 1000))
                count = count + 1

        self.assertListEqual(orig_x, saved_x)
    def test_halfmaps_mask(self):
        """ If the input is a 3D map with half volumes associated
            Save them to the output directory
            Also create a mask and export it
        """
        # first create the 3 3dmaps and the mask
        self.import_volume_halfmaps()
        # run the export protocol
        from pwem import Domain
        XmippProtResolution3D = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtResolution3D', doRaise=True)

        prot = self.newProtocol(XmippProtResolution3D)
        prot.inputVolume.set(self.half1)
        prot.referenceVolume.set(self.half2)
        self.launchProtocol(prot)

        protExp = self.newProtocol(emprot.ProtExportDataBases)
        protExp.setObjLabel("export to DB\ndir3")
        protExp.exportVolume.set(self.volume)
        protExp.additionalVolumesToExport.set(True)
        protExp.exportAdditionalVolumes.set([
            self.protImportHalf1.outputVolume,
            self.protImportHalf2.outputVolume
        ])
        protExp.exportFSC.set(prot.outputFSC)
        protExp.masksToExport.set(True)
        protExp.exportMasks.set([self.mask])
        protExp.exportAtomStruct.set(self._importAtomStructCIF())

        protExp.filesPath.set(os.getcwd() + "/dir3")
        self.launchProtocol(protExp)

        # Check the files were generated properly.
        # protExp._createFileNamesTemplates()
        dirName = protExp.filesPath.get()
        nameVolume = os.path.join(dirName, protExp.VOLUMENAME)
        nameFsc = os.path.join(dirName, "fsc_%02d.xml" % 1)
        nameAtomStruct = os.path.join(dirName, protExp.COORDINATEFILENAME)
        self.assertTrue(os.path.exists(nameVolume))
        self.assertTrue(os.path.exists(nameFsc))
        self.assertTrue(os.path.exists(nameAtomStruct))
Exemple #8
0
    def _compareMicPairs(self, micFn, uSet, tSet):
        # compare micFn input file and input micsSet
        micMd = md.MetaData(micFn)
        micFnDict = {}
        inputMicsDict = {}
        XmippMdRow = Domain.importFromPlugin('xmipp3', 'XmippMdRow')
        for objId in micMd:
            row = XmippMdRow()
            row.readFromMd(micMd, objId)
            micUFn = pwutils.removeBaseExt(row.getValue(md.MDL_MICROGRAPH))
            micTFn = pwutils.removeBaseExt(row.getValue(md.MDL_MICROGRAPH_TILTED))
            micFnDict[micUFn] = micTFn

        for micU, micT in izip(uSet, tSet):
            inputMicsDict[pwutils.removeBaseExt(micU.getFileName())] = pwutils.removeBaseExt(micT.getFileName())

        for micKey in inputMicsDict:
            if micKey not in micFnDict:
                return False
        return True
Exemple #9
0
    def relionRefine(self, protSubSetClass, vol, classid):

        ProtRelionRefine3D = Domain.importFromPlugin('relion.protocols',
                                                     'ProtRelionRefine3D',
                                                     doRaise=True)
        relionRefine = self.newProtocol(ProtRelionRefine3D,
                                        objLabel='relion_%s' % classid,
                                        doCTF=False, runMode=1,
                                        memoryPreThreads=1,
                                        maskDiameterA=424,
                                        angularSamplingDeg=1,
                                        localSearchAutoSamplingDeg=2,
                                        symmetryGroup="c1",
                                        numberOfMpi=3, numberOfThreads=1)

        relionRefine.inputParticles.set(protSubSetClass.outputParticles)
        relionRefine.referenceVolume.set(vol.outputVolume)

        self.launchProtocol(relionRefine)
        return relionRefine
Exemple #10
0
    def _readCoordinates(self, coordsDir, suffix=''):
        micTiltPairs = self.inputMicrographsTiltedPair.get()
        uSuffix = 'Untilted' + suffix
        tSuffix = 'Tilted' + suffix
        uSet = micTiltPairs.getUntilted()
        tSet = micTiltPairs.getTilted()
        # Create Untilted and Tilted SetOfCoordinates
        readSetOfCoordinates = Domain.importFromPlugin('xmipp3.convert',
                                                       'readSetOfCoordinates')
        uCoordSet = self._createSetOfCoordinates(uSet, suffix=uSuffix)
        readSetOfCoordinates(coordsDir, uSet, uCoordSet)
        uCoordSet.write()
        tCoordSet = self._createSetOfCoordinates(tSet, suffix=tSuffix)
        readSetOfCoordinates(coordsDir, tSet, tCoordSet)
        tCoordSet.write()
        boxSize = self._getBoxSize()
        if boxSize:
            uCoordSet.setBoxSize(boxSize)
            tCoordSet.setBoxSize(boxSize)

        return uCoordSet, tCoordSet
    def _validate(self):
        # Check base validation before the specific ones
        errors = ProtAlignMovies._validate(self)

        if self.doApplyDoseFilter and self.inputMovies.get():
            inputMovies = self.inputMovies.get()
            doseFrame = inputMovies.getAcquisition().getDosePerFrame()

            if doseFrame == 0.0 or doseFrame is None:
                errors.append('Dose per frame for input movies is 0 or not '
                              'set. You cannot apply dose filter.')

        if self.doComputeMicThumbnail or self.doComputePSD:
            try:
                from pwem import Domain
                eman2 = Domain.importFromPlugin('eman2', doRaise=True)
            except:
                errors.append(
                    "EMAN2 plugin not found!\nComputing thumbnails "
                    "or PSD requires EMAN2 plugin and binaries installed.")

        return errors
    def _displayModel(self, e=None):
        # bildFileName = os.path.abspath(self.protocol._getTmpPath(
        #    "axis_output.bild"))
        bildFileName = self.protocol._getTmpPath("axis_output.bild")

        # Axis Dim
        dim = 150.
        sampling = 1.
        Chimera.createCoordinateAxisFile(dim,
                                         bildFileName=bildFileName,
                                         sampling=sampling)

        fnCmd = self.protocol._getTmpPath("chimera_output.cxc")
        f = open(fnCmd, 'w')
        # change to workingDir
        # If we do not use cd and the project name has an space
        # the protocol fails even if we pass absolute paths
        f.write('cd %s\n' % os.getcwd())
        # reference axis model = 0
        f.write("open %s\n" % bildFileName)
        f.write("cofr 0,0,0\n")  # set center of coordinates
        # f.write("open %s\n" % os.path.abspath(
        #     self.protocol.pdbFileToBeRefined.get().getFileName()))
        f.write("open %s\n" %
                self.protocol.pdbFileToBeRefined.get().getFileName())

        if self.protocol.SYMMETRY.get() and \
                os.path.exists(self.protocol.getSymmetrizedModelName()):
            f.write("open %s\n" % self.protocol.getSymmetrizedModelName())
        f.close()
        # run in the background
        chimeraPlugin = Domain.importFromPlugin('chimera',
                                                'Plugin',
                                                doRaise=True)
        chimeraPlugin.runChimeraProgram(chimeraPlugin.getProgram(),
                                        fnCmd + "&",
                                        cwd=os.getcwd())
        return []
    def _visualize(self, obj, **args):
        # TODO if input volume is not mrc this will not work.
        # Construct the coordinate file and visualization
        bildFileName = os.path.abspath(
            self.protocol._getExtraPath("axis.bild"))
        dims = []
        samplings = []
        if len(self.protocol.inputVolumes) is 0:
            if self.protocol.pdbFileToBeRefined.get().getVolume() is not None:
                dim = self.protocol.pdbFileToBeRefined.get().getVolume(
                ).getDim()[0]
                sampling = self.protocol.pdbFileToBeRefined.get().getVolume().\
                    getSamplingRate()
                dims.append(dim)
                samplings.append(sampling)
        else:
            for i in range(len(self.protocol.inputVolumes)):
                dim = self.protocol.inputVolumes[i].get().getDim()[0]
                sampling = self.protocol.inputVolumes[i].get().\
                    getSamplingRate()
                dims.append(dim)
                samplings.append(sampling)
        if len(dims) != 0 and len(samplings) != 0:
            Chimera.createCoordinateAxisFile(max(dims),
                                             bildFileName=bildFileName,
                                             sampling=max(samplings))
        else:
            dim = 150.
            sampling = 1.
            Chimera.createCoordinateAxisFile(dim,
                                             bildFileName=bildFileName,
                                             sampling=sampling)

        fnCmd = self.protocol._getExtraPath("chimera.cxc")
        f = open(fnCmd, 'w')
        f.write("open %s\n" % bildFileName)
        f.write("cofr 0,0,0\n")

        outputsVol = []
        if len(self.protocol.inputVolumes) is 0:
            if self.protocol.pdbFileToBeRefined.get().getVolume() is not None:
                outputVol = self.protocol.pdbFileToBeRefined.get().getVolume()
                outputsVol.append(outputVol)
        else:
            for i in range(len(self.protocol.inputVolumes)):
                outputVol = self.protocol.inputVolumes[i].get()
                outputsVol.append(outputVol)

        count = 2
        if len(outputsVol) != 0:
            for outputVol in outputsVol:
                outputVolFileName = os.path.abspath(
                    ImageHandler.removeFileType(outputVol.getFileName()))
                x, y, z = outputVol.getOrigin(force=True).getShifts()
                f.write("open %s\n" % outputVolFileName)
                f.write("volume #%d  style surface voxelSize %f\n"
                        "volume #%d  origin %0.2f,%0.2f,%0.2f\n" %
                        (count, outputVol.getSamplingRate(), count, x, y, z))
                count += 1

        # counter = 1
        # template = self.protocol._getExtraPath(COOTPDBTEMPLATEFILENAME)
        databasePath = self.protocol._getExtraPath(
            OUTPUTDATABASENAMESWITHLABELS)
        conn = sqlite3.connect(databasePath)
        c = conn.cursor()
        sql = 'SELECT fileName FROM %s where saved = 1 order by id' % \
              DATABASETABLENAME
        c.execute(sql)
        for row in c:
            pdbFileName = os.path.abspath(row[0])
            if not pdbFileName.endswith(".mrc"):
                f.write("open %s\n" % pdbFileName)

        f.close()
        conn.close()
        # run in the background
        chimeraPlugin = Domain.importFromPlugin('chimera',
                                                'Plugin',
                                                doRaise=True)
        chimeraPlugin.runChimeraProgram(chimeraPlugin.getProgram(),
                                        fnCmd + "&")
        return []
Exemple #14
0
    def test_RefineEman(self):
        print(magentaStr("\n==> Importing data - micrograph pairs:"))
        protImportMicsPairs = self.newProtocol(ProtImportMicrographsTiltPairs,
                                               patternUntilted=self.micsUFn,
                                               patternTilted=self.micsTFn,
                                               samplingRate=1.88,
                                               voltage=200,
                                               sphericalAberration=2.0)
        self.launchProtocol(protImportMicsPairs)
        self.assertIsNotNone(
            protImportMicsPairs.outputMicrographsTiltPair,
            "There was a problem with the import of mic pairs")

        print(magentaStr("\n==> Importing data - coordinate pairs:"))
        protImportCoords = self.newProtocol(
            ProtImportCoordinatesPairs,
            importFrom=1,  # from eman
            patternUntilted=self.patternU,
            patternTilted=self.patternT,
            boxSize=256)
        protImportCoords.inputMicrographsTiltedPair.set(
            protImportMicsPairs.outputMicrographsTiltPair)
        self.launchProtocol(protImportCoords)
        self.assertIsNotNone(
            protImportCoords.outputCoordinatesTiltPair,
            "There was a problem with the import of coord pairs")

        print(magentaStr("\n==> Running xmipp3 - extract particles:"))
        XmippProtExtractParticlesPairs = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtExtractParticlesPairs')
        protExtractPairs = self.newProtocol(XmippProtExtractParticlesPairs,
                                            downFactor=2.0,
                                            boxSize=128,
                                            doInvert=True)

        protExtractPairs.inputCoordinatesTiltedPairs.set(
            protImportCoords.outputCoordinatesTiltPair)
        self.launchProtocol(protExtractPairs)
        self.assertIsNotNone(
            protExtractPairs.outputParticlesTiltPair,
            "There was a problem with particle pair extraction")

        print(magentaStr("\n==> Testing eman2 - tilt validate:"))
        protValidate = self.newProtocol(
            EmanProtTiltValidate,
            symmetry="c4",
            maxtilt=60.0,
            delta=2.0,
            shrink=2,
            quaternion=True,
            simcmpType=2,  # frc
            simcmpParams='maxres=60',
            simalignType=7,  # rotate_translate
            simralignType=1,  # refine
            numberOfThreads=4)
        protValidate.inputTiltPair.set(
            protExtractPairs.outputParticlesTiltPair)
        protValidate.inputVolume.set(self.protImportVol.outputVolume)
        protValidate._createFilenameTemplates()
        outputAngles = protValidate._getFileName('outputAngles')
        self.launchProtocol(protValidate)
        self.assertIsNotNone(outputAngles, "Missing some output files!")
 def __runXmippProgram(cls, program, args):
     """ Internal shortcut function to launch a Xmipp program. """
     from pwem import Domain
     xmipp3 = Domain.importFromPlugin('xmipp3')
     xmipp3.Plugin.runXmippProgram(program, args)
    def test_volume1_AtomStructPDB(self):
        from pwem import Domain
        XmippProtResolution3D = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtResolution3D', doRaise=True)
        prot = self.newProtocol(XmippProtResolution3D)
        prot.inputVolume.set(self.protImportHalf1.outputVolume)
        prot.referenceVolume.set(self.protImportHalf2.outputVolume)
        self.launchProtocol(prot)

        protExp = self.newProtocol(emprot.ProtExportDataBases)
        protExp.setObjLabel("export to DB\ndir2")
        protExp.exportVolume.set(self.protImportHalf1.outputVolume)
        protExp.exportFSC.set(prot.outputFSC)

        # run Chimera rigid fit ti get a PDB file
        extraCommands = ""
        extraCommands += "scipionwrite #3 prefix DONOTSAVESESSION_\n"
        extraCommands += "exit\n"

        args = {
            'extraCommands': extraCommands,
            'inputVolume': self.protImportHalf1.outputVolume,
            'pdbFileToBeRefined': self._importAtomStructCIF()
        }
        from pwem import Domain

        ChimeraProtRigidFit = Domain.importFromPlugin('chimera.protocols',
                                                      'ChimeraProtRigidFit',
                                                      doRaise=True)
        protChimera = self.newProtocol(ChimeraProtRigidFit, **args)
        protChimera.setObjLabel('chimera fit\n volume and PDB\n save model')
        self.launchProtocol(protChimera)

        outputAttr = "DONOTSAVESESSION_Atom_struct__3_%06d" % protChimera.getObjId(
        )
        protExp.exportAtomStruct.set(getattr(protChimera, outputAttr))

        protExp.filesPath.set(os.getcwd() + "/dir2")
        self.launchProtocol(protExp)

        # Check the files were generated properly.
        # protExp._createFileNamesTemplates()
        dirName = protExp.filesPath.get()
        nameVolume = os.path.join(dirName, protExp.VOLUMENAME)
        nameFsc = os.path.join(dirName, "fsc_%02d.xml" % 1)
        nameAtomStruct = os.path.join(dirName, protExp.COORDINATEFILENAME)
        self.assertTrue(os.path.exists(nameVolume))
        self.assertTrue(os.path.exists(nameFsc))
        self.assertTrue(os.path.exists(nameAtomStruct))

        # Check if the files have the correct data
        orig_list_x, orig_list_y = protExp.exportFSC.get().getData()
        fo = open(nameFsc, "rU")
        saved_x = []
        orig_x = []
        count = 0
        for line in fo:
            if line[0:3] == '<x>':
                saved_x.append(int(float(line[3:-5]) * 1000))
                orig_x.append(int(orig_list_x[count] * 1000))
                count = count + 1

        self.assertListEqual(orig_x, saved_x)
Exemple #17
0
 def __runXmippProgram(program, args):
     """ Internal function to launch a Xmipp program. """
     from pwem import Domain
     xmipp3 = Domain.importFromPlugin('xmipp3', doRaise=True)
     xmipp3.runXmippProgram(program, args)
    def testRisosome(self):
        # First, import a set of micrographs
        print("Importing a set of micrographs...")
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=os.path.abspath(
                                          self.proj.getTmpPath()),
                                      filesPattern="*%s" % self.ext,
                                      samplingRateMode=1,
                                      magnification=79096,
                                      scannedPixelSize=56,
                                      voltage=300,
                                      sphericalAberration=2.0,
                                      dataStreaming=True,
                                      fileTimeout=3,
                                      timeout=60)
        protImport.setObjLabel('import 20 mics (streaming)')
        self.proj.launchProtocol(protImport, wait=False)
        self._waitOutput(protImport, 'outputMicrographs')

        # Now estimate CTF on the micrographs with ctffind
        print("Performing CTFfind...")
        ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
                                              'CistemProtCTFFind',
                                              doRaise=True)
        protCTF = self.newProtocol(ProtCTFFind,
                                   minDefocus=12000,
                                   maxDefocus=30000,
                                   runMode=1,
                                   numberOfMpi=1,
                                   numberOfThreads=1)
        protCTF.inputMicrographs.set(protImport.outputMicrographs)
        protCTF.setObjLabel('CTF ctffind')
        self.proj.launchProtocol(protCTF, wait=False)
        self._waitOutput(protCTF, 'outputCTF')

        # Now pick particles on the micrographs with Relion
        print("Performing Relion Autopicking (LoG)...")

        ProtRelionAutopickLoG = Domain.importFromPlugin(
            'relion.protocols', 'ProtRelionAutopickLoG')

        protPick = self.newProtocol(
            ProtRelionAutopickLoG,
            objLabel='Streaming relion - autopic (LoG)',
            inputMicrographs=protImport.outputMicrographs,
            boxSize=60,
            minDiameter=260,
            maxDiameter=380)
        self.proj.launchProtocol(protPick, wait=False)
        self._waitOutput(protPick, 'outputCoordinates')

        ProtRelionExtractParticles = Domain.importFromPlugin(
            'relion.protocols', 'ProtRelionExtractParticles', doRaise=True)
        protExtract = self.newProtocol(ProtRelionExtractParticles,
                                       objLabel='extract box=64',
                                       boxSize=64,
                                       doInvert=True)
        protExtract.inputCoordinates.set(protPick.outputCoordinates)
        protExtract.ctfRelations.set(protCTF.outputCTF)
        self.launchProtocol(protExtract)

        x, y, _ = protExtract.outputParticles.getDim()
        self.assertEqual(
            protExtract.outputParticles.getDim(), (64, 64, 1),
            "Dimension of the particles should be 64 x 64 and it "
            "is %d x %d" % (x, y))
    def test_workflow(self):
        #First, import a set of micrographs
        print("Importing a set of micrographs...")
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=self.micsFn,
                                      samplingRateMode=1, magnification=79096,
                                      scannedPixelSize=56, voltage=300,
                                      sphericalAberration=2.0)
        protImport.setObjLabel('import 20 mics')
        self.launchProtocol(protImport)
        self.assertIsNotNone(protImport.outputMicrographs,
                             "There was a problem with the import")
        
        print("Importing a volume...")
        protImportVol = self.newProtocol(emprot.ProtImportVolumes, filesPath=self.vol,
                                         samplingRate=7.08)
        protImportVol.setObjLabel('import single vol')
        self.launchProtocol(protImportVol)
        self.assertIsNotNone(protImportVol.outputVolume, "There was a problem with the import")
        
        print("Preprocessing the micrographs...")
        xmippProtocols = Domain.importFromPlugin('xmipp3.protocols',
                                                 doRaise=True)
        protPreprocess = self.newProtocol(xmippProtocols.XmippProtPreprocessMicrographs,
                                          doCrop=True, cropPixels=25)
        protPreprocess.inputMicrographs.set(protImport.outputMicrographs)
        protPreprocess.setObjLabel('crop 50px')
        self.launchProtocol(protPreprocess)
        self.assertIsNotNone(protPreprocess.outputMicrographs,
                             "There was a problem with the downsampling")
        
        print("Running Eman import coordinates...")
        protPP = self.newProtocol(emprot.ProtImportCoordinates,
                                 importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_EMAN,
                                 filesPath=self.crdsEmanDir,
                                 filesPattern='info/*_info.json', boxSize=60)
        protPP.inputMicrographs.set(protPreprocess.outputMicrographs)
        protPP.setObjLabel('import from Eman boxing')
        self.launchProtocol(protPP)
        self.assertIsNotNone(protPP.outputCoordinates,
                             "There was a problem with the Eman import coordinates")

        # Now estimate CTF on the micrographs with ctffind 
        print("Performing CTFfind...")
        grigorieffLabProtocols = Domain.importFromPlugin(
            'cistem.protocols',
            doRaise=True)
        protCTF = self.newProtocol(grigorieffLabProtocols.CistemProtCTFFind,
                                   minDefocus=12000, maxDefocus=30000,
                                   runMode=1, numberOfMpi=1, numberOfThreads=16)
        protCTF.inputMicrographs.set(protPreprocess.outputMicrographs)
        protCTF.setObjLabel('CTF ctffind')
        self.launchProtocol(protCTF)

        print("Run extract particles with <Same as picking> option")
        protExtract = self.newProtocol(xmippProtocols.XmippProtExtractParticles,
                                       boxSize=60,
                                       downsampleType=emprot.SAME_AS_PICKING,
                                       doRemoveDust=False,
                                       doInvert=False,
                                       doFlip=False, backRadius=28, runMode=1)
        protExtract.inputCoordinates.set(protPP.outputCoordinates)
        protExtract.ctfRelations.set(protCTF.outputCTF)
        protExtract.setObjLabel('Extract particles')
        self.launchProtocol(protExtract)
        self.assertIsNotNone(protExtract.outputParticles,
                             "There was a problem with the extract particles")
        
        print("Run CL2D")
        protCL2D = xmippProtocols.XmippProtCL2D(numberOfClasses=32,
                                                numberOfInitialClasses=4,
                                                numberOfIterations=2,
                                                numberOfMpi=16)
        protCL2D.inputParticles.set(protExtract.outputParticles)
        protCL2D.setObjLabel('CL2D')
        self.launchProtocol(protCL2D)   
        self.assertIsNotNone(protCL2D.outputClasses,
                             "There was a problem with CL2D")
        
        # Now estimate CTF on the micrographs with xmipp
        print("Performing Xmipp CTF...")
        protCTF2 = self.newProtocol(xmippProtocols.XmippProtCTFMicrographs,
                                    doInitialCTF=True,
                                    minDefocus=12000, maxDefocus=30000,
                                    runMode=1, numberOfMpi=1, numberOfThreads=16)
        protCTF2.ctfRelations.set(protCTF.outputCTF)
        protCTF2.inputMicrographs.set(protPreprocess.outputMicrographs)
        protCTF2.setObjLabel('CTF xmipp')
        self.launchProtocol(protCTF2)
        
        print("Running Xmipp Import Coordinates")
        protPP2 = self.newProtocol(emprot.ProtImportCoordinates,
                                 importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_XMIPP,
                                 filesPath=self.crdsXmippDir,
                                 filesPattern='*.pos', boxSize=60)
        protPP2.inputMicrographs.set(protPreprocess.outputMicrographs)  
        protPP2.setObjLabel('Xmipp Import Coords')
        self.launchProtocol(protPP2)
        self.assertIsNotNone(protPP2.outputCoordinates,
                             "There was a problem with the Xmipp import coordinates")
        
        print("Run extract particles with <Same as picking> option")
        protExtract2 = self.newProtocol(xmippProtocols.XmippProtExtractParticles,
                                        boxSize=60,
                                        downsampleType=emprot.SAME_AS_PICKING,
                                        doRemoveDust=False, doInvert=True,
                                        doFlip=False, backRadius=28, runMode=1)
        protExtract2.inputCoordinates.set(protPP2.outputCoordinates)
        protExtract2.ctfRelations.set(protCTF2.outputCTF)
        protExtract2.setObjLabel('Extract particles')
        self.launchProtocol(protExtract2)
        self.assertIsNotNone(protExtract2.outputParticles,
                             "There was a problem with the extract particles")
        
        print("Run Relion Classification2d")
        relionProtocols = Domain.importFromPlugin('relion.protocols',
                                                  doRaise=True)
        prot2D = relionProtocols.ProtRelionClassify2D(regularisationParamT=2,
                                                      numberOfMpi=4,
                                                      numberOfThreads=4)
        prot2D.numberOfClasses.set(50)
        prot2D.numberOfIterations.set(25)
        prot2D.inputParticles.set(protExtract2.outputParticles)
        prot2D.setObjLabel('relion 2D')
        self.launchProtocol(prot2D)        
        self.assertIsNotNone(prot2D.outputClasses, "There was a problem with Relion 2D:\n" + (prot2D.getErrorMessage() or "No error set"))
        
        print("Run Relion Refine")
        proRef = relionProtocols.ProtRelionRefine3D()
        proRef.inputParticles.set(protExtract2.outputParticles)
        proRef.referenceVolume.set(protImportVol.outputVolume)
        proRef.setObjLabel('relion Refine')
        self.launchProtocol(proRef)        
        self.assertIsNotNone(proRef.outputVolume, "There was a problem with Relion Refine:\n" + (proRef.getErrorMessage() or "No error set"))
        self.assertIsNotNone(proRef.outputParticles, "There was a problem with Relion Refine:\n" + (proRef.getErrorMessage() or "No error set"))
    def test_workflow(self):
        # First, import a set of micrographs
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=self.micsFn,
                                      samplingRate=1.237, voltage=300)
        self.launchProtocol(protImport)
        self.assertIsNotNone(protImport.outputMicrographs,
                             "There was a problem with the import")
        #         self.validateFiles('protImport', protImport)

        # Import a set of volumes
        print("Import Volume")
        protImportVol = self.newProtocol(emprot.ProtImportVolumes, filesPath=self.vol1,
                                         samplingRate=9.896)
        self.launchProtocol(protImportVol)
        self.assertIsNotNone(protImportVol.getFiles(),
                             "There was a problem with the import")
        #        self.validateFiles('protImportVol', protImportVol)

        # Perform a downsampling on the micrographs
        print("Downsampling...")
        XmippProtPreprocessMicrographs = Domain.importFromPlugin(
            'xmipp3.protocols',
            'XmippProtPreprocessMicrographs',
            doRaise=True)
        protDownsampling = self.newProtocol(XmippProtPreprocessMicrographs,
                                            doDownsample=True, downFactor=5,
                                            doCrop=False, runMode=1)
        protDownsampling.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protDownsampling)
        self.assertIsNotNone(protDownsampling.outputMicrographs,
                             "There was a problem with the downsampling")
        #         self.validateFiles('protDownsampling', protDownsampling)

        # Estimate CTF on the downsampled micrographs
        print("Performing CTFfind...")
        ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
                                              'CistemProtCTFFind', doRaise=True)
        protCTF = self.newProtocol(ProtCTFFind, numberOfThreads=4,
                                   minDefocus=22000, maxDefocus=25000)
        protCTF.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protCTF)
        self.assertIsNotNone(protCTF.outputCTF,
                             "There was a problem with the CTF estimation")
        #         self.validateFiles('protCTF', protCTF)

        print("Running Eman import coordinates...")
        protPP = self.newProtocol(emprot.ProtImportCoordinates,
                                  importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_EMAN,
                                  filesPath=self.crdsDir,
                                  filesPattern='*_info.json', boxSize=110)
        protPP.inputMicrographs.set(protDownsampling.outputMicrographs)
        self.launchProtocol(protPP)
        self.assertIsNotNone(protPP.outputCoordinates,
                             "There was a problem with the Eman import coordinates")

        print("<Run extract particles with Same as picking>")
        XmippProtExtractParticles = Domain.importFromPlugin(
            'xmipp3.protocols',
            'XmippProtExtractParticles')

        SAME_AS_PICKING = Domain.importFromPlugin('xmipp3.constants',
                                                  'SAME_AS_PICKING')
        protExtract = self.newProtocol(XmippProtExtractParticles, boxSize=110,
                                       downsampleType=SAME_AS_PICKING,
                                       doFlip=True, doInvert=True, runMode=1)
        protExtract.inputCoordinates.set(protPP.outputCoordinates)
        protExtract.ctfRelations.set(protCTF.outputCTF)
        self.launchProtocol(protExtract)
        self.assertIsNotNone(protExtract.outputParticles,
                             "There was a problem with the extract particles")
        # self.validateFiles('protExtract', protExtract)

        print("Run Preprocess particles")
        XmippProtCropResizeParticles = Domain.importFromPlugin(
            'xmipp3.protocols',
            'XmippProtCropResizeParticles')
        protCropResize = self.newProtocol(XmippProtCropResizeParticles,
                                          doResize=True, resizeOption=1,
                                          resizeDim=110)
        protCropResize.inputParticles.set(protExtract.outputParticles)
        self.launchProtocol(protCropResize)

        self.assertIsNotNone(protCropResize.outputParticles,
                             "There was a problem with resize/crop the particles")

        print("Run ML2D")
        XmippProtML2D = Domain.importFromPlugin('xmipp3.protocols',
                                                'XmippProtML2D')
        protML2D = self.newProtocol(XmippProtML2D, numberOfClasses=8, maxIters=2,
                                    numberOfMpi=2, numberOfThreads=2)
        protML2D.inputParticles.set(protCropResize.outputParticles)
        self.launchProtocol(protML2D)
        self.assertIsNotNone(protML2D.outputClasses,
                             "There was a problem with ML2D")
        # self.validateFiles('protML2D', protML2D)

        #         #FIXME: Check the initial model with EMAn and restore the next step
        #         return

        print("Run Initial Model")

        EmanProtInitModel = Domain.importFromPlugin('eman2.protocols',
                                                    'EmanProtInitModel',
                                                    doRaise=True)
        protIniModel = self.newProtocol(EmanProtInitModel, numberOfIterations=1,
                                        numberOfModels=2,
                                        shrink=5, symmetry='icos',
                                        numberOfThreads=3)
        protIniModel.inputSet.set(protML2D.outputClasses)
        self.launchProtocol(protIniModel)
        self.assertIsNotNone(protIniModel.outputVolumes,
                             "There was a problem with Initial Model")
Exemple #21
0
    def _visualize(self, obj, **kwargs):
        cls = type(obj)

        if issubclass(cls, emobj.Volume):
            fn = emlib.image.ImageHandler.locationToXmipp(obj)
            self._addObjView(obj, fn,
                             {RENDER: 'image',
                              SAMPLINGRATE: obj.getSamplingRate()})

        elif issubclass(cls, emobj.Image):
            fn = emlib.image.ImageHandler.locationToXmipp(obj)
            self._addObjView(obj, fn)

        elif issubclass(cls, emobj.SetOfPDBs):
            fn = obj.getFileName()
            labels = 'id _filename '
            self._addObjView(obj, fn, {ORDER: labels,
                                       VISIBLE: labels,
                                       MODE: MODE_MD,
                                       RENDER: "no"})

        elif issubclass(cls, emobj.SetOfMovies):
            fn = obj.getFileName()
            # Enabled for the future has to be available
            labels = ('id _filename _samplingRate _acquisition._dosePerFrame '
                      '_acquisition._doseInitial ')
            moviesView = self._addObjView(obj, fn, {ORDER: labels,
                                                    VISIBLE: labels,
                                                    MODE: MODE_MD,
                                                    RENDER: "no"})
            # For movies increase the JVM memory by 1 GB, just in case
            moviesView.setMemory(getJvmMaxMemory() + 1)

        elif issubclass(cls, emobj.SetOfMicrographs):
            self._views.append(MicrographsView(self._project, obj, **kwargs))

        elif issubclass(cls, emobj.MicrographsTiltPair):
            labels = 'id enabled _untilted._filename _tilted._filename'
            renderLabels = '_untilted._filename _tilted._filename'
            self._addObjView(obj, obj.getFileName(), {ORDER: labels,
                                                      VISIBLE: labels,
                                                      MODE: MODE_MD,
                                                      RENDER: renderLabels})

        elif issubclass(cls, emobj.ParticlesTiltPair):
            labels = 'id enabled _untilted._filename _tilted._filename'
            renderLabels = '_untilted._filename _tilted._filename'
            self._addObjView(obj, obj.getFileName(), {ORDER: labels,
                                                      VISIBLE: labels,
                                                      RENDER: renderLabels,
                                                      MODE: MODE_MD})

        elif issubclass(cls, emobj.SetOfCoordinates):
            # FIXME: Remove dependency on xmipp3 plugin to visualize coordinates
            xmipp3 = Domain.importFromPlugin('xmipp3',
                                             errorMsg="xmipp3 plugin is required "
                                                      "now to visualize coordinates.")
            micSet = obj.getMicrographs()  # accessing mics to provide metadata file
            if micSet is None:
                raise Exception('visualize: SetOfCoordinates has no micrographs set.')

            mdFn = getattr(micSet, '_xmippMd', None)
            if mdFn:
                fn = mdFn.get()
            else:  # happens if protocol is not an xmipp one
                fn = self._getTmpPath(micSet.getName() + '_micrographs.xmd')
                xmipp3.convert.writeSetOfMicrographs(micSet, fn)
            tmpDir = self._getTmpPath(obj.getName())
            pwutils.cleanPath(tmpDir)
            pwutils.makePath(tmpDir)
            # FIXME: (JMRT) We are always writing the SetOfCoordinates and removing
            # the tmpDir, we need to take into account if the user have pick
            # some particles in the tmpDir and have not save them, that now
            # will loose all picked particles.
            # A possible solution could be to alert that changes have not been
            # written during modification of tmpDir or create a new Xmipp picking
            # protocol to continue picking later without loosing the coordinates.
            xmipp3.convert.writeSetOfCoordinates(tmpDir, obj)
            self._views.append(CoordinatesObjectView(self._project, fn,
                                                     tmpDir, self.protocol,
                                                     inTmpFolder=True))

        elif issubclass(cls, emobj.SetOfParticles):
            fn = obj.getFileName()
            labels = ('id enabled _index _filename _xmipp_zScore _xmipp_cumulativeSSNR '
                      '_sampling _xmipp_scoreByVariance _xmipp_scoreEmptiness '
                      '_ctfModel._defocusU _ctfModel._defocusV _ctfModel._defocusAngle '
                      '_transform._matrix')
            self._addObjView(obj, fn, {ORDER: labels,
                                       VISIBLE: labels,
                                       SORT_BY: '_xmipp_zScore asc',
                                       RENDER: '_filename'})

        elif issubclass(cls, emobj.SetOfVolumes):
            fn = obj.getFileName()
            labels = 'id enabled comment _filename '
            self._addObjView(obj, fn, {MODE: MODE_MD,
                                       ORDER: labels,
                                       VISIBLE: labels,
                                       RENDER: '_filename'})

        elif issubclass(cls, emobj.SetOfClasses2D):
            self._views.append(ClassesView(self._project, obj.strId(),
                                           obj.getFileName(), **kwargs))

        elif issubclass(cls, emobj.SetOfClasses3D):
            self._views.append(Classes3DView(self._project, obj.strId(),
                                             obj.getFileName()))

        elif issubclass(cls, emobj.SetOfImages):
            self._views.append(ObjectView(self._project, obj.strId(),
                                          obj.getFileName(), **kwargs))

        elif issubclass(cls, emobj.SetOfCTF):
            self._views.append(CtfView(self._project, obj))

        elif issubclass(cls, emobj.CoordinatesTiltPair):
            # FIXME: Remove dependency on xmipp3 plugin to visualize coordinates
            xmipp3 = Domain.importFromPlugin('xmipp3',
                                             errorMsg="xmipp3 plugin is required "
                                                      "now to visualize coordinates.")
            tmpDir = self._getTmpPath(obj.getName())
            pwutils.makePath(tmpDir)

            mdFn = os.path.join(tmpDir, 'input_micrographs.xmd')
            xmipp3.convert.writeSetOfMicrographsPairs(
                obj.getUntilted().getMicrographs(),
                obj.getTilted().getMicrographs(), mdFn)

            # TODO: Review this if ever a non Xmipp CoordinatesTiltPair is available
            xmipp3.convert.writeSetOfCoordinates(tmpDir, obj.getUntilted())
            xmipp3.convert.writeSetOfCoordinates(tmpDir, obj.getTilted())
            launchTiltPairPickerGUI(mdFn, tmpDir, self.protocol)

        elif issubclass(cls, emprot.ProtParticlePicking):
            if obj.getOutputsSize() >= 1:
                self._visualize(obj.getCoords())

        elif issubclass(cls, emprot.ProtImportMovies):
            movs = obj.outputMovies
            self._visualize(movs)
            gainFn = movs.getGain()
            if gainFn is not None:
                if os.path.exists(gainFn):
                    self._views.append(DataView(gainFn))

        return self._views
try:
    from itertools import izip
except:
    izip = zip

import urllib.request
import os

import pyworkflow.tests as tests
from pwem.protocols import ProtImportParticles, ProtSplitSet, ProtUnionSet
from pwem.emlib.image import ImageHandler
from pwem import Domain

from emxlib.protocols import ProtEmxExport

XmippProtExtractParticles = Domain.importFromPlugin(
    'xmipp3.protocols', 'XmippProtExtractParticles')
XmippProtFilterParticles = Domain.importFromPlugin('xmipp3.protocols',
                                                   'XmippProtFilterParticles')
XmippProtApplyAlignment = Domain.importFromPlugin('xmipp3.protocols',
                                                  'XmippProtApplyAlignment')
XmippProtReconstructFourier = Domain.importFromPlugin(
    'xmipp3.protocols', 'XmippProtReconstructFourier')


class TestEmxWeb(tests.BaseTest):
    """test emx web page """
    @classmethod
    def setUpClass(cls):
        tests.setupTestProject(cls)
        # cls.baseUrl = "http://i2pc.cnb.csic.es/emx/resourcesEmx/Tests/"
        cls.baseUrl = "http://heisenberg.cnb.csic.es:8080/emx/resourcesEmx/Tests/"
Exemple #23
0
    def readCoordinates(self):
        micTiltPairs = self.getInputMicrographs()
        ci = self.getImportClass()
        uSet = micTiltPairs.getUntilted()
        tSet = micTiltPairs.getTilted()
        # Create Untilted and Tilted SetOfCoordinates
        uCoordSet = self._createSetOfCoordinates(uSet, suffix='Untilted')
        tCoordSet = self._createSetOfCoordinates(tSet, suffix='Tilted')
        anglesSet = self._createSetOfAngles()

        def _importCoords(uCoordSet, tCoordSet):
            for micU, micT in izip(uSet, tSet):
                coordFnU, coordFnT = self.getMatchingCoord(micU, micT)
                if coordFnU and coordFnT:

                    def addCoordinateU(coord):
                        coord.setMicrograph(micU)
                        self.correctCoordinatePosition(coord)
                        uCoordSet.append(coord)

                    ci.importCoordinates(coordFnU,
                                         addCoordinate=addCoordinateU)

                    def addCoordinateT(coord):
                        coord.setMicrograph(micT)
                        self.correctCoordinatePosition(coord)
                        tCoordSet.append(coord)

                    ci.importCoordinates(coordFnT,
                                         addCoordinate=addCoordinateT)

                    def addAngles(ang):
                        anglesSet.append(ang)

                    if self.importFrom.get() == self.IMPORT_FROM_EMAN:
                        ci.importAngles(coordFnT, addAngles=addAngles)

        _importCoords(uCoordSet, tCoordSet)
        boxSize = self.boxSize.get()
        uCoordSet.setBoxSize(boxSize)
        tCoordSet.setBoxSize(boxSize)

        uCoordSet.write()
        tCoordSet.write()

        if self.importFrom.get() == self.IMPORT_FROM_XMIPP:
            readAnglesFromMicrographs = Domain.importFromPlugin(
                'xmipp3.convert', 'readAnglesFromMicrographs')
            if exists(self.xmippMdFn.get()):
                checkAngles = self._compareMicPairs(self.xmippMdFn, uSet, tSet)
                if checkAngles:
                    readAnglesFromMicrographs(self.xmippMdFn, anglesSet)
                else:
                    self.warning('Angles for some micrographs were not found, '
                                 'skipping angles import')
            else:
                self.warning(
                    'Micrograph xmd file not provided, so tilt angles will '
                    'not be imported')
        anglesSet.write()

        return uCoordSet, tCoordSet, anglesSet
Exemple #24
0
    def testXmippRCTWorkflowBasic(self):
        # First, import a set of micrographs
        protImport = self.newProtocol(emprot.ProtImportMicrographsTiltPairs,
                                      patternUntilted=self.micsUFn,
                                      patternTilted=self.micsTFn,
                                      samplingRate=2.28, voltage=100,
                                      sphericalAberration=2.9)
        self.launchProtocol(protImport)
        self.assertIsNotNone(protImport.outputMicrographsTiltPair,
                             "There was a problem with the import")
        # self.validateFiles('protImportRCT', protImport)

        # Then simulate a particle picking
        print("Running fake particle picking...")
        from pwem import Domain
        XmippProtParticlePickingPairs = Domain.importFromPlugin(
            'xmipp3.protocols',
            'XmippProtParticlePickingPairs',
            doRaise=True)
        protPicking = self.newProtocol(XmippProtParticlePickingPairs,
                                       importFolder=self.allCrdsDir)

        protPicking.inputMicrographsTiltedPair.set(protImport.outputMicrographsTiltPair)

        self.proj.launchProtocol(protPicking, wait=True)

        self.assertIsNotNone(protPicking.outputCoordinatesTiltPair,
                             "There was a problem with the faked picking")
        # self.validateFiles('protPicking', protPicking)

        # Extract particles
        print("Run extract particles with Same as picking")
        XmippProtExtractParticlesPairs = Domain.importFromPlugin(
            'xmipp3.protocols',
            'XmippProtExtractParticlesPairs')
        SAME_AS_PICKING = Domain.importFromPlugin('xmipp3.constants',
                                                  'SAME_AS_PICKING')
        protExtract = self.newProtocol(XmippProtExtractParticlesPairs,
                                       downFactor=2,
                                       boxSize=60,
                                       doInvert=False,
                                       downsampleType=SAME_AS_PICKING)
        protExtract.inputCoordinatesTiltedPairs.set(protPicking.outputCoordinatesTiltPair)

        self.proj.launchProtocol(protExtract, wait=True)

        # self.validateFiles('protExtract', protExtract)
        self.assertIsNotNone(protExtract.outputParticlesTiltPair,
                             "There was a problem with the extract particles")

        # Classify using Xmipp CL2D
        print("Run CL2D")
        XmippProtCL2D = Domain.importFromPlugin('xmipp3.protocols',
                                                'XmippProtCL2D')
        protCL2D = self.newProtocol(XmippProtCL2D,
                                    numberOfClasses=10,
                                    numberOfInitialClasses=1,
                                    numberOfIterations=3, numberOfMpi=2)
        protCL2D.inputParticles.set(protExtract.outputParticlesTiltPair.getUntilted())
        self.launchProtocol(protCL2D)
        self.assertIsNotNone(protCL2D.outputClasses,
                             "There was a problem with CL2D")
        # self.validateFiles('protCL2D', protCL2D)

        # Random Conical Tilt
        print("Run Random Conical Tilt")
        XmippProtRCT = Domain.importFromPlugin('xmipp3.protocols',
                                               'XmippProtRCT')
        protRCT = self.newProtocol(XmippProtRCT)
        protRCT.inputParticlesTiltPair.set(protExtract.outputParticlesTiltPair)
        protRCT.inputParticles.set(protCL2D.outputClasses)

        self.proj.launchProtocol(protRCT, wait=True)

        # self.validateFiles('protExtract', protExtract)
        self.assertIsNotNone(protRCT.outputVolumes, "There was a problem with the RCT")
Exemple #25
0
    def test_pattern(self):
        """ Import several Particles from a given pattern.
        """
        def checkOutputs(prot):

            t0 = time.time()

            while not (prot.isFinished() or prot.isFailed()):

                # Time out 6 minutes, just in case
                tdelta = time.time() - t0
                if tdelta > 6*60:
                    break

                time.sleep(10)

                prot = self._updateProtocol(prot)

                # Check if the protocol is still launched
                if prot.isLaunched():
                    continue
                elif prot.isScheduled():
                    continue

                if prot.hasAttribute("outputCTF"):
                    ctfSet = emobj.SetOfCTF(filename=prot._getPath(CTF_SQLITE))
                    baseFn = prot._getPath(CTF_SQLITE)
                    self.assertTrue(os.path.isfile(baseFn))
                    counter = 0
                    if ctfSet.getSize() > counter:
                        counter += 1
                        for ctf in ctfSet:
                            self.assertNotEqual(ctf._resolution.get(), None)
                            self.assertNotEqual(ctf._fitQuality.get(), None)
                            self.assertNotEqual(ctf.isEnabled(), None)
                            self.assertNotEqual(ctf._defocusU.get(), None)
                            self.assertNotEqual(ctf._defocusV.get(), None)
                            self.assertNotEqual(ctf._defocusRatio.get(), None)
                            if ctf.getPhaseShift():
                                self.assertNotEqual(ctf.getPhaseShift(), None)
            self.assertIsNotNone(prot.outputCTF,
                                 "Error: outputCTF is not produced "
                                 "in %s." % prot.getClassName())
            self.assertEqual(prot.outputCTF.getSize(), MICS)

        # Simulate streaming with create stream data protocol
        kwargs = {'xDim': 4096,
                  'yDim': 4096,
                  'nDim': MICS,
                  'samplingRate': 1.25,
                  'creationInterval': 15,
                  'delay': 0,
                  'setof': emprot.SET_OF_RANDOM_MICROGRAPHS}  # SetOfMicrographs

        # create mic in streaming mode
        protStream = self.newProtocol(emprot.ProtCreateStreamData, **kwargs)
        protStream.setObjLabel('create Stream Mic')
        self.proj.launchProtocol(protStream, wait=False)

        # 1st ctf - ctffind4 in streaming

        ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
                                              'CistemProtCTFFind', doRaise=True)
        protCTF = self.newProtocol(ProtCTFFind)
        protCTF.inputMicrographs.set(protStream)
        protCTF.inputMicrographs.setExtended('outputMicrographs')
        protCTF.findPhaseShift.set(True)
        protCTF.numberOfThreads.set(4)
        self.proj.scheduleProtocol(protCTF)

        # 2nd ctf - Xmipp CTF
        kwargs = {'ctfDownFactor': 2,
                  'numberOfThreads': 4
                  }

        XmippProtCTFMicrographs = Domain.importFromPlugin(
            'xmipp3.protocols.protocol_ctf_micrographs',
            'XmippProtCTFMicrographs', doRaise=True)

        protCTF2 = self.newProtocol(XmippProtCTFMicrographs, **kwargs)
        protCTF2.inputMicrographs.set(protStream)
        protCTF2.inputMicrographs.setExtended('outputMicrographs')
        self.proj.scheduleProtocol(protCTF2)

        # 3rd ctf - Gctf
        protCTF3 = None
        try:
            # check if box has nvidia cuda libs.
            pynvml.nvmlInit()  # fails if not GPU attached
            ProtGctf = Domain.importFromPlugin('gctf.protocols', 'ProtGctf',
                                               doRaise=True)
            protCTF3 = ProtGctf()
            protCTF3.inputMicrographs.set(protStream)
            protCTF3.inputMicrographs.setExtended('outputMicrographs')
            protCTF3.ctfDownFactor.set(2)
            self.proj.scheduleProtocol(protCTF3)

        except pynvml.NVMLError as err:
            print("Cannot find GPU."
                  "I assume that no GPU is connected to this machine")

        # Check first ctf output - Ctffind
        checkOutputs(protCTF)

        # Check 2nd ctf output - Xmipp
        checkOutputs(protCTF2)

        # If GPU and therefore GCTF protocl
        if protCTF3 is not None:
            checkOutputs(protCTF3)
    def test_workflow(self):
        try:
            from itertools import izip
        except ImportError:
            izip = zip
        # First, import a set of micrographs
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=self.micsFn,
                                      samplingRate=1.237, voltage=300)
        self.launchProtocol(protImport)
        self.assertIsNotNone(protImport.outputMicrographs,
                             "There was a problem with the import")
        #         self.validateFiles('protImport', protImport)

        # Import a set of volumes
        print("Import Volume")
        protImportVol = self.newProtocol(emprot.ProtImportVolumes, filesPath=self.vol1,
                                         samplingRate=9.896)
        self.launchProtocol(protImportVol)
        self.assertIsNotNone(protImportVol.getFiles(),
                             "There was a problem with the import")
        #        self.validateFiles('protImportVol', protImportVol)

        # Perform a downsampling on the micrographs
        print("Downsampling...")
        XmippProtPreprocessMicrographs = Domain.importFromPlugin('xmipp3.protocols',
                                                                 'XmippProtPreprocessMicrographs',
                                                                 doRaise=True)
        protDownsampling = self.newProtocol(XmippProtPreprocessMicrographs,
                                            doDownsample=True, downFactor=5,
                                            doCrop=False, runMode=1)
        protDownsampling.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protDownsampling)
        self.assertIsNotNone(protDownsampling.outputMicrographs,
                             "There was a problem with the downsampling")
        #         self.validateFiles('protDownsampling', protDownsampling)

        # Estimate CTF on the downsampled micrographs
        print("Performing CTFfind...")
        ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
                                              'CistemProtCTFFind', doRaise=True)
        protCTF = self.newProtocol(ProtCTFFind, numberOfThreads=4,
                                   minDefocus=22000, maxDefocus=25000)
        protCTF.inputMicrographs.set(protDownsampling.outputMicrographs)
        self.launchProtocol(protCTF)
        self.assertIsNotNone(protCTF.outputCTF,
                             "There was a problem with the CTF estimation")

        valuesList = [[24000, 24000], [22548, 22518], [23058, 23029]]
        for ctfModel, values in izip(protCTF.outputCTF, valuesList):
            self.assertAlmostEquals(ctfModel.getDefocusU(), values[0], delta=1000)
            self.assertAlmostEquals(ctfModel.getDefocusV(), values[1], delta=1000)
            self.assertAlmostEquals(ctfModel.getMicrograph().getSamplingRate(),
                                    6.185, delta=0.001)

        #         self.validateFiles('protCTF', protCTF)

        print("Running Eman import coordinates...")
        protPP = self.newProtocol(emprot.ProtImportCoordinates,
                                  importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_EMAN,
                                  filesPath=self.crdsDir,
                                  filesPattern='*_info.json', boxSize=110)
        protPP.inputMicrographs.set(protDownsampling.outputMicrographs)
        self.launchProtocol(protPP)
        self.assertIsNotNone(protPP.outputCoordinates,
                             "There was a problem with the Eman import coordinates")

        # Extract the SetOfParticles.
        print("Run extract particles with other downsampling factor")
        XmippProtExtractParticles = Domain.importFromPlugin(
            'xmipp3.protocols',
            'XmippProtExtractParticles')
        OTHER = Domain.importFromPlugin('xmipp3.constants', 'OTHER')
        protExtract = self.newProtocol(XmippProtExtractParticles,
                                       boxSize=64,
                                       downsampleType=OTHER,
                                       downFactor=8.0,
                                       doFlip=False, runMode=1, doInvert=False)
        protExtract.inputCoordinates.set(protPP.outputCoordinates)
        protExtract.ctfRelations.set(protCTF.outputCTF)
        protExtract.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protExtract)
        self.assertIsNotNone(protExtract.outputParticles,
                             "There was a problem with the extract particles")
    def test_workflow(self):
        #First, import a set of micrographs
        print("Importing a set of micrographs...")
        #TODO missing contrst amplitude ROB
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=self.micsFn,
                                      samplingRateMode=1, magnification=79096,
                                      scannedPixelSize=56, voltage=300,
                                      sphericalAberration=2.0)
        protImport.setObjLabel('import 20 mics')
        self.launchProtocol(protImport)
        self.assertIsNotNone(protImport.outputMicrographs,
                             "There was a problem with the import")
        
        print("Importing a volume...")
        protImportVol = self.newProtocol(emprot.ProtImportVolumes,
                                         filesPath=self.vol, samplingRate=7.08)
        protImportVol.setObjLabel('import single vol')
        self.launchProtocol(protImportVol)
        self.assertIsNotNone(protImportVol.outputVolume,
                             "There was a problem with the import")
        
        print("Preprocessing the micrographs...")
        xmippProtocols = Domain.importFromPlugin('xmipp3.protocols',
                                                 doRaise=True)
        protPreprocess = self.newProtocol(xmippProtocols.XmippProtPreprocessMicrographs,
                                          doCrop=True, cropPixels=25)
        protPreprocess.inputMicrographs.set(protImport.outputMicrographs)
        protPreprocess.setObjLabel('crop 50px')
        self.launchProtocol(protPreprocess)
        self.assertIsNotNone(protPreprocess.outputMicrographs,
                             "There was a problem with the downsampling")

        print("Running Eman import coordinates...")
        protPP = self.newProtocol(emprot.ProtImportCoordinates,
                                 importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_EMAN,
                                 filesPath=self.crdsEmanDir,
                                 filesPattern='info/*_info.json', boxSize=60)
        protPP.inputMicrographs.set(protPreprocess.outputMicrographs)
        protPP.setObjLabel('import from Eman boxing')
        self.launchProtocol(protPP)
        self.assertIsNotNone(protPP.outputCoordinates,
                             "There was a problem with the Eman import coordinates")

        # Now estimate CTF on the micrographs with ctffind 
        print("Performing CTFfind...")
        grigorieffLabProtocols = Domain.importFromPlugin(
            'cistem.protocols',
            doRaise=True)
        protCTF = self.newProtocol(grigorieffLabProtocols.CistemProtCTFFind,
                                   minDefocus=12000, maxDefocus=30000,
                                   runMode=1, numberOfMpi=1, numberOfThreads=16)
        protCTF.inputMicrographs.set(protPreprocess.outputMicrographs)
        protCTF.setObjLabel('CTF ctffind')
        self.launchProtocol(protCTF)
        
        print("Run extract particles with <Same as picking> option")
        protExtract = self.newProtocol(xmippProtocols.XmippProtExtractParticles,
                                       boxSize=60,
                                       downsampleType=emprot.SAME_AS_PICKING,
                                       doRemoveDust=False,
                                       doInvert=False,
                                       doFlip=False, backRadius=28, runMode=1)
        protExtract.inputCoordinates.set(protPP.outputCoordinates)
        protExtract.ctfRelations.set(protCTF.outputCTF)
        protExtract.setObjLabel('Extract particles')
        self.launchProtocol(protExtract)
        self.assertIsNotNone(protExtract.outputParticles,
                             "There was a problem with the extract particles")
    def test1(self):
        # Import a set of micrographs
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=self.micsFn,
                                      samplingRate=1.237,
                                      voltage=300)
        self.launchProtocol(protImport)
        self.assertIsNotNone(protImport.outputMicrographs,
                             "There was a problem with the import")

        # Create pure noise micrographs (to force a Discarded consensus set)
        protStream = self.newProtocol(
            emprot.ProtCreateStreamData,
            xDim=9216,
            yDim=9441,
            nDim=3,
            samplingRate=1.237,
            setof=2,  # 2 -> SetOfRandomMicrographs
            creationInterval=1)
        self.proj.launchProtocol(protStream, wait=False)

        # Computes the CTF with Xmipp
        XmippProtCTFMicrographs = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtCTFMicrographs', doRaise=True)
        protCTF1 = self.newProtocol(XmippProtCTFMicrographs)
        protCTF1.inputMicrographs.set(protImport.outputMicrographs)
        self.proj.launchProtocol(protCTF1, wait=False)

        # Computes the CTF with CTFFind4
        ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
                                              'CistemProtCTFFind',
                                              doRaise=True)
        protCTF2 = self.newProtocol(ProtCTFFind)
        protCTF2.inputMicrographs.set(protImport.outputMicrographs)
        self.proj.launchProtocol(protCTF2, wait=False)

        self._waitOutput(protStream, "outputMicrographs")
        # Computes the CTF with CTFFind4 for the noise mics
        protCTF3 = self.newProtocol(ProtCTFFind)
        protCTF3.inputMicrographs.set(protStream.outputMicrographs)
        self.proj.launchProtocol(protCTF3, wait=False)

        # Computes the Consensus of GOOD CTFs
        self._waitOutput(protCTF1, "outputCTF")
        self._waitOutput(protCTF2, "outputCTF")
        XmippProtCTFConsensus = Domain.importFromPlugin(
            'xmipp3.protocols', 'XmippProtCTFConsensus', doRaise=True)
        protCTFcons = self.newProtocol(XmippProtCTFConsensus,
                                       objLabel='default (pass)',
                                       useDefocus=False,
                                       useAstigmatism=False,
                                       useResolution=False,
                                       calculateConsensus=True,
                                       averageDefocus=False,
                                       includeSecondary=False)
        protCTFcons.inputCTF.set(protCTF1.outputCTF)
        protCTFcons.inputCTF2.set(protCTF2.outputCTF)
        self.launchProtocol(protCTFcons)

        protCTF1.outputCTF.load()  # Needed to update the setSize
        self.checkCTFs(protCTFcons,
                       refMics=protImport.outputMicrographs,
                       refCTFs=protCTF1.outputCTF)

        # Computes the Consensus comparing a good CTF to a RANDOM one
        self._waitOutput(protCTF3, "outputCTF")
        protCTFcons2 = self.newProtocol(XmippProtCTFConsensus,
                                        objLabel='default (block)',
                                        useDefocus=False,
                                        useAstigmatism=False,
                                        useResolution=False,
                                        calculateConsensus=True,
                                        averageDefocus=False,
                                        includeSecondary=False)

        protCTFcons2.inputCTF.set(protCTF1.outputCTF)
        protCTFcons2.inputCTF2.set(protCTF3.outputCTF)
        self.launchProtocol(protCTFcons2)
        self.checkCTFs(protCTFcons2,
                       refMics=protImport.outputMicrographs,
                       refCTFs=protCTF1.outputCTF,
                       label="Discarded")

        # Averaging CTF parameters
        protCTFcons3 = self.newProtocol(XmippProtCTFConsensus,
                                        objLabel='defocus average',
                                        useDefocus=False,
                                        useAstigmatism=False,
                                        useResolution=False,
                                        calculateConsensus=True,
                                        averageDefocus=True)
        protCTFcons3.inputCTF.set(protCTF1.outputCTF)
        protCTFcons3.inputCTF2.set(protCTF2.outputCTF)
        self.launchProtocol(protCTFcons3)

        protCTF1.outputCTF.load()  # Needed to update the set
        protCTF2.outputCTF.load()  # Needed to update the set
        ctfAveraged = {
            'defocusU': 24140.0898,
            'defocusV': 23569.0801,
            'defocusAngle': 58.3429
        }
        self.checkCTFs(protCTFcons3,
                       refMics=protImport.outputMicrographs,
                       refCTFs=protCTF1.outputCTF,
                       avgCTF=ctfAveraged)

        # merging Metadata columns
        protCTFcons4 = self.newProtocol(XmippProtCTFConsensus,
                                        objLabel='metadata merge',
                                        useDefocus=False,
                                        useAstigmatism=False,
                                        useResolution=False,
                                        calculateConsensus=True,
                                        includeSecondary=True)
        protCTFcons4.inputCTF.set(protCTF2.outputCTF)
        protCTFcons4.inputCTF2.set(protCTF1.outputCTF)
        self.launchProtocol(protCTFcons4)

        protCTF1.outputCTF.load()  # Needed to update the set
        protCTF2.outputCTF.load()  # Needed to update the set
        self.checkCTFs(protCTFcons4,
                       refMics=protImport.outputMicrographs,
                       refCTFs=protCTF2.outputCTF,
                       MDmerging=True)
Exemple #29
0
    def testXmippWorkflow(self):
        # First, import a set of micrographs
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      filesPath=self.micsFn,
                                      samplingRate=1.237,
                                      voltage=300)
        self.launchProtocol(protImport)
        self.assertSetSize(protImport.outputMicrographs, 3)
        self.assertIsNotNone(protImport.outputMicrographs.getFileName(),
                             "There was a problem with the import")
        self.validateFiles('protImport', protImport)

        # Import a set of volumes
        print("Import Volume")
        protImportVol = self.newProtocol(emprot.ProtImportVolumes,
                                         filesPath=self.vol1,
                                         samplingRate=9.896)
        self.launchProtocol(protImportVol)
        self.assertIsNotNone(protImportVol.getFiles(),
                             "There was a problem with the import")
        #        self.validateFiles('protImportVol', protImportVol)

        # Perform a downsampling on the micrographs
        print("Downsampling...")
        xmippProtcols = Domain.importFromPlugin('xmipp3.protocols',
                                                doRaise=True)
        protDownsampling = self.newProtocol(
            xmippProtcols.XmippProtPreprocessMicrographs,
            doDownsample=True,
            downFactor=5,
            doCrop=False,
            runMode=1)
        protDownsampling.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protDownsampling)
        self.assertSetSize(protDownsampling.outputMicrographs, 3)
        self.assertIsNotNone(protDownsampling.outputMicrographs,
                             "There was a problem with the downsampling")
        self.validateFiles('protDownsampling', protDownsampling)

        print("Importing coordinates")
        protPP = self.newProtocol(
            emprot.ProtImportCoordinates,
            importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_XMIPP,
            filesPath=self.allCrdsDir,
            filesPattern='*.pos',
            boxSize=110)

        protPP.inputMicrographs.set(protDownsampling.outputMicrographs)
        self.launchProtocol(protPP)
        self.protDict['protPicking'] = protPP
        self.assertSetSize(
            protPP.outputCoordinates, 143 + 138 + 122 + 110 + 138 + 122,
            "There was a problem with the import of coordinates")

        # Now estimate CTF on the downsampled micrographs
        print("Performing CTF...")
        protCTF = self.newProtocol(xmippProtcols.XmippProtCTFMicrographs,
                                   numberOfThreads=4,
                                   minDefocus=2.2,
                                   maxDefocus=2.5)  # Defocus is in microns
        protCTF.inputMicrographs.set(protDownsampling.outputMicrographs)
        self.launchProtocol(protCTF)
        self.assertSetSize(protCTF.outputCTF, 3)
        self.assertIsNotNone(protCTF.outputCTF,
                             "There was a problem with the CTF estimation")
        # After CTF estimation, the output micrograph should have CTF info
        self.validateFiles('protCTF', protCTF)

        print("Run extract particles with other downsampling factor")
        protExtract = self.newProtocol(xmippProtcols.XmippProtExtractParticles,
                                       boxSize=64,
                                       downsampleType=emprot.OTHER,
                                       doFlip=True,
                                       downFactor=8,
                                       runMode=1,
                                       doInvert=True)
        protExtract.inputCoordinates.set(protPP.outputCoordinates)
        protExtract.ctfRelations.set(protCTF.outputCTF)
        protExtract.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protExtract)
        self.assertIsNotNone(protExtract.outputParticles,
                             "There was a problem with the extract particles")
        self.validateFiles('protExtract', protExtract)

        print("Run Extract Coordinates without applying shifts")
        protExtractCoords = self.newProtocol(emprot.ProtExtractCoords)
        protExtractCoords.inputParticles.set(protExtract.outputParticles)
        protExtractCoords.inputMicrographs.set(protImport.outputMicrographs)
        self.launchProtocol(protExtractCoords)
        # The size of the set of coordinates must be the same as the input set of particles
        self.assertSetSize(
            protExtractCoords.outputCoordinates,
            size=protExtract.outputParticles.getSize(),
            msg="There was a problem with the coordinates extraction")
        # Check if the scaling factor is being calculated and applied correctly
        scale = protExtract.outputParticles.getSamplingRate(
        ) / protImport.outputMicrographs.getSamplingRate()
        inParticleCoord = protExtract.outputParticles.getFirstItem(
        ).getCoordinate()
        x, y = inParticleCoord.getPosition()
        self.assertAlmostEqual(
            protExtractCoords.outputCoordinates.getFirstItem().getPosition(),
            (int(x * scale), int(y * scale)))

        print("Run Screen Particles")
        protScreen = self.newProtocol(xmippProtcols.XmippProtScreenParticles,
                                      autoParRejection=xmippProtcols.
                                      XmippProtScreenParticles.REJ_MAXZSCORE,
                                      maxZscore=3.0)
        protScreen.inputParticles.set(protExtract.outputParticles)
        self.launchProtocol(protScreen)
        self.assertSetSize(protScreen.outputParticles,
                           msg="There was a problem with Screen Particles")

        print("Run ML2D")
        protML2D = self.newProtocol(xmippProtcols.XmippProtML2D,
                                    numberOfClasses=1,
                                    maxIters=4,
                                    doMlf=True,
                                    numberOfMpi=2,
                                    numberOfThreads=1)
        protML2D.inputParticles.set(protScreen.outputParticles)
        self.launchProtocol(protML2D)
        self.assertIsNotNone(protML2D.outputClasses,
                             "There was a problem with ML2D")
        self.validateFiles('protML2D', protML2D)

        print("Run CL2D")
        protCL2D = self.newProtocol(xmippProtcols.XmippProtCL2D,
                                    numberOfClasses=2,
                                    numberOfInitialClasses=1,
                                    numberOfIterations=4,
                                    numberOfMpi=2)
        protCL2D.inputParticles.set(protExtract.outputParticles)
        self.launchProtocol(protCL2D)
        self.assertIsNotNone(protCL2D.outputClasses,
                             "There was a problem with CL2D")
        self.validateFiles('protCL2D', protCL2D)

        print("Run Only Align2d")
        protOnlyAlign = self.newProtocol(xmippProtcols.XmippProtCL2DAlign,
                                         maximumShift=5,
                                         numberOfIterations=2,
                                         numberOfMpi=2,
                                         numberOfThreads=1,
                                         useReferenceImage=False)
        protOnlyAlign.inputParticles.set(protExtract.outputParticles)
        self.launchProtocol(protOnlyAlign)
        self.assertIsNotNone(protOnlyAlign.outputParticles,
                             "There was a problem with Only align2d")
        self.validateFiles('protOnlyAlign', protOnlyAlign)

        print("Run Extract Coordinates applying shifts")
        protExtractCoordsShifts = self.newProtocol(emprot.ProtExtractCoords,
                                                   applyShifts=True)
        protExtractCoordsShifts.setObjLabel(
            'Extract Coordinates applying shifts')
        inputParticles = protOnlyAlign.outputParticles
        inputMics = protDownsampling.outputMicrographs
        protExtractCoordsShifts.inputParticles.set(inputParticles)
        protExtractCoordsShifts.inputMicrographs.set(inputMics)
        self.launchProtocol(protExtractCoordsShifts)
        # The size of the set of coordinates must be the same as the input set of particles
        outputParticles = protExtractCoordsShifts.outputCoordinates
        self.assertSetSize(
            outputParticles,
            size=protExtract.outputParticles.getSize(),
            msg="There was a problem with the coordinates extraction")
        # Check if the scaling factor is being calculated and applied correctly
        scale = inputParticles.getSamplingRate(
        ) / protDownsampling.outputMicrographs.getSamplingRate()
        inParticleCoord = inputParticles.getFirstItem().getCoordinate()
        shifts = protExtractCoordsShifts.getShifts(
            inputParticles.getFirstItem().getTransform(),
            inputParticles.getAlignment())
        x, y = inParticleCoord.getPosition()
        xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
        self.assertAlmostEqual(
            protExtractCoordsShifts.outputCoordinates.getFirstItem().
            getPosition(), (int(xCoor * scale), int(yCoor * scale)))

        print("Run kerdensom")
        ProtKerdensom = self.newProtocol(xmippProtcols.XmippProtKerdensom,
                                         useMask=False,
                                         SomXdim=2,
                                         SomYdim=2,
                                         SomReg0=800,
                                         SomReg1=400,
                                         SomSteps=2)
        ProtKerdensom.inputParticles.set(protOnlyAlign.outputParticles)
        self.launchProtocol(ProtKerdensom)
        self.assertIsNotNone(ProtKerdensom.outputClasses,
                             "There was a problem with kerdensom")
        # self.validateFiles('ProtKerdensom', ProtKerdensom)

        print("Run Rotational Spectra")
        xmippProtRotSpectra = self.newProtocol(
            xmippProtcols.XmippProtRotSpectra,
            SomXdim=2,
            SomYdim=2,
            spectraInnerRadius=4,
            spectraOuterRadius=24)
        xmippProtRotSpectra.inputParticles.set(protOnlyAlign.outputParticles)
        self.launchProtocol(xmippProtRotSpectra)
        self.assertIsNotNone(xmippProtRotSpectra.outputClasses,
                             "There was a problem with Rotational Spectra")
    def testRisosome(self):
        print("Importing 2D averages (subset of 4)")
        ih = emlib.image.ImageHandler()
        classesFn = self.ds.getFile('import/classify2d/extra/'
                                    'relion_it015_classes.mrcs')

        outputName = 'input_averages.mrcs'
        inputTmp = os.path.abspath(self.proj.getTmpPath())
        outputFn = self.proj.getTmpPath(outputName)

        for i, index in enumerate([5, 16, 17, 18, 24]):
            ih.convert((index, classesFn), (i + 1, outputFn))

        protAvgs = self.newProtocol(emprot.ProtImportAverages,
                                    objLabel='avgs - 5',
                                    filesPath=inputTmp,
                                    filesPattern=outputName,
                                    samplingRate=7.08)
        self.launchProtocol(protAvgs)

        # First, import a set of micrographs
        print("Importing a set of micrographs...")
        protImport = self.newProtocol(emprot.ProtImportMicrographs,
                                      objLabel='import 20 mics (streaming)',
                                      filesPath=os.path.abspath(
                                          self.proj.getTmpPath()),
                                      filesPattern="*%s" % self.ext,
                                      samplingRateMode=1,
                                      magnification=79096,
                                      scannedPixelSize=56,
                                      voltage=300,
                                      sphericalAberration=2.0,
                                      dataStreaming=True,
                                      fileTimeout=10,
                                      timeout=60)
        self.proj.launchProtocol(protImport, wait=False)
        self._waitOutput(protImport, 'outputMicrographs')

        # Now estimate CTF on the micrographs with ctffind
        print("Performing CTFfind...")
        ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
                                              'CistemProtCTFFind',
                                              doRaise=True)
        protCtf = self.newProtocol(
            ProtCTFFind,
            objLabel='ctffind',
            inputMicrographs=protImport.outputMicrographs,
            minDefocus=12000,
            maxDefocus=30000,
            slowSearch=False)
        self.proj.launchProtocol(protCtf, wait=False)
        self._waitOutput(protCtf, 'outputCTF')

        self._waitUntilMinSize(protCtf.outputCTF)

        # Select some good averages from the iterations mrcs a

        ProtRelion2Autopick = Domain.importFromPlugin('relion.protocols',
                                                      'ProtRelion2Autopick')
        relion_RUN_COMPUTE = Domain.importFromPlugin('relion', 'RUN_COMPUTE')

        protPick = self.newProtocol(
            ProtRelion2Autopick,
            objLabel='autopick refs',
            inputMicrographs=protImport.outputMicrographs,
            ctfRelations=protCtf.outputCTF,
            runType=relion_RUN_COMPUTE,
            inputReferences=protAvgs.outputAverages,
            numberOfMpi=2)
        self.launchProtocol(protPick)