Beispiel #1
0
def readSetOfParticles(lstFile, partSet, copyOrLink, direc):
    for index, fn in iterLstFile(lstFile):
        item = Particle()
        # set full path to particles stack file
        abspath = os.path.abspath(lstFile)
        fn = abspath.replace('sets/%s' % os.path.basename(lstFile), '') + fn
        newFn = pwutils.join(direc, os.path.basename(fn))
        if not pwutils.exists(newFn):
            copyOrLink(fn, newFn)

        item.setLocation(index, newFn)
        partSet.append(item)
    def createSetOfParticles(self, setPartSqliteName, partFn,
                             doCtf=False):
        # create a set of particles

        self.partSet = SetOfParticles(filename=setPartSqliteName)
        self.partSet.setAlignment(ALIGN_PROJ)
        self.partSet.setAcquisition(Acquisition(voltage=300,
                                           sphericalAberration=2,
                                           amplitudeContrast=0.1,
                                           magnification=60000))
        self.partSet.setSamplingRate(samplingRate)
        self.partSet.setHasCTF(True)
        aList = [np.array(m) for m in mList]
        #defocus=15000 + 5000* random.random()
        for i, a in enumerate(aList):
            p = Particle()
            if doCtf:
                defocusU = defocusList[i]#+500.
                defocusV = defocusList[i]
                ctf = CTFModel(defocusU=defocusU,
                               defocusV=defocusV,
                               defocusAngle=defocusAngle[i])
                ctf.standardize()
                p.setCTF(ctf)

            p.setLocation(i + 1, partFn)
            p.setTransform(Transform(a))
            self.partSet.append(p)

        self.partSet.write()
Beispiel #3
0
    def _fillClasses(self, outputClasses):
        """ Create the SetOfClasses2D """
        inputSet = self.inputClasses.get().getImages()
        myRep = md.MetaData('classes@' +
                            self._getExtraPath('final_classes.xmd'))

        for row in md.iterRows(myRep):
            fn = row.getValue(md.MDL_IMAGE)
            rep = Particle(fn)
            repId = row.getObjId()
            newClass = Class2D(objId=repId)
            newClass.setAlignment2D()
            newClass.copyInfo(inputSet)
            newClass.setAcquisition(inputSet.getAcquisition())
            newClass.setRepresentative(rep)
            outputClasses.append(newClass)

        i = 1
        mdBlocks = md.getBlocksInMetaDataFile(
            self._getExtraPath('final_classes.xmd'))
        for block in mdBlocks:
            if block.startswith('class00'):
                mdClass = md.MetaData(block + "@" +
                                      self._getExtraPath('final_classes.xmd'))
                imgClassId = i
                newClass = outputClasses[imgClassId]
                newClass.enableAppend()
                for row in md.iterRows(mdClass):
                    part = rowToParticle(row)
                    newClass.append(part)
                i += 1
                newClass.setAlignment2D()
                outputClasses.update(newClass)
Beispiel #4
0
    def test_mrcsLink(self):
        """ In this case just a link with .mrcs extension 
        should be created
        """
        stackFile = self.dsEmx.getFile('particles/particles.mrc')
        partSet = SetOfParticles(filename=':memory:')

        for i in range(1, 10):
            particle = Particle()
            particle.setLocation(i, stackFile)
            partSet.append(particle)

        outputDir = self.getOutputPath()

        filesDict = convertBinaryFiles(partSet, outputDir)

        print filesDict
Beispiel #5
0
 def test_mrcsLink(self):
     """ In this case just a link with .mrcs extension 
     should be created
     """
     stackFile = self.dsEmx.getFile('particles/particles.mrc')
     partSet = SetOfParticles(filename=':memory:')
     
     for i in range(1, 10):
         particle = Particle()
         particle.setLocation(i, stackFile)
         partSet.append(particle)
         
     outputDir = self.getOutputPath()
     
     filesDict = convertBinaryFiles(partSet, outputDir)
     
     print filesDict        
Beispiel #6
0
    def test_particlesWithPhaseShiftToStar(self):
        """ Write a SetOfParticles to Relion star input file. """
        imgSet = SetOfParticles(filename=self.getOutputPath("particles_ph_sh.sqlite"))
        n = 10
        fn = self.getFile('particles_binary')
        ctfs = [CTFModel(defocusU=10000, defocusV=15000,
                         defocusAngle=15, phaseShift=90),
                CTFModel(defocusU=20000, defocusV=25000,
                         defocusAngle=25, phaseShift=60)
                ]
        acquisition = Acquisition(magnification=60000, voltage=300,
                                  sphericalAberration=2.,
                                  amplitudeContrast=0.07)
        imgSet.setAcquisition(acquisition)
        coord = Coordinate()
        coord.setMicId(1)

        for i in range(n):
            p = Particle()
            p.setLocation(i + 1, fn)
            ctf = ctfs[i % 2]
            p.setCTF(ctf)
            p.setAcquisition(acquisition)
            p._xmipp_zScore = Float(i)
            coord.setX(i * 10)
            coord.setY(i * 10)
            p.setCoordinate(coord)
            imgSet.append(p)

        fnStar = self.getOutputPath('particles_ph_sh.star')
        fnStk = self.getOutputPath('particles.stk')

        print (">>> Writing to file: %s" % fnStar)
        relion.writeSetOfParticles(imgSet, fnStar, fnStk)

        mdAll = md.MetaData(fnStar)
        self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
        self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
        self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
        self.assertTrue(mdAll.containsLabel(md.RLN_CTF_PHASESHIFT))
Beispiel #7
0
    def test_hdfToStk(self):
        """ In this case the hdf stack files should be converted
        to .stk spider files for Relion.
        """
        stackFiles = [
            'BPV_1386_ptcls.hdf', 'BPV_1387_ptcls.hdf', 'BPV_1388_ptcls.hdf'
        ]

        partSet = SetOfParticles(filename=':memory:')

        for fn in stackFiles:
            particle = Particle()
            particle.setLocation(1, self.ds.getFile('particles/%s' % fn))
            partSet.append(particle)

        outputDir = self.getOutputPath()

        filesDict = convertBinaryFiles(partSet, outputDir)

        partSet.close()

        print filesDict
Beispiel #8
0
 def test_hdfToStk(self):
     """ In this case the hdf stack files should be converted
     to .stk spider files for Relion.
     """
     stackFiles = ['BPV_1386_ptcls.hdf',
                   'BPV_1387_ptcls.hdf',
                   'BPV_1388_ptcls.hdf']
     
     partSet = SetOfParticles(filename=':memory:')
     
     for fn in stackFiles:
         particle = Particle()
         particle.setLocation(1, self.ds.getFile('particles/%s' % fn))
         partSet.append(particle)
         
     outputDir = self.getOutputPath()
     
     filesDict = convertBinaryFiles(partSet, outputDir)
     
     partSet.close()
     
     print filesDict
    def readPartsFromMics(self, micList, outputParts):
        """ Read the particles extract for the given list of micrographs
        and update the outputParts set with new items.
        """
        p = Particle()
        for mic in micList:
            # We need to make this dict because there is no ID in the .xmd file
            coordDict = {}
            for coord in self.coordDict[mic.getObjId()]:
                pos = self._getPos(coord)
                if pos in coordDict:
                    print(
                        "WARNING: Ignoring duplicated coordinate: %s, id=%s" %
                        (coord.getObjId(), pos))
                coordDict[pos] = coord

            added = set()  # Keep track of added coords to avoid duplicates
            for row in md.iterRows(self._getMicXmd(mic)):
                pos = (row.getValue(md.MDL_XCOOR), row.getValue(md.MDL_YCOOR))
                coord = coordDict.get(pos, None)
                if coord is not None and coord.getObjId() not in added:
                    # scale the coordinates according to particles dimension.
                    coord.scale(self.getBoxScale())
                    p.copyObjId(coord)
                    p.setLocation(xmippToLocation(row.getValue(md.MDL_IMAGE)))
                    p.setCoordinate(coord)
                    p.setMicId(mic.getObjId())
                    p.setCTF(mic.getCTF())
                    # adding the variance and Gini coeff. value of the mic zone
                    setXmippAttributes(p, row, md.MDL_SCORE_BY_VAR)
                    setXmippAttributes(p, row, md.MDL_SCORE_BY_GINI)
                    if row.containsLabel(md.MDL_ZSCORE_DEEPLEARNING1):
                        setXmippAttributes(p, row, md.MDL_ZSCORE_DEEPLEARNING1)

                    # disabled particles (in metadata) should not add to the
                    # final set
                    if row.getValue(md.MDL_ENABLED) > 0:
                        outputParts.append(p)
                        added.add(coord.getObjId())

            # Release the list of coordinates for this micrograph since it
            # will not be longer needed
            del self.coordDict[mic.getObjId()]
Beispiel #10
0
    def testOrderBy(self):
        """ create set of particles and orderby a given attribute
        """
        # This function was written by Roberto. It does things
        # differently, so let's keep it for reference.

        #create set of particles

        inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
        inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')

        imgSet = SetOfParticles(filename=inFileNameMetadata)
        imgSet.setSamplingRate(1.5)
        acq = Acquisition()
        acq.setAmplitudeContrast(0.1)
        acq.setMagnification(10000)
        acq.setVoltage(200)
        acq.setSphericalAberration(2.0)

        imgSet.setAcquisition(acq)
        img = Particle()

        for i in range(1, 10):
            img.setLocation(i, inFileNameData)
            img.setMicId(i % 3)
            img.setClassId(i % 5)
            imgSet.append(img)
            img.cleanObjId()

        imgSet.write()
        #now import the dataset
        prot1 = self.newProtocol(
            ProtImportParticles,
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata,
            magnification=10000,
            samplingRate=1.5)
        prot1.setObjLabel('from sqlite (test-sets)')
        self.launchProtocol(prot1)

        if prot1.outputParticles is None:
            raise Exception(
                'Import of images: %s, failed. outputParticles is None.' %
                inFileNameMetadata)

        protSplitSet = self.newProtocol(ProtSplitSet,
                                        inputSet=prot1.outputParticles,
                                        numberOfSets=2,
                                        randomize=True)
        self.launchProtocol(protSplitSet)

        inputSets = [
            protSplitSet.outputParticles01, protSplitSet.outputParticles02
        ]
        outputSet = SetOfParticles(
            filename=self.proj.getTmpPath('gold.sqlite'))
        for itemSet in inputSets:
            for obj in itemSet:
                outputSet.append(obj)

        for item1, item2 in izip(imgSet, outputSet):
            if not item1.equalAttributes(item2):
                print "Items differ:"
                prettyDict(item1.getObjDict())
                prettyDict(item2.getObjDict())
            self.assertTrue(item1.equalAttributes(item2), )
Beispiel #11
0
    def testMergeDifferentAttrs(self):
        """ Test merge from subsets with different attritubes.
        That is, M1(a,b,c) U M2(a,b,c,d)"""

        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        attrb3 = [31, 32]
        counter = 0
        # Test the join handles different attributes at a second level
        ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
        ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
        ctf2._myOwnQuality = Float(1.)
        img1.setCTF(ctf1)
        img2.setCTF(ctf2)

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            img1._attrb3 = Float(attrb3[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter += 1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set1',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata1,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set2',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata2,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                                   objLabel='join different attrs',
                                   ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)

        counter = 0

        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            self.assertFalse(hasattr(img, '_attrb3'),
                             "join should not have attrb3")
            self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
            ctf = img.getCTF()
            self.assertIsNotNone(ctf, "Image should have CTF after join")
            self.assertFalse(hasattr(ctf, '_myOwnQuality'),
                             "CTF should not have non common attributes")
            counter += 1
Beispiel #12
0
    def testMergeAlternateColumn(self):
        """Test that the union operation works as expected.
        Even if the order of the columns do not match.
        That is, M1(a,b,c) U M2(a,c,b)"""
        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        counter = 0

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter += 1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set1',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata1,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set2',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata2,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                                   objLabel='join diff column order',
                                   ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)
        #assert
        counter = 0
        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            counter += 1
Beispiel #13
0
def _particlesFromEmx(protocol,
                      emxData,
                      emxFile,
                      outputDir,
                      acquisition,
                      samplingRate,
                      copyOrLink,
                      alignType=ALIGN_NONE):
    """ Create the output SetOfCoordinates or SetOfParticles given an EMXData object.
    Add CTF information to the particles if present.
    """
    emxParticle = emxData.getFirstObject(emxlib.PARTICLE)
    partDir = dirname(emxFile)
    micSet = getattr(protocol, 'outputMicrographs', None)

    if emxParticle is not None:
        # Check if there are particles or coordinates
        fn = emxParticle.get(emxlib.FILENAME)
        if exists(join(
                partDir,
                fn)):  # if the particles has binary data, means particles case
            partSet = protocol._createSetOfParticles()
            partSet.setAcquisition(
                acquisition)  # this adquisition is per project
            #we need one per particle

            part = Particle()

            if _hasCtfLabels(emxParticle) or _hasCtfLabels(
                    emxParticle.getMicrograph()):
                part.setCTF(CTFModel())
                partSet.setHasCTF(True)
            if emxParticle.has('centerCoord__X'):
                part.setCoordinate(Coordinate())
            #if emxParticle.has('transformationMatrix__t11'):
            #    _particleFromEmx(emxParticle, part)
            #    #partSet.setAlignment3D()
            #    partSet.setAlignment(alignType)
            #else:
            #    partSet.setAlignment(alignType)
            partSet.setAlignment(alignType)
            if not samplingRate:
                samplingRate = part.getSamplingRate()
            partSet.setSamplingRate(samplingRate)
            partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
            particles = True
        else:  # if not binary data, the coordinate case
            if micSet is None:
                raise Exception(
                    'Could not import Coordinates from EMX, micrographs not imported.'
                )
            partSet = protocol._createSetOfCoordinates(micSet)
            part = Coordinate()
            particles = False

        copiedFiles = {}  # copied or linked

        for emxParticle in emxData.iterClasses(emxlib.PARTICLE):
            if particles:
                _particleFromEmx(emxParticle, part)
                i, fn = part.getLocation()
                partFn = join(partDir, fn)
                newFn = join(outputDir, basename(partFn))
                newLoc = (i, newFn)

                if not partFn in copiedFiles:
                    copyOrLink(partFn, newFn)
                    copiedFiles[partFn] = newFn

                part.setLocation(newLoc)

                if partSet.hasAlignment():
                    transform = Transform()
                    _transformFromEmx(emxParticle, part, transform, alignType)
                    part.setTransform(transform)

            else:
                _coordinateFromEmx(emxParticle, part)

            partSet.append(part)
            part.cleanObjId()
        if particles:
            protocol._defineOutputs(outputParticles=partSet)
        else:
            protocol._defineOutputs(outputCoordinates=partSet)

        if micSet is not None:
            protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
Beispiel #14
0
    def launchTest(self, fileKey, mList, alignType=None, **kwargs):
        """ Helper function to launch similar alignment tests
        give the EMX transformation matrix.
        Params:
            fileKey: the file where to grab the input stack images.
            mList: the matrix list of transformations
                (should be the same length of the stack of images)
        """
        print ("\n")
        print ("*" * 80)
        print ("* Launching test: ", fileKey)
        print ("*" * 80)

        is2D = alignType == ALIGN_2D

        stackFn = self.dataset.getFile(fileKey)
        partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
        mdFn = self.getOutputPath(fileKey + "_particles.star")
        partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")

        if self.IS_ALIGNMENT:
            outputFn = self.getOutputPath(fileKey + "_output.mrcs")
            outputFnRelion = self.getOutputPath(fileKey + "_output")
            goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
        else:
            outputFn = self.getOutputPath(fileKey + "_output.vol")
            goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')

        if PRINT_FILES:
            print("BINARY DATA: ", stackFn)
            print("SET1:        ", partFn1)
            print("  MD:        ", mdFn)
            print("SET2:        ", partFn2)
            print("OUTPUT:      ", outputFn)
            print("GOLD:        ", goldFn)

        if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
            partSet = SetOfParticles(filename=partFn1)
        else:
            partSet = SetOfVolumes(filename=partFn1)
        partSet.setAlignment(alignType)
        partSet.setAcquisition(Acquisition(voltage=300,
                                           sphericalAberration=2,
                                           amplitudeContrast=0.1,
                                           magnification=60000))
        # Populate the SetOfParticles with  images
        # taken from images.mrc file
        # and setting the previous alignment parameters
        aList = [numpy.array(m) for m in mList]
        for i, a in enumerate(aList):
            p = Particle()
            p.setLocation(i + 1, stackFn)
            p.setTransform(Transform(a))
            partSet.append(p)
        # Write out the .sqlite file and check that are correctly aligned
        print ("Parset", partFn1)
        partSet.printAll()
        partSet.write()
        # Convert to a Xmipp metadata and also check that the images are
        # aligned correctly
        if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
            relion.writeSetOfParticles(partSet, mdFn,"/tmp", alignType=alignType)
            partSet2 = SetOfParticles(filename=partFn2)
        else:
            relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
            partSet2 = SetOfVolumes(filename=partFn2)
        # Let's create now another SetOfImages reading back the written
        # Xmipp metadata and check one more time.
        partSet2.copyInfo(partSet)
        if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
            relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
        else:
            relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)

        partSet2.write()

        if PRINT_MATRIX:
            for i, img in enumerate(partSet2):
                m1 = aList[i]
                m2 = img.getTransform().getMatrix()
                print ("-" * 5)
                print (img.getFileName(), img.getIndex())
                print ('m1:\n', m1, relion.geometryFromMatrix(m1, False))

                print ('m2:\n', m2, relion.geometryFromMatrix(m2, False))
                # self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))

        # Launch apply transformation and check result images
        runRelionProgram(self.CMD % locals())

        if SHOW_IMAGES:
            runRelionProgram('scipion show %(outputFn)s' % locals())

        if os.path.exists(goldFn):
            self.assertTrue(
                ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
                "Different data files:\n>%s\n<%s" % (goldFn, outputFn))
    def testOrderBy(self):
        """ create set of particles and orderby a given attribute
        """
        # This function was written by Roberto. It does things
        # differently, so let's keep it for reference.

        #create set of particles

        inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
        inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')

        imgSet = SetOfParticles(filename=inFileNameMetadata)
        imgSet.setSamplingRate(1.5)
        acq = Acquisition()
        acq.setAmplitudeContrast(0.1)
        acq.setMagnification(10000)
        acq.setVoltage(200)
        acq.setSphericalAberration(2.0)
        
        imgSet.setAcquisition(acq)
        img = Particle()

        for i in range(1, 10):
            img.setLocation(i, inFileNameData)
            img.setMicId(i%3)
            img.setClassId(i%5)
            imgSet.append(img)
            img.cleanObjId()

        imgSet.write()
        #now import the dataset
        prot1 = self.newProtocol(ProtImportParticles,
                                 importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                 sqliteFile=inFileNameMetadata,
                                 magnification=10000,
                                 samplingRate=1.5
                                 )
        prot1.setObjLabel('from sqlite (test-sets)')
        self.launchProtocol(prot1)

        if prot1.outputParticles is None:
            raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata)
        
        protSplitSet   = self.newProtocol(ProtSplitSet,
                                          inputSet=prot1.outputParticles,
                                          numberOfSets=2,
                                          randomize=True)
        self.launchProtocol(protSplitSet)

        inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02]
        outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite'))
        for itemSet in inputSets:
            for obj in itemSet:
                outputSet.append(obj)

        for item1, item2 in izip(imgSet, outputSet):
            if not item1.equalAttributes(item2):
                print "Items differ:"
                prettyDict(item1.getObjDict())
                prettyDict(item2.getObjDict())
            self.assertTrue(item1.equalAttributes(item2),  )
Beispiel #16
0
    def launchTest(self, fileKey, mList, alignType=None, **kwargs):
        """ Helper function to launch similar alignment tests
        give the EMX transformation matrix.
        Params:
            fileKey: the file where to grab the input stack images.
            mList: the matrix list of transformations
                (should be the same length of the stack of images)
        """
        print "\n"
        print "*" * 80
        print "* Launching test: ", fileKey
        print "*" * 80

        is2D = alignType == ALIGN_2D

        stackFn = self.dataset.getFile(fileKey)
        partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
        mdFn = self.getOutputPath(fileKey + "_particles.star")
        partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")

        if self.IS_ALIGNMENT:
            outputFn = self.getOutputPath(fileKey + "_output.mrcs")
            outputFnRelion = self.getOutputPath(fileKey + "_output")
            goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
        else:
            outputFn = self.getOutputPath(fileKey + "_output.vol")
            goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')

        if PRINT_FILES:
            print "BINARY DATA: ", stackFn
            print "SET1:        ", partFn1
            print "  MD:        ", mdFn
            print "SET2:        ", partFn2
            print "OUTPUT:      ", outputFn
            print "GOLD:        ", goldFn

        if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
            partSet = SetOfParticles(filename=partFn1)
        else:
            partSet = SetOfVolumes(filename=partFn1)
        partSet.setAlignment(alignType)
        partSet.setAcquisition(
            Acquisition(voltage=300,
                        sphericalAberration=2,
                        amplitudeContrast=0.1,
                        magnification=60000))
        # Populate the SetOfParticles with  images
        # taken from images.mrc file
        # and setting the previous alignment parameters
        aList = [numpy.array(m) for m in mList]
        for i, a in enumerate(aList):
            p = Particle()
            p.setLocation(i + 1, stackFn)
            p.setTransform(Transform(a))
            partSet.append(p)
        # Write out the .sqlite file and check that are correctly aligned
        print "Parset", partFn1
        partSet.printAll()
        partSet.write()
        # Convert to a Xmipp metadata and also check that the images are
        # aligned correctly
        if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
            relion.writeSetOfParticles(partSet,
                                       mdFn,
                                       "/tmp",
                                       alignType=alignType)
            partSet2 = SetOfParticles(filename=partFn2)
        else:
            relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
            partSet2 = SetOfVolumes(filename=partFn2)
        # Let's create now another SetOfImages reading back the written
        # Xmipp metadata and check one more time.
        partSet2.copyInfo(partSet)
        if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
            relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
        else:
            relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)

        partSet2.write()

        if PRINT_MATRIX:
            for i, img in enumerate(partSet2):
                m1 = aList[i]
                m2 = img.getTransform().getMatrix()
                print "-" * 5
                print img.getFileName(), img.getIndex()
                print 'm1:\n', m1, relion.geometryFromMatrix(m1, False)

                print 'm2:\n', m2, relion.geometryFromMatrix(m2, False)
                # self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))

        # Launch apply transformation and check result images
        runRelionProgram(self.CMD % locals())

        if SHOW_IMAGES:
            runRelionProgram('scipion show %(outputFn)s' % locals())

        if os.path.exists(goldFn):
            self.assertTrue(
                ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
                "Different data files:\n>%s\n<%s" % (goldFn, outputFn))
Beispiel #17
0
    def testMergeAlternateColumn(self):
        """Test that the union operation works as expected.
        Even if the order of the columns do not match.
        That is, M1(a,b,c) U M2(a,c,b)"""
        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        counter = 0

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            imgSet1.append(img1)
            counter +=1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1= Float(attrb1[counter])
            img2._attrb2= Float(attrb2[counter])
            imgSet2.append(img2)
            counter +=1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set1',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata1,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set2',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata2,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                       objLabel='join diff column order',
                       ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)
        #assert
        counter=0
        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter],img._attrb1,4)
            self.assertAlmostEqual(attrb2[counter],img._attrb2,4)
            counter += 1
Beispiel #18
0
def _particlesFromEmx(protocol
                      , emxData
                      , emxFile
                      , outputDir
                      , acquisition
                      , samplingRate
                      , copyOrLink
                      , alignType=ALIGN_NONE):
    """ Create the output SetOfCoordinates or SetOfParticles given an EMXData object.
    Add CTF information to the particles if present.
    """
    emxParticle = emxData.getFirstObject(emxlib.PARTICLE)
    partDir = dirname(emxFile)
    micSet = getattr(protocol, 'outputMicrographs', None)

    if emxParticle is not None:     
        # Check if there are particles or coordinates
        fn = emxParticle.get(emxlib.FILENAME)
        if exists(join(partDir, fn)): # if the particles has binary data, means particles case
            partSet = protocol._createSetOfParticles()
            partSet.setAcquisition(acquisition)# this adquisition is per project
                                            #we need one per particle
            
            part = Particle()

            if _hasCtfLabels(emxParticle) or _hasCtfLabels(emxParticle.getMicrograph()):
                part.setCTF(CTFModel())
                partSet.setHasCTF(True)
            if emxParticle.has('centerCoord__X'):
                part.setCoordinate(Coordinate())
            #if emxParticle.has('transformationMatrix__t11'):
            #    _particleFromEmx(emxParticle, part)
            #    #partSet.setAlignment3D()
            #    partSet.setAlignment(alignType)
            #else:
            #    partSet.setAlignment(alignType)
            partSet.setAlignment(alignType)
            if not samplingRate:
                samplingRate = part.getSamplingRate()
            partSet.setSamplingRate(samplingRate) 
            partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
            particles = True
        else: # if not binary data, the coordinate case
            if micSet is None:
                raise Exception('Could not import Coordinates from EMX, micrographs not imported.')
            partSet = protocol._createSetOfCoordinates(micSet)
            part = Coordinate()
            particles = False

        copiedFiles = {} # copied or linked


        for emxParticle in emxData.iterClasses(emxlib.PARTICLE):
            if particles:
                _particleFromEmx(emxParticle, part)
                i, fn = part.getLocation()
                partFn = join(partDir, fn)
                newFn = join(outputDir, basename(partFn))
                newLoc = (i, newFn)
                
                if not partFn in copiedFiles:
                    copyOrLink(partFn, newFn)
                    copiedFiles[partFn] = newFn
                    
                part.setLocation(newLoc)
                
                if partSet.hasAlignment():
                    transform = Transform()
                    _transformFromEmx(emxParticle, part, transform, alignType)
                    part.setTransform(transform)

            else:
                _coordinateFromEmx(emxParticle, part)
                
            partSet.append(part)
            part.cleanObjId()
        if particles:
            protocol._defineOutputs(outputParticles=partSet)
        else:
            protocol._defineOutputs(outputCoordinates=partSet)
        
        if micSet is not None:
            protocol._defineSourceRelation(protocol.outputMicrographs, 
                                           partSet)
    def _checkNewItems(self, objSet):
        """ Check for already computed micrograph/movie and
        update the output set. """
        objDict = {}
        newObj = False
        for obj in objSet:
            objDict[obj.getFileName()] = True

        if objDict:
            if objSet.getSize():
                objSet.enableAppend()
                objSet.loadAllProperties()
        else:
            objSet.setStreamState(objSet.STREAM_OPEN)
            acquisition = Acquisition()
            if self.setof == SET_OF_MICROGRAPHS:
                acquisition.setMagnification(
                    self.inputMics.get().getAcquisition().getMagnification())
                acquisition.setVoltage(
                    self.inputMics.get().getAcquisition().getVoltage())
                acquisition.setSphericalAberration(self.inputMics.get(
                ).getAcquisition().getSphericalAberration())
                acquisition.setAmplitudeContrast(self.inputMics.get(
                ).getAcquisition().getAmplitudeContrast())
                objSet.setAcquisition(acquisition)
                objSet.setSamplingRate(self.inputMics.get().getSamplingRate())
            elif self.setof == SET_OF_MOVIES:
                acquisition.setMagnification(
                    self.inputMovies.get().getAcquisition().getMagnification())
                acquisition.setVoltage(
                    self.inputMovies.get().getAcquisition().getVoltage())
                acquisition.setSphericalAberration(self.inputMovies.get(
                ).getAcquisition().getSphericalAberration())
                acquisition.setAmplitudeContrast(self.inputMovies.get(
                ).getAcquisition().getAmplitudeContrast())
                objSet.setAcquisition(acquisition)
                objSet.setSamplingRate(
                    self.inputMovies.get().getSamplingRate())
            else:
                acquisition.setMagnification(self._magnification)
                acquisition.setVoltage(self._voltage)
                acquisition.setSphericalAberration(self._sphericalAberration)
                acquisition.setAmplitudeContrast(self._amplitudeContrast)
                objSet.setAcquisition(acquisition)
                if self.setof == SET_OF_PARTICLES:
                    objSet.setSamplingRate(
                        self.inputParticles.get().getSamplingRate())
                else:
                    objSet.setSamplingRate(self.samplingRate.get())

        if self.setof == SET_OF_MOVIES:
            obj = Movie()
        elif self.setof == SET_OF_MICROGRAPHS:
            obj = Micrograph()
        elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
            obj = Micrograph()
        elif self.setof == SET_OF_PARTICLES:
            obj = Particle()
        else:
            raise Exception('Unknown data type')

        for k, v in self.dictObj.iteritems():
            if (k not in objDict):
                self.counter += 1
                obj.setFileName(k)
                if self.setof != SET_OF_PARTICLES:
                    obj.setMicName(basename(k))
                obj.setObjId(self.counter)
                objSet.append(obj)
                newObj = True

        return objSet, newObj  # why a dictionary, a boolean may be enough
Beispiel #20
0
    def testMergeDifferentAttrs(self):
        """ Test merge from subsets with different attritubes.
        That is, M1(a,b,c) U M2(a,b,c,d)"""

        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        attrb3 = [31, 32]
        counter = 0
        # Test the join handles different attributes at a second level
        ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
        ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
        ctf2._myOwnQuality = Float(1.)
        img1.setCTF(ctf1)
        img2.setCTF(ctf2)

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            img1._attrb3 = Float(attrb3[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter +=1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set1',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata1,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set2',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata2,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                       objLabel='join different attrs',
                       ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)

        counter = 0

        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            self.assertFalse(hasattr(img, '_attrb3'),
                             "join should not have attrb3")
            self.assertTrue(hasattr(img, '_attrb2'),
                            "join should have attrb2")
            ctf = img.getCTF()
            self.assertIsNotNone(ctf, "Image should have CTF after join")
            self.assertFalse(hasattr(ctf, '_myOwnQuality'),
                             "CTF should not have non common attributes")
            counter += 1
    def testMergeDifferentAttrs(self):
        """ Test merge from subsets with different attritubes.
        That is, M1(a,b,c) U M2(a,b,c,d)"""

        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        attrb3 = [31, 32]
        counter = 0
        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            img1._attrb3 = Float(attrb3[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter += 1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set1',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata1,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set2',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata2,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                                   objLabel='join diff column order',
                                   ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)
        #assert
        counter = 0
        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            if hasattr(img, '_attrb3'):
                self.assertTrue(False, "join should not have attrb3")
            if not hasattr(img, '_attrb2'):
                self.assertTrue(False, "join should have attrb2")
            counter += 1
Beispiel #22
0
    def aaatest_particlesToStar(self):
        """ Write a SetOfParticles to Relion star input file. """
        imgSet = SetOfParticles(
            filename=self.getOutputPath("particles.sqlite"))
        n = 10
        fn = self.particles
        ctfs = [
            CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
            CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
        ]
        acquisition = Acquisition(magnification=60000,
                                  voltage=300,
                                  sphericalAberration=2.,
                                  amplitudeContrast=0.07)
        imgSet.setAcquisition(acquisition)
        coord = Coordinate()
        coord.setMicId(1)

        for i in range(n):
            p = Particle()
            p.setLocation(i + 1, fn)
            ctf = ctfs[i % 2]
            p.setCTF(ctf)
            p.setAcquisition(acquisition)
            p._xmipp_zScore = Float(i)
            coord.setX(i * 10)
            coord.setY(i * 10)
            p.setCoordinate(coord)
            imgSet.append(p)

        fnStar = self.getOutputPath('particles.star')
        fnStk = self.getOutputPath('particles.stk')

        print ">>> Writing to file: %s" % fnStar
        relion.writeSetOfParticles(imgSet, fnStar, fnStk)

        mdAll = md.MetaData(fnStar)
        self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
        self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
        self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))