Beispiel #1
0
    def testMergeDifferentAttrs(self):
        """ Test merge from subsets with different attritubes.
        That is, M1(a,b,c) U M2(a,b,c,d)"""

        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        attrb3 = [31, 32]
        counter = 0
        # Test the join handles different attributes at a second level
        ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
        ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
        ctf2._myOwnQuality = Float(1.)
        img1.setCTF(ctf1)
        img2.setCTF(ctf2)

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            img1._attrb3 = Float(attrb3[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter += 1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set1',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata1,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set2',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata2,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                                   objLabel='join different attrs',
                                   ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)

        counter = 0

        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            self.assertFalse(hasattr(img, '_attrb3'),
                             "join should not have attrb3")
            self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
            ctf = img.getCTF()
            self.assertIsNotNone(ctf, "Image should have CTF after join")
            self.assertFalse(hasattr(ctf, '_myOwnQuality'),
                             "CTF should not have non common attributes")
            counter += 1
Beispiel #2
0
    def testOrderBy(self):
        """ create set of particles and orderby a given attribute
        """
        # This function was written by Roberto. It does things
        # differently, so let's keep it for reference.

        #create set of particles

        inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
        inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')

        imgSet = SetOfParticles(filename=inFileNameMetadata)
        imgSet.setSamplingRate(1.5)
        acq = Acquisition()
        acq.setAmplitudeContrast(0.1)
        acq.setMagnification(10000)
        acq.setVoltage(200)
        acq.setSphericalAberration(2.0)

        imgSet.setAcquisition(acq)
        img = Particle()

        for i in range(1, 10):
            img.setLocation(i, inFileNameData)
            img.setMicId(i % 3)
            img.setClassId(i % 5)
            imgSet.append(img)
            img.cleanObjId()

        imgSet.write()
        #now import the dataset
        prot1 = self.newProtocol(
            ProtImportParticles,
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata,
            magnification=10000,
            samplingRate=1.5)
        prot1.setObjLabel('from sqlite (test-sets)')
        self.launchProtocol(prot1)

        if prot1.outputParticles is None:
            raise Exception(
                'Import of images: %s, failed. outputParticles is None.' %
                inFileNameMetadata)

        protSplitSet = self.newProtocol(ProtSplitSet,
                                        inputSet=prot1.outputParticles,
                                        numberOfSets=2,
                                        randomize=True)
        self.launchProtocol(protSplitSet)

        inputSets = [
            protSplitSet.outputParticles01, protSplitSet.outputParticles02
        ]
        outputSet = SetOfParticles(
            filename=self.proj.getTmpPath('gold.sqlite'))
        for itemSet in inputSets:
            for obj in itemSet:
                outputSet.append(obj)

        for item1, item2 in izip(imgSet, outputSet):
            if not item1.equalAttributes(item2):
                print "Items differ:"
                prettyDict(item1.getObjDict())
                prettyDict(item2.getObjDict())
            self.assertTrue(item1.equalAttributes(item2), )
Beispiel #3
0
    def testMergeAlternateColumn(self):
        """Test that the union operation works as expected.
        Even if the order of the columns do not match.
        That is, M1(a,b,c) U M2(a,c,b)"""
        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        counter = 0

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter += 1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set1',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata1,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set2',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata2,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                                   objLabel='join diff column order',
                                   ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)
        #assert
        counter = 0
        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            counter += 1
    def testOrderBy(self):
        """ create set of particles and orderby a given attribute
        """
        # This function was written by Roberto. It does things
        # differently, so let's keep it for reference.

        #create set of particles

        inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
        inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')

        imgSet = SetOfParticles(filename=inFileNameMetadata)
        imgSet.setSamplingRate(1.5)
        acq = Acquisition()
        acq.setAmplitudeContrast(0.1)
        acq.setMagnification(10000)
        acq.setVoltage(200)
        acq.setSphericalAberration(2.0)
        
        imgSet.setAcquisition(acq)
        img = Particle()

        for i in range(1, 10):
            img.setLocation(i, inFileNameData)
            img.setMicId(i%3)
            img.setClassId(i%5)
            imgSet.append(img)
            img.cleanObjId()

        imgSet.write()
        #now import the dataset
        prot1 = self.newProtocol(ProtImportParticles,
                                 importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                 sqliteFile=inFileNameMetadata,
                                 magnification=10000,
                                 samplingRate=1.5
                                 )
        prot1.setObjLabel('from sqlite (test-sets)')
        self.launchProtocol(prot1)

        if prot1.outputParticles is None:
            raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata)
        
        protSplitSet   = self.newProtocol(ProtSplitSet,
                                          inputSet=prot1.outputParticles,
                                          numberOfSets=2,
                                          randomize=True)
        self.launchProtocol(protSplitSet)

        inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02]
        outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite'))
        for itemSet in inputSets:
            for obj in itemSet:
                outputSet.append(obj)

        for item1, item2 in izip(imgSet, outputSet):
            if not item1.equalAttributes(item2):
                print "Items differ:"
                prettyDict(item1.getObjDict())
                prettyDict(item2.getObjDict())
            self.assertTrue(item1.equalAttributes(item2),  )
Beispiel #5
0
    def testMergeDifferentAttrs(self):
        """ Test merge from subsets with different attritubes.
        That is, M1(a,b,c) U M2(a,b,c,d)"""

        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        attrb3 = [31, 32]
        counter = 0
        # Test the join handles different attributes at a second level
        ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
        ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
        ctf2._myOwnQuality = Float(1.)
        img1.setCTF(ctf1)
        img2.setCTF(ctf2)

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            img1._attrb3 = Float(attrb3[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter +=1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set1',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata1,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set2',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata2,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                       objLabel='join different attrs',
                       ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)

        counter = 0

        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            self.assertFalse(hasattr(img, '_attrb3'),
                             "join should not have attrb3")
            self.assertTrue(hasattr(img, '_attrb2'),
                            "join should have attrb2")
            ctf = img.getCTF()
            self.assertIsNotNone(ctf, "Image should have CTF after join")
            self.assertFalse(hasattr(ctf, '_myOwnQuality'),
                             "CTF should not have non common attributes")
            counter += 1
Beispiel #6
0
    def testMergeAlternateColumn(self):
        """Test that the union operation works as expected.
        Even if the order of the columns do not match.
        That is, M1(a,b,c) U M2(a,c,b)"""
        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        counter = 0

        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            imgSet1.append(img1)
            counter +=1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1= Float(attrb1[counter])
            img2._attrb2= Float(attrb2[counter])
            imgSet2.append(img2)
            counter +=1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set1',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata1,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(ProtImportParticles,
                                      objLabel='import set2',
                                      importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
                                      sqliteFile=inFileNameMetadata2,
                                      magnification=10000,
                                      samplingRate=7.08,
                                      haveDataBeenPhaseFlipped=True
                                      )
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                       objLabel='join diff column order',
                       ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)
        #assert
        counter=0
        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter],img._attrb1,4)
            self.assertAlmostEqual(attrb2[counter],img._attrb2,4)
            counter += 1
    def testMergeDifferentAttrs(self):
        """ Test merge from subsets with different attritubes.
        That is, M1(a,b,c) U M2(a,b,c,d)"""

        #create two set of particles
        inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
        inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
        imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
        imgSet2 = SetOfParticles(filename=inFileNameMetadata2)

        inFileNameData = self.proj.getTmpPath('particles.stk')
        img1 = Particle()
        img2 = Particle()
        attrb1 = [11, 12, 13, 14]
        attrb2 = [21, 22, 23, 24]
        attrb3 = [31, 32]
        counter = 0
        for i in range(1, 3):
            img1.cleanObjId()
            img1.setLocation(i, inFileNameData)
            img1.setMicId(i % 3)
            img1.setClassId(i % 5)
            img1.setSamplingRate(1.)
            img1._attrb1 = Float(attrb1[counter])
            img1._attrb2 = Float(attrb2[counter])
            img1._attrb3 = Float(attrb3[counter])
            imgSet1.append(img1)
            counter += 1

        for i in range(1, 3):
            img2.cleanObjId()
            img2.setLocation(i, inFileNameData)
            img2.setClassId(i % 5)
            img2.setMicId(i % 3)
            img2.setSamplingRate(2.)
            img2._attrb1 = Float(attrb1[counter])
            img2._attrb2 = Float(attrb2[counter])
            imgSet2.append(img2)
            counter += 1

        imgSet1.write()
        imgSet2.write()

        #import them
        protImport1 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set1',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata1,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport1)

        protImport2 = self.newProtocol(
            ProtImportParticles,
            objLabel='import set2',
            importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
            sqliteFile=inFileNameMetadata2,
            magnification=10000,
            samplingRate=7.08,
            haveDataBeenPhaseFlipped=True)
        self.launchProtocol(protImport2)

        #create merge protocol
        p_union = self.newProtocol(ProtUnionSet,
                                   objLabel='join diff column order',
                                   ignoreExtraAttributes=True)
        p_union.inputSets.append(protImport1.outputParticles)
        p_union.inputSets.append(protImport2.outputParticles)
        self.proj.launchProtocol(p_union, wait=True)
        #assert
        counter = 0
        for img in p_union.outputSet:
            self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
            self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
            if hasattr(img, '_attrb3'):
                self.assertTrue(False, "join should not have attrb3")
            if not hasattr(img, '_attrb2'):
                self.assertTrue(False, "join should have attrb2")
            counter += 1