Example #1
0
    def testListFitting(self):
        true1, data1 = self.generate_data()
        true2, data2 = self.generate_data()

        model = DPMixtureModel(3, 2000, 100, 1, type='BEM')
        rs = model.fit([data1, data2])
        assert (len(rs) == 2)
        for r in rs:
            print 'mu ', r.mus
            diffs = {}
            for i in gen_mean:

                diffs[i] = np.min(np.abs(r.mus - gen_mean[i]), 0)
                # print i, gen_mean[i], diffs[i], np.vdot(diffs[i],diffs[i])
                assert (np.vdot(diffs[i], diffs[i]) < 2)

        fcm1 = FCMdata('test_fcm1', data1, ['fsc', 'ssc'], [0, 1])
        fcm2 = FCMdata('test_fcm2', data2, ['fsc', 'ssc'], [0, 1])

        c = FCMcollection('fcms', [fcm1, fcm2])

        rs = model.fit(c)
        assert (len(rs) == 2)
        for r in rs:
            print 'mu ', r.mus
            diffs = {}
            for i in gen_mean:

                diffs[i] = np.min(np.abs(r.mus - gen_mean[i]), 0)
                # print i, gen_mean[i], diffs[i], np.vdot(diffs[i],diffs[i])
                assert (np.vdot(diffs[i], diffs[i]) < 4)
 def testBoundaryEvents(self):
     pnts = array([[0, 1, 2], [3, 4, 5], [0, 2, 5]])
     fcm = FCMdata('test_fcm', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
     eps = 1e-10
     result = fcm.boundary_events()
     assert result['fsc'] - 1 < eps
     assert result['ssc'] - 2.0 / 3.0 < eps
     assert result['cd3'] - 1 < eps
Example #3
0
 def testBoundaryEvents(self):
     pnts = array([[0,1,2],[3,4,5],[0,2,5]])
     fcm = FCMdata('test_fcm', pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
     eps = 1e-10
     result = fcm.boundary_events()
 
     assert result['fsc'] - 1 < eps, str(result['fsc'])
     assert result['ssc'] - 2.0/3.0 < eps
     assert result['cd3'] - 1 < eps
Example #4
0
    def testCheckNames(self):
        pnts = array([[1, 1, 1], [5, 5, 5]])
        fcm1 = FCMdata('test_fcm1', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
        fcm2 = FCMdata('test_fcm2', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
        fcm3 = FCMdata('test_fcm3', pnts, ['fsc', 'ssc', 'cd4'], [0, 1])

        fcms1 = FCMcollection('fcms1', [fcm1, fcm2])
        fcms2 = FCMcollection('fcms2', [fcm1, fcm2, fcm3, fcms1])

        check1 = fcms1.check_names()
        assert check1[fcms1.name] == [True, True, True]
        check2 = fcms2.check_names()
        assert check2[fcms2.name] == [check1, True, True, False]
Example #5
0
class SubsampleTestCase(unittest.TestCase):
    def setUp(self):
        self.pnts = array([[0,1,2],[3,4,5]])
        self.fcm = FCMdata('test_fcm', self.pnts, ['fsc','ssc','cd3'], [0,1])
        self.samp = SubsampleFactory[:,1]

    def testSubSample(self):
        self.samp.subsample(self.fcm)
        assert self.fcm[0] == 1, 'subsample via subsample failed'
        assert self.fcm[1] == 4, 'subsample via subsample failed'
        self.fcm.visit('root')
        self.fcm.subsample(self.samp)
        assert self.fcm[0] == 1, 'subsample via fcm failed'
        assert self.fcm[1] == 4, 'subsample via fcm failed'
        
    def testChainSubSample(self):
        self.fcm.visit('root')
        sam2 = SubsampleFactory[0]
        self.fcm.subsample(self.samp).subsample(sam2)
        assert self.fcm.view() == 1, 'subsample chaining failed'
Example #6
0
class SubsampleTestCase(unittest.TestCase):
    def setUp(self):
        self.pnts = array([[0, 1, 2], [3, 4, 5]])
        self.fcm = FCMdata('test_fcm', self.pnts, ['fsc', 'ssc', 'cd3'],
                           [0, 1])
        self.samp = SubsampleFactory[:, 1]

    def testSubSample(self):
        self.samp.subsample(self.fcm)
        assert self.fcm[0] == 1, 'subsample via subsample failed'
        assert self.fcm[1] == 4, 'subsample via subsample failed'
        self.fcm.visit('root')
        self.fcm.subsample(self.samp)
        assert self.fcm[0] == 1, 'subsample via fcm failed'
        assert self.fcm[1] == 4, 'subsample via fcm failed'

    def testChainSubSample(self):
        self.fcm.visit('root')
        sam2 = SubsampleFactory[0]
        self.fcm.subsample(self.samp).subsample(sam2)
        assert self.fcm.view() == 1, 'subsample chaining failed'
Example #7
0
 def setUp(self):
     self.pnts = array([[0,1,2],[3,4,5]])
     self.fcm = FCMdata('test_fcm', self.pnts, ['fsc','ssc','cd3'], [0,1])
     self.samp = SubsampleFactory[:,1]
Example #8
0
    def get_FCMdata(self, auto_comp=False, **kwargs):
        """Return the next FCM data set stored in a FCS file"""

        # parse headers
        header = self.parse_header(self.cur_offset)

        # parse text
        text = self.parse_text(self.cur_offset, header['text_start'],
                               header['text_stop'])

        # parse annalysis
        try:
            astart = text['beginanalysis']
        except KeyError:
            astart = header['analysis_start']
        try:
            astop = text['endanalysis']
        except KeyError:
            astop = header['analysis_end']
        analysis = self.parse_analysis(self.cur_offset, astart, astop)
        # parse data
        try:
            dstart = int(text['begindata'])
        except KeyError:
            dstart = header['data_start']
        try:
            dstop = int(text['enddata'])
        except KeyError:
            dstop = header['data_end']

        # account for LMD reporting the wrong values for the size of the data
        # segment
        lmd = self.fix_lmd(self.cur_offset, header['text_start'],
                           header['text_stop'])
        dstop = dstop + lmd
        data = self.parse_data(self.cur_offset, dstart, dstop, text)

        # build fcmdata object
        channels = []
        scchannels = []
        scchannel_indexes = []
        to_transform = []
        base_chan_name = []
        for i in range(1, int(text['par']) + 1):
            base_chan_name.append(text['p%dn' % i])
            try:
                if text['p%ds' % i] not in ['', ' ']:
                    name = text['p%ds' % i]
                else:
                    name = text['p%dn' % i]
            except KeyError:
                name = text['p%dn' % i]
            channels.append(name)
            # if not name.lower().startswith('fl'):
            if not is_fl_channel(name):
                scchannels.append(name)
                if name != 'Time':
                    scchannel_indexes.append(i - 1)
            else:  # we're a FL channel
                try:
                    if text['p%dr' % i] == '262144':
                        to_transform.append(i - 1)
                except KeyError:
                    pass

        try:
            unused_path, name = os.path.split(self._fh.name)
        except AttributeError:
            name = 'InMemoryFile'
        name, unused_ext = os.path.splitext(name)
        tmpfcm = FCMdata(
            name, data, zip(base_chan_name, channels), scchannels,
            Annotation({
                'text': text,
                'header': header,
                'analysis': analysis,
            }))
        if self.sidx is not None and self.spill is not None:
            # if we're passed a spillover we assume we compensate
            auto_comp = True
        if auto_comp:
            if self.sidx is None and self.spill is None:
                if tmpfcm.get_spill():
                    spill, sidx = get_spill(tmpfcm.get_spill())
                    tmpfcm.compensate(sidx=sidx, spill=spill)
            else:
                tmpfcm.compensate(sidx=self.sidx, spill=self.spill)

        if self.transform == 'logicle':
            try:
                if isinstance(kwargs['r'], Number):
                    self.r = kwargs['r']
                elif numpy.all(numpy.isreal(kwargs['r'])):
                    self.r = numpy.zeros(data.shape[1])
            except KeyError:
                self.r = None
            if 'T' in kwargs.keys():
                T = kwargs['T']
            else:
                T = 262144

            if 'm' in kwargs.keys():
                m = kwargs['m']
            else:
                m = 4.5

            if 'scale_max' in kwargs.keys():
                scale_max = kwargs['scale_max']
            else:
                scale_max = 1e5

            if 'scale_min' in kwargs.keys():
                scale_min = kwargs['scale_min']
            else:
                scale_min = 0

            if 'rquant' in kwargs.keys():
                rquant = kwargs['rquant']
            else:
                rquant = None
            if 'w' in kwargs.keys():

                w = kwargs['w']
            else:
                w = 0.5
            if 'a' in kwargs.keys():
                a = kwargs['a']
            else:
                a = 0

            if to_transform:
                tmpfcm.logicle(to_transform,
                               T=T,
                               m=m,
                               r=self.r,
                               w=w,
                               a=a,
                               scale_max=scale_max,
                               scale_min=scale_min,
                               rquant=rquant)

        elif self.transform == 'log':
            if to_transform:
                tmpfcm.log(to_transform)

        try:
            tmpfcm._r = self.r
        except AttributeError:
            pass
        try:
            tmpfcm._w = self.w
        except AttributeError:
            pass

        if 'nextdata' in text:
            self.cur_offset = int(text['nextdata'])
        return tmpfcm
Example #9
0
 def setUp(self):
     pnts = array([[1, 1, 1], [5, 5, 5]])
     fcm1 = FCMdata('test_fcm1', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
     fcm2 = FCMdata('test_fcm2', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
     self.fcms = FCMcollection('fcms', [fcm1, fcm2])
Example #10
0
class FCMdataTestCase(unittest.TestCase):
    def setUp(self):
        self.pnts = array([[0, 1, 2], [3, 4, 5]])
        self.fcm = FCMdata('test_fcm', self.pnts, ['fsc', 'ssc', 'cd3'],
                           [0, 1])

    def testChannels(self):
        assert self.fcm.channels[0] == 'fsc', "channel property fails"

    def testGetPnts(self):
        a = randint(0, 1)
        b = randint(0, 2)
        assert self.fcm.view()[a, b] == self.pnts[
            a, b], "Data not consistent with initial data"

    def testGetChannelByName(self):
        assert self.fcm.get_channel_by_name(
            ['fsc'])[0] == 0, 'incorrect first column'
        assert self.fcm.get_channel_by_name(
            ['fsc'])[1] == 3, 'incorrect first column'

    def testGetMarkers(self):
        #print self.fcm.markers
        assert self.fcm.markers == [2], 'Marker CD3 not picked up'

    def testGetItem(self):
        a = randint(0, 1)
        b = randint(0, 2)
        assert type(self.fcm[a]) == type(
            self.pnts[a]), "__getitem__ failed to return array"

        print 'new', self.pnts[a, b]
        print 'new', self.fcm[a, b]

        assert self.fcm[a,
                        b] == self.pnts[a,
                                        b], '__getitem__ returned wrong value'
        assert self.fcm[:, 'fsc'][a] == self.pnts[:, 0][
            a], '__getitem__ with multiple strings failed'
        assert self.fcm[:, ['fsc', 'ssc']][a, 0] == self.pnts[
            a, 0], '__getitem__ with multiple strings failed'
        assert self.fcm[:, ['fsc', 1]][a, 0] == self.pnts[
            a, 0], '__getitem__ with mixed strings failed'

    def testDeligate(self):
        assert self.fcm.mean() == self.pnts.mean(), "deligation of mean failed"


#    def testSubSample(self):
#        self.fcm.subsample([2])
#        assert self.fcm.view()[0] == self.pnts[0,2], "subsample failed"
#        assert self.fcm.view()[1] == self.pnts[1,2], "subsample failed"

#    def testlogicle(self):
#        from numpy.random import normal, lognormal, shuffle
#        from numpy import concatenate
#        from core.fcmtransforms import quantile
#        from pylab import hist, show
#
#        d1 = normal(0, 50, (50000))
#        d2 = lognormal(8, 1, (50000))
#        d3 = array([concatenate([d1, d2])]).T
#
#        T = 262144
#        d = 4
#        m = d*log(10)
#        r = quantile(d3[d3<0], 0.05)
#        self.fcm = FCMdata(d3, ['a'])

    def testPolyGate(self):
        verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        assert all(self.fcm.view() == array([[0, 1, 2]
                                             ])), 'gate excluded wrong points'
        self.fcm.visit('root')
        self.fcm.gate(g)
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def testEmptyPolyGate(self):
        verts = array([[10, 10], [10, 11], [11, 11], [11, 10]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)), 'gated region not empty')

        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)), 'gated region not empty')

        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def testIntervalGate(self):
        verts = array([1.5, 4.5])
        cols = [0]
        g = IntervalGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(), array([[3, 4, 5]]),
                           'gate excluded wrong points')
        self.fcm.visit('root')
        self.fcm.gate(g)
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def testEmptyIntervalGate(self):
        verts = array([10.5, 40.5])
        cols = [0]
        g = IntervalGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)),
                           'gate excluded wrong points')
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)),
                           'gate excluded wrong points')
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def testBoundaryEvents(self):
        pnts = array([[0, 1, 2], [3, 4, 5], [0, 2, 5]])
        fcm = FCMdata('test_fcm', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
        eps = 1e-10
        result = fcm.boundary_events()
        assert result['fsc'] - 1 < eps
        assert result['ssc'] - 2.0 / 3.0 < eps
        assert result['cd3'] - 1 < eps

    def testChainOp(self):
        verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g).gate(g)
        self.assertTrue(all(self.fcm.view() == array([[0, 1, 2]])),
                        'gate excluded wrong points')

    def testGetAttr(self):
        assert self.fcm.shape == (2, 3), '__gettattr__ failed to deligate'

    def testSummary(self):
        tmp = self.fcm.summary()
        assert tmp.startswith('fsc:') == True, 'Summary failed'

    def testPickle(self):
        import pickle
        import StringIO
        buffer = StringIO.StringIO()
        pickle.dump(self.fcm, buffer)
        buffer.seek(0)
        tmp = pickle.load(buffer)
        self.assertTrue(all(self.fcm[:] == tmp[:]))
        for unused in range(3):
            buffer = StringIO.StringIO()
            pickle.dump(tmp, buffer)
            buffer.seek(0)
            tmp = pickle.load(buffer)
        self.assertTrue(all(self.fcm[:] == tmp[:]))

    def testCopy(self):
        cpy = self.fcm.copy()
        self.assertFalse(cpy is self.fcm,
                         "copy reproduced the exact same object")
        self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
                        "copy failed to reproduce the view tree")

        # make sure changes to object self.fcm don't show up on cpy
        verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        self.assertFalse(cpy.tree.pprint() == self.fcm.tree.pprint(),
                         "copy failed to reproduce the view tree")

        #make sure tree is actually copied
        cpy = self.fcm.copy()
        self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
                        "copy failed to reproduce the view tree")
Example #11
0
 def setUp(self):
     self.pnts = array([[0, 1, 2], [3, 4, 5]])
     self.fcm = FCMdata('test_fcm', self.pnts, ['fsc', 'ssc', 'cd3'],
                        [0, 1])
Example #12
0
 def setUp(self):
     self.pnts = array([[0,1,2],[3,4,5]])
     self.fcm = FCMdata('test_fcm', self.pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
Example #13
0
class FCMdataTestCase(unittest.TestCase):
    def setUp(self):
        self.pnts = array([[0,1,2],[3,4,5]])
        self.fcm = FCMdata('test_fcm', self.pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
        
    def testChannels(self):
        assert self.fcm.channels[0] == 'fsc', "channel property fails"
        
    def testLen(self):
        assert len(self.fcm) == 2, 'length wrong'
        
    def testShortNames(self):
        assert self.fcm.short_names[2] == 'fl-1', "channel property fails"
        
    def testLongNames(self):
        assert self.fcm.long_names[2] == 'fl-1::cd3', 'long names property fails: %s' % self.fcm.long_names[2]
        assert self.fcm.long_names[0] == 'fsc', 'long names property fails: %s' % self.fcm.long_names[0]
        
    def testGetPnts(self):
        a = randint(0,1)
        b = randint(0,2)
        assert self.fcm.view()[a,b] == self.pnts[a,b], "Data not consistent with initial data"
            
    def testGetChannelByName(self):
        assert self.fcm.get_channel_by_name(['fsc'])[0] == 0, 'incorrect first column'
        assert self.fcm.get_channel_by_name(['fsc'])[1] == 3, 'incorrect first column'
        assert self.fcm.get_channel_by_name(['fl-1'])[0] == 2, 'incorrect last column: %d' % self.fcm.get_channel_by_name(['fl-1'])[0]
        assert self.fcm.get_channel_by_name(['fl-1'])[1] == 5, 'incorrect first column: %d' % self.fcm.get_channel_by_name(['fl-1'])[1]
    def testGetMarkers(self):
        #print self.fcm.markers
        assert self.fcm.markers == [2], 'Marker CD3 not picked up'
    
    def testGetItem(self):
        a = randint(0,1)
        b = randint(0,2)
        assert type(self.fcm[a]) == type(self.pnts[a]), "__getitem__ failed to return array"
        
        assert self.fcm[a,b] == self.pnts[a,b], '__getitem__ returned wrong value'
        assert self.fcm[:,'fsc'][a] == self.pnts[:,0][a], '__getitem__ with multiple strings failed'
        assert self.fcm[:,['fsc','ssc']][a,0] == self.pnts[a,0], '__getitem__ with multiple strings failed'
        assert self.fcm[:,['fsc',1]][a,0] == self.pnts[a,0], '__getitem__ with mixed strings failed'
        
    def testDeligate(self):
        assert self.fcm.mean() == self.pnts.mean(), "deligation of mean failed"
        
#    def testSubSample(self):
#        self.fcm.subsample([2])
#        assert self.fcm.view()[0] == self.pnts[0,2], "subsample failed"
#        assert self.fcm.view()[1] == self.pnts[1,2], "subsample failed"
        
#    def testlogicle(self):
#        from numpy.random import normal, lognormal, shuffle
#        from numpy import concatenate
#        from core.fcmtransforms import quantile
#        from pylab import hist, show
#    
#        d1 = normal(0, 50, (50000))
#        d2 = lognormal(8, 1, (50000))
#        d3 = array([concatenate([d1, d2])]).T
# 
#        T = 262144
#        d = 4
#        m = d*log(10)
#        r = quantile(d3[d3<0], 0.05)
#        self.fcm = FCMdata(d3, ['a'])

    def testPolyGate(self):
        verts =  array([[-.1,-.1],[-.1,1.1],[1.1,1.1], [1.1,-.1]])
        cols = [0,1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        assert all(self.fcm.view() == array([[0,1,2]])), 'gate excluded wrong points'
        self.fcm.visit('root')
        self.fcm.gate(g)
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'
        
    def testEmptyPolyGate(self):
        verts =  array([[10,10],[10,11],[11,11], [11,10]])
        cols = [0,1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gated region not empty')

        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gated region not empty')

        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'
        
    def testIntervalGate(self):
        verts =  array([1.5,4.5])
        cols = [0]
        g = IntervalGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),array([[3,4,5]]), 'gate excluded wrong points')
        self.fcm.visit('root')
        self.fcm.gate(g)
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def testEmptyIntervalGate(self):
        verts =  array([10.5,40.5])
        cols = [0]
        g = IntervalGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gate excluded wrong points')
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gate excluded wrong points')
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def testBoundaryEvents(self):
        pnts = array([[0,1,2],[3,4,5],[0,2,5]])
        fcm = FCMdata('test_fcm', pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
        eps = 1e-10
        result = fcm.boundary_events()
    
        assert result['fsc'] - 1 < eps, str(result['fsc'])
        assert result['ssc'] - 2.0/3.0 < eps
        assert result['cd3'] - 1 < eps

    def testChainOp(self):
        verts =  array([[-.1,-.1],[-.1,1.1],[1.1,1.1], [1.1,-.1]])
        cols = [0,1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g).gate(g)
        self.assertTrue( all(self.fcm.view()== array([[0,1,2]])), 'gate excluded wrong points')
        
    def testGetAttr(self):
        assert self.fcm.shape == (2,3), '__gettattr__ failed to deligate'
        
    def testSummary(self):
        tmp =  self.fcm.summary()
        assert tmp.startswith('fsc:') == True, 'Summary failed'
        
    def testPickle(self):
        import pickle
        import StringIO
        buffer = StringIO.StringIO()
        pickle.dump(self.fcm, buffer)
        buffer.seek(0)
        tmp = pickle.load(buffer)
        self.assertTrue(all(self.fcm[:] == tmp[:]))
        for unused in range(3):
            buffer = StringIO.StringIO()
            pickle.dump(tmp, buffer)
            buffer.seek(0)
            tmp = pickle.load(buffer)
        self.assertTrue(all(self.fcm[:] == tmp[:]))
        
    def testCopy(self):
        cpy = self.fcm.copy()
        self.assertFalse(cpy is self.fcm, "copy reproduced the exact same object")
        self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(), "copy failed to reproduce the view tree")
        
        # make sure changes to object self.fcm don't show up on cpy
        verts =  array([[-.1,-.1],[-.1,1.1],[1.1,1.1], [1.1,-.1]])
        cols = [0,1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        self.assertFalse(cpy.tree.pprint() == self.fcm.tree.pprint(), "copy failed to reproduce the view tree")
        
        #make sure tree is actually copied
        cpy = self.fcm.copy()
        self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(), "copy failed to reproduce the view tree")

    def testRandomSubsample(self):
        self.fcm.subsample(1)
        
        self.assertEqual(1,self.fcm.shape[0], 'random subsampling failed')
        self.assertTrue(self.fcm[0] in self.pnts, 'random subsample geneterated non-existant point')

    def testAnomalySubsample(self):
        mu = array([0,1,2])
        sig = array([[1,0,0],[0,1,0],[0,0,1]])
        cluster = DPCluster(1.0, mu, sig)
        mix = DPMixture([cluster])
        self.fcm.subsample(1, 'anomaly', mix)
        
        self.assertEqual(1,self.fcm.shape[0], 'anomaly subsampling failed')
        self.assertTrue(self.fcm[0] in self.pnts, 'anomaly subsample geneterated non-existant point')

    def testBiasSubsample(self):
        neg_mu = array([0,1,2])
        pos_mu = array([3,4,5])
        sig = array([[1,0,0],[0,1,0],[0,0,1]])
        neg_cluster = DPCluster(1.0,neg_mu, sig)
        neg_mix = DPMixture([neg_cluster])
        pos_cluster0 = DPCluster(0.5, neg_mu, sig)
        pos_cluster1 = DPCluster(0.5, pos_mu, sig)
        pos_mix = DPMixture([pos_cluster0, pos_cluster1])
        self.fcm.subsample(1, 'bias', pos=pos_mix, neg=neg_mix)
        
        self.assertEqual(1,self.fcm.shape[0], 'bias subsampling failed')
        self.assertTrue(self.fcm[0] in self.pnts, 'bias subsample geneterated non-existant point')

    def testSubsamplebySlice(self):
        self.fcm.subsample(slice(1))
        self.assertEqual(1,self.fcm.shape[0])
        assert_array_equal(self.pnts[0], self.fcm[0], 'subsample by slice failed')
Example #14
0
    def get_FCMdata(self, auto_comp=False, **kwargs):
        """Return the next FCM data set stored in a FCS file"""

        # parse headers
        header = self.parse_header(self.cur_offset)

        # parse text
        text = self.parse_text(
            self.cur_offset,
            header['text_start'],
            header['text_stop'])

        # parse annalysis
        try:
            astart = text['beginanalysis']
        except KeyError:
            astart = header['analysis_start']
        try:
            astop = text['endanalysis']
        except KeyError:
            astop = header['analysis_end']
        analysis = self.parse_analysis(self.cur_offset, astart, astop)
        # parse data
        try:
            dstart = int(text['begindata'])
        except KeyError:
            dstart = header['data_start']
        try:
            dstop = int(text['enddata'])
        except KeyError:
            dstop = header['data_end']

        # account for LMD reporting the wrong values for the size of the data
        # segment
        lmd = self.fix_lmd(
            self.cur_offset,
            header['text_start'],
            header['text_stop'])
        dstop = dstop + lmd
        data = self.parse_data(self.cur_offset, dstart, dstop, text)

        # build fcmdata object
        channels = []
        scchannels = []
        scchannel_indexes = []
        to_transform = []
        base_chan_name = []
        for i in range(1, int(text['par']) + 1):
            base_chan_name.append(text['p%dn' % i])
            try:
                if text['p%ds' % i] not in ['', ' ']:
                    name = text['p%ds' % i]
                else:
                    name = text['p%dn' % i]
            except KeyError:
                name = text['p%dn' % i]
            channels.append(name)
            # if not name.lower().startswith('fl'):
            if not is_fl_channel(name):
                scchannels.append(name)
                if name != 'Time':
                    scchannel_indexes.append(i - 1)
            else:  # we're a FL channel
                try:
                    if text['p%dr' % i] == '262144':
                        to_transform.append(i - 1)
                except KeyError:
                    pass

        try:
            unused_path, name = os.path.split(self._fh.name)
        except AttributeError:
            name = 'InMemoryFile'
        name, unused_ext = os.path.splitext(name)
        tmpfcm = FCMdata(name, data, zip(base_chan_name, channels), scchannels,
                         Annotation({'text': text,
                                     'header': header,
                                     'analysis': analysis,
                                     }))
        if self.sidx is not None and self.spill is not None:
            # if we're passed a spillover we assume we compensate
            auto_comp = True
        if auto_comp:
            if self.sidx is None and self.spill is None:
                if tmpfcm.get_spill():
                    spill, sidx = get_spill(tmpfcm.get_spill())
                    tmpfcm.compensate(sidx=sidx, spill=spill)
            else:
                tmpfcm.compensate(sidx=self.sidx, spill=self.spill)

        if self.transform == 'logicle':
            try:
                if isinstance(kwargs['r'], Number):
                    self.r = kwargs['r']
                elif numpy.all(numpy.isreal(kwargs['r'])):
                    self.r = numpy.zeros(data.shape[1])
            except KeyError:
                self.r = None
            if 'T' in kwargs.keys():
                T = kwargs['T']
            else:
                T = 262144

            if 'm' in kwargs.keys():
                m = kwargs['m']
            else:
                m = 4.5

            if 'scale_max' in kwargs.keys():
                scale_max = kwargs['scale_max']
            else:
                scale_max = 1e5

            if 'scale_min' in kwargs.keys():
                scale_min = kwargs['scale_min']
            else:
                scale_min = 0

            if 'rquant' in kwargs.keys():
                rquant = kwargs['rquant']
            else:
                rquant = None
            if 'w' in kwargs.keys():

                w = kwargs['w']
            else:
                w = 0.5
            if 'a' in kwargs.keys():
                a = kwargs['a']
            else:
                a = 0

            if to_transform:
                tmpfcm.logicle(
                    to_transform,
                    T=T,
                    m=m,
                    r=self.r,
                    w=w,
                    a=a,
                    scale_max=scale_max,
                    scale_min=scale_min,
                    rquant=rquant)

        elif self.transform == 'log':
            if to_transform:
                tmpfcm.log(to_transform)

        try:
            tmpfcm._r = self.r
        except AttributeError:
            pass
        try:
            tmpfcm._w = self.w
        except AttributeError:
            pass

        if 'nextdata' in text:
            self.cur_offset = int(text['nextdata'])
        return tmpfcm
Example #15
0
 def setUp(self):
     self.pnts = array([[0, 1, 2], [3, 4, 5]])
     self.fcm = FCMdata('test_fcm',
                        self.pnts, ['fsc', 'ssc', 'fl-1'],
                        scatters=[0, 1])
Example #16
0
class FCMDataTestCase(unittest.TestCase):
    def setUp(self):
        self.pnts = array([[0, 1, 2], [3, 4, 5]])
        self.fcm = FCMdata('test_fcm',
                           self.pnts, ['fsc', 'ssc', 'fl-1'],
                           scatters=[0, 1])

    def test_channels(self):
        assert self.fcm.channels[0] == 'fsc', "channel property fails"

    def test_len(self):
        assert len(self.fcm) == 2, 'length wrong'

    def test_get_pnts(self):
        a = randint(0, 1)
        b = randint(0, 2)
        assert self.fcm.view()[a, b] == self.pnts[a, b], \
            "Data not consistent with initial data"

    def test_get_channel_by_name(self):
        assert self.fcm.get_channel_by_name(['fsc'])[0] == 0, \
            'incorrect first column'
        assert self.fcm.get_channel_by_name(['fsc'])[1] == 3, \
            'incorrect first column'
        assert self.fcm.get_channel_by_name(['fl-1'])[0] == 2, \
            'incorrect last column: %d' % \
            self.fcm.get_channel_by_name(['fl-1'])[0]
        assert self.fcm.get_channel_by_name(['fl-1'])[1] == 5, \
            'incorrect first column: %d' \
            % self.fcm.get_channel_by_name(['fl-1'])[1]

    def test_get_markers(self):
        assert self.fcm.markers == [2], "Marker CD3 not picked up"

    def test_get_item(self):
        a = randint(0, 1)
        b = randint(0, 2)
        assert type(self.fcm[a]) == type(self.pnts[a]), \
            "__getitem__ failed to return array"

        assert self.fcm[a, b] == self.pnts[a, b], \
            "__getitem__ returned wrong value"
        assert self.fcm[:, 'fsc'][a] == self.pnts[:, 0][a], \
            "__getitem__ with multiple strings failed"
        assert self.fcm[:, ['fsc', 'ssc']][a, 0] == self.pnts[a, 0], \
            "__getitem__ with multiple strings failed"
        assert self.fcm[:, ['fsc', 1]][a, 0] == self.pnts[a, 0], \
            "__getitem__ with mixed strings failed"

    def test_delegate(self):
        assert self.fcm.mean() == self.pnts.mean(), "delegation of mean failed"

    def test_poly_gate(self):
        verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        assert all(self.fcm.view() == array([[0, 1, 2]])), \
            "gate excluded wrong points"
        self.fcm.visit('root')
        self.fcm.gate(g)
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def test_empty_poly_gate(self):
        verts = array([[10, 10], [10, 11], [11, 11], [11, 10]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)), "gated region not empty")

        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)), 'gated region not empty')

        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def test_interval_gate(self):
        verts = array([1.5, 4.5])
        cols = [0]
        g = IntervalGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(), array([[3, 4, 5]]),
                           'gate excluded wrong points')
        self.fcm.visit('root')
        self.fcm.gate(g)
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def test_empty_interval_gate(self):
        verts = array([10.5, 40.5])
        cols = [0]
        g = IntervalGate(verts, cols)
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)),
                           'gate excluded wrong points')
        self.fcm.gate(g)
        assert_array_equal(self.fcm.view(),
                           array([]).reshape((0, 3)),
                           'gate excluded wrong points')
        nodes = self.fcm.tree.nodes.keys()
        assert 'g2' in nodes, 'gating name mangled'
        assert 'g1' in nodes, 'gating name mangled'

    def test_boundary_events(self):
        pnts = array([[0, 1, 2], [3, 4, 5], [0, 2, 5]])
        fcm = FCMdata('test_fcm', pnts, ['fsc', 'ssc', 'cd3'], scatters=[0, 1])
        eps = 1e-10
        result = fcm.boundary_events()

        assert result['fsc'] - 1 < eps, str(result['fsc'])
        assert result['ssc'] - 2.0 / 3.0 < eps
        assert result['cd3'] - 1 < eps

    def test_chain_op(self):
        verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g).gate(g)
        self.assertTrue(all(self.fcm.view() == array([[0, 1, 2]])),
                        'gate excluded wrong points')

    def test_getattr(self):
        assert self.fcm.shape == (2, 3), '__gettattr__ failed to deligate'

    def test_summary(self):
        tmp = self.fcm.summary()
        assert tmp.startswith('fsc:') is True, 'Summary failed'

    def test_pickle(self):
        import pickle
        import StringIO
        str_buffer = StringIO.StringIO()
        pickle.dump(self.fcm, str_buffer)
        str_buffer.seek(0)
        tmp = pickle.load(str_buffer)
        self.assertTrue(all(self.fcm[:] == tmp[:]))
        for unused in range(3):
            str_buffer = StringIO.StringIO()
            pickle.dump(tmp, str_buffer)
            str_buffer.seek(0)
            tmp = pickle.load(str_buffer)
        self.assertTrue(all(self.fcm[:] == tmp[:]))

    def test_copy(self):
        cpy = self.fcm.copy()
        self.assertFalse(cpy is self.fcm,
                         "copy reproduced the exact same object")
        self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
                        "copy failed to reproduce the view tree")

        # make sure changes to object self.fcm don't show up on cpy
        verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
        cols = [0, 1]
        g = PolyGate(verts, cols)
        self.fcm.gate(g)
        self.assertFalse(cpy.tree.pprint() == self.fcm.tree.pprint(),
                         "copy failed to reproduce the view tree")

        #make sure tree is actually copied
        cpy = self.fcm.copy()
        self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
                        "copy failed to reproduce the view tree")

    def test_random_subsample(self):
        self.fcm.subsample(1)

        self.assertEqual(1, self.fcm.shape[0], 'random subsampling failed')
        self.assertTrue(self.fcm[0] in self.pnts,
                        'random subsample geneterated non-existant point')

    def test_anomaly_subsample(self):
        mu = array([0, 1, 2])
        sig = array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
        cluster = DPCluster(1.0, mu, sig)
        mix = DPMixture([cluster])
        self.fcm.subsample(1, 'anomaly', mix)

        self.assertEqual(1, self.fcm.shape[0], 'anomaly subsampling failed')
        self.assertTrue(self.fcm[0] in self.pnts,
                        'anomaly subsample geneterated non-existant point')

    def test_bias_subsample(self):
        neg_mu = array([0, 1, 2])
        pos_mu = array([3, 4, 5])
        sig = array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
        neg_cluster = DPCluster(1.0, neg_mu, sig)
        neg_mix = DPMixture([neg_cluster])
        pos_cluster0 = DPCluster(0.5, neg_mu, sig)
        pos_cluster1 = DPCluster(0.5, pos_mu, sig)
        pos_mix = DPMixture([pos_cluster0, pos_cluster1])
        self.fcm.subsample(1, 'bias', pos=pos_mix, neg=neg_mix)

        self.assertEqual(1, self.fcm.shape[0], 'bias subsampling failed')
        self.assertTrue(self.fcm[0] in self.pnts,
                        'bias subsample geneterated non-existant point')

    def test_subsample_by_slice(self):
        self.fcm.subsample(slice(1))
        self.assertEqual(1, self.fcm.shape[0])
        assert_array_equal(self.pnts[0], self.fcm[0],
                           'subsample by slice failed')