def testListFitting(self):
true1, data1 = self.generate_data()
true2, data2 = self.generate_data()
model = DPMixtureModel(3, 2000, 100, 1, type='BEM')
rs = model.fit([data1, data2])
assert (len(rs) == 2)
for r in rs:
print 'mu ', r.mus
diffs = {}
for i in gen_mean:
diffs[i] = np.min(np.abs(r.mus - gen_mean[i]), 0)
# print i, gen_mean[i], diffs[i], np.vdot(diffs[i],diffs[i])
assert (np.vdot(diffs[i], diffs[i]) < 2)
fcm1 = FCMdata('test_fcm1', data1, ['fsc', 'ssc'], [0, 1])
fcm2 = FCMdata('test_fcm2', data2, ['fsc', 'ssc'], [0, 1])
c = FCMcollection('fcms', [fcm1, fcm2])
rs = model.fit(c)
assert (len(rs) == 2)
for r in rs:
print 'mu ', r.mus
diffs = {}
for i in gen_mean:
diffs[i] = np.min(np.abs(r.mus - gen_mean[i]), 0)
# print i, gen_mean[i], diffs[i], np.vdot(diffs[i],diffs[i])
assert (np.vdot(diffs[i], diffs[i]) < 4)
def testBoundaryEvents(self):
pnts = array([[0, 1, 2], [3, 4, 5], [0, 2, 5]])
fcm = FCMdata('test_fcm', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
eps = 1e-10
result = fcm.boundary_events()
assert result['fsc'] - 1 < eps
assert result['ssc'] - 2.0 / 3.0 < eps
assert result['cd3'] - 1 < eps
def testBoundaryEvents(self):
pnts = array([[0,1,2],[3,4,5],[0,2,5]])
fcm = FCMdata('test_fcm', pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
eps = 1e-10
result = fcm.boundary_events()
assert result['fsc'] - 1 < eps, str(result['fsc'])
assert result['ssc'] - 2.0/3.0 < eps
assert result['cd3'] - 1 < eps
def testCheckNames(self):
pnts = array([[1, 1, 1], [5, 5, 5]])
fcm1 = FCMdata('test_fcm1', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
fcm2 = FCMdata('test_fcm2', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
fcm3 = FCMdata('test_fcm3', pnts, ['fsc', 'ssc', 'cd4'], [0, 1])
fcms1 = FCMcollection('fcms1', [fcm1, fcm2])
fcms2 = FCMcollection('fcms2', [fcm1, fcm2, fcm3, fcms1])
check1 = fcms1.check_names()
assert check1[fcms1.name] == [True, True, True]
check2 = fcms2.check_names()
assert check2[fcms2.name] == [check1, True, True, False]
class SubsampleTestCase(unittest.TestCase):
def setUp(self):
self.pnts = array([[0,1,2],[3,4,5]])
self.fcm = FCMdata('test_fcm', self.pnts, ['fsc','ssc','cd3'], [0,1])
self.samp = SubsampleFactory[:,1]
def testSubSample(self):
self.samp.subsample(self.fcm)
assert self.fcm[0] == 1, 'subsample via subsample failed'
assert self.fcm[1] == 4, 'subsample via subsample failed'
self.fcm.visit('root')
self.fcm.subsample(self.samp)
assert self.fcm[0] == 1, 'subsample via fcm failed'
assert self.fcm[1] == 4, 'subsample via fcm failed'
def testChainSubSample(self):
self.fcm.visit('root')
sam2 = SubsampleFactory[0]
self.fcm.subsample(self.samp).subsample(sam2)
assert self.fcm.view() == 1, 'subsample chaining failed'
class SubsampleTestCase(unittest.TestCase):
def setUp(self):
self.pnts = array([[0, 1, 2], [3, 4, 5]])
self.fcm = FCMdata('test_fcm', self.pnts, ['fsc', 'ssc', 'cd3'],
[0, 1])
self.samp = SubsampleFactory[:, 1]
def testSubSample(self):
self.samp.subsample(self.fcm)
assert self.fcm[0] == 1, 'subsample via subsample failed'
assert self.fcm[1] == 4, 'subsample via subsample failed'
self.fcm.visit('root')
self.fcm.subsample(self.samp)
assert self.fcm[0] == 1, 'subsample via fcm failed'
assert self.fcm[1] == 4, 'subsample via fcm failed'
def testChainSubSample(self):
self.fcm.visit('root')
sam2 = SubsampleFactory[0]
self.fcm.subsample(self.samp).subsample(sam2)
assert self.fcm.view() == 1, 'subsample chaining failed'
def setUp(self):
self.pnts = array([[0,1,2],[3,4,5]])
self.fcm = FCMdata('test_fcm', self.pnts, ['fsc','ssc','cd3'], [0,1])
self.samp = SubsampleFactory[:,1]
def get_FCMdata(self, auto_comp=False, **kwargs):
"""Return the next FCM data set stored in a FCS file"""
# parse headers
header = self.parse_header(self.cur_offset)
# parse text
text = self.parse_text(self.cur_offset, header['text_start'],
header['text_stop'])
# parse annalysis
try:
astart = text['beginanalysis']
except KeyError:
astart = header['analysis_start']
try:
astop = text['endanalysis']
except KeyError:
astop = header['analysis_end']
analysis = self.parse_analysis(self.cur_offset, astart, astop)
# parse data
try:
dstart = int(text['begindata'])
except KeyError:
dstart = header['data_start']
try:
dstop = int(text['enddata'])
except KeyError:
dstop = header['data_end']
# account for LMD reporting the wrong values for the size of the data
# segment
lmd = self.fix_lmd(self.cur_offset, header['text_start'],
header['text_stop'])
dstop = dstop + lmd
data = self.parse_data(self.cur_offset, dstart, dstop, text)
# build fcmdata object
channels = []
scchannels = []
scchannel_indexes = []
to_transform = []
base_chan_name = []
for i in range(1, int(text['par']) + 1):
base_chan_name.append(text['p%dn' % i])
try:
if text['p%ds' % i] not in ['', ' ']:
name = text['p%ds' % i]
else:
name = text['p%dn' % i]
except KeyError:
name = text['p%dn' % i]
channels.append(name)
# if not name.lower().startswith('fl'):
if not is_fl_channel(name):
scchannels.append(name)
if name != 'Time':
scchannel_indexes.append(i - 1)
else: # we're a FL channel
try:
if text['p%dr' % i] == '262144':
to_transform.append(i - 1)
except KeyError:
pass
try:
unused_path, name = os.path.split(self._fh.name)
except AttributeError:
name = 'InMemoryFile'
name, unused_ext = os.path.splitext(name)
tmpfcm = FCMdata(
name, data, zip(base_chan_name, channels), scchannels,
Annotation({
'text': text,
'header': header,
'analysis': analysis,
}))
if self.sidx is not None and self.spill is not None:
# if we're passed a spillover we assume we compensate
auto_comp = True
if auto_comp:
if self.sidx is None and self.spill is None:
if tmpfcm.get_spill():
spill, sidx = get_spill(tmpfcm.get_spill())
tmpfcm.compensate(sidx=sidx, spill=spill)
else:
tmpfcm.compensate(sidx=self.sidx, spill=self.spill)
if self.transform == 'logicle':
try:
if isinstance(kwargs['r'], Number):
self.r = kwargs['r']
elif numpy.all(numpy.isreal(kwargs['r'])):
self.r = numpy.zeros(data.shape[1])
except KeyError:
self.r = None
if 'T' in kwargs.keys():
T = kwargs['T']
else:
T = 262144
if 'm' in kwargs.keys():
m = kwargs['m']
else:
m = 4.5
if 'scale_max' in kwargs.keys():
scale_max = kwargs['scale_max']
else:
scale_max = 1e5
if 'scale_min' in kwargs.keys():
scale_min = kwargs['scale_min']
else:
scale_min = 0
if 'rquant' in kwargs.keys():
rquant = kwargs['rquant']
else:
rquant = None
if 'w' in kwargs.keys():
w = kwargs['w']
else:
w = 0.5
if 'a' in kwargs.keys():
a = kwargs['a']
else:
a = 0
if to_transform:
tmpfcm.logicle(to_transform,
T=T,
m=m,
r=self.r,
w=w,
a=a,
scale_max=scale_max,
scale_min=scale_min,
rquant=rquant)
elif self.transform == 'log':
if to_transform:
tmpfcm.log(to_transform)
try:
tmpfcm._r = self.r
except AttributeError:
pass
try:
tmpfcm._w = self.w
except AttributeError:
pass
if 'nextdata' in text:
self.cur_offset = int(text['nextdata'])
return tmpfcm
def setUp(self):
pnts = array([[1, 1, 1], [5, 5, 5]])
fcm1 = FCMdata('test_fcm1', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
fcm2 = FCMdata('test_fcm2', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
self.fcms = FCMcollection('fcms', [fcm1, fcm2])
class FCMdataTestCase(unittest.TestCase):
def setUp(self):
self.pnts = array([[0, 1, 2], [3, 4, 5]])
self.fcm = FCMdata('test_fcm', self.pnts, ['fsc', 'ssc', 'cd3'],
[0, 1])
def testChannels(self):
assert self.fcm.channels[0] == 'fsc', "channel property fails"
def testGetPnts(self):
a = randint(0, 1)
b = randint(0, 2)
assert self.fcm.view()[a, b] == self.pnts[
a, b], "Data not consistent with initial data"
def testGetChannelByName(self):
assert self.fcm.get_channel_by_name(
['fsc'])[0] == 0, 'incorrect first column'
assert self.fcm.get_channel_by_name(
['fsc'])[1] == 3, 'incorrect first column'
def testGetMarkers(self):
#print self.fcm.markers
assert self.fcm.markers == [2], 'Marker CD3 not picked up'
def testGetItem(self):
a = randint(0, 1)
b = randint(0, 2)
assert type(self.fcm[a]) == type(
self.pnts[a]), "__getitem__ failed to return array"
print 'new', self.pnts[a, b]
print 'new', self.fcm[a, b]
assert self.fcm[a,
b] == self.pnts[a,
b], '__getitem__ returned wrong value'
assert self.fcm[:, 'fsc'][a] == self.pnts[:, 0][
a], '__getitem__ with multiple strings failed'
assert self.fcm[:, ['fsc', 'ssc']][a, 0] == self.pnts[
a, 0], '__getitem__ with multiple strings failed'
assert self.fcm[:, ['fsc', 1]][a, 0] == self.pnts[
a, 0], '__getitem__ with mixed strings failed'
def testDeligate(self):
assert self.fcm.mean() == self.pnts.mean(), "deligation of mean failed"
# def testSubSample(self):
# self.fcm.subsample([2])
# assert self.fcm.view()[0] == self.pnts[0,2], "subsample failed"
# assert self.fcm.view()[1] == self.pnts[1,2], "subsample failed"
# def testlogicle(self):
# from numpy.random import normal, lognormal, shuffle
# from numpy import concatenate
# from core.fcmtransforms import quantile
# from pylab import hist, show
#
# d1 = normal(0, 50, (50000))
# d2 = lognormal(8, 1, (50000))
# d3 = array([concatenate([d1, d2])]).T
#
# T = 262144
# d = 4
# m = d*log(10)
# r = quantile(d3[d3<0], 0.05)
# self.fcm = FCMdata(d3, ['a'])
def testPolyGate(self):
verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
assert all(self.fcm.view() == array([[0, 1, 2]
])), 'gate excluded wrong points'
self.fcm.visit('root')
self.fcm.gate(g)
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testEmptyPolyGate(self):
verts = array([[10, 10], [10, 11], [11, 11], [11, 10]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)), 'gated region not empty')
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)), 'gated region not empty')
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testIntervalGate(self):
verts = array([1.5, 4.5])
cols = [0]
g = IntervalGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(), array([[3, 4, 5]]),
'gate excluded wrong points')
self.fcm.visit('root')
self.fcm.gate(g)
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testEmptyIntervalGate(self):
verts = array([10.5, 40.5])
cols = [0]
g = IntervalGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)),
'gate excluded wrong points')
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)),
'gate excluded wrong points')
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testBoundaryEvents(self):
pnts = array([[0, 1, 2], [3, 4, 5], [0, 2, 5]])
fcm = FCMdata('test_fcm', pnts, ['fsc', 'ssc', 'cd3'], [0, 1])
eps = 1e-10
result = fcm.boundary_events()
assert result['fsc'] - 1 < eps
assert result['ssc'] - 2.0 / 3.0 < eps
assert result['cd3'] - 1 < eps
def testChainOp(self):
verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g).gate(g)
self.assertTrue(all(self.fcm.view() == array([[0, 1, 2]])),
'gate excluded wrong points')
def testGetAttr(self):
assert self.fcm.shape == (2, 3), '__gettattr__ failed to deligate'
def testSummary(self):
tmp = self.fcm.summary()
assert tmp.startswith('fsc:') == True, 'Summary failed'
def testPickle(self):
import pickle
import StringIO
buffer = StringIO.StringIO()
pickle.dump(self.fcm, buffer)
buffer.seek(0)
tmp = pickle.load(buffer)
self.assertTrue(all(self.fcm[:] == tmp[:]))
for unused in range(3):
buffer = StringIO.StringIO()
pickle.dump(tmp, buffer)
buffer.seek(0)
tmp = pickle.load(buffer)
self.assertTrue(all(self.fcm[:] == tmp[:]))
def testCopy(self):
cpy = self.fcm.copy()
self.assertFalse(cpy is self.fcm,
"copy reproduced the exact same object")
self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
"copy failed to reproduce the view tree")
# make sure changes to object self.fcm don't show up on cpy
verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
self.assertFalse(cpy.tree.pprint() == self.fcm.tree.pprint(),
"copy failed to reproduce the view tree")
#make sure tree is actually copied
cpy = self.fcm.copy()
self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
"copy failed to reproduce the view tree")
def setUp(self):
self.pnts = array([[0, 1, 2], [3, 4, 5]])
self.fcm = FCMdata('test_fcm', self.pnts, ['fsc', 'ssc', 'cd3'],
[0, 1])
def setUp(self):
self.pnts = array([[0,1,2],[3,4,5]])
self.fcm = FCMdata('test_fcm', self.pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
class FCMdataTestCase(unittest.TestCase):
def setUp(self):
self.pnts = array([[0,1,2],[3,4,5]])
self.fcm = FCMdata('test_fcm', self.pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
def testChannels(self):
assert self.fcm.channels[0] == 'fsc', "channel property fails"
def testLen(self):
assert len(self.fcm) == 2, 'length wrong'
def testShortNames(self):
assert self.fcm.short_names[2] == 'fl-1', "channel property fails"
def testLongNames(self):
assert self.fcm.long_names[2] == 'fl-1::cd3', 'long names property fails: %s' % self.fcm.long_names[2]
assert self.fcm.long_names[0] == 'fsc', 'long names property fails: %s' % self.fcm.long_names[0]
def testGetPnts(self):
a = randint(0,1)
b = randint(0,2)
assert self.fcm.view()[a,b] == self.pnts[a,b], "Data not consistent with initial data"
def testGetChannelByName(self):
assert self.fcm.get_channel_by_name(['fsc'])[0] == 0, 'incorrect first column'
assert self.fcm.get_channel_by_name(['fsc'])[1] == 3, 'incorrect first column'
assert self.fcm.get_channel_by_name(['fl-1'])[0] == 2, 'incorrect last column: %d' % self.fcm.get_channel_by_name(['fl-1'])[0]
assert self.fcm.get_channel_by_name(['fl-1'])[1] == 5, 'incorrect first column: %d' % self.fcm.get_channel_by_name(['fl-1'])[1]
def testGetMarkers(self):
#print self.fcm.markers
assert self.fcm.markers == [2], 'Marker CD3 not picked up'
def testGetItem(self):
a = randint(0,1)
b = randint(0,2)
assert type(self.fcm[a]) == type(self.pnts[a]), "__getitem__ failed to return array"
assert self.fcm[a,b] == self.pnts[a,b], '__getitem__ returned wrong value'
assert self.fcm[:,'fsc'][a] == self.pnts[:,0][a], '__getitem__ with multiple strings failed'
assert self.fcm[:,['fsc','ssc']][a,0] == self.pnts[a,0], '__getitem__ with multiple strings failed'
assert self.fcm[:,['fsc',1]][a,0] == self.pnts[a,0], '__getitem__ with mixed strings failed'
def testDeligate(self):
assert self.fcm.mean() == self.pnts.mean(), "deligation of mean failed"
# def testSubSample(self):
# self.fcm.subsample([2])
# assert self.fcm.view()[0] == self.pnts[0,2], "subsample failed"
# assert self.fcm.view()[1] == self.pnts[1,2], "subsample failed"
# def testlogicle(self):
# from numpy.random import normal, lognormal, shuffle
# from numpy import concatenate
# from core.fcmtransforms import quantile
# from pylab import hist, show
#
# d1 = normal(0, 50, (50000))
# d2 = lognormal(8, 1, (50000))
# d3 = array([concatenate([d1, d2])]).T
#
# T = 262144
# d = 4
# m = d*log(10)
# r = quantile(d3[d3<0], 0.05)
# self.fcm = FCMdata(d3, ['a'])
def testPolyGate(self):
verts = array([[-.1,-.1],[-.1,1.1],[1.1,1.1], [1.1,-.1]])
cols = [0,1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
assert all(self.fcm.view() == array([[0,1,2]])), 'gate excluded wrong points'
self.fcm.visit('root')
self.fcm.gate(g)
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testEmptyPolyGate(self):
verts = array([[10,10],[10,11],[11,11], [11,10]])
cols = [0,1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gated region not empty')
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gated region not empty')
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testIntervalGate(self):
verts = array([1.5,4.5])
cols = [0]
g = IntervalGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),array([[3,4,5]]), 'gate excluded wrong points')
self.fcm.visit('root')
self.fcm.gate(g)
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testEmptyIntervalGate(self):
verts = array([10.5,40.5])
cols = [0]
g = IntervalGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gate excluded wrong points')
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),array([]).reshape((0,3)), 'gate excluded wrong points')
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def testBoundaryEvents(self):
pnts = array([[0,1,2],[3,4,5],[0,2,5]])
fcm = FCMdata('test_fcm', pnts, [('fsc','fsc'),('ssc','ssc'),('fl-1','cd3')], [0,1])
eps = 1e-10
result = fcm.boundary_events()
assert result['fsc'] - 1 < eps, str(result['fsc'])
assert result['ssc'] - 2.0/3.0 < eps
assert result['cd3'] - 1 < eps
def testChainOp(self):
verts = array([[-.1,-.1],[-.1,1.1],[1.1,1.1], [1.1,-.1]])
cols = [0,1]
g = PolyGate(verts, cols)
self.fcm.gate(g).gate(g)
self.assertTrue( all(self.fcm.view()== array([[0,1,2]])), 'gate excluded wrong points')
def testGetAttr(self):
assert self.fcm.shape == (2,3), '__gettattr__ failed to deligate'
def testSummary(self):
tmp = self.fcm.summary()
assert tmp.startswith('fsc:') == True, 'Summary failed'
def testPickle(self):
import pickle
import StringIO
buffer = StringIO.StringIO()
pickle.dump(self.fcm, buffer)
buffer.seek(0)
tmp = pickle.load(buffer)
self.assertTrue(all(self.fcm[:] == tmp[:]))
for unused in range(3):
buffer = StringIO.StringIO()
pickle.dump(tmp, buffer)
buffer.seek(0)
tmp = pickle.load(buffer)
self.assertTrue(all(self.fcm[:] == tmp[:]))
def testCopy(self):
cpy = self.fcm.copy()
self.assertFalse(cpy is self.fcm, "copy reproduced the exact same object")
self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(), "copy failed to reproduce the view tree")
# make sure changes to object self.fcm don't show up on cpy
verts = array([[-.1,-.1],[-.1,1.1],[1.1,1.1], [1.1,-.1]])
cols = [0,1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
self.assertFalse(cpy.tree.pprint() == self.fcm.tree.pprint(), "copy failed to reproduce the view tree")
#make sure tree is actually copied
cpy = self.fcm.copy()
self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(), "copy failed to reproduce the view tree")
def testRandomSubsample(self):
self.fcm.subsample(1)
self.assertEqual(1,self.fcm.shape[0], 'random subsampling failed')
self.assertTrue(self.fcm[0] in self.pnts, 'random subsample geneterated non-existant point')
def testAnomalySubsample(self):
mu = array([0,1,2])
sig = array([[1,0,0],[0,1,0],[0,0,1]])
cluster = DPCluster(1.0, mu, sig)
mix = DPMixture([cluster])
self.fcm.subsample(1, 'anomaly', mix)
self.assertEqual(1,self.fcm.shape[0], 'anomaly subsampling failed')
self.assertTrue(self.fcm[0] in self.pnts, 'anomaly subsample geneterated non-existant point')
def testBiasSubsample(self):
neg_mu = array([0,1,2])
pos_mu = array([3,4,5])
sig = array([[1,0,0],[0,1,0],[0,0,1]])
neg_cluster = DPCluster(1.0,neg_mu, sig)
neg_mix = DPMixture([neg_cluster])
pos_cluster0 = DPCluster(0.5, neg_mu, sig)
pos_cluster1 = DPCluster(0.5, pos_mu, sig)
pos_mix = DPMixture([pos_cluster0, pos_cluster1])
self.fcm.subsample(1, 'bias', pos=pos_mix, neg=neg_mix)
self.assertEqual(1,self.fcm.shape[0], 'bias subsampling failed')
self.assertTrue(self.fcm[0] in self.pnts, 'bias subsample geneterated non-existant point')
def testSubsamplebySlice(self):
self.fcm.subsample(slice(1))
self.assertEqual(1,self.fcm.shape[0])
assert_array_equal(self.pnts[0], self.fcm[0], 'subsample by slice failed')
def get_FCMdata(self, auto_comp=False, **kwargs):
"""Return the next FCM data set stored in a FCS file"""
# parse headers
header = self.parse_header(self.cur_offset)
# parse text
text = self.parse_text(
self.cur_offset,
header['text_start'],
header['text_stop'])
# parse annalysis
try:
astart = text['beginanalysis']
except KeyError:
astart = header['analysis_start']
try:
astop = text['endanalysis']
except KeyError:
astop = header['analysis_end']
analysis = self.parse_analysis(self.cur_offset, astart, astop)
# parse data
try:
dstart = int(text['begindata'])
except KeyError:
dstart = header['data_start']
try:
dstop = int(text['enddata'])
except KeyError:
dstop = header['data_end']
# account for LMD reporting the wrong values for the size of the data
# segment
lmd = self.fix_lmd(
self.cur_offset,
header['text_start'],
header['text_stop'])
dstop = dstop + lmd
data = self.parse_data(self.cur_offset, dstart, dstop, text)
# build fcmdata object
channels = []
scchannels = []
scchannel_indexes = []
to_transform = []
base_chan_name = []
for i in range(1, int(text['par']) + 1):
base_chan_name.append(text['p%dn' % i])
try:
if text['p%ds' % i] not in ['', ' ']:
name = text['p%ds' % i]
else:
name = text['p%dn' % i]
except KeyError:
name = text['p%dn' % i]
channels.append(name)
# if not name.lower().startswith('fl'):
if not is_fl_channel(name):
scchannels.append(name)
if name != 'Time':
scchannel_indexes.append(i - 1)
else: # we're a FL channel
try:
if text['p%dr' % i] == '262144':
to_transform.append(i - 1)
except KeyError:
pass
try:
unused_path, name = os.path.split(self._fh.name)
except AttributeError:
name = 'InMemoryFile'
name, unused_ext = os.path.splitext(name)
tmpfcm = FCMdata(name, data, zip(base_chan_name, channels), scchannels,
Annotation({'text': text,
'header': header,
'analysis': analysis,
}))
if self.sidx is not None and self.spill is not None:
# if we're passed a spillover we assume we compensate
auto_comp = True
if auto_comp:
if self.sidx is None and self.spill is None:
if tmpfcm.get_spill():
spill, sidx = get_spill(tmpfcm.get_spill())
tmpfcm.compensate(sidx=sidx, spill=spill)
else:
tmpfcm.compensate(sidx=self.sidx, spill=self.spill)
if self.transform == 'logicle':
try:
if isinstance(kwargs['r'], Number):
self.r = kwargs['r']
elif numpy.all(numpy.isreal(kwargs['r'])):
self.r = numpy.zeros(data.shape[1])
except KeyError:
self.r = None
if 'T' in kwargs.keys():
T = kwargs['T']
else:
T = 262144
if 'm' in kwargs.keys():
m = kwargs['m']
else:
m = 4.5
if 'scale_max' in kwargs.keys():
scale_max = kwargs['scale_max']
else:
scale_max = 1e5
if 'scale_min' in kwargs.keys():
scale_min = kwargs['scale_min']
else:
scale_min = 0
if 'rquant' in kwargs.keys():
rquant = kwargs['rquant']
else:
rquant = None
if 'w' in kwargs.keys():
w = kwargs['w']
else:
w = 0.5
if 'a' in kwargs.keys():
a = kwargs['a']
else:
a = 0
if to_transform:
tmpfcm.logicle(
to_transform,
T=T,
m=m,
r=self.r,
w=w,
a=a,
scale_max=scale_max,
scale_min=scale_min,
rquant=rquant)
elif self.transform == 'log':
if to_transform:
tmpfcm.log(to_transform)
try:
tmpfcm._r = self.r
except AttributeError:
pass
try:
tmpfcm._w = self.w
except AttributeError:
pass
if 'nextdata' in text:
self.cur_offset = int(text['nextdata'])
return tmpfcm
def setUp(self):
self.pnts = array([[0, 1, 2], [3, 4, 5]])
self.fcm = FCMdata('test_fcm',
self.pnts, ['fsc', 'ssc', 'fl-1'],
scatters=[0, 1])
class FCMDataTestCase(unittest.TestCase):
def setUp(self):
self.pnts = array([[0, 1, 2], [3, 4, 5]])
self.fcm = FCMdata('test_fcm',
self.pnts, ['fsc', 'ssc', 'fl-1'],
scatters=[0, 1])
def test_channels(self):
assert self.fcm.channels[0] == 'fsc', "channel property fails"
def test_len(self):
assert len(self.fcm) == 2, 'length wrong'
def test_get_pnts(self):
a = randint(0, 1)
b = randint(0, 2)
assert self.fcm.view()[a, b] == self.pnts[a, b], \
"Data not consistent with initial data"
def test_get_channel_by_name(self):
assert self.fcm.get_channel_by_name(['fsc'])[0] == 0, \
'incorrect first column'
assert self.fcm.get_channel_by_name(['fsc'])[1] == 3, \
'incorrect first column'
assert self.fcm.get_channel_by_name(['fl-1'])[0] == 2, \
'incorrect last column: %d' % \
self.fcm.get_channel_by_name(['fl-1'])[0]
assert self.fcm.get_channel_by_name(['fl-1'])[1] == 5, \
'incorrect first column: %d' \
% self.fcm.get_channel_by_name(['fl-1'])[1]
def test_get_markers(self):
assert self.fcm.markers == [2], "Marker CD3 not picked up"
def test_get_item(self):
a = randint(0, 1)
b = randint(0, 2)
assert type(self.fcm[a]) == type(self.pnts[a]), \
"__getitem__ failed to return array"
assert self.fcm[a, b] == self.pnts[a, b], \
"__getitem__ returned wrong value"
assert self.fcm[:, 'fsc'][a] == self.pnts[:, 0][a], \
"__getitem__ with multiple strings failed"
assert self.fcm[:, ['fsc', 'ssc']][a, 0] == self.pnts[a, 0], \
"__getitem__ with multiple strings failed"
assert self.fcm[:, ['fsc', 1]][a, 0] == self.pnts[a, 0], \
"__getitem__ with mixed strings failed"
def test_delegate(self):
assert self.fcm.mean() == self.pnts.mean(), "delegation of mean failed"
def test_poly_gate(self):
verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
assert all(self.fcm.view() == array([[0, 1, 2]])), \
"gate excluded wrong points"
self.fcm.visit('root')
self.fcm.gate(g)
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def test_empty_poly_gate(self):
verts = array([[10, 10], [10, 11], [11, 11], [11, 10]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)), "gated region not empty")
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)), 'gated region not empty')
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def test_interval_gate(self):
verts = array([1.5, 4.5])
cols = [0]
g = IntervalGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(), array([[3, 4, 5]]),
'gate excluded wrong points')
self.fcm.visit('root')
self.fcm.gate(g)
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def test_empty_interval_gate(self):
verts = array([10.5, 40.5])
cols = [0]
g = IntervalGate(verts, cols)
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)),
'gate excluded wrong points')
self.fcm.gate(g)
assert_array_equal(self.fcm.view(),
array([]).reshape((0, 3)),
'gate excluded wrong points')
nodes = self.fcm.tree.nodes.keys()
assert 'g2' in nodes, 'gating name mangled'
assert 'g1' in nodes, 'gating name mangled'
def test_boundary_events(self):
pnts = array([[0, 1, 2], [3, 4, 5], [0, 2, 5]])
fcm = FCMdata('test_fcm', pnts, ['fsc', 'ssc', 'cd3'], scatters=[0, 1])
eps = 1e-10
result = fcm.boundary_events()
assert result['fsc'] - 1 < eps, str(result['fsc'])
assert result['ssc'] - 2.0 / 3.0 < eps
assert result['cd3'] - 1 < eps
def test_chain_op(self):
verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g).gate(g)
self.assertTrue(all(self.fcm.view() == array([[0, 1, 2]])),
'gate excluded wrong points')
def test_getattr(self):
assert self.fcm.shape == (2, 3), '__gettattr__ failed to deligate'
def test_summary(self):
tmp = self.fcm.summary()
assert tmp.startswith('fsc:') is True, 'Summary failed'
def test_pickle(self):
import pickle
import StringIO
str_buffer = StringIO.StringIO()
pickle.dump(self.fcm, str_buffer)
str_buffer.seek(0)
tmp = pickle.load(str_buffer)
self.assertTrue(all(self.fcm[:] == tmp[:]))
for unused in range(3):
str_buffer = StringIO.StringIO()
pickle.dump(tmp, str_buffer)
str_buffer.seek(0)
tmp = pickle.load(str_buffer)
self.assertTrue(all(self.fcm[:] == tmp[:]))
def test_copy(self):
cpy = self.fcm.copy()
self.assertFalse(cpy is self.fcm,
"copy reproduced the exact same object")
self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
"copy failed to reproduce the view tree")
# make sure changes to object self.fcm don't show up on cpy
verts = array([[-.1, -.1], [-.1, 1.1], [1.1, 1.1], [1.1, -.1]])
cols = [0, 1]
g = PolyGate(verts, cols)
self.fcm.gate(g)
self.assertFalse(cpy.tree.pprint() == self.fcm.tree.pprint(),
"copy failed to reproduce the view tree")
#make sure tree is actually copied
cpy = self.fcm.copy()
self.assertTrue(cpy.tree.pprint() == self.fcm.tree.pprint(),
"copy failed to reproduce the view tree")
def test_random_subsample(self):
self.fcm.subsample(1)
self.assertEqual(1, self.fcm.shape[0], 'random subsampling failed')
self.assertTrue(self.fcm[0] in self.pnts,
'random subsample geneterated non-existant point')
def test_anomaly_subsample(self):
mu = array([0, 1, 2])
sig = array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
cluster = DPCluster(1.0, mu, sig)
mix = DPMixture([cluster])
self.fcm.subsample(1, 'anomaly', mix)
self.assertEqual(1, self.fcm.shape[0], 'anomaly subsampling failed')
self.assertTrue(self.fcm[0] in self.pnts,
'anomaly subsample geneterated non-existant point')
def test_bias_subsample(self):
neg_mu = array([0, 1, 2])
pos_mu = array([3, 4, 5])
sig = array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
neg_cluster = DPCluster(1.0, neg_mu, sig)
neg_mix = DPMixture([neg_cluster])
pos_cluster0 = DPCluster(0.5, neg_mu, sig)
pos_cluster1 = DPCluster(0.5, pos_mu, sig)
pos_mix = DPMixture([pos_cluster0, pos_cluster1])
self.fcm.subsample(1, 'bias', pos=pos_mix, neg=neg_mix)
self.assertEqual(1, self.fcm.shape[0], 'bias subsampling failed')
self.assertTrue(self.fcm[0] in self.pnts,
'bias subsample geneterated non-existant point')
def test_subsample_by_slice(self):
self.fcm.subsample(slice(1))
self.assertEqual(1, self.fcm.shape[0])
assert_array_equal(self.pnts[0], self.fcm[0],
'subsample by slice failed')