def test_basic_collectable():
c = Collectable()
# empty by default
assert_equal(c.name, None)
assert_equal(c.value, None)
assert_equal(c.__doc__, None)
# late assignment
c.name = 'somename'
c.value = 12345
assert_equal(c.name, 'somename')
assert_equal(c.value, 12345)
# immediate content
c = Collectable('value', 'myname', "This is a test")
assert_equal(c.name, 'myname')
assert_equal(c.value, 'value')
assert_equal(c.__doc__, "This is a test")
assert_equal(str(c), 'myname')
# repr
e = eval(repr(c))
assert_equal(e.name, 'myname')
assert_equal(e.value, 'value')
assert_equal(e.__doc__, "This is a test")
# shallow copy does not create a view of value array
c.value = np.arange(5)
d = copy.copy(c)
assert_false(d.value.base is c.value)
# names starting with _ are not allowed
assert_raises(ValueError, c._set_name, "_underscore")
def test_basic_collectable():
c = Collectable()
# empty by default
assert_equal(c.name, None)
assert_equal(c.value, None)
assert_equal(c.__doc__, None)
# late assignment
c.name = 'somename'
c.value = 12345
assert_equal(c.name, 'somename')
assert_equal(c.value, 12345)
# immediate content
c = Collectable('value', 'myname', "This is a test")
assert_equal(c.name, 'myname')
assert_equal(c.value, 'value')
assert_equal(c.__doc__, "This is a test")
assert_equal(str(c), 'myname')
# repr
e = eval(repr(c))
assert_equal(e.name, 'myname')
assert_equal(e.value, 'value')
assert_equal(e.__doc__, "This is a test")
# shallow copy does not create a view of value array
c.value = np.arange(5)
d = copy.copy(c)
assert_false(d.value.base is c.value)
# names starting with _ are not allowed
assert_raises(ValueError, c._set_name, "_underscore")
def test_forward_dense_array_mapper():
mask = np.ones((3, 2), dtype="bool")
map_ = mask_mapper(mask)
# test shape reports
assert_equal(map_.forward1(mask).shape, (6,))
# test 1sample mapping
assert_array_equal(map_.forward1(np.arange(6).reshape(3, 2)), [0, 1, 2, 3, 4, 5])
# test 4sample mapping
foursample = map_.forward(np.arange(24).reshape(4, 3, 2))
assert_array_equal(
foursample, [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11], [12, 13, 14, 15, 16, 17], [18, 19, 20, 21, 22, 23]]
)
# check incomplete masks
mask[1, 1] = 0
map_ = mask_mapper(mask)
assert_equal(map_.forward1(mask).shape, (5,))
assert_array_equal(map_.forward1(np.arange(6).reshape(3, 2)), [0, 1, 2, 4, 5])
# check that it doesn't accept wrong dataspace
assert_raises(ValueError, map_.forward, np.arange(4).reshape(2, 2))
# check fail if neither mask nor shape
assert_raises(ValueError, mask_mapper)
# check that a full mask is automatically created when providing shape
m = mask_mapper(shape=(2, 3, 4))
mp = m.forward1(np.arange(24).reshape(2, 3, 4))
assert_array_equal(mp, np.arange(24))
def test_sphere_scaled():
s1 = ne.Sphere(3)
s = ne.Sphere(3, element_sizes=(1, 1))
# Should give exactly the same results since element_sizes are 1s
for p in ((0, 0), (-23, 1)):
assert_array_equal(s1(p), s(p))
ok_(len(s(p)) == len(set(s(p))))
# Raise exception if query dimensionality does not match element_sizes
assert_raises(ValueError, s, (1,))
s = ne.Sphere(3, element_sizes=(1.5, 2))
assert_array_equal(s((0, 0)),
[(-2, 0), (-1, -1), (-1, 0), (-1, 1),
(0, -1), (0, 0), (0, 1),
(1, -1), (1, 0), (1, 1), (2, 0)])
s = ne.Sphere(1.5, element_sizes=(1.5, 1.5, 1.5))
res = s((0, 0, 0))
ok_(np.all([np.sqrt(np.sum(np.array(x)**2)) <= 1.5 for x in res]))
ok_(len(res) == 7)
# all neighbors so no more than 1 voxel away -- just a cube, for
# some "sphere" effect radius had to be 3.0 ;)
td = np.sqrt(3*1.5**2)
s = ne.Sphere(td, element_sizes=(1.5, 1.5, 1.5))
res = s((0, 0, 0))
ok_(np.all([np.sqrt(np.sum(np.array(x)**2)) <= td for x in res]))
ok_(np.all([np.sum(np.abs(x) > 1) == 0 for x in res]))
ok_(len(res) == 27)
def test_subset():
data = np.array(
[[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]])
# float array doesn't work
sm = FeatureSliceMapper(np.ones(16))
assert_raises(IndexError, sm.forward, data)
# full mask
sm = FeatureSliceMapper(slice(None))
# should not change single samples
assert_array_equal(sm.forward(data[0:1].copy()), data[0:1])
# or multi-samples
assert_array_equal(sm.forward(data.copy()), data)
sm.train(data)
# same on reverse
assert_array_equal(sm.reverse(data[0:1].copy()), data[0:1])
# or multi-samples
assert_array_equal(sm.reverse(data.copy()), data)
# identical mappers
sm_none = FeatureSliceMapper(slice(None))
sm_int = FeatureSliceMapper(np.arange(16))
sm_bool = FeatureSliceMapper(np.ones(16, dtype='bool'))
sms = [sm_none, sm_int, sm_bool]
# test subsets
sids = [3,4,5,6]
bsubset = np.zeros(16, dtype='bool')
bsubset[sids] = True
subsets = [sids, slice(3,7), bsubset, [3,3,4,4,6,6,6,5]]
# all test subset result in equivalent masks, hence should do the same to
# the mapper and result in identical behavior
for st in sms:
for i, sub in enumerate(subsets):
# shallow copy
orig = copy(st)
subsm = FeatureSliceMapper(sub)
# should do copy-on-write for all important stuff!!
assert_true(orig.is_mergable(subsm))
orig += subsm
# test if selection did its job
if i == 3:
# special case of multiplying features
assert_array_equal(orig.forward1(data[0].copy()), subsets[i])
else:
assert_array_equal(orig.forward1(data[0].copy()), sids)
## all of the above shouldn't change the original mapper
#assert_array_equal(sm.get_mask(), np.arange(16))
# check for some bug catcher
# no 3D input
#assert_raises(IndexError, sm.forward, np.ones((3,2,1)))
# no input of wrong length
if __debug__:
# checked only in __debug__
assert_raises(ValueError, sm.forward, np.ones(4))
def test_chainmapper():
# the chain needs at lest one mapper
assert_raises(ValueError, ChainMapper, [])
# a typical first mapper is to flatten
cm = ChainMapper([FlattenMapper()])
# few container checks
assert_equal(len(cm), 1)
assert_true(isinstance(cm[0], FlattenMapper))
# now training
# come up with data
samples_shape = (2, 2, 4)
data_shape = (4,) + samples_shape
data = np.arange(np.prod(data_shape)).reshape(data_shape)
pristinedata = data.copy()
target = [
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63],
]
target = np.array(target)
# if it is not trained it knows nothing
cm.train(data)
# a new mapper should appear when doing feature selection
cm.append(FeatureSliceMapper(range(1, 16)))
assert_equal(cm.forward1(data[0]).shape, (15,))
assert_equal(len(cm), 2)
# multiple slicing
cm.append(FeatureSliceMapper([9, 14]))
assert_equal(cm.forward1(data[0]).shape, (2,))
assert_equal(len(cm), 3)
# check reproduction
cm_clone = eval(repr(cm))
assert_equal(repr(cm_clone), repr(cm))
# what happens if we retrain the whole beast an same data as before
cm.train(data)
assert_equal(cm.forward1(data[0]).shape, (2,))
assert_equal(len(cm), 3)
# let's map something
mdata = cm.forward(data)
assert_array_equal(mdata, target[:, [10, 15]])
# and back
rdata = cm.reverse(mdata)
# original shape
assert_equal(rdata.shape, data.shape)
# content as far it could be restored
assert_array_equal(rdata[rdata > 0], data[rdata > 0])
assert_equal(np.sum(rdata > 0), 8)
def test_attrmap_conflicts():
am_n = AttributeMap({'a':1, 'b':2, 'c':1})
am_t = AttributeMap({'a':1, 'b':2, 'c':1}, collisions_resolution='tuple')
am_l = AttributeMap({'a':1, 'b':2, 'c':1}, collisions_resolution='lucky')
q_f = ['a', 'b', 'a', 'c']
# should have no effect on forward mapping
ok_(np.all(am_n.to_numeric(q_f) == am_t.to_numeric(q_f)))
ok_(np.all(am_t.to_numeric(q_f) == am_l.to_numeric(q_f)))
assert_raises(ValueError, am_n.to_literal, [2])
r_t = am_t.to_literal([2, 1])
r_l = am_l.to_literal([2, 1])
def test_splitter():
ds = give_data()
# split with defaults
spl1 = Splitter('chunks')
assert_raises(NotImplementedError, spl1, ds)
splits = list(spl1.generate(ds))
assert_equal(len(splits), len(ds.sa['chunks'].unique))
for split in splits:
# it should have perform basic slicing!
assert_true(split.samples.base is ds.samples)
assert_equal(len(split.sa['chunks'].unique), 1)
assert_true('lastsplit' in split.a)
assert_true(splits[-1].a.lastsplit)
# now again, more customized
spl2 = Splitter('targets', attr_values = [0,1,1,2,3,3,3], count=4,
noslicing=True)
splits = list(spl2.generate(ds))
assert_equal(len(splits), 4)
for split in splits:
# it should NOT have perform basic slicing!
assert_false(split.samples.base is ds.samples)
assert_equal(len(split.sa['targets'].unique), 1)
assert_equal(len(split.sa['chunks'].unique), 10)
assert_true(splits[-1].a.lastsplit)
# two should be identical
assert_array_equal(splits[1].samples, splits[2].samples)
# now go wild and split by feature attribute
ds.fa['roi'] = np.repeat([0,1], 5)
# splitter should auto-detect that this is a feature attribute
spl3 = Splitter('roi')
splits = list(spl3.generate(ds))
assert_equal(len(splits), 2)
for split in splits:
assert_true(split.samples.base is ds.samples)
assert_equal(len(split.fa['roi'].unique), 1)
assert_equal(split.shape, (100, 5))
# and finally test chained splitters
cspl = ChainNode([spl2, spl3, spl1])
splits = list(cspl.generate(ds))
# 4 target splits and 2 roi splits each and 10 chunks each
assert_equal(len(splits), 80)
def test_collections():
sa = SampleAttributesCollection()
assert_equal(len(sa), 0)
assert_raises(ValueError, sa.__setitem__, 'test', 0)
l = range(5)
sa['test'] = l
# auto-wrapped
assert_true(isinstance(sa['test'], ArrayCollectable))
assert_equal(len(sa), 1)
# names which are already present in dict interface
assert_raises(ValueError, sa.__setitem__, 'values', range(5))
sa_c = copy.deepcopy(sa)
assert_equal(len(sa), len(sa_c))
assert_array_equal(sa.test, sa_c.test)
def test_cached_query_engine():
"""Test cached query engine
"""
sphere = ne.Sphere(1)
# dataset with just one "space"
ds = datasets['3dlarge']
qe0 = ne.IndexQueryEngine(myspace=sphere)
qec = ne.CachedQueryEngine(qe0)
# and ground truth one
qe = ne.IndexQueryEngine(myspace=sphere)
results_ind = []
results_kw = []
def cmp_res(res1, res2):
comp = [x == y for x, y in zip(res1, res2)]
ok_(np.all(comp))
for iq, q in enumerate((qe, qec)):
q.train(ds)
# sequential train on the same should be ok in both cases
q.train(ds)
res_ind = [q[fid] for fid in xrange(ds.nfeatures)]
res_kw = [q(myspace=x) for x in ds.fa.myspace]
# test if results match
cmp_res(res_ind, res_kw)
results_ind.append(res_ind)
results_kw.append(res_kw)
# now check if results of cached were the same as of regular run
cmp_res(results_ind[0], results_ind[1])
# Now do sanity checks
assert_raises(ValueError, qec.train, ds[:, :-1])
assert_raises(ValueError, qec.train, ds.copy())
ds2 = ds.copy()
qec.untrain()
qec.train(ds2)
# should be the same results on the copy
cmp_res(results_ind[0], [qec[fid] for fid in xrange(ds.nfeatures)])
cmp_res(results_kw[0], [qec(myspace=x) for x in ds.fa.myspace])
ok_(qec.train(ds2) is None)
def test_reverse_dense_array_mapper():
mask = np.ones((3, 2), dtype="bool")
mask[1, 1] = 0
map_ = mask_mapper(mask)
rmapped = map_.reverse1(np.arange(1, 6))
assert_equal(rmapped.shape, (3, 2))
assert_equal(rmapped[1, 1], 0)
assert_equal(rmapped[2, 1], 5)
# check that it doesn't accept wrong dataspace
assert_raises(ValueError, map_.forward, np.arange(6))
rmapped2 = map_.reverse(np.arange(1, 11).reshape(2, 5))
assert_equal(rmapped2.shape, (2, 3, 2))
assert_equal(rmapped2[0, 1, 1], 0)
assert_equal(rmapped2[1, 1, 1], 0)
assert_equal(rmapped2[0, 2, 1], 5)
assert_equal(rmapped2[1, 2, 1], 10)
def test_query_engine():
data = np.arange(54)
# indices in 3D
ind = np.transpose((np.ones((3, 3, 3)).nonzero()))
# sphere generator for 3 elements diameter
sphere = ne.Sphere(1)
# dataset with just one "space"
ds = Dataset([data, data], fa={'s_ind': np.concatenate((ind, ind))})
# and the query engine attaching the generator to the "index-space"
qe = ne.IndexQueryEngine(s_ind=sphere)
# cannot train since the engine does not know about the second space
assert_raises(ValueError, qe.train, ds)
# now do it again with a full spec
ds = Dataset([data, data], fa={'s_ind': np.concatenate((ind, ind)),
't_ind': np.repeat([0,1], 27)})
qe = ne.IndexQueryEngine(s_ind=sphere, t_ind=None)
qe.train(ds)
# internal representation check
# YOH: invalid for new implementation with lookup tables (dictionaries)
#assert_array_equal(qe._searcharray,
# np.arange(54).reshape(qe._searcharray.shape) + 1)
# should give us one corner, collapsing the 't_ind'
assert_array_equal(qe(s_ind=(0, 0, 0)),
[0, 1, 3, 9, 27, 28, 30, 36])
# directly specifying an index for 't_ind' without having an ROI
# generator, should give the same corner, but just once
assert_array_equal(qe(s_ind=(0, 0, 0), t_ind=0), [0, 1, 3, 9])
# just out of the mask -- no match
assert_array_equal(qe(s_ind=(3, 3, 3)), [])
# also out of the mask -- but single match
assert_array_equal(qe(s_ind=(2, 2, 3), t_ind=1), [53])
# query by id
assert_array_equal(qe(s_ind=(0, 0, 0), t_ind=0), qe[0])
assert_array_equal(qe(s_ind=(0, 0, 0), t_ind=[0, 1]),
qe(s_ind=(0, 0, 0)))
# should not fail if t_ind is outside
assert_array_equal(qe(s_ind=(0, 0, 0), t_ind=[0, 1, 10]),
qe(s_ind=(0, 0, 0)))
# should fail if asked about some unknown thing
assert_raises(ValueError, qe.__call__, s_ind=(0, 0, 0), buga=0)
# Test by using some literal feature atttribute
ds.fa['lit'] = ['roi1', 'ro2', 'r3']*18
# should work as well as before
assert_array_equal(qe(s_ind=(0, 0, 0)), [0, 1, 3, 9, 27, 28, 30, 36])
# should fail if asked about some unknown (yet) thing
assert_raises(ValueError, qe.__call__, s_ind=(0,0,0), lit='roi1')
# Create qe which can query literals as well
qe_lit = ne.IndexQueryEngine(s_ind=sphere, t_ind=None, lit=None)
qe_lit.train(ds)
# should work as well as before
assert_array_equal(qe_lit(s_ind=(0, 0, 0)), [0, 1, 3, 9, 27, 28, 30, 36])
# and subselect nicely -- only /3 ones
assert_array_equal(qe_lit(s_ind=(0, 0, 0), lit='roi1'),
[0, 3, 9, 27, 30, 36])
assert_array_equal(qe_lit(s_ind=(0, 0, 0), lit=['roi1', 'ro2']),
[0, 1, 3, 9, 27, 28, 30, 36])
def test_mapper_vs_zscore():
"""Test by comparing to results of elderly z-score function
"""
# data: 40 sample feature line in 20d space (40x20; samples x features)
dss = [
dataset_wizard(np.concatenate(
[np.arange(40) for i in range(20)]).reshape(20,-1).T,
targets=1, chunks=1),
] + datasets.values()
for ds in dss:
ds1 = deepcopy(ds)
ds2 = deepcopy(ds)
zsm = ZScoreMapper(chunks_attr=None)
assert_raises(RuntimeError, zsm.forward, ds1.samples)
zsm.train(ds1)
ds1z = zsm.forward(ds1.samples)
zscore(ds2, chunks_attr=None)
assert_array_almost_equal(ds1z, ds2.samples)
assert_array_equal(ds1.samples, ds.samples)
def test_sphere():
# test sphere initialization
s = ne.Sphere(1)
center0 = (0, 0, 0)
center1 = (1, 1, 1)
assert_equal(len(s(center0)), 7)
target = array([array([-1, 0, 0]),
array([ 0, -1, 0]),
array([ 0, 0, -1]),
array([0, 0, 0]),
array([0, 0, 1]),
array([0, 1, 0]),
array([1, 0, 0])])
# test of internals -- no recomputation of increments should be done
prev_increments = s._increments
assert_array_equal(s(center0), target)
ok_(prev_increments is s._increments)
# query lower dimensionality
_ = s((0, 0))
ok_(not prev_increments is s._increments)
# test Sphere call
target = [array([0, 1, 1]),
array([1, 0, 1]),
array([1, 1, 0]),
array([1, 1, 1]),
array([1, 1, 2]),
array([1, 2, 1]),
array([2, 1, 1])]
res = s(center1)
assert_array_equal(array(res), target)
# They all should be tuples
ok_(np.all([isinstance(x, tuple) for x in res]))
# test for larger diameter
s = ne.Sphere(4)
assert_equal(len(s(center1)), 257)
# test extent keyword
#s = ne.Sphere(4,extent=(1,1,1))
#assert_array_equal(array(s((0,0,0))), array([[0,0,0]]))
# test Errors during initialisation and call
#assert_raises(ValueError, ne.Sphere, 2)
#assert_raises(ValueError, ne.Sphere, 1.0)
# no longer extent available
assert_raises(TypeError, ne.Sphere, 1, extent=(1))
assert_raises(TypeError, ne.Sphere, 1, extent=(1.0, 1.0, 1.0))
s = ne.Sphere(1)
#assert_raises(ValueError, s, (1))
if __debug__:
# No float coordinates allowed for now...
# XXX might like to change that ;)
#
assert_raises(ValueError, s, (1.0, 1.0, 1.0))
def test_array_collectable():
c = ArrayCollectable()
# empty by default
assert_equal(c.name, None)
assert_equal(c.value, None)
# late assignment
c.name = 'somename'
assert_raises(ValueError, c._set, 12345)
assert_equal(c.value, None)
c.value = np.arange(5)
assert_equal(c.name, 'somename')
assert_array_equal(c.value, np.arange(5))
# immediate content
data = np.random.random(size=(3,10))
c = ArrayCollectable(data.copy(), 'myname',
"This is a test", length=3)
assert_equal(c.name, 'myname')
assert_array_equal(c.value, data)
assert_equal(c.__doc__, "This is a test")
assert_equal(str(c), 'myname')
# repr
from numpy import array
e = eval(repr(c))
assert_equal(e.name, 'myname')
assert_array_almost_equal(e.value, data)
assert_equal(e.__doc__, "This is a test")
# cannot assign array of wrong length
assert_raises(ValueError, c._set, np.arange(5))
assert_equal(len(c), 3)
# shallow copy DOES create a view of value array
c.value = np.arange(3)
d = copy.copy(c)
assert_true(d.value.base is c.value)
# names starting with _ are not allowed
assert_raises(ValueError, c._set_name, "_underscore")
def test_array_collectable():
c = ArrayCollectable()
# empty by default
assert_equal(c.name, None)
assert_equal(c.value, None)
# late assignment
c.name = 'somename'
assert_raises(ValueError, c._set, 12345)
assert_equal(c.value, None)
c.value = np.arange(5)
assert_equal(c.name, 'somename')
assert_array_equal(c.value, np.arange(5))
# immediate content
data = np.random.random(size=(3, 10))
c = ArrayCollectable(data.copy(), 'myname', "This is a test", length=3)
assert_equal(c.name, 'myname')
assert_array_equal(c.value, data)
assert_equal(c.__doc__, "This is a test")
assert_equal(str(c), 'myname')
# repr
from numpy import array
e = eval(repr(c))
assert_equal(e.name, 'myname')
assert_array_almost_equal(e.value, data)
assert_equal(e.__doc__, "This is a test")
# cannot assign array of wrong length
assert_raises(ValueError, c._set, np.arange(5))
assert_equal(len(c), 3)
# shallow copy DOES create a view of value array
c.value = np.arange(3)
d = copy.copy(c)
assert_true(d.value.base is c.value)
# names starting with _ are not allowed
assert_raises(ValueError, c._set_name, "_underscore")
def test_attrmap():
map_default = {'eins': 0, 'zwei': 2, 'sieben': 1}
map_custom = {'eins': 11, 'zwei': 22, 'sieben': 33}
literal = ['eins', 'zwei', 'sieben', 'eins', 'sieben', 'eins']
literal_nonmatching = ['uno', 'dos', 'tres']
num_default = [0, 2, 1, 0, 1, 0]
num_custom = [11, 22, 33, 11, 33, 11]
# no custom mapping given
am = AttributeMap()
assert_false(am)
ok_(len(am) == 0)
assert_array_equal(am.to_numeric(literal), num_default)
assert_array_equal(am.to_literal(num_default), literal)
ok_(am)
ok_(len(am) == 3)
#
# Tests for recursive mapping + preserving datatype
class myarray(np.ndarray):
pass
assert_raises(KeyError, am.to_literal, [(1, 2), 2, 0])
literal_fancy = [(1, 2), 2, [0], np.array([0, 1]).view(myarray)]
literal_fancy_tuple = tuple(literal_fancy)
literal_fancy_array = np.array(literal_fancy, dtype=object)
for l in (literal_fancy, literal_fancy_tuple,
literal_fancy_array):
res = am.to_literal(l, recurse=True)
assert_equal(res[0], ('sieben', 'zwei'))
assert_equal(res[1], 'zwei')
assert_equal(res[2], ['eins'])
assert_array_equal(res[3], ['eins', 'sieben'])
# types of result and subsequences should be preserved
ok_(isinstance(res, l.__class__))
ok_(isinstance(res[0], tuple))
ok_(isinstance(res[1], str))
ok_(isinstance(res[2], list))
ok_(isinstance(res[3], myarray))
# yet another example
a = np.empty(1, dtype=object)
a[0] = (0, 1)
res = am.to_literal(a, recurse=True)
ok_(isinstance(res[0], tuple))
#
# with custom mapping
am = AttributeMap(map=map_custom)
assert_array_equal(am.to_numeric(literal), num_custom)
assert_array_equal(am.to_literal(num_custom), literal)
# if not numeric nothing is mapped
assert_array_equal(am.to_numeric(num_custom), num_custom)
# even if the map doesn't fit
assert_array_equal(am.to_numeric(num_default), num_default)
# need to_numeric first
am = AttributeMap()
assert_raises(RuntimeError, am.to_literal, [1,2,3])
# stupid args
assert_raises(ValueError, AttributeMap, map=num_custom)
# map mismatch
am = AttributeMap(map=map_custom)
if __debug__:
# checked only in __debug__
assert_raises(KeyError, am.to_numeric, literal_nonmatching)
# needs reset and should work afterwards
am.clear()
assert_array_equal(am.to_numeric(literal_nonmatching), [2, 0, 1])
# and now reverse
am = AttributeMap(map=map_custom)
assert_raises(KeyError, am.to_literal, num_default)
# dict-like interface
am = AttributeMap()
ok_([(k, v) for k, v in am.iteritems()] == [])
def test_zscore():
"""Test z-scoring transformation
"""
# dataset: mean=2, std=1
samples = np.array((0, 1, 3, 4, 2, 2, 3, 1, 1, 3, 3, 1, 2, 2, 2, 2)).\
reshape((16, 1))
data = dataset_wizard(samples.copy(), targets=range(16), chunks=[0] * 16)
assert_equal(data.samples.mean(), 2.0)
assert_equal(data.samples.std(), 1.0)
data_samples = data.samples.copy()
zscore(data, chunks_attr='chunks')
# copy should stay intact
assert_equal(data_samples.mean(), 2.0)
assert_equal(data_samples.std(), 1.0)
# we should be able to operate on ndarrays
# But we can't change type inplace for an array, can't we?
assert_raises(TypeError, zscore, data_samples, chunks_attr=None)
# so lets do manually
data_samples = data_samples.astype(float)
zscore(data_samples, chunks_attr=None)
assert_array_equal(data.samples, data_samples)
print data_samples
# check z-scoring
check = np.array([-2, -1, 1, 2, 0, 0, 1, -1, -1, 1, 1, -1, 0, 0, 0, 0],
dtype='float64').reshape(16, 1)
assert_array_equal(data.samples, check)
data = dataset_wizard(samples.copy(), targets=range(16), chunks=[0] * 16)
zscore(data, chunks_attr=None)
assert_array_equal(data.samples, check)
# check z-scoring taking set of labels as a baseline
data = dataset_wizard(samples.copy(),
targets=[0, 2, 2, 2, 1] + [2] * 11,
chunks=[0] * 16)
zscore(data, param_est=('targets', [0, 1]))
assert_array_equal(samples, data.samples + 1.0)
# check that zscore modifies in-place; only guaranteed if no upcasting is
# necessary
samples = samples.astype('float')
data = dataset_wizard(samples,
targets=[0, 2, 2, 2, 1] + [2] * 11,
chunks=[0] * 16)
zscore(data, param_est=('targets', [0, 1]))
assert_array_equal(samples, data.samples)
# these might be duplicating code above -- but twice is better than nothing
# dataset: mean=2, std=1
raw = np.array((0, 1, 3, 4, 2, 2, 3, 1, 1, 3, 3, 1, 2, 2, 2, 2))
# dataset: mean=12, std=1
raw2 = np.array((0, 1, 3, 4, 2, 2, 3, 1, 1, 3, 3, 1, 2, 2, 2, 2)) + 10
# zscore target
check = [-2, -1, 1, 2, 0, 0, 1, -1, -1, 1, 1, -1, 0, 0, 0, 0]
ds = dataset_wizard(raw.copy(), targets=range(16), chunks=[0] * 16)
pristine = dataset_wizard(raw.copy(), targets=range(16), chunks=[0] * 16)
zm = ZScoreMapper()
# should do global zscore by default
zm.train(ds) # train
assert_array_almost_equal(zm.forward(ds), np.transpose([check]))
# should not modify the source
assert_array_equal(pristine, ds)
# if we tell it a different mean it should obey the order
zm = ZScoreMapper(params=(3,1))
zm.train(ds)
assert_array_almost_equal(zm.forward(ds), np.transpose([check]) - 1 )
assert_array_equal(pristine, ds)
# let's look at chunk-wise z-scoring
ds = dataset_wizard(np.hstack((raw.copy(), raw2.copy())),
targets=range(32),
chunks=[0] * 16 + [1] * 16)
# by default chunk-wise
zm = ZScoreMapper()
zm.train(ds) # train
assert_array_almost_equal(zm.forward(ds), np.transpose([check + check]))
# we should be able to do that same manually
zm = ZScoreMapper(params={0: (2,1), 1: (12,1)})
zm.train(ds) # train
assert_array_almost_equal(zm.forward(ds), np.transpose([check + check]))
def test_flatten():
samples_shape = (2, 2, 4)
data_shape = (4,) + samples_shape
data = np.arange(np.prod(data_shape)).reshape(data_shape).view(myarray)
pristinedata = data.copy()
target = [[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]]
target = np.array(target).view(myarray)
index_target = np.array([[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 0, 3],
[0, 1, 0], [0, 1, 1], [0, 1, 2], [0, 1, 3],
[1, 0, 0], [1, 0, 1], [1, 0, 2], [1, 0, 3],
[1, 1, 0], [1, 1, 1], [1, 1, 2], [1, 1, 3]])
# array subclass survives
ok_(isinstance(data, myarray))
# actually, there should be no difference between a plain FlattenMapper and
# a chain that only has a FlattenMapper as the one element
for fm in [FlattenMapper(space='voxel'),
ChainMapper([FlattenMapper(space='voxel'),
StaticFeatureSelection(slice(None))])]:
# not working if untrained
assert_raises(RuntimeError,
fm.forward1,
np.arange(np.sum(samples_shape) + 1))
fm.train(data)
ok_(isinstance(fm.forward(data), myarray))
ok_(isinstance(fm.forward1(data[2]), myarray))
assert_array_equal(fm.forward(data), target)
assert_array_equal(fm.forward1(data[2]), target[2])
assert_raises(ValueError, fm.forward, np.arange(4))
# all of that leaves that data unmodified
assert_array_equal(data, pristinedata)
# reverse mapping
ok_(isinstance(fm.reverse(target), myarray))
ok_(isinstance(fm.reverse1(target[0]), myarray))
ok_(isinstance(fm.reverse(target[1:2]), myarray))
assert_array_equal(fm.reverse(target), data)
assert_array_equal(fm.reverse1(target[0]), data[0])
assert_array_equal(fm.reverse(target[1:2]), data[1:2])
assert_raises(ValueError, fm.reverse, np.arange(14))
# check one dimensional data, treated as scalar samples
oned = np.arange(5)
fm.train(Dataset(oned))
# needs 2D
assert_raises(ValueError, fm.forward, oned)
# doesn't match mapper, since Dataset turns `oned` into (5,1)
assert_raises(ValueError, fm.forward, oned)
assert_equal(Dataset(oned).nfeatures, 1)
# try dataset mode, with some feature attribute
fattr = np.arange(np.prod(samples_shape)).reshape(samples_shape)
ds = Dataset(data, fa={'awesome': fattr.copy()})
assert_equal(ds.samples.shape, data_shape)
fm.train(ds)
dsflat = fm.forward(ds)
ok_(isinstance(dsflat, Dataset))
ok_(isinstance(dsflat.samples, myarray))
assert_array_equal(dsflat.samples, target)
assert_array_equal(dsflat.fa.awesome, np.arange(np.prod(samples_shape)))
assert_true(isinstance(dsflat.fa['awesome'], ArrayCollectable))
# test index creation
assert_array_equal(index_target, dsflat.fa.voxel)
# and back
revds = fm.reverse(dsflat)
ok_(isinstance(revds, Dataset))
ok_(isinstance(revds.samples, myarray))
assert_array_equal(revds.samples, data)
assert_array_equal(revds.fa.awesome, fattr)
assert_true(isinstance(revds.fa['awesome'], ArrayCollectable))
assert_false('voxel' in revds.fa)
def test_simpleboxcar():
data = np.atleast_2d(np.arange(10)).T
sp = np.arange(10)
# check if stupid thing don't work
assert_raises(ValueError, BoxcarMapper, sp, 0)
# now do an identity transformation
bcm = BoxcarMapper(sp, 1)
trans = bcm.forward(data)
# ,0 is a feature below, so we get explicit 2D out of 1D
assert_array_equal(trans[:,0], data)
# now check for illegal boxes
if __debug__:
# condition is checked only in __debug__
assert_raises(ValueError, BoxcarMapper(sp, 2).train, data)
# now something that should work
nbox = 9
boxlength = 2
sp = np.arange(nbox)
bcm = BoxcarMapper(sp, boxlength)
trans = bcm(data)
# check that is properly upcasts the dimensionality
assert_equal(trans.shape, (nbox, boxlength) + data.shape[1:])
# check actual values, squeezing the last dim for simplicity
assert_array_equal(trans.squeeze(), np.vstack((np.arange(9), np.arange(9)+1)).T)
# now test for proper data shape
data = np.ones((10,3,4,2))
sp = [ 2, 4, 3, 5 ]
trans = BoxcarMapper(sp, 4)(data)
assert_equal(trans.shape, (4,4,3,4,2))
# test reverse
data = np.arange(240).reshape(10, 3, 4, 2)
sp = [ 2, 4, 3, 5 ]
boxlength = 2
m = BoxcarMapper(sp, boxlength)
m.train(data)
mp = m.forward(data)
assert_equal(mp.shape, (4, 2, 3, 4, 2))
# try full reconstruct
mr = m.reverse(mp)
# shape has to match
assert_equal(mr.shape, (len(sp) * boxlength,) + data.shape[1:])
# only known samples are part of the results
assert_true((mr >= 24).all())
assert_true((mr < 168).all())
# check proper reconstruction of non-conflicting sample
assert_array_equal(mr[0].ravel(), np.arange(48, 72))
# check proper reconstruction of samples being part of multiple
# mapped samples
assert_array_equal(mr[1].ravel(), np.arange(72, 96))
# test reverse of a single sample
singlesample = np.arange(48).reshape(2, 3, 4, 2)
assert_array_equal(singlesample, m.reverse1(singlesample))
# should not work for shape mismatch, but it does work and is useful when
# reverse mapping sample attributes
#assert_raises(ValueError, m.reverse, singlesample[0])
# check broadcasting of 'raw' samples into proper boxcars on forward()
bc = m.forward1(np.arange(24).reshape(3, 4, 2))
assert_array_equal(bc, np.array(2 * [np.arange(24).reshape(3, 4, 2)]))
def test_chainmapper():
# the chain needs at lest one mapper
assert_raises(ValueError, ChainMapper, [])
# a typical first mapper is to flatten
cm = ChainMapper([FlattenMapper()])
# few container checks
assert_equal(len(cm), 1)
assert_true(isinstance(cm[0], FlattenMapper))
# now training
# come up with data
samples_shape = (2, 2, 4)
data_shape = (4,) + samples_shape
data = np.arange(np.prod(data_shape)).reshape(data_shape)
pristinedata = data.copy()
target = [
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63],
]
target = np.array(target)
# if it is not trained it knows nothing
cm.train(data)
# a new mapper should appear when doing feature selection
cm.append(StaticFeatureSelection(range(1, 16)))
assert_equal(cm.forward1(data[0]).shape, (15,))
assert_equal(len(cm), 2)
# multiple slicing
cm.append(StaticFeatureSelection([9, 14]))
assert_equal(cm.forward1(data[0]).shape, (2,))
assert_equal(len(cm), 3)
# check reproduction
if __debug__:
# debug mode needs special test as it enhances the repr output
# with module info and id() appendix for objects
import mvpa
cm_clone = eval(repr(cm))
assert_equal("#".join(repr(cm_clone).split("#")[:-1]), "#".join(repr(cm).split("#")[:-1]))
else:
cm_clone = eval(repr(cm))
assert_equal(repr(cm_clone), repr(cm))
# what happens if we retrain the whole beast an same data as before
cm.train(data)
assert_equal(cm.forward1(data[0]).shape, (2,))
assert_equal(len(cm), 3)
# let's map something
mdata = cm.forward(data)
assert_array_equal(mdata, target[:, [10, 15]])
# and back
rdata = cm.reverse(mdata)
# original shape
assert_equal(rdata.shape, data.shape)
# content as far it could be restored
assert_array_equal(rdata[rdata > 0], data[rdata > 0])
assert_equal(np.sum(rdata > 0), 8)
# Lets construct a dataset with mapper assigned and see
# if sub-selecting a feature adjusts trailing StaticFeatureSelection
# appropriately
ds_subsel = Dataset.from_wizard(data, mapper=cm)[:, 1]
tail_sfs = ds_subsel.a.mapper[-1]
assert_equal(repr(tail_sfs), "StaticFeatureSelection(slicearg=array([14]))")
