def annot2rawniml(a):
'''Converts annotation to raw NIML format'''
a = a.copy()
t = a.pop('AFNI_labeltable')
t['labels'] = ['R', 'G', 'B', 'A', 'key', 'name']
t['dset_type'] = 'LabelTableObject_data'
t['node_indices'] = None
r = dset.dset2rawniml(t).copy()
_fix_rawniml_table_output(r)
t = dset.dset2rawniml(a)
_fix_rawniml_main_output(t)
# add the table to the nodes
t['nodes'].insert(2, r)
return t
def annot2rawniml(a):
'''Converts annotation to raw NIML format'''
a = a.copy()
t = a.pop('AFNI_labeltable')
t['labels'] = ['R', 'G', 'B', 'A', 'key', 'name']
t['dset_type'] = 'LabelTableObject_data'
t['node_indices'] = None
r = dset.dset2rawniml(t).copy()
_fix_rawniml_table_output(r)
t = dset.dset2rawniml(a)
_fix_rawniml_main_output(t)
# add the table to the nodes
t['nodes'].insert(2, r)
return t
def test_surface_dset_niml_io_with_unicode(self, fn):
ds = dataset_wizard(np.arange(20).reshape((4, 5)), targets=1, chunks=1)
ds.sa['unicode'] = [u'u1', u'uu2', u'uuu3', u'uuuu4']
ds.sa['str'] = ['s1', 'ss2', 'sss3', 'ssss4']
ds.fa['node_indices'] = np.arange(5)
# ensure sample attributes are of String type (not array)
niml_dict = afni_niml_dset.dset2rawniml(niml.to_niml(ds))
expected_dtypes = dict(PYMVPA_SA_unicode='String',
PYMVPA_SA_str='String',
PYMVPA_SA_targets='1*int32')
def assert_has_expected_datatype(name, expected_dtype, niml):
"""helper function"""
nodes = niml_dict['nodes']
for node in nodes:
if node['name'] == name:
assert_equal(node['ni_type'], expected_dtype)
return
raise ValueError('not found: %s', name)
for name, expected_dtype in expected_dtypes.iteritems():
assert_has_expected_datatype(name, expected_dtype, niml)
# test NIML I/O
niml.write(fn, ds)
# remove extra fields added when reading the file
ds2 = niml.from_any(fn)
ds2.a.pop('history')
ds2.a.pop('filename')
ds2.sa.pop('labels')
ds2.sa.pop('stats')
# NIML does not support int64, only int32;
# compare equality of values in samples by setting the
# datatype the same as in the input (int32 or int64 depending
# on the platform)
ds2.samples = np.asarray(ds2.samples, dtype=ds.samples.dtype)
assert_datasets_equal(ds, ds2)
def test_surface_dset_niml_io_with_unicode(self, fn):
ds = dataset_wizard(np.arange(20).reshape((4, 5)), targets=1, chunks=1)
ds.sa['unicode'] = ['u1', 'uu2', 'uuu3', 'uuuu4']
ds.sa['str'] = ['s1', 'ss2', 'sss3', 'ssss4']
ds.fa['node_indices'] = np.arange(5)
# ensure sample attributes are of String type (not array)
niml_dict = afni_niml_dset.dset2rawniml(niml.to_niml(ds))
expected_dtypes = dict(PYMVPA_SA_unicode='String',
PYMVPA_SA_str='String',
PYMVPA_SA_targets='1*int32')
def assert_has_expected_datatype(name, expected_dtype, niml):
"""helper function"""
nodes = niml_dict['nodes']
for node in nodes:
if node['name'] == name:
assert_equal(node['ni_type'], expected_dtype)
return
raise ValueError('not found: %s', name)
for name, expected_dtype in expected_dtypes.items():
assert_has_expected_datatype(name, expected_dtype, niml)
# test NIML I/O
niml.write(fn, ds)
# remove extra fields added when reading the file
ds2 = niml.from_any(fn)
ds2.a.pop('history')
ds2.a.pop('filename')
ds2.sa.pop('labels')
ds2.sa.pop('stats')
# NIML does not support int64, only int32;
# compare equality of values in samples by setting the
# datatype the same as in the input (int32 or int64 depending
# on the platform)
ds2.samples = np.asarray(ds2.samples, dtype=ds.samples.dtype)
assert_datasets_equal(ds, ds2)
def test_afni_niml_dset(self):
sz = (100, 45) # dataset size
rng = self._get_rng() # generate random data
expected_vals = {
(0, 0): -2.13856,
(sz[0] - 1, sz[1] - 1): -1.92434,
(sz[0], sz[1] - 1): None,
(sz[0] - 1, sz[1]): None,
sz: None
}
# test for different formats in which the data is stored
fmts = ['text', 'binary', 'base64']
# also test for different datatypes
tps = [np.int32, np.int64, np.float32, np.float64]
# generated random data
data = rng.normal(size=sz)
# set labels for samples, and set node indices
labels = [
'lab_%d' % round(rng.uniform() * 1000) for _ in xrange(sz[1])
]
node_indices = np.argsort(rng.uniform(size=(sz[0], )))
node_indices = np.reshape(node_indices, (sz[0], 1))
eps = .00001
# test I/O
_, fn = tempfile.mkstemp('data.niml.dset', 'test')
# depending on the mode we do different tests (but on the same data)
modes = ['normal', 'skipio', 'sparse2full']
for fmt in fmts:
for tp in tps:
for mode in modes:
# make a dataset
dset = dict(data=np.asarray(data, tp),
labels=labels,
node_indices=node_indices)
dset_keys = dset.keys()
if mode == 'skipio':
# try conversion to/from raw NIML
# do not write to disk
r = afni_niml_dset.dset2rawniml(dset)
s = afni_niml.rawniml2string(r)
r2 = afni_niml.string2rawniml(s)
dset2 = afni_niml_dset.rawniml2dset(r2)[0]
else:
# write and read from disk
afni_niml_dset.write(fn, dset, fmt)
dset2 = afni_niml_dset.read(fn)
os.remove(fn)
# data in dset and dset2 should be identical
for k in dset_keys:
# general idea is to test whether v is equal to v2
v = dset[k]
v2 = dset2[k]
if k == 'data':
if mode == 'sparse2full':
# test the sparse2full feature
# this changes the order of the data over columns
# so we skip testing whether dset2 is equal to dset
nfull = 2 * sz[0]
dset3 = afni_niml_dset.sparse2full(
dset2, pad_to_node=nfull)
assert_equal(dset3['data'].shape[0], nfull)
idxs = dset['node_indices'][:, 0]
idxs3 = dset3['node_indices'][:, 0]
vbig = np.zeros((nfull, sz[1]))
vbig[idxs, :] = v[np.arange(sz[0]), :]
v = vbig
v2 = dset3['data'][idxs3, :]
else:
# check that data is as expected
for pos, val in expected_vals.iteritems():
if val is None:
assert_raises(IndexError,
lambda x: x[pos], v2)
else:
val2 = np.asarray(val, tp)
assert_true(abs(v2[pos] - val2) < eps)
if type(v) is list:
assert_equal(v, v2)
else:
eps_dec = 4
if mode != 'sparse2full' or k == 'data':
assert_array_almost_equal(v, v2, eps_dec)
def test_afni_niml_dset(self, fn):
sz = (100, 45) # dataset size
rng = self._get_rng() # generate random data
expected_vals = {(0, 0):-2.13856 , (sz[0] - 1, sz[1] - 1):-1.92434,
(sz[0], sz[1] - 1):None, (sz[0] - 1, sz[1]):None,
sz:None}
# test for different formats in which the data is stored
fmts = ['text', 'binary', 'base64']
# also test for different datatypes
tps = [np.int32, np.int64, np.float32, np.float64]
# generated random data
data = rng.normal(size=sz)
# set labels for samples, and set node indices
labels = ['lab_%d' % round(rng.uniform() * 1000)
for _ in xrange(sz[1])]
node_indices = np.argsort(rng.uniform(size=(sz[0],)))
node_indices = np.reshape(node_indices, (sz[0], 1))
eps = .00001
# test I/O
# depending on the mode we do different tests (but on the same data)
modes = ['normal', 'skipio', 'sparse2full']
for fmt in fmts:
for tp in tps:
for mode in modes:
# make a dataset
dset = dict(data=np.asarray(data, tp),
labels=labels,
node_indices=node_indices)
dset_keys = dset.keys()
if mode == 'skipio':
# try conversion to/from raw NIML
# do not write to disk
r = afni_niml_dset.dset2rawniml(dset)
s = afni_niml.rawniml2string(r)
r2 = afni_niml.string2rawniml(s)
dset2 = afni_niml_dset.rawniml2dset(r2)[0]
else:
# write and read from disk
afni_niml_dset.write(fn, dset, fmt)
dset2 = afni_niml_dset.read(fn)
os.remove(fn)
# data in dset and dset2 should be identical
for k in dset_keys:
# general idea is to test whether v is equal to v2
v = dset[k]
v2 = dset2[k]
if k == 'data':
if mode == 'sparse2full':
# test the sparse2full feature
# this changes the order of the data over columns
# so we skip testing whether dset2 is equal to dset
nfull = 2 * sz[0]
dset3 = afni_niml_dset.sparse2full(dset2,
pad_to_node=nfull)
assert_equal(dset3['data'].shape[0], nfull)
idxs = dset['node_indices'][:, 0]
idxs3 = dset3['node_indices'][:, 0]
vbig = np.zeros((nfull, sz[1]))
vbig[idxs, :] = v[np.arange(sz[0]), :]
v = vbig
v2 = dset3['data'][idxs3, :]
else:
# check that data is as expected
for pos, val in expected_vals.iteritems():
if val is None:
assert_raises(IndexError, lambda x:x[pos], v2)
else:
val2 = np.asarray(val, tp)
assert_true(abs(v2[pos] - val2) < eps)
if type(v) is list:
assert_equal(v, v2)
else:
eps_dec = 4
if mode != 'sparse2full' or k == 'data':
assert_array_almost_equal(v, v2, eps_dec)
