def test_vstack_successive(self):
x0_data = np.random.rand(10, 20)
x0_label = np.random.rand(10, 1)
x0_run = np.arange(10).reshape(10, 1) + 1
x1_data = np.random.rand(10, 20)
x1_label = np.random.rand(10, 1)
x1_run = np.arange(10).reshape(10, 1) + 1
bdata0 = bdpy.BData()
bdata0.add(x0_data, 'Data')
bdata0.add(x0_label, 'Label')
bdata0.add(x0_run, 'Run')
bdata1 = bdpy.BData()
bdata1.add(x1_data, 'Data')
bdata1.add(x1_label, 'Label')
bdata1.add(x0_run, 'Run')
bdata_merged = vstack([bdata0, bdata1], successive=['Run'])
np.testing.assert_array_equal(bdata_merged.select('Data'),
np.vstack([x0_data, x1_data]))
np.testing.assert_array_equal(bdata_merged.select('Label'),
np.vstack([x0_label, x1_label]))
np.testing.assert_array_equal(
bdata_merged.select('Run'),
np.vstack([x0_run, x1_run + len(x0_run)]))
def test_vstack_minimal(self):
x0_data = np.random.rand(5, 10)
x0_label = np.random.rand(5, 1)
x1_data = np.random.rand(5, 10)
x1_label = np.random.rand(5, 1)
bdata0 = bdpy.BData()
bdata0.add(x0_data, 'Data')
bdata0.add(x0_label, 'Label')
bdata0.add_metadata('key shared',
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, np.nan],
'Shared meta-data')
bdata0.add_metadata('key only in 0', np.random.rand(11),
'Meta-data only in bdata0')
bdata1 = bdpy.BData()
bdata1.add(x1_data, 'Data')
bdata1.add(x1_label, 'Label')
bdata1.add_metadata('key shared',
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, np.nan],
'Shared meta-data')
bdata0.add_metadata('key only in 1', np.random.rand(11),
'Meta-data only in bdata1')
bdata_merged = vstack([bdata0, bdata1], metadata_merge='minimal')
np.testing.assert_array_equal(bdata_merged.select('Data'),
np.vstack([x0_data, x1_data]))
np.testing.assert_array_equal(bdata_merged.select('Label'),
np.vstack([x0_label, x1_label]))
np.testing.assert_array_equal(bdata_merged.get_metadata('key shared'),
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, np.nan])
self.assertFalse('key only in 0' in bdata_merged.metadata.key)
self.assertFalse('key only in 1' in bdata_merged.metadata.key)
def test_vstack_vmap(self):
x0_data = np.random.rand(10, 20)
x0_label = np.random.permutation(np.arange(10)).reshape(10, 1) + 1
x0_label_map = {k: 'label_%04d' % k for k in x0_label.flatten()}
x1_data = np.random.rand(10, 20)
x1_label = np.random.permutation(np.arange(10)).reshape(10, 1) + 1
x1_label_map = {k: 'label_%04d' % k for k in x1_label.flatten()}
bdata0 = bdpy.BData()
bdata0.add(x0_data, 'Data')
bdata0.add(x0_label, 'Label')
bdata0.add_vmap('Label', x0_label_map)
bdata1 = bdpy.BData()
bdata1.add(x1_data, 'Data')
bdata1.add(x1_label, 'Label')
bdata1.add_vmap('Label', x1_label_map)
bdata_merged = vstack([bdata0, bdata1])
np.testing.assert_array_equal(bdata_merged.select('Data'),
np.vstack([x0_data, x1_data]))
np.testing.assert_array_equal(bdata_merged.select('Label'),
np.vstack([x0_label, x1_label]))
# Check vmap
assert bdata0.get_vmap('Label') == bdata1.get_vmap('Label')
assert bdata_merged.get_vmap('Label') == bdata0.get_vmap('Label')
def test_metadata_equal(self):
bdata0 = bdpy.BData()
bdata1 = bdpy.BData()
bdata0.add_metadata('key 0', [1, 1, 0], 'Test key 0')
bdata1.add_metadata('key 0', [1, 1, 0], 'Test key 0')
bdata0.add_metadata('key 1', [0, 0, 1], 'Test key 1')
bdata1.add_metadata('key 1', [0, 0, 1], 'Test key 1')
self.assertTrue(metadata_equal(bdata0, bdata1))
self.assertTrue(metadata_equal(bdata0, bdata1, strict=False))
self.assertTrue(metadata_equal(bdata0, bdata1, strict=True))
def test_metadata_equal_loose_notequal_value(self):
bdata0 = bdpy.BData()
bdata1 = bdpy.BData()
bdata0.add_metadata('key 0', [1, 1, 0], 'Test key 0')
bdata0.add_metadata('key 1', [0, 0, 1], 'Test key 1')
bdata1.add_metadata('key 1', [0, 0, 2], 'Test key 1')
bdata1.add_metadata('key 0', [1, 1, 0], 'Test key 0')
self.assertFalse(metadata_equal(bdata0, bdata1))
self.assertFalse(metadata_equal(bdata0, bdata1, strict=False))
self.assertFalse(metadata_equal(bdata0, bdata1, strict=False))
def test_add_dataset_pass0003(self):
"""Test for add_dataset (pass case 0003)"""
b = bdpy.BData()
attr_A = 'TestAttr_A'
data_A = np.random.rand(10, 10)
b.add_dataset(data_A, attr_A)
attr_B = 'TestAttr_B'
data_B = np.random.rand(10, 2)
b.add_dataset(data_B, attr_B)
attr_A2 = 'TestAttr_A'
data_A2 = np.random.rand(10, 3)
b.add_dataset(data_A2, attr_A2)
np.testing.assert_array_equal(b.dataSet, np.hstack((data_A, data_B, data_A2)))
self.assertEqual(b.metaData[0]['key'], attr_A)
#self.assertEqual(b.metaData[0]['description'], 'Attribute: TestAttr_A = 1')
np.testing.assert_array_equal(b.metaData[0]['value'],
np.hstack((np.ones(10, dtype = int),
[np.nan, np.nan],
np.ones(3, dtype = int))))
def test_vmap_errorcases(self):
n_sample = 4
bdata = bdpy.BData()
bdata.add(np.random.rand(n_sample, 3), 'MainData')
bdata.add(np.arange(n_sample) + 1, 'Label')
label_map = {(i + 1): 'label-%04d' % (i + 1) for i in range(n_sample)}
bdata.add_vmap('Label', label_map)
# Vmap not found
with self.assertRaises(ValueError):
bdata.get_label('MainData')
# Invalid vmap (map is not a dict)
label_map_invalid = range(n_sample)
with self.assertRaises(TypeError):
bdata.add_vmap('Label', label_map_invalid)
# Invalid vmap (key is str)
label_map_invalid = {'label-%04d' % i: i for i in range(n_sample)}
with self.assertRaises(TypeError):
bdata.add_vmap('Label', label_map_invalid)
# Inconsistent vmap
label_map_inconsist = {
i: 'label-%04d-inconsist' % i
for i in range(n_sample)
}
with self.assertRaises(ValueError):
bdata.add_vmap('Label', label_map_inconsist)
def test_add_metadata_pass0002(self):
"""Test for add_metadata (pass case 0001)"""
b = bdpy.BData()
md_key = 'MetaData_A'
md_desc_1 = 'Metadata for test'
md_val_1 = np.zeros(10)
md_desc_2 = 'Metadata for test (overwriting)'
md_val_2 = np.ones(10)
b.add_dataset(np.random.rand(10, 10), 'Test data')
b.add_metadata(md_key, md_val_1, md_desc_1)
b.add_metadata(md_key, md_val_2, md_desc_2)
exp_metaData = [ { 'key' : 'Test data',
'description' : '',
'value' : np.ones(10) },
{ 'key' : md_key,
'description' : md_desc_2,
'value' : md_val_2 } ]
for m, e in zip(b.metaData, exp_metaData):
self.assertEqual(m['key'], e['key'])
#self.assertEqual(m['description'], e['description'])
np.testing.assert_array_equal(m['value'], e['value'])
def test_add_metadata_5(self):
"""Test for add_metadata."""
md_key = 'TestMetaData'
md_desc = 'Metadata for test'
md_val = np.random.rand(10)
testdata_a = np.random.rand(10, 10)
testdata_b = np.random.rand(10, 10)
b = bdpy.BData()
b.add(testdata_a, 'TestDataA')
b.add(testdata_b, 'TestDataB')
b.add_metadata(md_key,
md_val,
where='TestDataA',
attribute='TestDataB')
assert_array_equal(b.dataSet, np.hstack((testdata_a, testdata_b)))
assert_array_equal(b.metadata.get('TestDataA', 'value'),
np.array([1] * 10 + [np.nan] * 10))
assert_array_equal(b.metadata.get('TestMetaData', 'value'),
np.hstack((md_val, [np.nan] * 10)))
def load_features(self, datfile, dattype=None, feature_list=[]):
if dattype == None:
# TODO: add data (file) type guessing
pass
if dattype == 'matconvnet':
self.datatype = dattype
self.layers = [
'cnn1', 'cnn2', 'cnn3', 'cnn4', 'cnn5', 'cnn6', 'cnn7', 'cnn8',
'hmax1', 'hmax2', 'hmax3', 'gist', 'sift'
]
self.data_raw = bdpy.BData(datfile)
self.num_units = {
feat: self.data_raw.select_feature(feat + ' = 1').shape[1]
for feat in self.layers
}
elif dattype == 'caffe':
self.datatype = dattype
self.layers = [
'conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6-conv',
'fc7-conv', 'fc8-conv'
]
with open(datfile, 'rb') as f:
self.data_raw = pickle.load(f)
self.num_units = {
feat: self.data_raw[feat][0].shape[0]
for feat in self.layers
}
def __init__(self, *args, **kwargs):
super(TestBdata, self).__init__(*args, **kwargs)
self.data = bdpy.BData()
x = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
[10, 11, 12, 13, 14, 15, 16, 17, 18, 19],
[20, 21, 22, 23, 24, 25, 26, 27, 28, 29],
[30, 31, 32, 33, 34, 35, 36, 37, 38, 39],
[40, 41, 42, 43, 44, 45, 46, 47, 48, 49]])
g = np.array([1, 2, 3, 4, 5])
self.data.add(x, 'VoxelData')
self.data.add(g, 'Group')
self.data.add_metadata('Mask_0:3', [1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
where='VoxelData')
self.data.add_metadata('Mask_3:3', [0, 0, 0, 1, 1, 1, 0, 0, 0, 0],
where='VoxelData')
self.data.add_metadata('Mask_6:3', [0, 0, 0, 0, 0, 0, 1, 1, 1, 0],
where='VoxelData')
self.data.add_metadata('Mask_0:5', [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
where='VoxelData')
self.data.add_metadata('Val_A', [9, 7, 5, 3, 1, 0, 2, 4, 6, 8],
where='VoxelData')
def test_select(self):
'''Test for BData.select.'''
data_x = np.random.rand(5, 10)
data_y = np.random.rand(5, 5)
b = bdpy.BData()
b.add(data_x, 'Data_X')
b.add(data_y, 'Data_Y')
b.add_metadata('ROI_0:5', [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
where='Data_X')
b.add_metadata('ROI_3:8', [0, 0, 0, 1, 1, 1, 1, 1, 0, 0],
where='Data_X')
b.add_metadata('ROI_4:9', [0, 0, 0, 0, 1, 1, 1, 1, 1, 0],
where='Data_X')
assert_array_equal(b.select('Data_X'), data_x)
assert_array_equal(b.select('Data_X = 1'), data_x)
assert_array_equal(b.select('ROI_0:5'), data_x[:, 0:5])
assert_array_equal(b.select('ROI_0:5 & ROI_3:8'), data_x[:, 3:5])
assert_array_equal(b.select('ROI_0:5 = 1 & ROI_3:8 = 1'), data_x[:,
3:5])
assert_array_equal(b.select('ROI_0:5 | ROI_3:8'), data_x[:, 0:8])
assert_array_equal(b.select('ROI_0:5 = 1 | ROI_3:8 = 1'), data_x[:,
0:8])
assert_array_equal(b.select('(ROI_0:5 | ROI_3:8) & ROI_4:9'),
data_x[:, 4:8])
assert_array_equal(b.select('ROI_0:5 | (ROI_3:8 & ROI_4:9)'),
data_x[:, 0:8])
assert_array_equal(b.select('ROI_*'), data_x[:, 0:9])
assert_array_equal(b.select('ROI_0:5 + ROI_3:8'), data_x[:, 0:8])
assert_array_equal(b.select('ROI_0:5 - ROI_3:8'), data_x[:, 0:3])
def test_add_get(self):
'''Test for BData.add and get.'''
data_x = np.random.rand(5, 10)
data_y = np.random.rand(5, 8)
data_z = np.random.rand(5, 20)
b = bdpy.BData()
b.add(data_x, 'Data_X')
b.add(data_y, 'Data_Y')
b.add(data_z, 'Data_Z')
# dataset
assert_array_equal(b.get('Data_X'), data_x)
assert_array_equal(b.get('Data_Y'), data_y)
assert_array_equal(b.get('Data_Z'), data_z)
# metadata
assert_array_equal(b.metadata.get('Data_X', 'value'),
np.array([1] * 10 + [np.nan] * 8 + [np.nan] * 20))
assert_array_equal(b.metadata.get('Data_Y', 'value'),
np.array([np.nan] * 10 + [1] * 8 + [np.nan] * 20))
assert_array_equal(b.metadata.get('Data_Z', 'value'),
np.array([np.nan] * 10 + [np.nan] * 8 + [1] * 20))
# metadata (BData.get_metadata)
assert_array_equal(b.get_metadata('Data_X'),
np.array([1] * 10 + [np.nan] * 8 + [np.nan] * 20))
assert_array_equal(b.get_metadata('Data_Y'),
np.array([np.nan] * 10 + [1] * 8 + [np.nan] * 20))
assert_array_equal(b.get_metadata('Data_Z'),
np.array([np.nan] * 10 + [np.nan] * 8 + [1] * 20))
def test_metadata_add_get_where(self):
'''Test for add/get_metadata with where option.'''
data_x = np.random.rand(5, 10)
data_y = np.random.rand(5, 8)
metadata_a = np.random.rand(10)
metadata_b = np.random.rand(8)
b = bdpy.BData()
b.add(data_x, 'Data_X')
b.add(data_y, 'Data_Y')
b.add_metadata('Metadata_A', metadata_a, where='Data_X')
b.add_metadata('Metadata_B', metadata_b, where='Data_Y')
assert_array_equal(b.get_metadata('Metadata_A'),
np.hstack([metadata_a,
np.array([np.nan] * 8)]))
assert_array_equal(b.get_metadata('Metadata_B'),
np.hstack([np.array([np.nan] * 10), metadata_b]))
assert_array_equal(b.get_metadata('Metadata_A', where='Data_X'),
metadata_a)
assert_array_equal(b.get_metadata('Metadata_B', where='Data_Y'),
metadata_b)
def test_vmap_add_invalid_name_vmap(self):
bdata = bdpy.BData()
bdata.add(np.random.rand(4, 3), 'MainData')
bdata.add(np.arange(4) + 1, 'Label')
label_map = {1: 'label-1', 2: 'label-2', 3: 'label-3', 4: 'label-4'}
with self.assertRaises(ValueError):
bdata.add_vmap('InvalidLabel', label_map)
def test_add(self):
"""Test for add."""
colname = 'TestData'
data = np.random.rand(5, 10)
b = bdpy.BData()
b.add(data, colname)
np.testing.assert_array_equal(b.dataSet, data)
np.testing.assert_array_equal(b.metadata.get(colname, 'value'),
np.array([1] * 10))
def test_vstack(self):
x0_data = np.random.rand(10, 20)
x0_label = np.random.rand(10, 1)
x1_data = np.random.rand(10, 20)
x1_label = np.random.rand(10, 1)
bdata0 = bdpy.BData()
bdata0.add(x0_data, 'Data')
bdata0.add(x0_label, 'Label')
bdata1 = bdpy.BData()
bdata1.add(x1_data, 'Data')
bdata1.add(x1_label, 'Label')
bdata_merged = vstack([bdata0, bdata1])
np.testing.assert_array_equal(bdata_merged.select('Data'),
np.vstack([x0_data, x1_data]))
np.testing.assert_array_equal(bdata_merged.select('Label'),
np.vstack([x0_label, x1_label]))
def test_vmap_add_get(self):
bdata = bdpy.BData()
bdata.add(np.random.rand(4, 3), 'MainData')
bdata.add(np.arange(4) + 1, 'Label')
label_map = {1: 'label-1', 2: 'label-2', 3: 'label-3', 4: 'label-4'}
label = ['label-1', 'label-2', 'label-3', 'label-4']
bdata.add_vmap('Label', label_map)
assert bdata.get_vmap('Label') == label_map
# Get labels
np.testing.assert_array_equal(bdata.get_label('Label'), label)
def test_add_dataset_pass0001(self):
"""Test for add_dataset (pass case 0001)"""
b = bdpy.BData()
attr = 'TestAttr'
data = np.random.rand(5, 10)
b.add_dataset(data, attr)
np.testing.assert_array_equal(b.dataSet, data)
self.assertEqual(b.metaData[0]['key'], attr)
#self.assertEqual(b.metaData[0]['description'], 'Attribute: TestAttr = 1')
np.testing.assert_array_equal(b.metaData[0]['value'], np.ones(10))
def test_vstack_vmap_inconsistent_vmap(self):
x0_data = np.random.rand(10, 20)
x0_label = np.random.permutation(np.arange(10)).reshape(10, 1) + 1
x0_label_map = {k: 'label_%04d' % k for k in x0_label.flatten()}
x1_data = np.random.rand(10, 20)
x1_label = np.random.permutation(np.arange(10)).reshape(10, 1) + 1
x1_label_map = {k: 'label_%04d_diff' % k for k in x1_label.flatten()}
bdata0 = bdpy.BData()
bdata0.add(x0_data, 'Data')
bdata0.add(x0_label, 'Label')
bdata0.add_vmap('Label', x0_label_map)
bdata1 = bdpy.BData()
bdata1.add(x1_data, 'Data')
bdata1.add(x1_label, 'Label')
bdata1.add_vmap('Label', x1_label_map)
with self.assertRaises(ValueError):
vstack([bdata0, bdata1], successive=['Label'])
def test_get_2(self):
"""Test for get."""
b = bdpy.BData()
attr_A = 'TestAttr_A'
data_A = np.random.rand(10, 10)
b.add(data_A, attr_A)
attr_B = 'TestAttr_B'
data_B = np.random.rand(10, 2)
b.add(data_B, attr_B)
exp_output = data_A
test_output = b.get('TestAttr_A')
np.testing.assert_array_equal(test_output, exp_output)
def test_add_metadata_1(self):
"""Test for add_metadata."""
md_key = 'TestMetaData'
md_desc = 'Metadata for test'
md_val = np.random.rand(10)
testdata = np.random.rand(10, 10)
b = bdpy.BData()
b.add(testdata, 'TestData')
b.add_metadata(md_key, md_val, md_desc)
assert_array_equal(b.dataSet, testdata)
assert_array_equal(b.metadata.get('TestData', 'value'),
np.array([1] * 10))
assert_array_equal(b.metadata.get('TestMetaData', 'value'), md_val)
def test_get_dataset_pass0001(self):
"""Test for get_dataset (pass case 0001)"""
b = bdpy.BData()
attr_A = 'TestAttr_A'
data_A = np.random.rand(10, 10)
b.add_dataset(data_A, attr_A)
attr_B = 'TestAttr_B'
data_B = np.random.rand(10, 2)
b.add_dataset(data_B, attr_B)
exp_output = np.hstack((data_A, data_B))
test_output = b.get_dataset()
np.testing.assert_array_equal(test_output, exp_output)
def test_add_2(self):
"""Test for add."""
colnames = ['TestData1', 'TestData2']
datalist = [np.random.rand(5, 10), np.random.rand(5, 3)]
b = bdpy.BData()
for c, d in zip(colnames, datalist):
b.add(d, c)
# Test
np.testing.assert_array_equal(b.dataSet, np.hstack(datalist))
np.testing.assert_array_equal(b.metadata.get(colnames[0], 'value'),
np.array([1] * 10 + [np.nan] * 3))
np.testing.assert_array_equal(b.metadata.get(colnames[1], 'value'),
np.array([np.nan] * 10 + [1] * 3))
def test_add_3(self):
"""Test for add."""
b = bdpy.BData()
data_a1 = np.random.rand(10, 10)
data_b = np.random.rand(10, 3)
data_a2 = np.random.rand(10, 5)
b.add(data_a1, 'TestDataA')
b.add(data_b, 'TestDataB')
b.add(data_a2, 'TestDataA')
np.testing.assert_array_equal(b.dataSet,
np.hstack((data_a1, data_b, data_a2)))
np.testing.assert_array_equal(
b.metadata.get('TestDataA', 'value'),
np.array([1] * 10 + [np.nan] * 3 + [1] * 5))
np.testing.assert_array_equal(
b.metadata.get('TestDataB', 'value'),
np.array([np.nan] * 10 + [1] * 3 + [np.nan] * 5))
def test_set_metadatadescription_1(self):
'''Test for set_metadatadescription.'''
data_x = np.random.rand(5, 10)
data_y = np.random.rand(5, 8)
metadata_a = np.random.rand(10)
metadata_b = np.random.rand(8)
b = bdpy.BData()
b.add(data_x, 'Data_X')
b.add(data_y, 'Data_Y')
b.add_metadata('Metadata_A', metadata_a, where='Data_X')
b.add_metadata('Metadata_B', metadata_b, where='Data_Y')
metadata_desc = 'Test metadata description'
b.set_metadatadescription('Metadata_A', metadata_desc)
self.assertEqual(b.metadata.get('Metadata_A', 'description'),
metadata_desc)
def test_vmap_add_unnecessary_vmap(self):
bdata = bdpy.BData()
bdata.add(np.random.rand(4, 3), 'MainData')
bdata.add(np.arange(4) + 1, 'Label')
label_map = {
1: 'label-1',
2: 'label-2',
3: 'label-3',
4: 'label-4',
5: 'label-5'
}
label_map_ture = {
1: 'label-1',
2: 'label-2',
3: 'label-3',
4: 'label-4'
}
bdata.add_vmap('Label', label_map)
assert bdata.get_vmap('Label') == label_map_ture
def test_add_metadata_2(self):
"""Test for add_metadata."""
md_key_1 = 'TestMetaData1'
md_desc_1 = 'Metadata for test 1'
md_val_1 = np.random.rand(10)
md_key_2 = 'TestMetaData2'
md_desc_2 = 'Metadata for test 2'
md_val_2 = np.random.rand(10)
testdata = np.random.rand(10, 10)
b = bdpy.BData()
b.add(testdata, 'TestData')
b.add_metadata(md_key_1, md_val_1, md_desc_1)
b.add_metadata(md_key_2, md_val_2, md_desc_2)
assert_array_equal(b.dataSet, testdata)
assert_array_equal(b.metadata.get('TestData', 'value'),
np.array([1] * 10))
assert_array_equal(b.metadata.get('TestMetaData1', 'value'), md_val_1)
assert_array_equal(b.metadata.get('TestMetaData2', 'value'), md_val_2)
def test_metadata_add_get(self):
'''Test for add/get_metadata.'''
data_x = np.random.rand(5, 10)
data_y = np.random.rand(5, 8)
n_col = data_x.shape[1] + data_y.shape[1]
metadata_a = np.random.rand(n_col)
metadata_b = np.random.rand(n_col)
b = bdpy.BData()
b.add(data_x, 'Data_X')
b.add(data_y, 'Data_Y')
b.add_metadata('Metadata_A', metadata_a)
b.add_metadata('Metadata_B', metadata_b)
assert_array_equal(b.metadata.get('Metadata_A', 'value'), metadata_a)
assert_array_equal(b.metadata.get('Metadata_B', 'value'), metadata_b)
assert_array_equal(b.get_metadata('Metadata_A'), metadata_a)
assert_array_equal(b.get_metadata('Metadata_B'), metadata_b)
def _load_file(self, fpath):
try:
return bdpy.BData(fpath)
except:
raise RuntimeError('Invalid data: %s' % fpath)
