def test_ModelTraining_fastl2lir_bdmodel_nochunk(self):
# Init
results_dir_root = './results/ml'
if os.path.exists(results_dir_root):
shutil.rmtree(results_dir_root)
os.makedirs(results_dir_root)
# Data setup
X = np.random.rand(100, 500)
Y = np.random.rand(100, 50)
# Test
model = FastL2LiR()
model_param = {'alpha' : 100,
'n_feat' : 100}
train = ModelTraining(model, X, Y)
train.id = os.path.splitext(os.path.basename(__file__))[0] + '-fastl2lir-nochunk-bdmodel'
train.model_parameters = model_param
train.dtype = np.float32
train.save_format = 'bdmodel'
train.save_path = os.path.join(results_dir_root, 'fastl2lir-nochunk-bdmodel', 'model')
train.run()
self.assertTrue(os.path.isfile(os.path.join(results_dir_root, 'fastl2lir-nochunk-bdmodel', 'model', 'W.mat')))
self.assertTrue(os.path.isfile(os.path.join(results_dir_root, 'fastl2lir-nochunk-bdmodel', 'model', 'b.mat')))
W = load_array(os.path.join(results_dir_root, 'fastl2lir-nochunk-bdmodel', 'model', 'W.mat'), key='W')
b = load_array(os.path.join(results_dir_root, 'fastl2lir-nochunk-bdmodel', 'model', 'b.mat'), key='b')
self.assertEqual(W.shape, (500, 50))
self.assertEqual(b.shape, (1, 50))
def test_load_save_dense_array(self):
# ndim = 1
data = np.random.rand(10)
save_array('./tmp/test_array_dense_ndim1.mat', data, key='testdata')
testdata = load_array('./tmp/test_array_dense_ndim1.mat',
key='testdata')
np.testing.assert_array_equal(data, testdata)
# ndim = 2
data = np.random.rand(3, 2)
save_array('./tmp/test_array_dense_ndim2.mat', data, key='testdata')
testdata = load_array('./tmp/test_array_dense_ndim2.mat',
key='testdata')
np.testing.assert_array_equal(data, testdata)
# ndim = 3
data = np.random.rand(4, 3, 2)
save_array('./tmp/test_array_dense_ndim3.mat', data, key='testdata')
testdata = load_array('./tmp/test_array_dense_ndim3.mat',
key='testdata')
np.testing.assert_array_equal(data, testdata)
def test_load_array_jl(self):
data = np.array([[1, 0, 0, 0], [2, 2, 0, 0], [3, 3, 3, 0]])
testdata = load_array('data/array_jl_dense_v1.mat', key='a')
np.testing.assert_array_equal(data, testdata)
testdata = load_array('data/array_jl_sparse_v1.mat', key='a')
np.testing.assert_array_equal(data, testdata)
def test_fastl2lir_div(model_store, x, chunk_axis=1):
# W: shape = (n_voxels, shape_features)
if os.path.isdir(os.path.join(model_store, 'W')):
W_files = sorted(glob.glob(os.path.join(model_store, 'W', '*.mat')))
elif os.path.isfile(os.path.join(model_store, 'W.mat')):
W_files = [os.path.join(model_store, 'W.mat')]
else:
raise RuntimeError('W not found.')
# b: shape = (1, shape_features)
if os.path.isdir(os.path.join(model_store, 'b')):
b_files = sorted(glob.glob(os.path.join(model_store, 'b', '*.mat')))
elif os.path.isfile(os.path.join(model_store, 'b.mat')):
b_files = [os.path.join(model_store, 'b.mat')]
else:
raise RuntimeError('b not found.')
x_mean = hdf5storage.loadmat(os.path.join(
model_store, 'x_mean.mat'))['x_mean'] # shape = (1, n_voxels)
x_norm = hdf5storage.loadmat(os.path.join(
model_store, 'x_norm.mat'))['x_norm'] # shape = (1, n_voxels)
y_mean = hdf5storage.loadmat(os.path.join(
model_store, 'y_mean.mat'))['y_mean'] # shape = (1, shape_features)
y_norm = hdf5storage.loadmat(os.path.join(
model_store, 'y_norm.mat'))['y_norm'] # shape = (1, shape_features)
# Normalize X
# This normalization is turned off in NCconveter prediction, because the output of NCconverter
# already take this into account.
# x = (x - x_mean) / x_norm
# Prediction
y_pred_list = []
for i, (Wf, bf) in enumerate(zip(W_files, b_files)):
print('Chunk %d' % i)
start_time = time()
W_tmp = load_array(Wf, key='W')
b_tmp = load_array(bf, key='b')
model = FastL2LiR(W=W_tmp, b=b_tmp)
y_pred_tmp = model.predict(x)
# Denormalize Y
if y_mean.ndim == 2:
y_pred_tmp = y_pred_tmp * y_norm + y_mean
else:
y_pred_tmp = y_pred_tmp * y_norm[:, [i], :] + y_mean[:, [i], :]
y_pred_list.append(y_pred_tmp)
print('Elapsed time: %f s' % (time() - start_time))
return np.concatenate(y_pred_list, axis=chunk_axis)
def test_load_save_sparse_array(self):
# ndim = 1
data = np.random.rand(10)
data[data < 0.8] = 0
save_array('./tmp/test_array_sparse_ndim1.mat',
data,
key='testdata',
sparse=True)
testdata = load_array('./tmp/test_array_sparse_ndim1.mat',
key='testdata')
np.testing.assert_array_equal(data, testdata)
# ndim = 2
data = np.random.rand(3, 2)
data[data < 0.8] = 0
save_array('./tmp/test_array_sparse_ndim2.mat',
data,
key='testdata',
sparse=True)
testdata = load_array('./tmp/test_array_sparse_ndim2.mat',
key='testdata')
np.testing.assert_array_equal(data, testdata)
# ndim = 3
data = np.random.rand(4, 3, 2)
data[data < 0.8] = 0
save_array('./tmp/test_array_sparse_ndim3.mat',
data,
key='testdata',
sparse=True)
testdata = load_array('./tmp/test_array_sparse_ndim3.mat',
key='testdata')
np.testing.assert_array_equal(data, testdata)
# Averaging brain data
x_labels_unique = np.unique(x_labels)
x = np.vstack([
np.mean(x[(np.array(x_labels) == lb).flatten(), :], axis=0)
for lb in x_labels_unique
])
print('Elapsed time (data preparation): %f' % (time() - start_time))
# Model directory
# ---------------
model_dir = os.path.join(models_dir_root, network, feat, sbj, roi, 'model')
# Preprocessing
# -------------
x_mean = load_array(os.path.join(model_dir, 'x_mean.mat'),
key='x_mean') # shape = (1, n_voxels)
x_norm = load_array(os.path.join(model_dir, 'x_norm.mat'),
key='x_norm') # shape = (1, n_voxels)
y_mean = load_array(os.path.join(model_dir, 'y_mean.mat'),
key='y_mean') # shape = (1, shape_features)
y_norm = load_array(os.path.join(model_dir, 'y_norm.mat'),
key='y_norm') # shape = (1, shape_features)
x = (x - x_mean) / x_norm
# Prediction
# ----------
print('Prediction')
start_time = time()
def run(self):
'''Run test.'''
if self.dtype is not None:
self.X = self.X.astype(self.dtype)
if self.model_path is None:
y_pred = self.model.predict(self.X, **self.model_parameters)
return y_pred
if self.model_format == 'pickle':
if os.path.isfile(self.model_path):
model_files = [self.model_path]
elif os.path.isdir(self.model_path):
model_files = sorted(
glob.glob(os.path.join(self.model_path, '*.pkl')))
else:
raise ValueError('Invalid model path: %s' % self.model_path)
elif self.model_format == 'bdmodel':
if os.path.isfile(self.model_path):
raise ValueError(
'BDmodel should be specified as a directory, not a file')
# W: shape = (n_voxels, shape_features)
if os.path.isdir(os.path.join(self.model_path, 'W')):
W_files = sorted(
glob.glob(os.path.join(self.model_path, 'W', '*.mat')))
elif os.path.isfile(os.path.join(self.model_path, 'W.mat')):
W_files = [os.path.join(self.model_path, 'W.mat')]
else:
raise RuntimeError('W not found.')
# b: shape = (1, shape_features)
if os.path.isdir(os.path.join(self.model_path, 'b')):
b_files = sorted(
glob.glob(os.path.join(self.model_path, 'b', '*.mat')))
elif os.path.isfile(os.path.join(self.model_path, 'b.mat')):
b_files = [os.path.join(self.model_path, 'b.mat')]
else:
raise RuntimeError('b not found.')
model_files = [(w, b) for w, b in zip(W_files, b_files)]
else:
raise ValueError('Unknown model format: %s' % self.model_format)
# Prediction loop
y_pred_list = []
for i, model_file in enumerate(model_files):
print('Chunk %d' % i)
start_time = time()
if self.model_format == 'pickle':
with open(model_file, 'rb') as f:
model = pickle.load(f)
elif self.model_format == 'bdmodel':
W = load_array(model_file[0], key='W')
b = load_array(model_file[1], key='b')
model = self.model
model.W = W
model.b = b
else:
raise ValueError('Unknown model format: %s' %
self.model_format)
y_pred = model.predict(self.X, **self.model_parameters)
y_pred_list.append(y_pred)
print('Elapsed time: %f s' % (time() - start_time))
if self.chunk_axis is None:
return y_pred_list[0]
else:
return np.concatenate(y_pred_list, axis=self.chunk_axis)