def mapper(key, output_collector):
import mapreduce as GLOBAL # access to global variables:
model_name, global_pen, gn_ratio, l1_ratio = key
lgn = global_pen * gn_ratio
ll1 = l1_ratio * global_pen * (1 - gn_ratio)
ll2 = (1 - l1_ratio) * global_pen * (1 - gn_ratio)
assert(np.allclose(ll1 + ll2 + lgn, global_pen))
X_train = GLOBAL.DATA_RESAMPLED["X"][0]
X_test = GLOBAL.DATA_RESAMPLED["X"][1]
Agn = GLOBAL.A
N_COMP = GLOBAL.N_COMP
model = pca_struct.PCAGraphNet(n_components=N_COMP,
l1=ll1, l2=ll2, lgn=lgn,
Agn=Agn,
criterion="frobenius",
eps=1e-6,
max_iter=10000,
output=False)
model.fit(X_train)
V = model.V
X_train_transform, _ = model.transform(X_train)
X_test_transform, _ = model.transform(X_test)
X_train_predict = model.predict(X_train)
X_test_predict = model.predict(X_test)
# Compute Frobenius norm between original and recontructed datasets
frobenius_train = np.linalg.norm(X_train - X_train_predict, 'fro')
frobenius_test = np.linalg.norm(X_test - X_test_predict, 'fro')
print("frobenius_test")
print(frobenius_test)
# Remove predicted values (they are huge)
del X_train_predict, X_test_predict
ret = dict(frobenius_train=frobenius_train,
frobenius_test=frobenius_test,
components=V,
X_train_transform=X_train_transform,
X_test_transform=X_test_transform)
output_collector.collect(key, ret)
def mapper(key, output_collector):
import mapreduce as GLOBAL # access to global variables:
model_name, global_pen, struct_ratio, l1_ratio = key
if model_name == 'pca':
global_pen = struct_ratio = l1_ratio = 0
if model_name == 'sparse_pca':
global_pen = struct_ratio = 0
ll1 = l1_ratio
if model_name == 'struct_pca':
ltv = global_pen * struct_ratio
ll1 = l1_ratio * global_pen * (1 - struct_ratio)
ll2 = (1 - l1_ratio) * global_pen * (1 - struct_ratio)
assert (np.allclose(ll1 + ll2 + ltv, global_pen))
if model_name == 'graphNet_pca':
lgn = global_pen * struct_ratio
ll1 = l1_ratio * global_pen * (1 - struct_ratio)
ll2 = (1 - l1_ratio) * global_pen * (1 - struct_ratio)
assert (np.allclose(ll1 + ll2 + lgn, global_pen))
X_train = GLOBAL.DATA_RESAMPLED["X"][0]
n, p = X_train.shape
X_test = GLOBAL.DATA_RESAMPLED["X"][1]
# A matrices
A = GLOBAL.A
N_COMP = GLOBAL.N_COMP
# Fit model
if model_name == 'pca':
model = sklearn.decomposition.PCA(n_components=N_COMP)
if model_name == 'sparse_pca':
model = sklearn.decomposition.SparsePCA(n_components=N_COMP, alpha=ll1)
if model_name == 'struct_pca':
model = pca_tv.PCA_L1_L2_TV(n_components=N_COMP,
l1=ll1,
l2=ll2,
ltv=ltv,
Atv=A,
criterion="frobenius",
eps=1e-6,
max_iter=100,
inner_max_iter=int(1e3),
output=False)
if model_name == 'graphNet_pca':
A = sparse.vstack(A)
model = pca_struct.PCAGraphNet(n_components=N_COMP,
l1=ll1,
l2=ll2,
lgn=lgn,
Agn=A,
criterion="frobenius",
eps=1e-6,
max_iter=500,
output=False)
model.fit(X_train)
# Save the projectors
if (model_name == 'pca') or (model_name == 'sparse_pca'):
V = model.components_.T
if (model_name == 'struct_pca') or (model_name == 'graphNet_pca'):
V = model.V
# Project train & test data
if (model_name == 'pca') or (model_name == 'sparse_pca'):
X_train_transform = model.transform(X_train)
X_test_transform = model.transform(X_test)
if (model_name == 'struct_pca') or (model_name == 'graphNet_pca'):
X_train_transform, _ = model.transform(X_train)
X_test_transform, _ = model.transform(X_test)
# Reconstruct train & test data
# For SparsePCA or PCA, the formula is: UV^t (U is given by transform)
# For StructPCA this is implemented in the predict method (which uses
# transform)
if (model_name == 'pca') or (model_name == 'sparse_pca'):
X_train_predict = np.dot(X_train_transform, V.T)
X_test_predict = np.dot(X_test_transform, V.T)
if (model_name == 'struct_pca') or (model_name == 'graphNet_pca'):
X_train_predict = model.predict(X_train)
X_test_predict = model.predict(X_test)
# Compute Frobenius norm between original and recontructed datasets
frobenius_train = np.linalg.norm(X_train - X_train_predict, 'fro')
frobenius_test = np.linalg.norm(X_test - X_test_predict, 'fro')
print(frobenius_test)
del X_train_predict, X_test_predict
ret = dict(frobenius_train=frobenius_train,
frobenius_test=frobenius_test,
components=V,
X_train_transform=X_train_transform,
X_test_transform=X_test_transform)
output_collector.collect(key, ret)
ll1 = l1_ratio * global_pen * (1 - gn_ratio)
ll2 = (1 - l1_ratio) * global_pen * (1 - gn_ratio)
assert(np.allclose(ll1 + ll2 + lgn, global_pen))
nib_mask = nib.load(INPUT_MASK)
Agn = sparse.vstack(tv_helper.linear_operator_from_mask(nib_mask.get_data()))
################################################################################
snapshot = AlgorithmSnapshot('/neurospin/brainomics/2014_pca_struct/fmri/fmri_time/gn_1e-8/').save_nipals
mod = pca_struct.PCAGraphNet(n_components=3,
l1=ll1, l2=ll2, lgn=lgn,
Agn=Agn,
criterion="frobenius",
eps=1e-8,
max_iter=500,
output=False,callback = snapshot)
mod.fit(X)
comp1 = np.load('/neurospin/brainomics/2014_pca_struct/fmri/fmri_time/gn_1e-8/component:1.npz')
comp2 = np.load('/neurospin/brainomics/2014_pca_struct/fmri/fmri_time/gn_1e-8/component:2.npz')
comp3 = np.load('/neurospin/brainomics/2014_pca_struct/fmri/fmri_time/gn_1e-8/component:3.npz')
comp1_V = (comp1['v'] - comp1['v'][-1,:])
eps_comp1 = np.zeros(comp1_V.shape[0]-1)