def mapper(key, output_collector): import mapreduce as GLOBAL # access to global variables: # GLOBAL.DATA # GLOBAL.DATA ::= {"X":[Xtrain, ytrain], "y":[Xtest, ytest]} # key: list of parameters model_name, global_pen, tv_ratio, l1_ratio = key if model_name == 'pca': # Force the key global_pen = tv_ratio = l1_ratio = 0 if model_name == 'sparse_pca': # Force the key tv_ratio = 0 l1_ratio = 1 ll1 = global_pen if model_name == 'struct_pca': ltv = global_pen * tv_ratio ll1 = l1_ratio * global_pen * (1 - tv_ratio) ll2 = (1 - l1_ratio) * global_pen * (1 - tv_ratio) assert (np.allclose(ll1 + ll2 + ltv, global_pen)) X_train = GLOBAL.DATA_RESAMPLED["X"][0] n, p = X_train.shape X_test = GLOBAL.DATA_RESAMPLED["X"][1] # A matrices Atv = GLOBAL.Atv Al1 = scipy.sparse.eye(p, p) # Fit model if model_name == 'pca': model = sklearn.decomposition.PCA(n_components=N_COMP) if model_name == 'sparse_pca': model = sklearn.decomposition.SparsePCA(n_components=N_COMP, alpha=ll1) if model_name == 'struct_pca': model = pca_tv.PCA_SmoothedL1_L2_TV(n_components=N_COMP, l1=ll1, l2=ll2, ltv=ltv, Atv=Atv, Al1=Al1, criterion="frobenius", eps=1e-6, max_iter=100, inner_max_iter=int(1e4), output=False) t0 = time.clock() model.fit(X_train) t1 = time.clock() _time = t1 - t0 #print "X_test", GLOBAL.DATA["X"][1].shape # Save the projectors if (model_name == 'pca') or (model_name == 'sparse_pca'): V = model.components_.T if model_name == 'struct_pca': V = model.V # Project train & test data if (model_name == 'pca') or (model_name == 'sparse_pca'): X_train_transform = model.transform(X_train) X_test_transform = model.transform(X_test) if (model_name == 'struct_pca'): X_train_transform, _ = model.transform(X_train) X_test_transform, _ = model.transform(X_test) # Reconstruct train & test data # For SparsePCA or PCA, the formula is: UV^t (U is given by transform) # For StructPCA this is implemented in the predict method (which uses # transform) if (model_name == 'pca') or (model_name == 'sparse_pca'): X_train_predict = np.dot(X_train_transform, V.T) X_test_predict = np.dot(X_test_transform, V.T) if (model_name == 'struct_pca'): X_train_predict = model.predict(X_train) X_test_predict = model.predict(X_test) # Compute Frobenius norm between original and recontructed datasets frobenius_train = np.linalg.norm(X_train - X_train_predict, 'fro') frobenius_test = np.linalg.norm(X_test - X_test_predict, 'fro') # Compute explained variance ratio evr_train = metrics.adjusted_explained_variance(X_train_transform) evr_train /= np.var(X_train, axis=0).sum() evr_test = metrics.adjusted_explained_variance(X_test_transform) evr_test /= np.var(X_test, axis=0).sum() # Compute geometric metrics and norms of components TV = parsimony.functions.nesterov.tv.TotalVariation(1, A=Atv) l0 = np.zeros((N_COMP, )) l1 = np.zeros((N_COMP, )) l2 = np.zeros((N_COMP, )) tv = np.zeros((N_COMP, )) for i in range(N_COMP): # Norms l0[i] = np.linalg.norm(V[:, i], 0) l1[i] = np.linalg.norm(V[:, i], 1) l2[i] = np.linalg.norm(V[:, i], 2) tv[i] = TV.f(V[:, i]) ret = dict(frobenius_train=frobenius_train, frobenius_test=frobenius_test, components=V, X_train_predict=X_train_predict, X_test_predict=X_test_predict, X_train_transform=X_train_transform, X_test_transform=X_test_transform, evr_train=evr_train, evr_test=evr_test, l0=l0, l1=l1, l2=l2, tv=tv, time=_time) output_collector.collect(key, ret)
def mapper(key, output_collector): import mapreduce as GLOBAL # access to global variables: model_name, global_pen, tv_ratio, l1_ratio = key if model_name == 'pca': # Force the key global_pen = tv_ratio = l1_ratio = 0 if model_name == 'sparse_pca': global_pen = tv_ratio = 0 ll1 = l1_ratio if model_name == 'struct_pca': ltv = global_pen * tv_ratio ll1 = l1_ratio * global_pen * (1 - tv_ratio) ll2 = (1 - l1_ratio) * global_pen * (1 - tv_ratio) assert (np.allclose(ll1 + ll2 + ltv, global_pen)) X_train = GLOBAL.DATA_RESAMPLED["X"][0] n, p = X_train.shape X_test = GLOBAL.DATA_RESAMPLED["X"][1] # A matrices Atv = GLOBAL.A N_COMP = GLOBAL.N_COMP # Fit model if model_name == 'pca': model = sklearn.decomposition.PCA(n_components=N_COMP) if model_name == 'sparse_pca': model = sklearn.decomposition.SparsePCA(n_components=N_COMP, alpha=ll1) if model_name == 'struct_pca': model = pca_tv.PCA_L1_L2_TV(n_components=N_COMP, l1=ll1, l2=ll2, ltv=ltv, Atv=Atv, criterion="frobenius", eps=1e-6, max_iter=100, inner_max_iter=int(1e4), output=False) model.fit(X_train) # Save the projectors if (model_name == 'pca') or (model_name == 'sparse_pca'): V = model.components_.T if model_name == 'struct_pca': V = model.V # Project train & test data if (model_name == 'pca') or (model_name == 'sparse_pca'): X_train_transform = model.transform(X_train) X_test_transform = model.transform(X_test) if (model_name == 'struct_pca'): X_train_transform, _ = model.transform(X_train) X_test_transform, _ = model.transform(X_test) # Reconstruct train & test data # For SparsePCA or PCA, the formula is: UV^t (U is given by transform) # For StructPCA this is implemented in the predict method (which uses # transform) if (model_name == 'pca') or (model_name == 'sparse_pca'): X_train_predict = np.dot(X_train_transform, V.T) X_test_predict = np.dot(X_test_transform, V.T) if (model_name == 'struct_pca'): X_train_predict = model.predict(X_train) X_test_predict = model.predict(X_test) # Compute Frobenius norm between original and recontructed datasets frobenius_train = np.linalg.norm(X_train - X_train_predict, 'fro') frobenius_test = np.linalg.norm(X_test - X_test_predict, 'fro') print(frobenius_test) # Compute explained variance ratio evr_train = metrics.adjusted_explained_variance(X_train_transform) evr_train /= np.var(X_train, axis=0).sum() evr_test = metrics.adjusted_explained_variance(X_test_transform) evr_test /= np.var(X_test, axis=0).sum() # Remove predicted values (they are huge) del X_train_predict, X_test_predict ret = dict(frobenius_train=frobenius_train, frobenius_test=frobenius_test, components=V, X_train_transform=X_train_transform, X_test_transform=X_test_transform, evr_train=evr_train, evr_test=evr_test) output_collector.collect(key, ret)
def mapper(key, output_collector): import mapreduce as GLOBAL # access to global variables: # GLOBAL.DATA # GLOBAL.DATA ::= {"X":[Xtrain, ytrain], "y":[Xtest, ytest]} # key: list of parameters model_name, global_pen, tv_ratio, l1_ratio = key if model_name == 'pca': # Force the key global_pen = tv_ratio = l1_ratio = 0 if model_name == 'struct_pca': ll1, ll2, ltv = compute_coefs_from_ratios(global_pen, tv_ratio, l1_ratio) # This should not happen if ll1 > GLOBAL.l1_max: raise ValueError X_train = GLOBAL.DATA_RESAMPLED["X"][0] n, p = X_train.shape X_test = GLOBAL.DATA_RESAMPLED["X"][1] # A matrices Atv = GLOBAL.Atv # Fit model if model_name == 'pca': model = sklearn.decomposition.PCA(n_components=GLOBAL.N_COMP) if model_name == 'struct_pca': model = pca_tv.PCA_L1_L2_TV(n_components=GLOBAL.N_COMP, l1=ll1, l2=ll2, ltv=ltv, Atv=Atv, criterion="frobenius", eps=1e-6, max_iter=100, inner_max_iter=int(1e4), output=False) t0 = time.clock() model.fit(X_train) t1 = time.clock() _time = t1 - t0 #print "X_test", GLOBAL.DATA["X"][1].shape # Save the projectors if (model_name == 'pca'): components = model.components_.T if model_name == 'struct_pca': components = model.V # Threshold components thresh_components = np.empty(components.shape) thresholds = np.empty((GLOBAL.N_COMP, )) for k in range(GLOBAL.N_COMP): thresh_comp, t = array_utils.arr_threshold_from_norm2_ratio( components[:, k], .99) thresh_components[:, k] = thresh_comp thresholds[k] = t # Project train & test data if (model_name == 'pca'): X_train_transform = model.transform(X_train) X_test_transform = model.transform(X_test) if (model_name == 'struct_pca'): X_train_transform, _ = model.transform(X_train) X_test_transform, _ = model.transform(X_test) # Reconstruct train & test data # For PCA, the formula is: UV^t (U is given by transform) # For StructPCA this is implemented in the predict method (which uses # transform) if (model_name == 'pca'): X_train_predict = np.dot(X_train_transform, components.T) X_test_predict = np.dot(X_test_transform, components.T) if (model_name == 'struct_pca'): X_train_predict = model.predict(X_train) X_test_predict = model.predict(X_test) # Compute Frobenius norm between original and recontructed datasets frobenius_train = np.linalg.norm(X_train - X_train_predict, 'fro') frobenius_test = np.linalg.norm(X_test - X_test_predict, 'fro') # Compute geometric metrics and norms of components TV = parsimony.functions.nesterov.tv.TotalVariation(1, A=Atv) l0 = np.zeros((GLOBAL.N_COMP, )) l1 = np.zeros((GLOBAL.N_COMP, )) l2 = np.zeros((GLOBAL.N_COMP, )) tv = np.zeros((GLOBAL.N_COMP, )) recall = np.zeros((GLOBAL.N_COMP, )) precision = np.zeros((GLOBAL.N_COMP, )) fscore = np.zeros((GLOBAL.N_COMP, )) for i in range(GLOBAL.N_COMP): # Norms l0[i] = np.linalg.norm(components[:, i], 0) l1[i] = np.linalg.norm(components[:, i], 1) l2[i] = np.linalg.norm(components[:, i], 2) tv[i] = TV.f(components[:, i]) # Compute explained variance ratio evr_train = metrics.adjusted_explained_variance(X_train_transform) evr_train /= np.var(X_train, axis=0).sum() evr_test = metrics.adjusted_explained_variance(X_test_transform) evr_test /= np.var(X_test, axis=0).sum() ret = dict(frobenius_train=frobenius_train, frobenius_test=frobenius_test, components=components, thresh_components=thresh_components, thresholds=thresholds, X_train_transform=X_train_transform, X_test_transform=X_test_transform, X_train_predict=X_train_predict, X_test_predict=X_test_predict, recall=recall, precision=precision, fscore=fscore, evr_train=evr_train, evr_test=evr_test, l0=l0, l1=l1, l2=l2, tv=tv, time=_time) output_collector.collect(key, ret)