def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) STRUCTURE = np.load(config["structure"]) A = tv_helper.A_from_mask(STRUCTURE) N_COMP = config["N_COMP"] GLOBAL.A, GLOBAL.STRUCTURE, GLOBAL.N_COMP = A, STRUCTURE, N_COMP
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) STRUCTURE = np.load(config["structure"]) A = tv_helper.A_from_mask(STRUCTURE) GLOBAL.Atv = A GLOBAL.FULL_RESAMPLE = config['full_resample']
def load_globals(config): import mapreduce as GLOBAL STRUCTURE = nibabel.load(config["structure"]) A = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.Atv = A GLOBAL.N_FOLDS = config['n_folds'] GLOBAL.FULL_RESAMPLE = config['full_resample']
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) MASK = nibabel.load(config["mask"]) A, _ = tv_helper.A_from_mask(MASK.get_data()) GLOBAL.A, GLOBAL.MASK = A, MASK GLOBAL.PENALTY_START = config['penalty_start']
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) GLOBAL.DATA_IMA = GLOBAL.load_data(config["data_IMA"]) STRUCTURE = nibabel.load(config["structure"]) A = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.A, GLOBAL.STRUCTURE = A, STRUCTURE
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) GLOBAL.PENALTY_START = config["penalty_start"] STRUCTURE = nibabel.load(config["structure"]) GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.STRUCTURE = STRUCTURE
def mapper(key, output_collector): import mapreduce as GLOBAL # access to global variables: #raise ImportError("could not import ") # GLOBAL.DATA, GLOBAL.STRUCTURE, GLOBAL.A # GLOBAL.DATA ::= {"X":[Xtrain, Xtest], "y":[ytrain, ytest]} # key: list of parameters Xtr = GLOBAL.DATA_RESAMPLED["X"][0] Xte = GLOBAL.DATA_RESAMPLED["X"][1] ytr = GLOBAL.DATA_RESAMPLED["y"][0] yte = GLOBAL.DATA_RESAMPLED["y"][1] print key, "Data shape:", Xtr.shape, Xte.shape, ytr.shape, yte.shape STRUCTURE = GLOBAL.STRUCTURE n_voxels = np.count_nonzero(STRUCTURE.get_data()) penalty_start = GLOBAL.PENALTY_START alpha = float(key[0]) l1, l2 = alpha * float(key[1]), alpha * float(key[2]) tv, k_ratio = alpha * float(key[3]), key[4] print "l1:%f, l2:%f, tv:%f, k_ratio:%f" % (l1, l2, tv, k_ratio) if k_ratio != 1: k = n_voxels * k_ratio k = int(k) aov = SelectKBest(k=k) aov.fit(Xtr[..., penalty_start:], ytr.ravel()) mask = STRUCTURE.get_data() != 0 mask[mask] = aov.get_support() #print mask.sum() A, _ = tv_helper.A_from_mask(mask) Xtr_r = np.hstack([ Xtr[:, :penalty_start], Xtr[:, penalty_start:][:, aov.get_support()] ]) Xte_r = np.hstack([ Xte[:, :penalty_start], Xte[:, penalty_start:][:, aov.get_support()] ]) else: mask = STRUCTURE.get_data() != 0 Xtr_r = Xtr Xte_r = Xte A = GLOBAL.A info = [Info.num_iter] mod = LinearRegressionL1L2TV(l1, l2, tv, A, penalty_start=penalty_start, algorithm_params={'info': info}) mod.fit(Xtr_r, ytr) y_pred = mod.predict(Xte_r) beta = mod.beta ret = dict(y_pred=y_pred, y_true=yte, beta=beta, n_iter=mod.get_info()['num_iter']) if output_collector: output_collector.collect(key, ret) else: return ret
def mapper(key, output_collector): import mapreduce as GLOBAL # access to global variables: #raise ImportError("could not import ") # GLOBAL.DATA, GLOBAL.STRUCTURE, GLOBAL.A # GLOBAL.DATA ::= {"X":[Xtrain, ytrain], "y":[Xtest, ytest]} # key: list of parameters Xtr = GLOBAL.DATA_RESAMPLED["X"][0] Xte = GLOBAL.DATA_RESAMPLED["X"][1] ytr = GLOBAL.DATA_RESAMPLED["y"][0] yte = GLOBAL.DATA_RESAMPLED["y"][1] print key, "Data shape:", Xtr.shape, Xte.shape, ytr.shape, yte.shape STRUCTURE = GLOBAL.STRUCTURE #alpha, ratio_l1, ratio_l2, ratio_tv, k = key #key = np.array(key) penalty_start = GLOBAL.CONFIG["penalty_start"] class_weight = "auto" # unbiased alpha = float(key[0]) l1, l2, tv, k = alpha * float(key[1]), alpha * float( key[2]), alpha * float(key[3]), key[4] print "l1:%f, l2:%f, tv:%f, k:%i" % (l1, l2, tv, k) if k != -1: k = int(k) aov = SelectKBest(k=k) aov.fit(Xtr[..., penalty_start:], ytr.ravel()) mask = STRUCTURE.get_data() != 0 mask[mask] = aov.get_support() #print mask.sum() A, _ = tv_helper.A_from_mask(mask) Xtr_r = np.hstack([ Xtr[:, :penalty_start], Xtr[:, penalty_start:][:, aov.get_support()] ]) Xte_r = np.hstack([ Xte[:, :penalty_start], Xte[:, penalty_start:][:, aov.get_support()] ]) else: mask = np.ones(Xtr.shape[0], dtype=bool) Xtr_r = Xtr Xte_r = Xte A = GLOBAL.A mod = LogisticRegressionL1L2TV(l1, l2, tv, A, penalty_start=penalty_start, class_weight=class_weight) mod.fit(Xtr_r, ytr) y_pred = mod.predict(Xte_r) proba_pred = mod.predict_probability(Xte_r) ret = dict(y_pred=y_pred, proba_pred=proba_pred, y_true=yte, beta=mod.beta, mask=mask) if output_collector: output_collector.collect(key, ret) else: return ret
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) STRUCTURE = nibabel.load(config["mask_filename"]) try: A = tv_helper.linear_operator_from_mask(STRUCTURE.get_data()) except: A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.A, GLOBAL.STRUCTURE, GLOBAL.CONFIG = A, STRUCTURE, config
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) GLOBAL.PARAMS = config["params"] GLOBAL.MAP_OUTPUT = config['map_output'] GLOBAL.OUTPUT_SUMMARY = config['output_summary'] GLOBAL.PENALTY_START = config["penalty_start"] STRUCTURE = nibabel.load(config["structure"]) GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.STRUCTURE = STRUCTURE GLOBAL.ROI = config["roi"] GLOBAL.OUTPUT_PATH = config["output_path"]
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) GLOBAL.CRITERIA = config["params"] GLOBAL.MAP_OUTPUT = config['map_output'] GLOBAL.PROB_CLASS1 = config["prob_class1"] GLOBAL.PENALTY_START = config["penalty_start"] STRUCTURE = nibabel.load(config["structure"]) GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.STRUCTURE = STRUCTURE GLOBAL.ROI = config["roi"] GLOBAL.MODEL = config["model"]
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) GLOBAL.CRITERIA = config["params"] GLOBAL.MAP_OUTPUT = config["map_output"] GLOBAL.OUTPUT_PATH = config["output_path"] GLOBAL.OUTPUT_PERMUTATIONS = config["output_permutations"] GLOBAL.PENALTY_START = config["penalty_start"] STRUCTURE = nibabel.load(config["structure"]) GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) GLOBAL.STRUCTURE = STRUCTURE GLOBAL.ROI = config["roi"] GLOBAL.SELECTION = pd.read_csv(os.path.join(config["output_path"], config["output_summary"]))
def load_globals(config): import mapreduce as GLOBAL # access to global variables GLOBAL.DATA = GLOBAL.load_data(config["data"]) GLOBAL.PROB_CLASS1 = config["prob_class1"] MODALITY = config["modality"] penalty_start = config["penalty_start"] if np.logical_or(MODALITY == "MRI", MODALITY == "PET"): STRUCTURE = nibabel.load(config["structure"]) A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) elif MODALITY == "MRI+PET": STRUCTURE = nibabel.load(config["structure"]) A1, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) # construct matrix A # Ax, Ay, Az are block diagonale matrices and diagonal elements # are elements of A1 # eg: Ax = diagblock(A1x, A1x) A = [] for i in range(3): a = sparse.bmat([[A1[i], None], [None, A1[i]]]) A.append(a) GLOBAL.A, GLOBAL.STRUCTURE, GLOBAL.MODALITY = A, STRUCTURE, MODALITY GLOBAL.PENALTY_START = penalty_start
def A_from_structure(structure_filepath): # Input: structure_filepath. Output: A, structure STRUCTURE = nibabel.load(structure_filepath) A, _ = tv_helper.A_from_mask(STRUCTURE.get_data()) return A, STRUCTURE
def mapper(key, output_collector): import mapreduce as GLOBAL # access to global variables: #raise ImportError("could not import ") # GLOBAL.DATA, GLOBAL.STRUCTURE, GLOBAL.A # GLOBAL.DATA ::= {"X":[Xtrain, Xtest], "y":[ytrain, ytest]} # key: list of parameters MODALITY = GLOBAL.MODALITY Xtr = GLOBAL.DATA_RESAMPLED["X"][0] Xte = GLOBAL.DATA_RESAMPLED["X"][1] ytr = GLOBAL.DATA_RESAMPLED["y"][0] yte = GLOBAL.DATA_RESAMPLED["y"][1] print key, "Data shape:", Xtr.shape, Xte.shape, ytr.shape, yte.shape STRUCTURE = GLOBAL.STRUCTURE n_voxels = np.count_nonzero(STRUCTURE.get_data()) #alpha, ratio_l1, ratio_l2, ratio_tv, k = key #key = np.array(key) penalty_start = GLOBAL.PENALTY_START class_weight = "auto" # unbiased alpha = float(key[0]) l1, l2 = alpha * float(key[1]), alpha * float(key[2]) tv, k_ratio = alpha * float(key[3]), key[4] print "l1:%f, l2:%f, tv:%f, k_ratio:%f" % (l1, l2, tv, k_ratio) if np.logical_or(MODALITY == "MRI", MODALITY == "PET"): if k_ratio != -1: k = n_voxels * k_ratio k = int(k) aov = SelectKBest(k=k) aov.fit(Xtr[..., penalty_start:], ytr.ravel()) mask = STRUCTURE.get_data() != 0 mask[mask] = aov.get_support() #print mask.sum() A, _ = tv_helper.A_from_mask(mask) Xtr_r = np.hstack([ Xtr[:, :penalty_start], Xtr[:, penalty_start:][:, aov.get_support()] ]) Xte_r = np.hstack([ Xte[:, :penalty_start], Xte[:, penalty_start:][:, aov.get_support()] ]) else: mask = STRUCTURE.get_data() != 0 Xtr_r = Xtr Xte_r = Xte A = GLOBAL.A elif MODALITY == "MRI+PET": if k_ratio != -1: k = 2 * n_voxels * k_ratio k = int(k) aov = SelectKBest(k=k) aov.fit(Xtr[..., penalty_start:], ytr.ravel()) support_mask = aov.get_support() # Create 3D mask for MRI mask_MRI = STRUCTURE.get_data() != 0 mask_MRI[mask_MRI] = support_mask[:n_voxels] mask_PET = STRUCTURE.get_data() != 0 mask_PET[mask_PET] = support_mask[n_voxels:] # We construct matrix A, it size is k*k # If k_MRI and k_PET are both different to 0 we construct # a matrix A for each modality and then concatenate them # If one of the modality is empty, the matrix A is constructed # from the other modality only k_MRI = np.count_nonzero(mask_MRI) k_PET = np.count_nonzero(mask_PET) # k_MRI and k_Pet can not be simultaneously equal to zero assert (k_MRI + k_PET == k) if (k_MRI == 0) and (k_PET != 0): A, _ = tv_helper.A_from_mask(mask_PET) if (k_PET == 0) and (k_MRI != 0): A, _ = tv_helper.A_from_mask(mask_MRI) if (k_MRI != 0) and (k_PET != 0): A1, _ = tv_helper.A_from_mask(mask_MRI) A2, _ = tv_helper.A_from_mask(mask_PET) A = [] for i in range(3): a = sparse.bmat([[A1[i], None], [None, A2[i]]]) A.append(a) Xtr_r = np.hstack([ Xtr[:, :penalty_start], Xtr[:, penalty_start:][:, support_mask] ]) Xte_r = np.hstack([ Xte[:, :penalty_start], Xte[:, penalty_start:][:, support_mask] ]) else: k_MRI = n_voxels k_PET = n_voxels mask_MRI = STRUCTURE.get_data() != 0 mask_PET = STRUCTURE.get_data() != 0 Xtr_r = Xtr Xte_r = Xte A = GLOBAL.A info = [Info.num_iter] mod = LogisticRegressionL1L2TV(l1, l2, tv, A, penalty_start=penalty_start, class_weight=class_weight, algorithm_params={'info': info}) mod.fit(Xtr_r, ytr) y_pred = mod.predict(Xte_r) proba_pred = mod.predict_probability(Xte_r) # a posteriori probability beta = mod.beta if (MODALITY == "MRI") or (MODALITY == "PET"): ret = dict(y_pred=y_pred, proba_pred=proba_pred, y_true=yte, beta=beta, mask=mask, n_iter=mod.get_info()['num_iter']) elif MODALITY == "MRI+PET": beta_MRI = beta[:(penalty_start + k_MRI)] beta_PET = np.vstack( [beta[:penalty_start], beta[(penalty_start + k_MRI):]]) ret = dict(y_pred=y_pred, proba_pred=proba_pred, y_true=yte, beta=beta, beta_MRI=beta_MRI, beta_PET=beta_PET, mask_MRI=mask_MRI, mask_PET=mask_PET, n_iter=mod.get_info()['num_iter']) if output_collector: output_collector.collect(key, ret) else: return ret
assert Xroi_train.shape == (150, 184) assert Xroi_test.shape == (100, 184) assert Xgm_train.shape == (150, 379778) assert Xgm_test.shape == (100, 379778) # enettv for GM arg = [float(p) for p in WHICH.split("_")] if len(arg) == 4: alpha, l1, l2, tv = arg else: alpha, l1, l2, tv, k = arg l1, l2, tv = alpha * l1, alpha * l2, alpha * tv mask_im = nibabel.load(os.path.join(GM, "mask.nii")) A, _ = tv_helper.A_from_mask(mask_im.get_data()) enettv = LogisticRegressionL1L2TV(l1, l2, tv, A, penalty_start=penalty_start, class_weight="auto") # lr l2 for roi p_lr_l2 = Pipeline([ ('scaler', StandardScaler()), # ('classifier', LogisticRegression(C=0.005, penalty='l2')), ('classifier', LogisticRegression(C=0.0022, penalty='l2')), ]) p_lr_l2 = LogisticRegression(C=0.0022, penalty='l2')