def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
STRUCTURE = np.load(config["structure"])
A = tv_helper.A_from_mask(STRUCTURE)
N_COMP = config["N_COMP"]
GLOBAL.A, GLOBAL.STRUCTURE, GLOBAL.N_COMP = A, STRUCTURE, N_COMP
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
STRUCTURE = np.load(config["structure"])
A = tv_helper.A_from_mask(STRUCTURE)
GLOBAL.Atv = A
GLOBAL.FULL_RESAMPLE = config['full_resample']
def load_globals(config):
import mapreduce as GLOBAL
STRUCTURE = nibabel.load(config["structure"])
A = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.Atv = A
GLOBAL.N_FOLDS = config['n_folds']
GLOBAL.FULL_RESAMPLE = config['full_resample']
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
MASK = nibabel.load(config["mask"])
A, _ = tv_helper.A_from_mask(MASK.get_data())
GLOBAL.A, GLOBAL.MASK = A, MASK
GLOBAL.PENALTY_START = config['penalty_start']
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
GLOBAL.DATA_IMA = GLOBAL.load_data(config["data_IMA"])
STRUCTURE = nibabel.load(config["structure"])
A = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.A, GLOBAL.STRUCTURE = A, STRUCTURE
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
GLOBAL.PENALTY_START = config["penalty_start"]
STRUCTURE = nibabel.load(config["structure"])
GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.STRUCTURE = STRUCTURE
def mapper(key, output_collector):
import mapreduce as GLOBAL # access to global variables:
#raise ImportError("could not import ")
# GLOBAL.DATA, GLOBAL.STRUCTURE, GLOBAL.A
# GLOBAL.DATA ::= {"X":[Xtrain, Xtest], "y":[ytrain, ytest]}
# key: list of parameters
Xtr = GLOBAL.DATA_RESAMPLED["X"][0]
Xte = GLOBAL.DATA_RESAMPLED["X"][1]
ytr = GLOBAL.DATA_RESAMPLED["y"][0]
yte = GLOBAL.DATA_RESAMPLED["y"][1]
print key, "Data shape:", Xtr.shape, Xte.shape, ytr.shape, yte.shape
STRUCTURE = GLOBAL.STRUCTURE
n_voxels = np.count_nonzero(STRUCTURE.get_data())
penalty_start = GLOBAL.PENALTY_START
alpha = float(key[0])
l1, l2 = alpha * float(key[1]), alpha * float(key[2])
tv, k_ratio = alpha * float(key[3]), key[4]
print "l1:%f, l2:%f, tv:%f, k_ratio:%f" % (l1, l2, tv, k_ratio)
if k_ratio != 1:
k = n_voxels * k_ratio
k = int(k)
aov = SelectKBest(k=k)
aov.fit(Xtr[..., penalty_start:], ytr.ravel())
mask = STRUCTURE.get_data() != 0
mask[mask] = aov.get_support()
#print mask.sum()
A, _ = tv_helper.A_from_mask(mask)
Xtr_r = np.hstack([
Xtr[:, :penalty_start], Xtr[:, penalty_start:][:,
aov.get_support()]
])
Xte_r = np.hstack([
Xte[:, :penalty_start], Xte[:, penalty_start:][:,
aov.get_support()]
])
else:
mask = STRUCTURE.get_data() != 0
Xtr_r = Xtr
Xte_r = Xte
A = GLOBAL.A
info = [Info.num_iter]
mod = LinearRegressionL1L2TV(l1,
l2,
tv,
A,
penalty_start=penalty_start,
algorithm_params={'info': info})
mod.fit(Xtr_r, ytr)
y_pred = mod.predict(Xte_r)
beta = mod.beta
ret = dict(y_pred=y_pred,
y_true=yte,
beta=beta,
n_iter=mod.get_info()['num_iter'])
if output_collector:
output_collector.collect(key, ret)
else:
return ret
def mapper(key, output_collector):
import mapreduce as GLOBAL # access to global variables:
#raise ImportError("could not import ")
# GLOBAL.DATA, GLOBAL.STRUCTURE, GLOBAL.A
# GLOBAL.DATA ::= {"X":[Xtrain, ytrain], "y":[Xtest, ytest]}
# key: list of parameters
Xtr = GLOBAL.DATA_RESAMPLED["X"][0]
Xte = GLOBAL.DATA_RESAMPLED["X"][1]
ytr = GLOBAL.DATA_RESAMPLED["y"][0]
yte = GLOBAL.DATA_RESAMPLED["y"][1]
print key, "Data shape:", Xtr.shape, Xte.shape, ytr.shape, yte.shape
STRUCTURE = GLOBAL.STRUCTURE
#alpha, ratio_l1, ratio_l2, ratio_tv, k = key
#key = np.array(key)
penalty_start = GLOBAL.CONFIG["penalty_start"]
class_weight = "auto" # unbiased
alpha = float(key[0])
l1, l2, tv, k = alpha * float(key[1]), alpha * float(
key[2]), alpha * float(key[3]), key[4]
print "l1:%f, l2:%f, tv:%f, k:%i" % (l1, l2, tv, k)
if k != -1:
k = int(k)
aov = SelectKBest(k=k)
aov.fit(Xtr[..., penalty_start:], ytr.ravel())
mask = STRUCTURE.get_data() != 0
mask[mask] = aov.get_support()
#print mask.sum()
A, _ = tv_helper.A_from_mask(mask)
Xtr_r = np.hstack([
Xtr[:, :penalty_start], Xtr[:, penalty_start:][:,
aov.get_support()]
])
Xte_r = np.hstack([
Xte[:, :penalty_start], Xte[:, penalty_start:][:,
aov.get_support()]
])
else:
mask = np.ones(Xtr.shape[0], dtype=bool)
Xtr_r = Xtr
Xte_r = Xte
A = GLOBAL.A
mod = LogisticRegressionL1L2TV(l1,
l2,
tv,
A,
penalty_start=penalty_start,
class_weight=class_weight)
mod.fit(Xtr_r, ytr)
y_pred = mod.predict(Xte_r)
proba_pred = mod.predict_probability(Xte_r)
ret = dict(y_pred=y_pred,
proba_pred=proba_pred,
y_true=yte,
beta=mod.beta,
mask=mask)
if output_collector:
output_collector.collect(key, ret)
else:
return ret
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
STRUCTURE = nibabel.load(config["mask_filename"])
try:
A = tv_helper.linear_operator_from_mask(STRUCTURE.get_data())
except:
A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.A, GLOBAL.STRUCTURE, GLOBAL.CONFIG = A, STRUCTURE, config
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
GLOBAL.PARAMS = config["params"]
GLOBAL.MAP_OUTPUT = config['map_output']
GLOBAL.OUTPUT_SUMMARY = config['output_summary']
GLOBAL.PENALTY_START = config["penalty_start"]
STRUCTURE = nibabel.load(config["structure"])
GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.STRUCTURE = STRUCTURE
GLOBAL.ROI = config["roi"]
GLOBAL.OUTPUT_PATH = config["output_path"]
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
GLOBAL.CRITERIA = config["params"]
GLOBAL.MAP_OUTPUT = config['map_output']
GLOBAL.PROB_CLASS1 = config["prob_class1"]
GLOBAL.PENALTY_START = config["penalty_start"]
STRUCTURE = nibabel.load(config["structure"])
GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.STRUCTURE = STRUCTURE
GLOBAL.ROI = config["roi"]
GLOBAL.MODEL = config["model"]
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
GLOBAL.CRITERIA = config["params"]
GLOBAL.MAP_OUTPUT = config["map_output"]
GLOBAL.OUTPUT_PATH = config["output_path"]
GLOBAL.OUTPUT_PERMUTATIONS = config["output_permutations"]
GLOBAL.PENALTY_START = config["penalty_start"]
STRUCTURE = nibabel.load(config["structure"])
GLOBAL.A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
GLOBAL.STRUCTURE = STRUCTURE
GLOBAL.ROI = config["roi"]
GLOBAL.SELECTION = pd.read_csv(os.path.join(config["output_path"],
config["output_summary"]))
def load_globals(config):
import mapreduce as GLOBAL # access to global variables
GLOBAL.DATA = GLOBAL.load_data(config["data"])
GLOBAL.PROB_CLASS1 = config["prob_class1"]
MODALITY = config["modality"]
penalty_start = config["penalty_start"]
if np.logical_or(MODALITY == "MRI", MODALITY == "PET"):
STRUCTURE = nibabel.load(config["structure"])
A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
elif MODALITY == "MRI+PET":
STRUCTURE = nibabel.load(config["structure"])
A1, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
# construct matrix A
# Ax, Ay, Az are block diagonale matrices and diagonal elements
# are elements of A1
# eg: Ax = diagblock(A1x, A1x)
A = []
for i in range(3):
a = sparse.bmat([[A1[i], None], [None, A1[i]]])
A.append(a)
GLOBAL.A, GLOBAL.STRUCTURE, GLOBAL.MODALITY = A, STRUCTURE, MODALITY
GLOBAL.PENALTY_START = penalty_start
def A_from_structure(structure_filepath):
# Input: structure_filepath. Output: A, structure
STRUCTURE = nibabel.load(structure_filepath)
A, _ = tv_helper.A_from_mask(STRUCTURE.get_data())
return A, STRUCTURE
def mapper(key, output_collector):
import mapreduce as GLOBAL # access to global variables:
#raise ImportError("could not import ")
# GLOBAL.DATA, GLOBAL.STRUCTURE, GLOBAL.A
# GLOBAL.DATA ::= {"X":[Xtrain, Xtest], "y":[ytrain, ytest]}
# key: list of parameters
MODALITY = GLOBAL.MODALITY
Xtr = GLOBAL.DATA_RESAMPLED["X"][0]
Xte = GLOBAL.DATA_RESAMPLED["X"][1]
ytr = GLOBAL.DATA_RESAMPLED["y"][0]
yte = GLOBAL.DATA_RESAMPLED["y"][1]
print key, "Data shape:", Xtr.shape, Xte.shape, ytr.shape, yte.shape
STRUCTURE = GLOBAL.STRUCTURE
n_voxels = np.count_nonzero(STRUCTURE.get_data())
#alpha, ratio_l1, ratio_l2, ratio_tv, k = key
#key = np.array(key)
penalty_start = GLOBAL.PENALTY_START
class_weight = "auto" # unbiased
alpha = float(key[0])
l1, l2 = alpha * float(key[1]), alpha * float(key[2])
tv, k_ratio = alpha * float(key[3]), key[4]
print "l1:%f, l2:%f, tv:%f, k_ratio:%f" % (l1, l2, tv, k_ratio)
if np.logical_or(MODALITY == "MRI", MODALITY == "PET"):
if k_ratio != -1:
k = n_voxels * k_ratio
k = int(k)
aov = SelectKBest(k=k)
aov.fit(Xtr[..., penalty_start:], ytr.ravel())
mask = STRUCTURE.get_data() != 0
mask[mask] = aov.get_support()
#print mask.sum()
A, _ = tv_helper.A_from_mask(mask)
Xtr_r = np.hstack([
Xtr[:, :penalty_start], Xtr[:,
penalty_start:][:,
aov.get_support()]
])
Xte_r = np.hstack([
Xte[:, :penalty_start], Xte[:,
penalty_start:][:,
aov.get_support()]
])
else:
mask = STRUCTURE.get_data() != 0
Xtr_r = Xtr
Xte_r = Xte
A = GLOBAL.A
elif MODALITY == "MRI+PET":
if k_ratio != -1:
k = 2 * n_voxels * k_ratio
k = int(k)
aov = SelectKBest(k=k)
aov.fit(Xtr[..., penalty_start:], ytr.ravel())
support_mask = aov.get_support()
# Create 3D mask for MRI
mask_MRI = STRUCTURE.get_data() != 0
mask_MRI[mask_MRI] = support_mask[:n_voxels]
mask_PET = STRUCTURE.get_data() != 0
mask_PET[mask_PET] = support_mask[n_voxels:]
# We construct matrix A, it size is k*k
# If k_MRI and k_PET are both different to 0 we construct
# a matrix A for each modality and then concatenate them
# If one of the modality is empty, the matrix A is constructed
# from the other modality only
k_MRI = np.count_nonzero(mask_MRI)
k_PET = np.count_nonzero(mask_PET)
# k_MRI and k_Pet can not be simultaneously equal to zero
assert (k_MRI + k_PET == k)
if (k_MRI == 0) and (k_PET != 0):
A, _ = tv_helper.A_from_mask(mask_PET)
if (k_PET == 0) and (k_MRI != 0):
A, _ = tv_helper.A_from_mask(mask_MRI)
if (k_MRI != 0) and (k_PET != 0):
A1, _ = tv_helper.A_from_mask(mask_MRI)
A2, _ = tv_helper.A_from_mask(mask_PET)
A = []
for i in range(3):
a = sparse.bmat([[A1[i], None], [None, A2[i]]])
A.append(a)
Xtr_r = np.hstack([
Xtr[:, :penalty_start], Xtr[:, penalty_start:][:, support_mask]
])
Xte_r = np.hstack([
Xte[:, :penalty_start], Xte[:, penalty_start:][:, support_mask]
])
else:
k_MRI = n_voxels
k_PET = n_voxels
mask_MRI = STRUCTURE.get_data() != 0
mask_PET = STRUCTURE.get_data() != 0
Xtr_r = Xtr
Xte_r = Xte
A = GLOBAL.A
info = [Info.num_iter]
mod = LogisticRegressionL1L2TV(l1,
l2,
tv,
A,
penalty_start=penalty_start,
class_weight=class_weight,
algorithm_params={'info': info})
mod.fit(Xtr_r, ytr)
y_pred = mod.predict(Xte_r)
proba_pred = mod.predict_probability(Xte_r) # a posteriori probability
beta = mod.beta
if (MODALITY == "MRI") or (MODALITY == "PET"):
ret = dict(y_pred=y_pred,
proba_pred=proba_pred,
y_true=yte,
beta=beta,
mask=mask,
n_iter=mod.get_info()['num_iter'])
elif MODALITY == "MRI+PET":
beta_MRI = beta[:(penalty_start + k_MRI)]
beta_PET = np.vstack(
[beta[:penalty_start], beta[(penalty_start + k_MRI):]])
ret = dict(y_pred=y_pred,
proba_pred=proba_pred,
y_true=yte,
beta=beta,
beta_MRI=beta_MRI,
beta_PET=beta_PET,
mask_MRI=mask_MRI,
mask_PET=mask_PET,
n_iter=mod.get_info()['num_iter'])
if output_collector:
output_collector.collect(key, ret)
else:
return ret
assert Xroi_train.shape == (150, 184)
assert Xroi_test.shape == (100, 184)
assert Xgm_train.shape == (150, 379778)
assert Xgm_test.shape == (100, 379778)
# enettv for GM
arg = [float(p) for p in WHICH.split("_")]
if len(arg) == 4:
alpha, l1, l2, tv = arg
else:
alpha, l1, l2, tv, k = arg
l1, l2, tv = alpha * l1, alpha * l2, alpha * tv
mask_im = nibabel.load(os.path.join(GM, "mask.nii"))
A, _ = tv_helper.A_from_mask(mask_im.get_data())
enettv = LogisticRegressionL1L2TV(l1,
l2,
tv,
A,
penalty_start=penalty_start,
class_weight="auto")
# lr l2 for roi
p_lr_l2 = Pipeline([
('scaler', StandardScaler()),
# ('classifier', LogisticRegression(C=0.005, penalty='l2')),
('classifier', LogisticRegression(C=0.0022, penalty='l2')),
])
p_lr_l2 = LogisticRegression(C=0.0022, penalty='l2')
