os.makedirs(output)
except FileExistsError:
pass
output = os.path.join(output, map_basename)
output_figure_filename = output + "_clust_info.pdf"
output_csv_clusters_info_filename = output + "_clust_info.csv"
output_clusters_labels_filename = output + "_clust_labels.nii.gz"
output_clusters_values_filename = output + "_clust_values.nii.gz"
print("Outputs:", output, output_figure_filename)
##########################################################################
# Find clusters (connected component above a given threshold)
print(thresh_neg_low, thresh_neg_high, thresh_pos_low, thresh_pos_high)
if thresh_norm_ratio < 1:
map_arr, thres = arr_threshold_from_norm2_ratio(
map_arr, thresh_norm_ratio)
print("Threshold image as %f" % thres)
clust_bool = np.zeros(map_arr.shape, dtype=bool)
clust_bool[((map_arr > thresh_neg_low) & (map_arr < thresh_neg_high)) |
((map_arr > thresh_pos_low) &
(map_arr < thresh_pos_high))] = True
map_arr[np.logical_not(clust_bool)] = False
map_clustlabels_arr, n_clusts = scipy.ndimage.label(clust_bool)
clust_sizes = scipy.ndimage.measurements.histogram(map_clustlabels_arr, 1,
n_clusts, n_clusts)
labels = np.unique(map_clustlabels_arr)[1:]
centers = scipy.ndimage.center_of_mass(clust_bool, map_clustlabels_arr,
labels)
# _clusters.nii.gz
img = nibabel.Nifti1Image(map_arr, map_img.affine)
with open(models_filename, 'rb') as fd:
key_vals = pickle.load(fd)
# filter out "ALL" folds
key_vals = {k:v for k, v in key_vals.items() if k[2] != "ALL"}
# Agregate maps by key[0]
by_param = {k:[] for k in set([k[0] for k in key_vals])}
for k, v in key_vals.items():
by_param[k[0]].append(v['coef_img'].ravel())
maps_similarity_l = list()
# Compute similarity measures
for k, v in by_param.items():
maps = np.array(v)
maps_t = np.vstack([arr_threshold_from_norm2_ratio(maps[i, :], .99)[0] for i in range(maps.shape[0])])
r_bar, dice_bar, fleiss_kappa_stat = maps_similarity(maps_t)
prop_non_zeros_mean = np.count_nonzero(maps_t) / np.prod(maps_t.shape)
maps_similarity_l.append([k, prop_non_zeros_mean, r_bar, dice_bar, fleiss_kappa_stat])
map_sim = pd.DataFrame(maps_similarity_l, columns=['param_0', 'prop_non_zeros_mean', 'r_bar', 'dice_bar', 'fleiss_kappa_stat'])
# Update excel file with similariy measures
xls_filename = OUTPUT(DATASET_TRAIN, scaling=scaling, harmo=harmo, type="models-5cv-enettv", ext="xlsx")
cv_scores_all = pd.read_excel(xls_filename, sheet_name='folds')
cv_scores = cv_scores_all[cv_scores_all.fold != "ALL"]
pred_score_mean_ = cv_scores.groupby(["param_0"]).mean()
pred_score_mean_ = pred_score_mean_.reset_index()
pred_score_mean = pd.merge(pred_score_mean_, map_sim)
assert pred_score_mean_.shape[0] == map_sim.shape[0] == pred_score_mean.shape[0]
def plot_coefmap_stats(coef_map, coef_maps, ref_img, vmax, prefix):
arr_threshold_from_norm2_ratio(coef_map, .99)[0]
coef_maps_t = np.vstack([
arr_threshold_from_norm2_ratio(coef_maps[i, :], .99)[0]
for i in range(coef_maps.shape[0])
])
w_selectrate = np.sum(coef_maps_t != 0, axis=0) / coef_maps_t.shape[0]
w_zscore = np.nan_to_num(
np.mean(coef_maps, axis=0) / np.std(coef_maps, axis=0))
w_mean = np.mean(coef_maps, axis=0)
w_std = np.std(coef_maps, axis=0)
val_arr = np.zeros(ref_img.get_fdata().shape)
val_arr[mask_arr] = coef_map
coefmap_img = nibabel.Nifti1Image(val_arr, affine=ref_img.affine)
coefmap_img.to_filename(prefix + "coefmap.nii.gz")
val_arr = np.zeros(ref_img.get_fdata().shape)
val_arr[mask_arr] = w_mean
coefmap_cvmean_img = nibabel.Nifti1Image(val_arr, affine=ref_img.affine)
coefmap_cvmean_img.to_filename(prefix + "coefmap_mean.nii.gz")
val_arr = np.zeros(ref_img.get_fdata().shape)
val_arr[mask_arr] = w_std
coefmap_cvstd_img = nibabel.Nifti1Image(val_arr, affine=ref_img.affine)
coefmap_cvstd_img.to_filename(prefix + "coefmap_std.nii.gz")
val_arr = np.zeros(ref_img.get_fdata().shape)
val_arr[mask_arr] = w_zscore
coefmap_cvzscore_img = nibabel.Nifti1Image(val_arr, affine=ref_img.affine)
coefmap_cvzscore_img.to_filename(prefix + "coefmap_zscore.nii.gz")
val_arr = np.zeros(ref_img.get_fdata().shape)
val_arr[mask_arr] = w_selectrate
coefmap_cvselectrate_img = nibabel.Nifti1Image(val_arr,
affine=ref_img.affine)
coefmap_cvselectrate_img.to_filename(prefix + "coefmap_selectrate.nii.gz")
# Plot
pdf = PdfPages(prefix + "coefmap.pdf")
fig = plt.figure(figsize=(11.69, 3 * 8.27))
ax = fig.add_subplot(511)
plotting.plot_glass_brain(coefmap_img,
threshold=1e-6,
plot_abs=False,
colorbar=True,
cmap=plt.cm.bwr,
vmax=vmax,
figure=fig,
axes=ax,
title="Coef")
ax = fig.add_subplot(512)
plotting.plot_glass_brain(coefmap_cvmean_img,
threshold=1e-6,
plot_abs=False,
colorbar=True,
cmap=plt.cm.bwr,
vmax=vmax,
figure=fig,
axes=ax,
title="Mean")
ax = fig.add_subplot(513)
plotting.plot_glass_brain(coefmap_cvstd_img,
threshold=1e-6,
colorbar=True,
figure=fig,
axes=ax,
title="Std")
ax = fig.add_subplot(514)
plotting.plot_glass_brain(coefmap_cvzscore_img,
threshold=1e-6,
plot_abs=False,
colorbar=True,
cmap=plt.cm.bwr,
figure=fig,
axes=ax,
title="Zscore")
ax = fig.add_subplot(515)
plotting.plot_glass_brain(coefmap_cvselectrate_img,
threshold=1e-6,
colorbar=True,
figure=fig,
axes=ax,
title="Select. Rate")
pdf.savefig()
plt.close(fig)
pdf.close()
maps = {
"coefmap": coefmap_img,
"coefmap_mean": coefmap_cvmean_img,
"coefmap_cvstd": coefmap_cvstd_img,
"coefmap_cvzscore": coefmap_cvzscore_img,
"coefmap_cvselectrate": coefmap_cvselectrate_img
}
return maps
key_vals = pickle.load(fd)
# filter out "ALL" folds
key_vals = {k: v for k, v in key_vals.items() if k[2] != "ALL"}
# Agregate maps by key[0]
by_param = {k: [] for k in set([k[0] for k in key_vals])}
for k, v in key_vals.items():
by_param[k[0]].append(v['coef_img'].ravel())
maps_similarity_l = list()
# Compute similarity measures
for k, v in by_param.items():
maps = np.array(v)
maps_t = np.vstack([
arr_threshold_from_norm2_ratio(maps[i, :], .99)[0]
for i in range(maps.shape[0])
])
r_bar, dice_bar, fleiss_kappa_stat = maps_similarity(maps_t)
prop_non_zeros_mean = np.count_nonzero(maps_t) / np.prod(maps_t.shape)
maps_similarity_l.append(
[k, prop_non_zeros_mean, r_bar, dice_bar, fleiss_kappa_stat])
map_sim = pd.DataFrame(maps_similarity_l,
columns=[
'param_0', 'prop_non_zeros_mean', 'r_bar',
'dice_bar', 'fleiss_kappa_stat'
])
# Update excel file with similariy measures
xls_filename = OUTPUT(DATASET_TRAIN,
# filter out "ALL" folds
key_vals = {k: v for k, v in key_vals.items() if k[2] != "ALL"}
# Agregate maps all key except the last whici is the fold
nkey_beforefold = len(list(key_vals.keys())[0]) - 1
by_param = {k: [] for k in set([k[:nkey_beforefold] for k in key_vals])}
for k, v in key_vals.items():
by_param[k[:nkey_beforefold]].append(v['coef_img'].ravel())
maps_similarity_l = list()
# Compute similarity measures
for k, v in by_param.items():
maps = np.array(v)
maps_t = np.vstack([
arr_threshold_from_norm2_ratio(maps[i, :], .99)[0]
for i in range(maps.shape[0])
])
r_bar, dice_bar, fleiss_kappa_stat = maps_similarity(maps_t)
prop_non_zeros_mean = np.count_nonzero(maps_t) / np.prod(maps_t.shape)
maps_similarity_l.append(
list(k) +
[prop_non_zeros_mean, r_bar, dice_bar, fleiss_kappa_stat])
map_sim = pd.DataFrame(
maps_similarity_l,
columns=['param_%i' % i for i in range(nkey_beforefold)] +
['prop_non_zeros_mean', 'r_bar', 'dice_bar', 'fleiss_kappa_stat'])
# Write new sheet 'mean_img' in excel file
sheets_ = pd.read_excel(xls_filename, sheet_name=None)