def plot_sliding_window(series, width, step):
'''
Plot a sliding-window graph
Parameters
----------
series: time-series, array , 3-D
width: window width
step: window step size, in samples. If not provided, window and step size are equal.
'''
from nilearn import plotting
import numpy as np
from nilearn.connectome import sym_matrix_to_vec
from nilearn.connectome import ConnectivityMeasure
cut = sliding_window_2d(series, width, step)
cut_matrix = np.zeros((cut.shape[0], cut.shape[2], cut.shape[2]))
correlation_measure = ConnectivityMeasure(kind='correlation')
for i in range(cut.shape[0]):
matrix = correlation_measure.fit_transform([cut[i]])[0]
cut_matrix[i, :, :] = matrix
vectors = np.zeros(
(cut_matrix.shape[0], sym_matrix_to_vec(cut_matrix[1]).shape[0]))
for i in range(cut_matrix.shape[0]):
vec = sym_matrix_to_vec(cut_matrix[i])
vectors[i, :] = vec
ax = np.corrcoef(vectors)
plotting.plot_matrix(ax, title="width={} step={}".format(width, step))
def CrossValidation(model, measure, y, skf, atalas):
from sklearn.metrics import make_scorer
from sklearn.metrics import accuracy_score, recall_score
import numpy as np
scoring = {
'accuracy': make_scorer(accuracy_score),
'sensitivity': make_scorer(recall_score),
'specificity': make_scorer(recall_score, pos_label=0)
}
from nilearn.connectome import ConnectivityMeasure
from nilearn.connectome import sym_matrix_to_vec
conn_est = ConnectivityMeasure(kind=measure)
conn_matrices = conn_est.fit_transform(atalas)
X = sym_matrix_to_vec(conn_matrices)
from sklearn.model_selection import cross_validate
scores = cross_validate(model,
X,
y,
cv=skf,
scoring=scoring,
return_train_score=True)
return [X, scores]
def plot_rdm(rdm, mat=0, cmap=None):
'''function to visualize RDM based rank transformed and scaled similarity values
(only for plotting, raw/initial values remain unchanged'''
from os.path import join as opj
from scipy.io.matlab import loadmat
from nilearn.connectome import sym_matrix_to_vec
from scipy.stats import rankdata
from nilearn.connectome import vec_to_sym_matrix
from sklearn import preprocessing
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
if mat == 1:
matfile = loadmat(rdm)
rdm = matfile['rdm'][0][0]
if cmap == None:
cmap = 'Spectral_r'
else:
cmap = cmap
rdm = pd.read_csv(rdm, sep=',')
if 'Unnamed: 0' in rdm:
del rdm['Unnamed: 0']
categories = list(rdm.columns)
rdm = rdm.as_matrix()
rdm_vec = sym_matrix_to_vec(rdm)
rdm_vec = rankdata(rdm_vec)
min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0, 1),
copy=True)
rdm_array = rdm_vec.reshape(-1, 2)
rdm_array = min_max_scaler.fit_transform(rdm_array)
rdm_array = rdm_array.flatten()
rdm_rank_scale = vec_to_sym_matrix(rdm_array)
ax = sns.heatmap(rdm_rank_scale,
xticklabels=categories,
yticklabels=categories,
cmap=cmap)
ax.set_yticklabels(categories, rotation=0)
ax.xaxis.tick_top()
ax.set_xticklabels(categories, rotation=90)
ax.collections[0].colorbar.set_label(
"pairwise similarities (iMDS), rank transformed & scaled [0,1]")
plt.tight_layout()
def test_fc_fd_vec(self):
corr_vec: dict = self.result.outputs.fc_fd_corr_values
self.assertIsInstance(corr_vec, dict)
self.assertTrue(all(isinstance(key, str) for key in corr_vec.keys()))
self.assertTrue(
all(isinstance(value, np.ndarray) for value in corr_vec.values()))
# value check
from nilearn.connectome import sym_matrix_to_vec
vec = sym_matrix_to_vec(self.group_corr_mat)
corr, _ = QualityMeasures.calculate_fc_fd_correlations(
self.group_conf_summary, vec) # TODO: Replace method call
assert_array_almost_equal(corr_vec[self.pipeline['name']], corr)
def plot_rdm(rdm, mat=False, cmap="Spectral_r"):
'''
function to visualize RDM based rank transformed and scaled similarity values
(only for plotting, raw/initial values remain unchanged)
'''
from os.path import join as opj
from scipy.io.matlab import loadmat
from scipy.stats import rankdata
import matplotlib.pyplot as plt
from sklearn.preprocessing import minmax_scale
import pandas as pd
import seaborn as sns
from nilearn.connectome import sym_matrix_to_vec, vec_to_sym_matrix
if mat is True:
matfile = loadmat(rdm)
rdm = matfile['rdm'][0][0]
if isinstance(rdm, str) is True:
rdm = pd.read_csv(rdm, sep=',')
if 'Unnamed: 0' in rdm:
del rdm['Unnamed: 0']
else:
rdm = rdm
categories = list(rdm.columns)
rdm = rdm.as_matrix()
rdm_vec = sym_matrix_to_vec(rdm)
rdm_vec = rankdata(rdm_vec)
rdm_array = rdm_vec.reshape(-1, 1)
rdm_array = minmax_scale(rdm_array, (0, 1))
rdm_array = rdm_array.flatten()
rdm_rank_scale = vec_to_sym_matrix(rdm_array)
y_categories = list(reversed(categories))
ax = sns.heatmap(rdm_rank_scale,
xticklabels=categories,
yticklabels=y_categories,
cmap=cmap)
ax.set_yticklabels(y_categories, rotation=0)
ax.xaxis.tick_top()
ax.set_xticklabels(categories, rotation=90)
ax.collections[0].colorbar.set_label(
"pairwise similarities, rank transformed & scaled [0,1]")
plt.tight_layout()
def test_fc_fd_vec_clean(self):
corr_vec: dict = self.result.outputs.fc_fd_corr_values_clean
self.assertIsInstance(corr_vec, dict)
self.assertTrue(all(isinstance(key, str) for key in corr_vec.keys()))
self.assertTrue(
all(isinstance(value, np.ndarray) for value in corr_vec.values()))
# value check
from nilearn.connectome import sym_matrix_to_vec
group_corr_mat = self.group_corr_mat[[0, 2], :, :]
vec: np.ndarray = sym_matrix_to_vec(group_corr_mat)
corr, _ = QualityMeasures.calculate_fc_fd_correlations(
self.group_conf_summary[self.group_conf_summary['include'] ==
True], vec)
assert_array_almost_equal(corr_vec[self.pipeline['name']], corr)
def test_edges_weight(self):
edges_weight: dict = self.result.outputs.edges_weight
self.assertIsInstance(edges_weight, dict)
self.assertTrue(
all(isinstance(key, str) for key in edges_weight.keys()))
self.assertTrue(
all(
isinstance(value, np.ndarray)
for value in edges_weight.values()))
# value check
from nilearn.connectome import sym_matrix_to_vec
vec: np.ndarray = sym_matrix_to_vec(self.group_corr_mat)
tested_edges_weight = vec.mean(axis=0)
assert_array_equal(edges_weight[self.pipeline['name']],
tested_edges_weight)
def map_tangent(data, diag=False):
"""Transform to tangent space.
Parameters
----------
data: list of numpy.ndarray of shape(n_features, n_features)
List of semi-positive definite matrices.
diag: bool
Whether to discard the diagonal elements before vectorizing. Default is
False.
Returns
-------
tangent: numpy.ndarray, shape(n_features * (n_features - 1) / 2)
"""
mean_ = _geometric_mean(data, max_iter=30, tol=1e-7)
whitening_ = _map_eigenvalues(lambda x: 1. / np.sqrt(x),
mean_)
tangent = [_map_eigenvalues(np.log, whitening_.dot(c).dot(whitening_))
for c in data]
tangent = np.array(tangent)
return sym_matrix_to_vec(tangent, discard_diagonal=diag)
clusterings.append(clustering)
Node_strengths = np.stack(Node_strengths)
eig_cens=np.stack(eig_cens)
clusterings=np.stack(clusterings)
from nilearn.connectome import sym_matrix_to_vec
mat_connectivity= []
matrix=connectivity_biomarkers['tangent']
for mat in matrix:
mat_connectivity.append(sym_matrix_to_vec(mat,discard_diagonal=True))
mat_connectivity = np.stack(mat_connectivity)
print(mat_connectivity.shape)
# ### Projection of statistical features
# In[6]:
## structural connectivity
from pathlib import Path
from scipy import io as sio
from pygsp import graphs
y=[]
for i in subjects_timeseries.keys():
y.append(labels[i])
type(y)
# In[ ]:
for measure in measures:
conn_est = ConnectivityMeasure(kind=measure)
conn_matrices = conn_est.fit_transform(subjects_timeseries.values())
X = sym_matrix_to_vec(conn_matrices)
for name, predictor in predictors:
# print (measure)
# print (name)
print(measure, name, np.mean(cross_val_score(predictor, X, np.array(y), cv=stratified_shuffle_split.split(X,y))))
# print( cross_val_score(predictor, X, np.array(y), cv=stratified_shuffle_split.split(X,y)))
# classifier = predictor
# for train_index,test_index in stratified_shuffle_split.split(X,y):
# predictor.fit(X[train_index],y[train_index])
# y_score = predictor.decision_function(X[test_index])
# average_precision = average_precision_score(y[test_index], y_score)
# print('Average precision-recall score: {0:0.2f}'.format(
# average_precision))
# disp = plot_precision_recall_curve(predictor, X[test_index], y[test_index])
where each row is a participant and each column is a connection'''
vector_motor = np.zeros((N_SUBJECTS, 64620))
vector_wm = np.zeros((N_SUBJECTS, 64620))
vector_gambling = np.zeros((N_SUBJECTS, 64620))
vector_emotion = np.zeros((N_SUBJECTS, 64620))
vector_language = np.zeros((N_SUBJECTS, 64620))
vector_relational = np.zeros((N_SUBJECTS, 64620))
vector_social = np.zeros((N_SUBJECTS, 64620))
'''import a package to extract the diagonal of the correlation matrix, as well as
initializing a list of the subset of subjects. It is a neccesary step in appending the list
of subjects to the connection data'''
from nilearn.connectome import sym_matrix_to_vec
subject_list = np.array(np.unique(range(339)))
for subject in range(subject_list.shape[0]):
vector_motor[subject, :] = sym_matrix_to_vec(
fc_matrix_motor[subject, :, :], discard_diagonal=True)
vector_motor[subject, :] = fc_matrix_motor[subject][np.triu_indices_from(
fc_matrix_motor[subject], k=1)]
for subject in range(subject_list.shape[0]):
vector_wm[subject, :] = sym_matrix_to_vec(fc_matrix_wm[subject, :, :],
discard_diagonal=True)
vector_wm[subject, :] = fc_matrix_wm[subject][np.triu_indices_from(
fc_matrix_wm[subject], k=1)]
for subject in range(subject_list.shape[0]):
vector_gambling[subject, :] = sym_matrix_to_vec(
fc_matrix_gambling[subject, :, :], discard_diagonal=True)
vector_gambling[subject, :] = fc_matrix_gambling[subject][
np.triu_indices_from(fc_matrix_gambling[subject], k=1)]
for subject in range(subject_list.shape[0]):
vector_emotion[subject, :] = sym_matrix_to_vec(
fc_matrix_emotion[subject, :, :], discard_diagonal=True)
def compare_motifs(struct_mat, func_mat, name, namer_dir, bins=20, N=4):
'''
Compare motif structure and population across structural and functional
graphs to achieve a homeostatic absolute threshold of each that optimizes
multiplex community detection and analysis.
Parameters
----------
in_mat : ndarray
M x M Connectivity matrix
thr : float
Absolute threshold [0, 1].
mlib : list
List of motif classes.
Returns
-------
mf : ndarray
1D vector listing the total motifs of size N for each
class of mlib.
References
----------
.. [1] Battiston, F., Nicosia, V., Chavez, M., & Latora, V. (2017).
Multilayer motif analysis of brain networks. Chaos.
https://doi.org/10.1063/1.4979282
'''
from pynets.stats.netmotifs import adaptivethresh
from pynets.core.thresholding import threshold_absolute
from pynets.core.thresholding import standardize
from scipy import spatial
from nilearn.connectome import sym_matrix_to_vec
import pandas as pd
import gc
mlib = ['1113', '1122', '1223', '2222', '2233', '3333']
# Standardize structural graph
struct_mat = standardize(struct_mat)
dims_struct = struct_mat.shape[0]
struct_mat[range(dims_struct), range(dims_struct)] = 0
at_struct = adaptivethresh(struct_mat, float(0.0), mlib, N)
print("%s%s%s" %
('Layer 1 (structural) has: ', np.sum(at_struct), ' total motifs'))
# Functional graph threshold window
func_mat = standardize(func_mat)
dims_func = func_mat.shape[0]
func_mat[range(dims_func), range(dims_func)] = 0
tmin_func = func_mat.min()
tmax_func = func_mat.max()
threshes_func = np.linspace(tmin_func, tmax_func, bins)
assert np.all(
struct_mat == struct_mat.T), "Structural Matrix must be symmetric"
assert np.all(
func_mat == func_mat.T), "Functional Matrix must be symmetric"
# Count motifs
print("%s%s%s%s" % ('Mining ', N, '-node motifs: ', mlib))
motif_dict = {}
motif_dict['struct'] = {}
motif_dict['func'] = {}
mat_dict = {}
mat_dict['struct'] = sym_matrix_to_vec(struct_mat, discard_diagonal=True)
mat_dict['funcs'] = {}
for thr_func in threshes_func:
# Count
at_func = adaptivethresh(func_mat, float(thr_func), mlib, N)
motif_dict['struct']["%s%s" %
('thr-', np.round(thr_func, 4))] = at_struct
motif_dict['func']["%s%s" % ('thr-', np.round(thr_func, 4))] = at_func
mat_dict['funcs']["%s%s" %
('thr-', np.round(thr_func, 4))] = sym_matrix_to_vec(
threshold_absolute(func_mat, thr_func),
discard_diagonal=True)
print("%s%s%s%s%s" %
('Layer 2 (functional) with absolute threshold of: ',
np.round(thr_func,
2), ' yields ', np.sum(at_func), ' total motifs'))
gc.collect()
df = pd.DataFrame(motif_dict)
for idx in range(len(df)):
df.set_value(
df.index[idx], 'motif_dist',
spatial.distance.cosine(df['struct'][idx], df['func'][idx]))
df = df[pd.notnull(df['motif_dist'])]
for idx in range(len(df)):
df.set_value(
df.index[idx], 'graph_dist_cosine',
spatial.distance.cosine(
mat_dict['struct'].reshape(-1, 1),
mat_dict['funcs'][df.index[idx]].reshape(-1, 1)))
df.set_value(
df.index[idx], 'graph_dist_correlation',
spatial.distance.correlation(
mat_dict['struct'].reshape(-1, 1),
mat_dict['funcs'][df.index[idx]].reshape(-1, 1)))
df['struct_func_3333'] = np.zeros(len(df))
df['struct_func_2233'] = np.zeros(len(df))
df['struct_func_2222'] = np.zeros(len(df))
df['struct_func_1223'] = np.zeros(len(df))
df['struct_func_1122'] = np.zeros(len(df))
df['struct_func_1113'] = np.zeros(len(df))
df['struct_3333'] = np.zeros(len(df))
df['func_3333'] = np.zeros(len(df))
df['struct_2233'] = np.zeros(len(df))
df['func_2233'] = np.zeros(len(df))
df['struct_2222'] = np.zeros(len(df))
df['func_2222'] = np.zeros(len(df))
df['struct_1223'] = np.zeros(len(df))
df['func_1223'] = np.zeros(len(df))
df['struct_1122'] = np.zeros(len(df))
df['func_1122'] = np.zeros(len(df))
df['struct_1113'] = np.zeros(len(df))
df['func_1113'] = np.zeros(len(df))
for idx in range(len(df)):
df.set_value(df.index[idx], 'struct_3333', df['struct'][idx][-1])
df.set_value(df.index[idx], 'func_3333', df['func'][idx][-1])
df.set_value(df.index[idx], 'struct_2233', df['struct'][idx][-2])
df.set_value(df.index[idx], 'func_2233', df['func'][idx][-2])
df.set_value(df.index[idx], 'struct_2222', df['struct'][idx][-3])
df.set_value(df.index[idx], 'func_2222', df['func'][idx][-3])
df.set_value(df.index[idx], 'struct_1223', df['struct'][idx][-4])
df.set_value(df.index[idx], 'func_1223', df['func'][idx][-4])
df.set_value(df.index[idx], 'struct_1122', df['struct'][idx][-5])
df.set_value(df.index[idx], 'func_1122', df['func'][idx][-5])
df.set_value(df.index[idx], 'struct_1113', df['struct'][idx][-6])
df.set_value(df.index[idx], 'func_1113', df['func'][idx][-6])
df['struct_func_3333'] = np.abs(df['struct_3333'] - df['func_3333'])
df['struct_func_2233'] = np.abs(df['struct_2233'] - df['func_2233'])
df['struct_func_2222'] = np.abs(df['struct_2222'] - df['func_2222'])
df['struct_func_1223'] = np.abs(df['struct_1223'] - df['func_1223'])
df['struct_func_1122'] = np.abs(df['struct_1122'] - df['func_1122'])
df['struct_func_1113'] = np.abs(df['struct_1113'] - df['func_1113'])
df = df.drop(columns=['struct', 'func'])
df = df.loc[~(df == 0).all(axis=1)]
df = df.sort_values(by=[
'motif_dist', 'graph_dist_cosine', 'graph_dist_correlation',
'struct_func_3333', 'struct_func_2233', 'struct_func_2222',
'struct_func_1223', 'struct_func_1122', 'struct_func_1113',
'struct_3333', 'func_3333', 'struct_2233', 'func_2233', 'struct_2222',
'func_2222', 'struct_1223', 'func_1223', 'struct_1122', 'func_1122',
'struct_1113', 'func_1113'
],
ascending=[
True, True, False, False, False, False, False,
False, False, False, False, False, False, False,
False, False, False, False, False, False, False
])
# Take the top 25th percentile
df = df.head(int(0.25 * len(df)))
best_threshes = []
best_mats = []
best_multigraphs = []
for key in list(df.index):
func_mat_tmp = func_mat.copy()
struct_mat_tmp = struct_mat.copy()
struct_thr = float(key.split('-')[-1])
func_thr = float(key.split('-')[-1])
best_threshes.append(str(func_thr))
func_mat_tmp[func_mat_tmp < func_thr] = 0
struct_mat_tmp[struct_mat_tmp < struct_thr] = 0
best_mats.append((func_mat_tmp, struct_mat_tmp))
mG = build_mx_multigraph(func_mat, struct_mat, f"{name}_{key}",
namer_dir)
best_multigraphs.append(mG)
mg_dict = dict(zip(best_threshes, best_multigraphs))
g_dict = dict(zip(best_threshes, best_mats))
return mg_dict, g_dict
def rdm_compare(rdms, models, comp=None, plot=None):
'''function to compare target and model
rmds'''
global dict_rdms
global DefaultListOrderedDict
from glob import glob
import pandas as pd
from collections import OrderedDict
from scipy.spatial import distance
from nilearn.connectome import sym_matrix_to_vec, vec_to_sym_matrix
from scipy.stats import rankdata, spearmanr, kendalltau, pearsonr, mstats
import numpy as np
from itertools import combinations
import pickle
import seaborn as sns
import matplotlib.pyplot as plt
import copy
class DefaultListOrderedDict(OrderedDict):
def __missing__(self,k):
self[k] = []
return self[k]
if isinstance(rdms, str) is True:
with open(rdms, 'rb') as f:
dict_rdms = pickle.load(f)
target_rdms = copy.deepcopy(dict_rdms['rdm'])
target_conds = target_rdms[0].keys()
else:
target_rdms = rdms
target_conds = rdms[0].keys()
if isinstance(models, str) is True:
with open(models, 'rb') as f:
dict_models = pickle.load(f)
models = dict_models['rdm']
model_ids = dict_models['id']
else:
models = models
for rdm in dict_models['rdm']:
if 'Unnamed: 0' in rdm:
del rdm['Unnamed: 0']
for index, rdm in enumerate(target_rdms):
target_rdms[index] = target_rdms[index].as_matrix()
global rdm_avg
list_cor_rdm=list(range(0,len(target_rdms)))
list_p=list(range(0,len(target_rdms)))
target_rdms_trans=list(range(0,len(target_rdms)))
if comp is None:
print('rdm values will not be transformed')
rdm_avg = pd.DataFrame(np.mean(target_rdms, axis=0), columns=target_conds)
for index, part_rdm in enumerate(target_rdms):
list_cor_rdm[index], list_p[index] = kendalltau(part_rdm.flatten(), rdm_avg.as_matrix().flatten())
list_cor_sub = list()
list_cor_rdm_sub = list()
list_p_sub = list()
for index, part in enumerate(target_rdms):
tmp_rdms = target_rdms.copy()
tmp_part = target_rdms[index]
tmp_rdms.pop(index)
tmp_rdm_avg = np.mean(tmp_rdms, axis=0)
list_cor_sub.append(kendalltau(tmp_part.flatten(), tmp_rdm_avg.flatten()))
for i, cor in enumerate(list_cor_sub):
list_cor_rdm_sub.append(cor.correlation)
list_p_sub.append(cor.pvalue)
elif comp == 'spearman':
for index, rdm in enumerate(target_rdms):
target_rdms_trans[index] = vec_to_sym_matrix(rankdata(sym_matrix_to_vec(rdm)))
rdm_avg = pd.DataFrame(np.mean(target_rdms_trans, axis=0), columns=target_conds)
for index, part_rdm in enumerate(target_rdms_trans):
list_cor_rdm[index], list_p[index] = spearmanr(part_rdm.flatten(), rdm_avg.as_matrix().flatten())
list_cor_sub = list()
list_cor_rdm_sub = list()
list_p_sub = list()
for index, part in enumerate(target_rdms_trans):
tmp_rdms = target_rdms_trans.copy()
tmp_part = target_rdms_trans[index]
tmp_rdms.pop(index)
tmp_rdm_avg = np.mean(tmp_rdms, axis=0)
list_cor_sub.append(spearmanr(tmp_part.flatten(), tmp_rdm_avg.flatten()))
for i, cor in enumerate(list_cor_sub):
list_cor_rdm_sub.append(cor.correlation)
list_p_sub.append(cor.pvalue)
elif comp == 'pearson':
for index, rdm in enumerate(target_rdms):
target_rdms_trans[index] = vec_to_sym_matrix(mstats.zscore(sym_matrix_to_vec(rdm)))
rdm_avg = pd.DataFrame(np.mean(target_rdms_trans, axis=0), columns=target_conds)
for index, part_rdm in enumerate(target_rdms_trans):
list_cor_rdm[index], list_p[index] = pearsonr(part_rdm.flatten(), rdm_avg.as_matrix().flatten())
list_cor_sub = list()
list_cor_rdm_sub = list()
list_p_sub = list()
for index, part in enumerate(target_rdms_trans):
tmp_rdms = target_rdms_trans.copy()
tmp_part = target_rdms_trans[index]
tmp_rdms.pop(index)
tmp_rdm_avg = np.mean(tmp_rdms, axis=0)
list_cor_sub.append(pearsonr(tmp_part.flatten(), tmp_rdm_avg.flatten()))
for i, cor in enumerate(list_cor_sub):
list_cor_rdm_sub.append(cor[0])
list_p_sub.append(cor[1])
upper_noise_ceiling = np.mean(list_cor_rdm)
lower_noise_ceiling=np.mean(list_cor_rdm_sub)
model_comp = pd.DataFrame(columns=['participant', 'models', 'cor'], index=np.arange(len(dict_models['id'])*len(dict_rdms['id'])))
model_comp['participant']=dict_rdms['id']*len(dict_models['id'])
model_comp['models']=sorted(dict_models['id']*len(dict_rdms['id']))
list_cor_models=list()
snd_rdms=list()
snd_rdms.append(rdm_avg.as_matrix())
for mod_rdm in models:
snd_rdms.append(mod_rdm.as_matrix())
ids_rdms=list()
ids_rdms.append('group average')
for mod_ids in model_ids:
ids_rdms.append(mod_ids)
if comp is None:
for index, model_rdm in enumerate(dict_models['rdm']):
for i, sub_rdm in enumerate(target_rdms):
list_cor_models.append(kendalltau(sub_rdm.flatten(), model_rdm.as_matrix().flatten()).correlation)
rdms_dist = [kendalltau(x.flatten(), y.flatten()).correlation for x, y in combinations(snd_rdms, 2)]
rdms_dist = pd.DataFrame(distance.squareform(rdms_dist), columns=ids_rdms)
elif comp == 'spearman':
for index, model_rdm in enumerate(dict_models['rdm']):
for i, sub_rdm in enumerate(target_rdms_trans):
list_cor_models.append(spearmanr(sub_rdm.flatten(), model_rdm.as_matrix().flatten()).correlation)
rdms_dist = [spearmanr(x.flatten(), y.flatten()).correlation for x, y in combinations(snd_rdms, 2)]
rdms_dist = pd.DataFrame(distance.squareform(rdms_dist), columns=ids_rdms)
elif comp == 'pearson':
for index, model_rdm in enumerate(dict_models['rdm']):
for i, sub_rdm in enumerate(target_rdms_trans):
list_cor_models.append(pearsonr(sub_rdm.flatten(), model_rdm.as_matrix().flatten())[0])
rdms_dist = [pearsonr(x.flatten(), y.flatten())[0] for x, y in combinations(snd_rdms, 2)]
rdms_dist = pd.DataFrame(distance.squareform(rdms_dist), columns=ids_rdms)
model_comp['cor']=list_cor_models
if plot is None:
print('results will no be plotted')
elif plot == 'bar':
ax=sns.barplot(x=model_comp['models'], y=model_comp['cor'], data=model_comp)
plt.plot(np.linspace(-20, 120, 1000), [upper_noise_ceiling] * 1000, 'r', alpha=0.1)
plt.plot(np.linspace(-20, 120, 1000), [lower_noise_ceiling] * 1000, 'r', alpha=0.1)
rect = plt.Rectangle((-20, lower_noise_ceiling), 10000, (upper_noise_ceiling - lower_noise_ceiling), color='r',
alpha=0.5)
ax.set_xticklabels(labels=list(dict_models['id']))
if comp is None or 'spearman':
ax.set(ylabel='spearman correlation with target RDM')
if comp == 'pearson':
ax.set(ylabel='pearson correlation with target RDM')
if comp == 'kendalltau':
ax.set(ylabel='kendall tau a correlation with target RDM')
ax.add_patch(rect)
plt.tight_layout()
elif plot == 'violin':
ax=sns.violinplot(x=model_comp['models'], y=model_comp['cor'], data=model_comp)
plt.plot(np.linspace(-20, 120, 1000), [upper_noise_ceiling] * 1000, 'r', alpha=0.1)
plt.plot(np.linspace(-20, 120, 1000), [lower_noise_ceiling] * 1000, 'r', alpha=0.1)
rect = plt.Rectangle((-20, lower_noise_ceiling), 10000, (upper_noise_ceiling - lower_noise_ceiling), color='r',
alpha=0.5)
ax.set_xticklabels(labels=list(dict_models['id']))
if comp is None or 'spearman':
ax.set(ylabel='spearman correlation with target RDM')
if comp == 'pearson':
ax.set(ylabel='pearson correlation with target RDM')
if comp == 'kendalltau':
ax.set(ylabel='kendall tau a correlation with target RDM')
ax.add_patch(rect)
plt.tight_layout()
return rdm_avg, model_comp, rdms_dist
def _run_interface(self, runtime):
# Loading data
group_corr_mat = np.load(
self.inputs.group_corr_mat) # array with matrices for all runs
group_conf_summary = pd.read_csv(
self.inputs.group_conf_summary,
sep='\t') # motion summary for all runs
pipeline_name = self.inputs.pipeline_name
distance_vector = sym_matrix_to_vec(
np.load(self.inputs.distance_matrix)) # load distance matrix
# Plotting motion
colour = ["#fe6863", "#00a074"]
sns.set_palette(colour)
fig = motion_plot(group_conf_summary)
fig.savefig(join(self.inputs.output_dir,
f"motion_criterion_{pipeline_name}.svg"),
dpi=300)
# Creating vectors with subject filter
all_sub_no = len(group_conf_summary)
icluded_sub = group_conf_summary["include"]
excluded_sub_no = all_sub_no - sum(
icluded_sub) # number of subjects excluded from analyses
# Create dictionary describing full sampple and sample after exluding highly motion runs
included = {
f"All subjects (n = {all_sub_no})":
[np.ones((all_sub_no), dtype=bool), False, all_sub_no, "All"],
f"After excluding {excluded_sub_no} high motion subjects (n = {all_sub_no - excluded_sub_no})":
[
group_conf_summary["include"].values.astype("bool"), True,
all_sub_no - excluded_sub_no, "No_high_motion"
]
}
group_corr_vec = sym_matrix_to_vec(group_corr_mat)
n_edges = group_corr_vec.shape[1]
fc_fd_corr, fc_fd_pval = (np.zeros(n_edges) for _ in range(2))
fc_fd_summary = []
edges_weight = {}
edges_weight_clean = {}
for key, value in included.items():
for i in range(n_edges):
corr = pearsonr(group_corr_vec[value[0], i],
group_conf_summary['mean_fd'].values[value[0]])
fc_fd_corr[i] = corr[0] # Pearson's r values
fc_fd_pval[i] = corr[1] # p-values
fc_fd_corr = np.nan_to_num(fc_fd_corr) # TODO: write exception
# Calculate correlation between FC-FD r values and distance vector
distance_dependence = pearsonr(fc_fd_corr, distance_vector)[0]
# Store summary measure
fc_fd_summary.append({
"pipeline":
pipeline_name,
"perc_fc_fd_uncorr":
np.sum(fc_fd_pval < 0.5) / len(fc_fd_pval) * 100,
"pearson_fc_fd":
np.median(fc_fd_corr),
"distance_dependence":
distance_dependence,
"tdof_loss":
group_conf_summary["n_conf"].mean(),
"cleaned":
value[1],
"subjects":
value[3],
"sub_no":
value[2]
})
# For cleaned dataset
if value[1]:
edges_weight_clean = {
pipeline_name: group_corr_vec[value[0]].mean(axis=0)
}
# For full dataset
if not value[1]:
edges_weight = {
pipeline_name: group_corr_vec[value[0]].mean(axis=0)
}
# Plotting FC and FC-FD correlation matrices
fc_fd_corr_mat = vec_to_sym_matrix(fc_fd_corr)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
fig1 = ax1.imshow(group_corr_mat[value[0]].mean(axis=0),
vmin=-1,
vmax=1,
cmap="RdBu_r")
ax1.set_title(f"{pipeline_name}: mean FC")
fig.colorbar(fig1, ax=ax1)
fig2 = ax2.imshow(fc_fd_corr_mat, vmin=-1, vmax=1, cmap="RdBu_r")
ax2.set_title(f"{pipeline_name}: FC-FD correlation")
fig.colorbar(fig2, ax=ax2)
fig.suptitle(f"{pipeline_name}: {key}")
fig.savefig(join(
self.inputs.output_dir,
f"FC_FD_corr_mat_{pipeline_name}_{value[3].lower()}.png"),
dpi=300)
exclude_list = [
f"sub-{x + 1:02}" for x in group_conf_summary[
group_conf_summary['include'] == 1]['subject']
]
self._results["fc_fd_summary"] = fc_fd_summary
self._results["edges_weight"] = edges_weight
self._results["edges_weight_clean"] = edges_weight_clean
self._results["exclude_list"] = exclude_list
return runtime
'''Assemble all FC matrices into one object'''
from nilearn.connectome import sym_matrix_to_vec
all_fc = np.stack((sz_matrix, bp_matrix, cn_matrix, adhd_matrix, test_sz_matrix, test_bp_matrix, test_adhd_matrix, test_cn_matrix))
#concat all the TS into one object
concat_ts = np.stack((rest_cn, rest_sz,rest_bp,rest_adhd,
test_rest_cn, test_rest_sz, test_rest_bp, test_rest_adhd))
concat_ts = np.asarray(concat_ts)
connectivity = ConnectivityMeasure(kind='correlation', vectorize=True)
connectivity.fit_transform(concat_ts)
for subject in all_fc:
sym_matrix_to_vec(subject, discard_diagonal=True)
sz_vector = sym_matrix_to_vec(sz_matrix, discard_diagonal=True)
bp_vector = sym_matrix_to_vec(bp_matrix, discard_diagonal=True)
adhd_vector = sym_matrix_to_vec(adhd_matrix, discard_diagonal=True)
cn_vector = sym_matrix_to_vec(cn_matrix, discard_diagonal=True)
test_sz_vector = sym_matrix_to_vec(test_sz_matrix, discard_diagonal=True)
test_bp_vector = sym_matrix_to_vec(test_bp_matrix, discard_diagonal=True)
test_adhd_vector = sym_matrix_to_vec(test_adhd_matrix, discard_diagonal=True)
test_cn_vector = sym_matrix_to_vec(test_cn_matrix, discard_diagonal=True)
concat_vector = np.stack((sz_vector, bp_vector,adhd_vector ,cn_vector,
test_sz_vector, test_bp_vector, test_adhd_vector, test_cn_vector))
for sub in range(sub_n):
for i, cond in enumerate(conditions):
task = timeseries_all[sub, design_matrix[cond].astype("bool").astype("bool"), :]
correlation_measure = ConnectivityMeasure(kind="correlation")
fc = correlation_measure.fit_transform([task])[0]
np.fill_diagonal(fc, 0)
correlation_matrices[sub, i, :, :] = fc
correlation_matrices.shape
# Calculate edgewise distances.
zero_back = sym_matrix_to_vec(correlation_matrices[:,0,:,:], discard_diagonal = True)
two_back = sym_matrix_to_vec(correlation_matrices[:,1,:,:], discard_diagonal = True)
stat, pvalues = stats.ttest_rel(zero_back, two_back)
import statsmodels.stats.multitest as ssm
_, pvals_corrected, _, _ = ssm.multipletests(pvalues, alpha = 0.05, method = "fdr_bh")
pvals_corrected_thr = np.zeros((len(pvals_corrected)))
pvals = np.array([0 if p >= 0.05 else 1 for p in pvals_corrected])
sum(pvals)
wei_vector = stat * pvals
diag = np.zeros((264))
def rdm_dist(rdms, comp=None, order=None):
'''function to compute distances between all
RDMs in a given dictionary'''
#global DefaultListOrderedDict
from collections import OrderedDict
class DefaultListOrderedDict(OrderedDict):
def __missing__(self, k):
self[k] = []
return self[k]
import pandas as pd
from collections import OrderedDict
import pickle
from scipy.spatial import distance
from scipy.stats import pearsonr, spearmanr, rankdata, mstats
from itertools import combinations
from nilearn.connectome import sym_matrix_to_vec
import numpy as np
if isinstance(rdms, str) is True:
with open(rdms, 'rb') as f:
dict_rdms = pickle.load(f)
rdms = dict_rdms['rdm']
ids = dict_rdms['id']
else:
dict_rdms = rdms
rdms = dict_rdms['rdm']
ids = dict_rdms['id']
if order is None:
print(
'RDM comparisons will be written to the results data frame in the order they are found in the pkl file'
)
elif order is not None:
print(
'RDM comparisons will be written to the results data frame in the order specified by the user'
)
order = order
df_order = pd.DataFrame()
df_order['rdms'] = rdms
df_order['rdm_id'] = ids
df_order.index = ids
df_order_user = pd.DataFrame(df_order.reindex(order))
rdms = df_order_user['rdms']
ids = df_order_user['rdm_id']
global rdms_dist
if comp is None or comp == 'euclidean':
rdms_dist = [
distance.euclidean(
sym_matrix_to_vec(x.to_numpy(), discard_diagonal=True),
sym_matrix_to_vec(y.to_numpy(), discard_diagonal=True))
for x, y in combinations(rdms, 2)
]
rdms_dist = pd.DataFrame(distance.squareform(rdms_dist), columns=ids)
elif comp == 'spearman':
for index, rdm in enumerate(rdms):
rdms[index] = rankdata(
sym_matrix_to_vec(rdm.values, discard_diagonal=True))
rdms_dist = [
spearmanr(x, y).correlation for x, y in combinations(rdms, 2)
]
rdms_dist = pd.DataFrame(distance.squareform(rdms_dist), columns=ids)
np.fill_diagonal(rdms_dist.values, 1)
#rdms_dist = rdms_dist.mask(rdms_dist.values > -1.05, 1 - rdms_dist.values)
elif comp == 'pearson':
for index, rdm in enumerate(rdms):
rdms[index] = mstats.zscore(
sym_matrix_to_vec(rdm.to_numpy(), discard_diagonal=True))
rdms_dist = [pearsonr(x, y)[0] for x, y in combinations(rdms, 2)]
rdms_dist = pd.DataFrame(distance.squareform(rdms_dist), columns=ids)
np.fill_diagonal(rdms_dist.values, 1)
#rdms_dist = rdms_dist.mask(rdms_dist.values > -1.05, 1 - rdms_dist.values)
return rdms_dist
def plot_rdm(rdm, mat=False, model=False, level=None, comp=None, cmap="Spectral_r"):
'''
function to visualize RDM based rank transformed and scaled similarity values
(only for plotting, raw/initial values remain unchanged)
'''
from scipy.io.matlab import loadmat
from scipy.stats import rankdata
import matplotlib.pyplot as plt
from sklearn.preprocessing import minmax_scale
import pandas as pd
import seaborn as sns
from nilearn.connectome import sym_matrix_to_vec, vec_to_sym_matrix
if mat is True:
matfile = loadmat(rdm)
rdm = matfile['rdm'][0][0]
if isinstance(rdm, str) is True:
rdm = pd.read_csv(rdm, sep=',')
if 'Unnamed: 0' in rdm:
del rdm['Unnamed: 0']
else:
rdm=rdm
categories = list(rdm.columns)
y_categories = list(categories)
if model is False and level == '2nd':
ax = sns.heatmap(rdm, xticklabels=categories, yticklabels=y_categories, cmap=cmap, vmin=-1, vmax=1)
ax.set_yticklabels(y_categories, rotation=0)
ax.xaxis.tick_top()
ax.set_xticklabels(categories, rotation=90)
if comp is None:
ax.collections[0].colorbar.set_label("correlations between RDMs")
if comp == 'kendalltaua':
ax.collections[0].colorbar.set_label("correlations between RDMs [kendall tau]")
if comp == 'spearman':
ax.collections[0].colorbar.set_label("correlations between RDMs [spearman]")
if comp == 'pearson':
ax.collections[0].colorbar.set_label("correlations between RDMs [pearson]")
plt.tight_layout()
if model is False and level is None:
rdm = rdm.to_numpy()
rdm_vec = sym_matrix_to_vec(rdm)
rdm_vec = rankdata(rdm_vec)
rdm_array = rdm_vec.reshape(-1, 1)
rdm_array = minmax_scale(rdm_array, (0, 1))
rdm_array = rdm_array.flatten()
rdm_rank_scale = vec_to_sym_matrix(rdm_array)
ax = sns.heatmap(rdm_rank_scale, xticklabels=categories, yticklabels=y_categories, cmap=cmap)
ax.set_yticklabels(y_categories, rotation=0)
ax.xaxis.tick_top()
ax.set_xticklabels(categories, rotation=90)
ax.collections[0].colorbar.set_label("pairwise similarities, rank transformed & scaled [0,1]")
plt.tight_layout()
if model is True:
rdm = rdm.to_numpy()
rdm_vec = sym_matrix_to_vec(rdm)
rdm_array = rdm_vec.reshape(-1, 1)
rdm_array = minmax_scale(rdm_array, (0, 1))
rdm_array = rdm_array.flatten()
rdm_scale = vec_to_sym_matrix(rdm_array)
ax = sns.heatmap(rdm_scale, xticklabels=categories, yticklabels=y_categories, cmap=cmap)
ax.set_yticklabels(y_categories, rotation=0)
ax.xaxis.tick_top()
ax.set_xticklabels(categories, rotation=90)
ax.collections[0].colorbar.set_label("pairwise similarities, scaled [0,1]")
plt.tight_layout()
def plot_conn_hist(matrix_file, modules, atlas=False, output_file="hist.png"):
from matplotlib import pyplot as plt
from matplotlib import gridspec as mgs
import matplotlib.cm as cm
from matplotlib.colors import ListedColormap
import PUMI.utils.globals as glb
import pandas as pd
import numpy as np
import nibabel as nb
from nilearn.plotting import plot_img
import os
from nilearn.connectome import sym_matrix_to_vec
# load matrix file
mat = pd.read_csv(matrix_file, sep="\t")
mat.set_index('Unnamed: 0', inplace=True)
regnum = mat.shape[0]
mat = mat.values
histdata=[]
# create a list of modular connectivities
for mod in pd.Series(modules).unique():
idx = np.array(np.where([m == mod for m in modules])).flatten()
submat = mat[np.ix_(idx, idx)]
mat[np.ix_(idx, idx)] = None
histdata.insert(0, np.array(sym_matrix_to_vec(submat)))
histdata.insert(0, np.array([0])) # temporary hack to make nicer inter-modular colors
histdata.insert(0, np.array(sym_matrix_to_vec(mat)))
mycolors = ListedColormap(cm.get_cmap('Set1').colors[:7][::-1])
modules = pd.Series(modules).values
lut = pd.factorize(modules)[0] + 1
legend = False
if atlas:
legend = True
# Define nested GridSpec
wratios = [100, 20]
subplot = mgs.GridSpec(1, 1)[0]
gs = mgs.GridSpecFromSubplotSpec(1, 1 + int(legend), subplot_spec=subplot,
width_ratios=wratios[:2 + int(legend)],
wspace=0.0)
ax0 = plt.subplot(gs[0])
# colorbars are very much hardcoded
# TODO: fix colorbars (make it work for other atlases)
cols = cm.get_cmap('Set1').colors[:9][::-1]
plt.hist(histdata, stacked=True, normed=True, color=cols)
if legend:
gslegend = mgs.GridSpecFromSubplotSpec(
5, 1, subplot_spec=gs[1], wspace=0.0, hspace=0.0)
background_file = glb._FSLDIR_ + "/data/standard/MNI152_T1_3mm_brain.nii.gz" #TODO: works only for 3mm atlas
background = nb.load(background_file)
atlas = nb.load(atlas)
nslices = background.shape[-1]
#coords = np.linspace(int(0 * nslices), int(0.99 * nslices), 5).astype(np.uint8)
coords = [-40, 20, 0, 20, 40] #works in MNI space
lut2 = lut
lut2 = np.array([0] + lut2.tolist())
relabeled = lut2[np.array(atlas.get_data(), dtype=int)]
atl = nb.Nifti1Image(relabeled, atlas.get_affine())
for i, c in enumerate(coords):
ax2 = plt.subplot(gslegend[i])
plot_img(atl, bg_img=background, axes=ax2, display_mode='z',
annotate=False, cut_coords=[c], threshold=0.1, cmap=mycolors,
interpolation='nearest', vmin=1, vmax=7)
if output_file is not None:
figure = plt.gcf()
figure.savefig(output_file, bbox_inches='tight')
plt.close(figure)
return os.getcwd() + '/' + output_file
return 0
