def embedding(upper_corr, full_shape, n_components):
'''
Diffusion embedding on connectivity matrix using mapaling package:
https://github.com/satra/mapalign
'''
# reconstruct full matrix
print('...full matrix')
full_corr = np.zeros(tuple(full_shape))
full_corr[np.triu_indices_from(full_corr, k=1)] = np.nan_to_num(upper_corr)
del upper_corr
gc.collect()
full_corr += full_corr.T
gc.collect()
# run actual embedding
print('...embed')
K = (full_corr + 1) / 2.
del full_corr
gc.collect()
K[np.where(np.eye(K.shape[0]) == 1)] = 1.0
embedding_results, \
embedding_dict = embed.compute_diffusion_map(K,
n_components=n_components,
overwrite=True,
return_result=True)
return embedding_results, embedding_dict
def get_gradients(connectmat, ncomp):
threshMat = connectmat.copy()
np.fill_diagonal(threshMat, 0)
# Generate percentile thresholds for 90th percentile
perc = np.array([np.percentile(x, 90) for x in threshMat])
# Threshold each row of the matrix by setting values below 90th perc to 0
for i in range(threshMat.shape[0]):
threshMat[i, threshMat[i, :] < perc[i]] = 0
# Count negative values per row
neg_values = np.array(
[sum(threshMat[i, :] < 0) for i in range(threshMat.shape[0])])
print('Negative values occur in %d rows' % sum(neg_values > 0))
# remove negative ones
threshMat[threshMat < 0] = 0
cosSimilarity = sklearn.metrics.pairwise.cosine_similarity(threshMat)
# de = DiffusionMapEmbedding().fit_transform(cosSimilarity)
dme = compute_diffusion_map(cosSimilarity,
n_components=ncomp,
return_result=True)
# lambdas
lambdas = dme[1]['lambdas']
# gradients
grads = dme[0]
return grads, lambdas
def dme(network, threshold=90, n_components=10, return_result=False, **kwargs):
"""
Threshold, cosine similarity, and diffusion map embed `network`
Parameters
----------
network : (N, N) array_like
Symmetric network on which to perform diffusion map embedding
threshold : [0, 100] float, optional
Threshold used to "sparsify" `network` prior to embedding. Default: 90
n_components : int, optional
Number of components to retain from embedding of `network`. Default: 10
return_result : bool, optional
Whether to return result dictionary including eigenvalues, original
eigenvectors, etc. from embedding. Default: False
kwargs : key-value pairs, optional
Passed directly to :func:`mapalign.embed.compute_diffusion_map`
Returns
-------
embedding : (N, C) numpy.ndarray
Embedding of `N` samples in `C`-dimensional spaces
res : dict
Only if `return_result=True`
"""
from mapalign import embed
from sklearn import metrics
from sklearn.utils.extmath import _deterministic_vector_sign_flip
# threshold
network = network.copy()
threshold = np.percentile(network, threshold, axis=1, keepdims=True)
network[network < threshold] = 0
# cosine similarity
network = metrics.pairwise.cosine_similarity(network)
# embed (and ensure consistent output with regard to sign flipping)
emb, res = embed.compute_diffusion_map(network,
n_components=n_components,
return_result=True,
**kwargs)
emb = _deterministic_vector_sign_flip(emb.T).T
if return_result:
return emb, res
return emb
def embed_dense_connectome(aff):
"""
This function takes the affinity matrix and returns the low-dimensional
embedding.
"""
from mapalign import embed
# Embedding using mapalign
emb, res = embed.compute_diffusion_map(aff, alpha=0.5, n_components=300)
# Divide each non-trivial eigenvector by the first one and remove the latter
X = res['vectors']
X = (X.T / X[:, 0]).T[:, 1:]
return X
def main(s):
import os.path
if os.path.isfile(
'/network/lustre/iss01/home/daniel.margulies/data/lsd/embedding/embedding_dense_emb.%s.npy'
% s):
emb = np.load(
'/network/lustre/iss01/home/daniel.margulies/data/lsd/embedding/embedding_dense_emb.%s.npy'
% s)
else:
K = load_fs(s)
K[np.isnan(K)] = 0.0
A_mA = K - K.mean(1)[:, None]
ssA = (A_mA**2).sum(1)
Asq = np.sqrt(np.dot(ssA[:, None], ssA[None]))
Adot = A_mA.dot(A_mA.T)
K = Adot / Asq
del A_mA, ssA, Asq, Adot
K = run_perc(K, 90)
norm = (K * K).sum(0, keepdims=True)**.5
K = K.T @ K
aff = K / norm / norm.T
del norm, K
#aff = sparse.csr_matrix(aff)
emb, res = embed.compute_diffusion_map(aff,
alpha=0.5,
n_components=5,
skip_checks=True,
overwrite=True,
eigen_solver=eigsh,
return_result=True)
del aff
np.save(
'/network/lustre/iss01/home/daniel.margulies/data/lsd/embedding/embedding_dense_emb.%s.npy'
% s, emb)
np.save(
'/network/lustre/iss01/home/daniel.margulies/data/lsd/embedding/embedding_dense_res.%s.npy'
% s, res)
def embedding(upper_corr, full_shape, mask, n_components):
'''
Diffusion embedding on connectivity matrix using mapaling package:
https://github.com/satra/mapalign
'''
# reconstruct full matrix
print '...full matrix'
full_corr = np.zeros(tuple(full_shape))
full_corr[np.triu_indices_from(full_corr, k=1)] = np.nan_to_num(upper_corr)
full_corr += full_corr.T
all_vertex = range(full_corr.shape[0])
# apply mask
print '...mask'
masked_corr = np.delete(full_corr, mask, 0)
del full_corr
masked_corr = np.delete(masked_corr, mask, 1)
cortex = np.delete(all_vertex, mask)
# run actual embedding
print '...embed'
K = (masked_corr + 1) / 2.
del masked_corr
K[np.where(np.eye(K.shape[0]) == 1)] = 1.0
#v = np.sqrt(np.sum(K, axis=1))
#A = K/(v[:, None] * v[None, :])
#del K, v
#A = np.squeeze(A * [A > 0])
#embedding_results = runEmbed(A, n_components_embedding)
embedding_results, embedding_dict = embed.compute_diffusion_map(
K, n_components=n_components, overwrite=True)
# reconstruct masked vertices as zeros
embedding_recort = np.zeros((len(all_vertex), embedding_results.shape[1]))
for e in range(embedding_results.shape[1]):
embedding_recort[:, e] = recort(len(all_vertex),
embedding_results[:, e], cortex, 0)
return embedding_recort, embedding_dict
def t1embedding(upper_corr, full_shape, mask, n_components):
import numpy as np
from mapalign import embed
# reconstruct full matrix
print '...full matrix'
full_corr = np.zeros(tuple(full_shape))
full_corr[np.triu_indices_from(full_corr, k=1)] = np.nan_to_num(upper_corr)
full_corr += full_corr.T
all_vertex=range(full_corr.shape[0])
# apply mask
print '...mask'
masked_corr = np.delete(full_corr, mask, 0)
del full_corr
masked_corr = np.delete(masked_corr, mask, 1)
cortex=np.delete(all_vertex, mask)
# run actual embedding
print '...embed'
K=1-(masked_corr/masked_corr.max())
#K = (masked_corr + 1) / 2.
del masked_corr
K[np.where(np.eye(K.shape[0])==1)]=1.0
#v = np.sqrt(np.sum(K, axis=1))
#A = K/(v[:, None] * v[None, :])
#del K, v
#A = np.squeeze(A * [A > 0])
#embedding_results = runEmbed(A, n_components_embedding)
embedding_results, embedding_dict = embed.compute_diffusion_map(K, n_components=n_components, overwrite=True)
# reconstruct masked vertices as zeros
embedding_recort=np.zeros((len(all_vertex),embedding_results.shape[1]))
for e in range(embedding_results.shape[1]):
embedding_recort[:,e]=recort(len(all_vertex), embedding_results[:,e], cortex, 0)
return embedding_recort, embedding_dict
def main(
workdir,
outdir,
atlas,
kernel,
sparsity,
affinity,
approach,
gradients,
subcort,
neurosynth,
neurosynth_file,
sleuth_file,
nimare_dataset,
roi_mask,
term,
topic,
):
workdir = op.join(workdir, "tmp")
if op.isdir(workdir):
shutil.rmtree(workdir)
os.makedirs(workdir)
atlas_name = "atlas-{0}".format(atlas)
kernel_name = "kernel-{0}".format(kernel)
sparsity_name = "sparsity-{0}".format(sparsity)
affinity_name = "affinity-{0}".format(affinity)
approach_name = "approach-{0}".format(approach)
gradients_name = "gradients-{0}".format(gradients)
dset = None
# handle neurosynth dataset, if called
if neurosynth:
if neurosynth_file is None:
ns_data_dir = op.join(workdir, "neurosynth")
dataset_file = op.join(ns_data_dir, "neurosynth_dataset.pkl.gz")
# download neurosynth dataset if necessary
if not op.isfile(dataset_file):
neurosynth_download(ns_data_dir)
else:
dataset_file = neurosynth_file
dset = Dataset.load(dataset_file)
dataset_name = "dataset-neurosynth"
# handle sleuth text file, if called
if sleuth_file is not None:
dset = convert_sleuth_to_dataset(sleuth_file, target="mni152_2mm")
dataset_name = "dataset-{0}".format(op.basename(sleuth_file).split(".")[0])
if nimare_dataset is not None:
dset = Dataset.load(nimare_dataset)
dataset_name = "dataset-{0}".format(op.basename(nimare_dataset).split(".")[0])
if dset:
# slice studies, if needed
if roi_mask is not None:
roi_ids = dset.get_studies_by_mask(roi_mask)
print(
"{}/{} studies report at least one coordinate in the "
"ROI".format(len(roi_ids), len(dset.ids))
)
dset_sel = dset.slice(roi_ids)
dset = dset_sel
dataset_name = "dataset-neurosynth_mask-{0}".format(
op.basename(roi_mask).split(".")[0]
)
if term is not None:
labels = ["Neurosynth_TFIDF__{label}".format(label=label) for label in [term]]
term_ids = dset.get_studies_by_label(labels=labels, label_threshold=0.1)
print(
"{}/{} studies report association "
"with the term {}".format(len(term_ids), len(dset.ids), term)
)
dset_sel = dset.slice(term_ids)
dset = dset_sel
# img_inds = np.nonzero(dset.masker.mask_img.get_fdata()) # unused
# vox_locs = np.unravel_index(img_inds, dset.masker.mask_img.shape) # unused
dataset_name = "dataset-neurosynth_term-{0}".format(term)
if topic is not None:
topics = [
"Neurosynth_{version}__{topic}".format(version=topic[0], topic=topic)
for topic in topic[1:]
]
topics_ids = []
for topic in topics:
topic_ids = dset.annotations.id[np.where(dset.annotations[topic])[0]].tolist()
topics_ids.extend(topic_ids)
print(
"{}/{} studies report association "
"with the term {}".format(len(topic_ids), len(dset.ids), topic)
)
topics_ids_unique = np.unique(topics_ids)
print("{} unique ids".format(len(topics_ids_unique)))
dset_sel = dset.slice(topics_ids_unique)
dset = dset_sel
# img_inds = np.nonzero(dset.masker.mask_img.get_fdata()) # unused
# vox_locs = np.unravel_index(img_inds, dset.masker.mask_img.shape) # unused
dataset_name = "dataset-neurosynth_topic-{0}".format("_".join(topic[1:]))
if (
neurosynth
or (sleuth_file is not None)
or (nimare_dataset is not None)
):
# set kernel for MA smoothing
if kernel == "peaks2maps":
print("Running peak2maps")
k = Peaks2MapsKernel(resample_to_mask=True)
elif kernel == "alekernel":
print("Running alekernel")
k = ALEKernel(fwhm=15)
if atlas is not None:
if atlas == "harvard-oxford":
print("Parcellating using the Harvard Oxford Atlas")
# atlas_labels = atlas.labels[1:] # unused
atlas_shape = atlas.maps.shape
atlas_affine = atlas.maps.affine
atlas_data = atlas.maps.get_fdata()
elif atlas == "aal":
print("Parcellating using the AAL Atlas")
atlas = datasets.fetch_atlas_aal()
# atlas_labels = atlas.labels # unused
atlas_shape = nib.load(atlas.maps).shape
atlas_affine = nib.load(atlas.maps).affine
atlas_data = nib.load(atlas.maps).get_fdata()
elif atlas == "craddock-2012":
print("Parcellating using the Craddock-2012 Atlas")
atlas = datasets.fetch_atlas_craddock_2012()
elif atlas == "destrieux-2009":
print("Parcellating using the Destrieux-2009 Atlas")
atlas = datasets.fetch_atlas_destrieux_2009(lateralized=True)
# atlas_labels = atlas.labels[3:] # unused
atlas_shape = nib.load(atlas.maps).shape
atlas_affine = nib.load(atlas.maps).affine
atlas_data = nib.load(atlas.maps).get_fdata()
elif atlas == "msdl":
print("Parcellating using the MSDL Atlas")
atlas = datasets.fetch_atlas_msdl()
elif atlas == "surface":
print("Generating surface vertices")
if atlas != "fsaverage5" and atlas != "hcp":
imgs = k.transform(dset, return_type="image")
masker = NiftiLabelsMasker(
labels_img=atlas.maps, standardize=True, memory="nilearn_cache"
)
time_series = masker.fit_transform(imgs)
else:
# change to array for other approach
imgs = k.transform(dset, return_type="image")
print(np.shape(imgs))
if atlas == "fsaverage5":
fsaverage = fetch_surf_fsaverage(mesh="fsaverage5")
pial_left = fsaverage.pial_left
pial_right = fsaverage.pial_right
medial_wall_inds_left = surface.load_surf_data(
"./templates/lh.Medial_wall.label"
)
print(np.shape(medial_wall_inds_left))
medial_wall_inds_right = surface.load_surf_data(
"./templates/rh.Medial_wall.label"
)
print(np.shape(medial_wall_inds_right))
sulc_left = fsaverage.sulc_left
sulc_right = fsaverage.sulc_right
elif atlas == "hcp":
pial_left = "./templates/S1200.L.pial_MSMAll.32k_fs_LR.surf.gii"
pial_right = "./templates/S1200.R.pial_MSMAll.32k_fs_LR.surf.gii"
medial_wall_inds_left = np.where(
nib.load("./templates/hcp.tmp.lh.dscalar.nii").get_fdata()[0] == 0
)[0]
medial_wall_inds_right = np.where(
nib.load("./templates/hcp.tmp.rh.dscalar.nii").get_fdata()[0] == 0
)[0]
left_verts = 32492 - len(medial_wall_inds_left)
sulc_left = nib.load(
"./templates/S1200.sulc_MSMAll.32k_fs_LR.dscalar.nii"
).get_fdata()[0][0:left_verts]
sulc_left = np.insert(
sulc_left,
np.subtract(
medial_wall_inds_left, np.arange(len(medial_wall_inds_left))
),
0,
)
sulc_right = nib.load(
"./templates/S1200.sulc_MSMAll.32k_fs_LR.dscalar.nii"
).get_fdata()[0][left_verts:]
sulc_right = np.insert(
sulc_right,
np.subtract(
medial_wall_inds_right, np.arange(len(medial_wall_inds_right))
),
0,
)
surf_lh = surface.vol_to_surf(
imgs,
pial_left,
radius=6.0,
interpolation="nearest",
kind="ball",
n_samples=None,
mask_img=dset.masker.mask_img,
)
surf_rh = surface.vol_to_surf(
imgs,
pial_right,
radius=6.0,
interpolation="nearest",
kind="ball",
n_samples=None,
mask_img=dset.masker.mask_img,
)
surfs = np.transpose(np.vstack((surf_lh, surf_rh)))
del surf_lh, surf_rh
# handle cortex first
coords_left = surface.load_surf_data(pial_left)[0]
coords_left = np.delete(coords_left, medial_wall_inds_left, axis=0)
coords_right = surface.load_surf_data(pial_right)[0]
coords_right = np.delete(coords_right, medial_wall_inds_right, axis=0)
print("Left Hemipshere Vertices")
surface_macms_lh, inds_discard_lh = build_macms(dset, surfs, coords_left)
print(np.shape(surface_macms_lh))
print(inds_discard_lh)
print("Right Hemipshere Vertices")
surface_macms_rh, inds_discard_rh = build_macms(dset, surfs, coords_right)
print(np.shape(surface_macms_rh))
print(len(inds_discard_rh))
lh_vertices_total = np.shape(surface_macms_lh)[0]
rh_vertices_total = np.shape(surface_macms_rh)[0]
time_series = np.transpose(np.vstack((surface_macms_lh, surface_macms_rh)))
print(np.shape(time_series))
del surface_macms_lh, surface_macms_rh
if subcort:
subcort_img = nib.load("templates/rois-subcortical_mni152_mask.nii.gz")
subcort_vox = np.asarray(np.where(subcort_img.get_fdata()))
subcort_mm = vox2mm(subcort_vox.T, subcort_img.affine)
print("Subcortical Voxels")
subcort_macm, inds_discard_subcort = build_macms(dset, surfs, subcort_mm)
num_subcort_vox = np.shape(subcort_macm)[0]
print(inds_discard_subcort)
time_series = np.hstack((time_series, np.asarray(subcort_macm).T))
print(np.shape(time_series))
time_series = time_series.astype("float32")
print("calculating correlation matrix")
correlation = ConnectivityMeasure(kind="correlation")
time_series = correlation.fit_transform([time_series])[0]
print(np.shape(time_series))
if affinity == "cosine":
time_series = calculate_affinity(time_series, 10 * sparsity)
else:
time_series = np.transpose(k.transform(dset, return_type="array"))
print("Performing gradient analysis")
gradients, statistics = embed.compute_diffusion_map(
time_series, alpha=0.5, return_result=True, overwrite=True
)
pickle.dump(statistics, open(op.join(workdir, "statistics.p"), "wb"))
# if subcortical included in gradient decomposition, remove gradient scores
if subcort:
subcort_grads = gradients[np.shape(gradients)[0] - num_subcort_vox :, :]
subcort_grads = insert(subcort_grads, inds_discard_subcort)
gradients = gradients[0 : np.shape(gradients)[0] - num_subcort_vox, :]
# get left hemisphere gradient scores, and insert 0's where medial wall is
gradients_lh = gradients[0:lh_vertices_total, :]
if len(inds_discard_lh) > 0:
gradients_lh = insert(gradients_lh, inds_discard_lh)
gradients_lh = insert(gradients_lh, medial_wall_inds_left)
# get right hemisphere gradient scores and insert 0's where medial wall is
gradients_rh = gradients[-rh_vertices_total:, :]
if len(inds_discard_rh) > 0:
gradients_rh = insert(gradients_rh, inds_discard_rh)
gradients_rh = insert(gradients_rh, medial_wall_inds_right)
grad_dict = {
"grads_lh": gradients_lh,
"grads_rh": gradients_rh,
"pial_left": pial_left,
"sulc_left": sulc_left,
"pial_right": pial_right,
"sulc_right": sulc_right,
}
if subcort:
grad_dict["subcort_grads"] = subcort_grads
pickle.dump(grad_dict, open(op.join(workdir, "gradients.p"), "wb"))
# map the gradient to the parcels
for i in range(np.shape(gradients)[1]):
if atlas is not None:
if atlas == "fsaverage5" or atlas == "hcp":
plot_surfaces(grad_dict, i, workdir)
if subcort:
tmpimg = masking.unmask(subcort_grads[:, i], subcort_img)
nib.save(tmpimg, op.join(workdir, "gradient-{0}.nii.gz".format(i)))
else:
tmpimg = np.zeros(atlas_shape)
for j, n in enumerate(np.unique(atlas_data)[1:]):
inds = atlas_data == n
tmpimg[inds] = gradients[j, i]
nib.save(
nib.Nifti1Image(tmpimg, atlas_affine),
op.join(workdir, "gradient-{0}.nii.gz".format(i)),
)
else:
tmpimg = np.zeros(np.prod(dset.masker.mask_img.shape))
inds = np.ravel_multi_index(
np.nonzero(dset.masker.mask_img.get_fdata()), dset.masker.mask_img.shape
)
tmpimg[inds] = gradients[:, i]
nib.save(
nib.Nifti1Image(
np.reshape(tmpimg, dset.masker.mask_img.shape), dset.masker.mask_img.affine
),
op.join(workdir, "gradient-{0}.nii.gz".format(i)),
)
os.system(
"python3 /Users/miriedel/Desktop/GitHub/surflay/make_figures.py "
"-f {grad_image} --colormap jet".format(
grad_image=op.join(workdir, "gradient-{0}.nii.gz".format(i))
)
)
output_dir = op.join(
outdir,
(
f"{dataset_name}_{atlas_name}_{kernel_name}_{sparsity_name}_{gradients_name}_"
f"{affinity_name}_{approach_name}"
)
)
shutil.copytree(workdir, output_dir)
shutil.rmtree(workdir)
def load_network(kind,
parcel,
data="lau",
weights='log',
hemi="both",
version=1,
subset="all",
path=None):
'''
Function to load a dictionary containing information about the specified
brain network.
Parameters
----------
kind : string
Either 'SC' or 'FC'.
hemi : string
Either "both", "L" or "R".
weights " string
The weights of the edges. The options "normal", "log" or "binary".
The default is "log".
data : string
Either "HCP" or "lau".
parcel : string
Either "68", "114", ... [if 'lau'] / "s400", "s800" [if "HCP"]
version : int
Version of the network.
subset : string
Either 'discov', 'valid' or 'all'
path : string
path to the "data" folder in which the data will be stored. If
none, then assumes that path is current folder.
Returns
-------
Network : dictionary
Dictionary storing relevant attributes about the network
'''
# Initialize dictionary + store basic information about the network
Network = {}
Network["info"] = {}
Network["info"]["kind"] = kind
Network["info"]["parcel"] = parcel
Network["info"]["data"] = data
Network["info"]["hemi"] = hemi
Network["info"]["weights"] = weights
Network["info"]["version"] = version
Network["info"]["subset"] = subset
# Modify parameter names to what they are in file names
version = '' if version == 1 else '_v' + str(version)
subset = '' if subset == 'all' else subset
hemi = '' if hemi == 'both' else hemi
# Store important paths for loading the relevant data
main_path = path + "/brainNetworks/" + data + "/"
network_path = (main_path + "matrices/consensus/" + subset + kind +
parcel + hemi + version + "/")
matrix_path = network_path + "/" + weights
# Store general information about the network's parcellation
parcel_info = get_general_parcellation_info(parcel)
Network['order'] = parcel_info[0]
Network['noplot'] = parcel_info[1]
Network['lhannot'] = parcel_info[2]
Network['rhannot'] = parcel_info[3]
Network['atlas'] = parcel_info[4]
# Load the cammoun_id of the parcellation, if Cammoun (i.e. 033, 060, etc.)
if parcel[0] != 's':
Network['cammoun_id'] = parcel_to_n(parcel)
# masks
masks = get_node_masks(Network, path=main_path)
Network['node_mask'] = masks[0]
Network['hemi_mask'] = masks[1]
Network['subcortex_mask'] = masks[2]
# hemisphere
Network['hemi'] = get_hemi(Network, path=main_path)
# coordinates
Network['coords'] = get_coordinates(Network, path=main_path)
# Adjacency matrix
Network['adj'], last_modified = get_adjacency(Network,
matrix_path,
minimal_processing=True,
return_last_modified=True)
# Test whether the network is connected. Raise a warning if not...
if not np.all(bct.reachdist(Network['adj'])[0]):
warnings.warn(("This brain network appears to be disconnected. This "
"might cause problem for the computation of the other "
"measures"))
# node strength
Network["str"] = np.sum(Network['adj'], axis=0)
# Inverse of adjacency matrix
inv = Network['adj'].copy()
inv[Network['adj'] > 0] = 1 / inv[Network['adj'] > 0]
Network["inv_adj"] = inv
# distance
Network["dist"] = cdist(Network["coords"], Network["coords"])
# clustering coefficient
Network["cc"] = bct.clustering_coef_wu(Network['adj'])
# shortest path
Network['sp'] = get_shortest_path(Network,
matrix_path=matrix_path,
last_modified=last_modified)
# diffusion embedding
de = compute_diffusion_map(Network['adj'],
n_components=10,
return_result=True,
skip_checks=True)
Network["de"] = de[0]
Network["de_extra"] = de[1]
# Principal components
Network['PCs'], Network['PCs_ev'] = getPCs(Network['adj'])
# eigenvector centrality
Network["ec"] = bct.eigenvector_centrality_und(Network['adj'])
# mean first passage time
Network["mfpt"] = bct.mean_first_passage_time(Network['adj'])
# betweenness centrality
Network['bc'] = get_betweenness(Network,
matrix_path=matrix_path,
last_modified=last_modified)
# routing efficiency
Network["r_eff"] = efficiency(Network)
# diffusion efficiency
Network["d_eff"] = efficiency_diffusion(Network)
# subgraph centrality
Network["subc"] = bct.subgraph_centrality(Network["adj"])
# closeness centrality
Network['clo'] = 1 / np.mean(Network['sp'], axis=0)
# communities + participation coefficient
path = matrix_path + "/communities/"
if os.path.exists(path):
files = []
for i in os.listdir(path):
if os.path.isfile(os.path.join(path, i)) and 'ci_' in i:
files.append(i)
if len(files) > 0:
Network["ci"] = []
for file in files:
Network["ci"].append(np.load(os.path.join(path, file)))
Network["ppc"] = []
for i in range(len(files)):
ppc = bct.participation_coef(Network['adj'], Network["ci"][i])
Network["ppc"].append(ppc)
# Edge lengths
if (data == "HCP") and (kind == "SC"):
path = main_path + "matrices/" + subset + kind + parcel + hemi + "_lengths.npy"
if os.path.exists(path):
Network["lengths"] = np.load(path)
# streamline connection lengths
path = network_path + "/len.npy"
if os.path.exists(path):
Network['len'] = np.load(path)
# ROI names
if parcel[0] != "s":
Network['ROInames'] = get_ROInames(Network)
# geodesic distances between nodes
if parcel[0] == "s":
n = parcel[1:]
fname_l = n + "Parcels7Networks_lh_dist.csv"
fname_r = n + "Parcels7Networks_rh_dist.csv"
else:
fname_l = "scale" + Network['cammoun_id'] + "_lh_dist.csv"
fname_r = "scale" + Network['cammoun_id'] + "_rh_dist.csv"
Network['geo_dist_L'] = pd.read_csv(main_path + "/geodesic/medial/" +
fname_l,
header=None).values
Network['geo_dist_R'] = pd.read_csv(main_path + "/geodesic/medial/" +
fname_r,
header=None).values
return Network
#### threshold at 90th percentile
print('thresholding each row at its 90th percentile...')
perc = np.array([np.percentile(x, 90) for x in indiv_matrix])
N = indiv_matrix.shape[0]
for i in range(N):
indiv_matrix[i, indiv_matrix[i,:] < perc[i]] = 0
indiv_matrix[indiv_matrix < 0] = 0
#### compute the affinity matrix
print('calculating affinity matrix with scipy...')
indiv_matrix = spatial.distance.pdist(indiv_matrix, metric = 'cosine')
indiv_matrix = spatial.distance.squareform(indiv_matrix)
indiv_matrix = 1.0 - indiv_matrix
print('affinity shape ', np.shape(indiv_matrix))
#### get gradients
print('computing gradients...')
emb, res = embed.compute_diffusion_map(indiv_matrix, alpha = 0.5,
n_components = 10,
return_result=True)
#### save
out_name_emb = os.path.join(out_prfx + '_dense_emb.npy')
out_name_res = os.path.join(out_prfx + '_dense_res.npy')
print(out_name_emb)
np.save(out_name_emb, emb)
np.save(out_name_res, res['lambdas'])
X = np.random.randint(0,1000,size=(500,200))
corr = np.corrcoef(X)
# correlation matrix with ones on diagonal and positive version
corr1 = corr.copy()
L1 = (corr1+1)/2
# correlation matrix with zeros on diagonal, positive version (0.5 on diagonal), correction back to zero diagonal
corr0 = corr.copy()
corr0[np.where(np.eye(corr0.shape[0])==1)]=0.0
L05 = (corr0+1)/2
L0 = L05.copy()
L0[np.where(np.eye(L0.shape[0])==1)]=0.0
# embedding of all threee versions
E1, R1 = compute_diffusion_map(L1, n_components=10)
E0, R0 = compute_diffusion_map(L0, n_components=10)
E05, R05 = compute_diffusion_map(L05, n_components=10)
E0_E1 = False
E0_E05 = False
E05_E1 = False
tol = 0.000001
while any([E0_E1, E0_E05, E05_E1])==False:
E1_E0 = np.allclose(E0,E1,atol=tol)
if E1_E0 == True:
print "E0 = E1 with tolerance %s"%str(tol)
E0_E05 = np.allclose(E0,E05,atol=tol)
def __init__(self,
kind,
parcel,
data='lau',
hemi='both',
binary=False,
version=1,
subset='all',
path=None):
mainPath = path + "/brainNetworks/" + data + "/"
home = os.path.expanduser("~")
self.info = {}
self.info["kind"] = kind
self.info["parcel"] = parcel
self.info["data"] = data
self.info["hemi"] = hemi
self.info["binary"] = binary
self.info["version"] = version
self.info["subset"] = subset
if version == 1:
version = ''
else:
version = "_v2"
if binary is True:
binary = "b"
else:
binary = ''
if subset == "all":
subset = ''
if hemi == "both":
hemi = ''
matrxPath = mainPath + "matrices/" + subset + kind + parcel + hemi + binary + version
# hemisphere
self.hemi = np.load(matrxPath + "/hemi.npy")
# Adjacency matrix
path = matrxPath + ".npy"
A = np.load(path)
# Look at time when file was last modified
last_modified = os.path.getmtime(path)
# set negative values to 0, fill diagonal, make symmetric
A[A < 0] = 0
np.fill_diagonal(A, 0)
A = (A + A.T) / 2
self.adj = A
# Number of nodes in the network
self.n = len(self.adj)
# coordinates
path = mainPath + "coords/coords" + parcel + hemi + ".npy"
self.coords = np.load(path)
# Inverse of adjacency matrix
inv = A.copy()
inv[A > 0] = 1 / inv[A > 0]
self.inv_adj = inv
# distance
self.dist = cdist(self.coords, self.coords)
# shortest path
#
# Loaded from saved file...
# IF file not found OR Adjacency was modified after creation,
# then recompute measure
path = matrxPath + "/sp.npy"
if os.path.exists(path) is False:
print("shortest path not found")
print("computing shortest path...")
self.sp = bct.distance_wei(self.inv_adj)[0]
np.save(matrxPath + "/sp.npy", self.sp)
elif os.path.getmtime(path) < last_modified:
print("new adjacency matrix was found")
print("computing shortest paths...")
self.sp = bct.distance_wei(self.inv_adj)[0]
np.save(matrxPath + "/sp.npy", self.sp)
else:
self.sp = np.load(path)
# diffusion embedding
de = compute_diffusion_map(A, n_components=10, return_result=True)
self.de = de[0]
self.de_extra = de[1]
# Principal components
self.PCs, self.PCs_ev = load_data.getPCs(self.adj)
# betweenness centrality
#
# Loaded from saved file...
# IF file not found OR Adjacency was modified after creation,
# then recompute measure
path = matrxPath + "/bc.npy"
if os.path.exists(path) is False:
print("betweenness centrality not found")
print("computing betweenness centrality...")
self.bc = bct.betweenness_wei(self.inv_adj)
np.save(matrxPath + "/bc.npy", self.bc)
elif os.path.getmtime(path) < last_modified:
print("new adjacency matrix was found")
print("recomputing betweeness centrality...")
self.bc = bct.betweenness_wei(self.inv_adj)
np.save(matrxPath + "/bc.npy", self.bc)
else:
self.bc = np.load(path)
# communities + participation coefficient
path = matrxPath + "/communities/"
if os.path.exists(path):
files = []
for i in os.listdir(path):
if os.path.isfile(os.path.join(path, i)) and 'ci_' in i:
files.append(i)
if len(files) > 0:
self.ci = []
for file in files:
self.ci.append(np.load(os.path.join(path, file)))
self.ppc = []
for i in range(len(files)):
ppc = bct.participation_coef(A, self.ci[i])
self.ppc.append(ppc)
if (data == "HCP") and (kind == "SC"):
path = mainPath + "matrices/" + subset + kind + parcel + hemi + "_lengths.npy"
self.lengths = np.load(path)
# streamline connection lengths
path = matrxPath + "/len.npy"
if os.path.exists(path) is True:
self.len = np.load(path)
# network information
if parcel[0] == "s":
nb = parcel[1:]
self.order = "LR"
self.noplot = [b'Background+FreeSurfer_Defined_Medial_Wall', b'']
self.lhannot = (home + "/"
"nnt-data/"
"atl-schaefer2018/"
"fsaverage/"
"atl-Schaefer2018_space-fsaverage_"
"hemi-L_desc-" + nb + "Parcels7Networks_"
"deterministic.annot")
self.rhannot = (home + "/"
"nnt-data/"
"atl-schaefer2018/"
"fsaverage/"
"atl-Schaefer2018_space-fsaverage_"
"hemi-R_desc-" + nb + "Parcels7Networks_"
"deterministic.annot")
else:
nb = _parcel_to_n(parcel)
self.order = "RL"
self.noplot = None
self.lhannot = (home + "/"
"nnt-data/"
"atl-cammoun2012/"
"fsaverage/"
"atl-Cammoun2012_space-fsaverage_"
"res-" + nb + "_hemi-L_deterministic.annot")
self.rhannot = (home + "/"
"nnt-data/"
"atl-cammoun2012/"
"fsaverage/"
"atl-Cammoun2012_space-fsaverage_"
"res-" + nb + "_hemi-R_deterministic.annot")
self.cammoun_id = nb
import numpy as np
# Load affinitity matrix
aff = np.load('gradient_data/conn_matrices/cosine_affinity.npy')
import sys
sys.path.append("../mapalign")
from mapalign import embed
emb, res = embed.compute_diffusion_map(aff, alpha=0.5)
print("Minimum value is %f" % fcmatrix.min())
# how many nodes have negative values
# Count negative values per row
neg_values = np.array([sum(fcmatrix[i,:] < 0) for i in range(N)])
print("Negative values occur in %d rows" % sum(neg_values > 0))
# example subject 1 had no negative values survive, but imagine other subjects might we set these to zero
fcmatrix[fcmatrix < 0] = 0
# Now we are dealing with sparse vectors. Cosine similarity is used as affinity metric
aff = 1 - pairwise_distances(fcmatrix, metric = 'cosine')
### Commented out saving this: cubic can't hang with multiple 17734x17734 matrices saved per subject (few GB each)
# Save affinity matrix
# savepath= "/cbica/projects/pinesParcels/data/CombinedData/" + str(sid) + "/vertexwise_cos_affinmat.npy"
# np.save(savepath, aff)
# save checkpoint reached, now compute dmap
from mapalign import embed
emb, res = embed.compute_diffusion_map(aff, alpha = 0.5, return_result=True)
savepathe= "/cbica/projects/pinesParcels/data/CombinedData/" + str(sid) + "/vertexwise_emb.npy"
savepathr= "/cbica/projects/pinesParcels/data/CombinedData/" + str(sid) + "/vertexwise_res.npy"
np.save(savepathe, emb)
np.save(savepathr, res)
X = np.random.randint(0, 1000, size=(500, 200))
corr = np.corrcoef(X)
# correlation matrix with ones on diagonal and positive version
corr1 = corr.copy()
L1 = (corr1 + 1) / 2
# correlation matrix with zeros on diagonal, positive version (0.5 on diagonal), correction back to zero diagonal
corr0 = corr.copy()
corr0[np.where(np.eye(corr0.shape[0]) == 1)] = 0.0
L05 = (corr0 + 1) / 2
L0 = L05.copy()
L0[np.where(np.eye(L0.shape[0]) == 1)] = 0.0
# embedding of all threee versions
E1, R1 = compute_diffusion_map(L1, n_components=10)
E0, R0 = compute_diffusion_map(L0, n_components=10)
E05, R05 = compute_diffusion_map(L05, n_components=10)
E0_E1 = False
E0_E05 = False
E05_E1 = False
tol = 0.000001
while any([E0_E1, E0_E05, E05_E1]) == False:
E1_E0 = np.allclose(E0, E1, atol=tol)
if E1_E0 == True:
print "E0 = E1 with tolerance %s" % str(tol)
E0_E05 = np.allclose(E0, E05, atol=tol)
if E0_E05 == True: