def visualize(self, i, savedir=""):
"""
Visualize the ith graph of self.graphs, passed-to-ranks.
Parameters
----------
i : int
Graph to visualize.
savedir : str, optional
Directory to save graph into.
If left empty, do not save.
"""
nmax = np.max(self.graphs)
if isinstance(i, int):
graph = pass_to_ranks(self.graphs[i])
sub = self.subjects[i]
sesh = "" # TODO
elif isinstance(i, np.ndarray):
graph = pass_to_ranks(i)
sub = ""
sesh = ""
else:
raise TypeError("Passed value must be integer or np.ndarray.")
viz = heatmap(
graph,
title=f"sub-{sub}_session-{sesh}",
xticklabels=True,
yticklabels=True,
vmin=0,
vmax=1,
)
# set color of title
viz.set_title(viz.get_title(), color="black")
# set color of colorbar ticks
viz.collections[0].colorbar.ax.yaxis.set_tick_params(color="black")
# set font size and color of heatmap ticks
for item in viz.get_xticklabels() + viz.get_yticklabels():
item.set_color("black")
item.set_fontsize(7)
if savedir:
p = Path(savedir).resolve()
if not p.is_dir():
p.mkdir()
plt.savefig(
p / f"sub-{sub}_sesh-{sesh}.png",
facecolor="white",
bbox_inches="tight",
dpi=300,
)
else:
plt.show()
plt.cla()
def discriminability(self, PTR=True, **kwargs):
"""
Attach discriminability functionality to the object.
See `discr_stat` for full documentation.
Returns
-------
stat : float
Discriminability statistic.
"""
if PTR:
graphs = np.copy(self.graphs)
# No need to pass to ranks and normalize if it's a correlation??
#for graph in graphs:
# ranked = np.array([pass_to_ranks(graph)])
# print('oof')
graphs = np.array([pass_to_ranks(graph) for graph in graphs])
X = self._X(graphs)
z, x, y = graphs.shape
spread = np.zeros((x, 0))
maxsort = np.argmax(graphs, axis=2)
for qq in range(z):
spread = np.insert(spread, 0, maxsort[qq][0:x], axis=1)
return spread
#return discr_stat(X, self.Y, **kwargs)
return discr_stat(self.X, self.Y, **kwargs)
def ase(adj, n_components, ptr=True):
if ptr:
adj = pass_to_ranks(adj)
ase = AdjacencySpectralEmbed(n_components=n_components)
latent = ase.fit_transform(adj)
latent = np.concatenate(latent, axis=-1)
return latent
def omni(adjs, n_components):
if PTR:
adjs = [pass_to_ranks(a) for a in adjs]
omni = OmnibusEmbed(n_components=n_components // len(adjs))
latent = omni.fit_transform(adjs)
latent = np.concatenate(latent, axis=-1) # first is for in/out
latent = np.concatenate(latent, axis=-1) # second is for concat. each graph
return latent
def preprocess_adjs(adjs, method="ase"):
adjs = [pass_to_ranks(a) for a in adjs]
adjs = [a + 1 / a.size for a in adjs]
if method == "ase":
adjs = [augment_diagonal(a) for a in adjs]
elif method == "lse":
adjs = [to_laplace(a) for a in adjs]
return adjs
def lse(adj, n_components, regularizer=None):
if PTR:
adj = pass_to_ranks(adj)
lap = to_laplace(adj, form="R-DAD")
ase = AdjacencySpectralEmbed(n_components=n_components)
latent = ase.fit_transform(lap)
latent = np.concatenate(latent, axis=-1)
return latent
def lateral_omni(adj, lp_inds, rp_inds):
left_left_adj = pass_to_ranks(adj[np.ix_(lp_inds, lp_inds)])
right_right_adj = pass_to_ranks(adj[np.ix_(rp_inds, rp_inds)])
omni = OmnibusEmbed(n_components=3, n_elbows=2, check_lcc=False, n_iter=10)
ipsi_embed = omni.fit_transform([left_left_adj, right_right_adj])
ipsi_embed = np.concatenate(ipsi_embed, axis=-1)
ipsi_embed = np.concatenate(ipsi_embed, axis=0)
left_right_adj = pass_to_ranks(adj[np.ix_(lp_inds, rp_inds)])
right_left_adj = pass_to_ranks(adj[np.ix_(rp_inds, lp_inds)])
omni = OmnibusEmbed(n_components=3, n_elbows=2, check_lcc=False, n_iter=10)
contra_embed = omni.fit_transform([left_right_adj, right_left_adj])
contra_embed = np.concatenate(contra_embed, axis=-1)
contra_embed = np.concatenate(contra_embed, axis=0)
embed = np.concatenate((ipsi_embed, contra_embed), axis=1)
return embed
def ase_concatenate(adjs, n_components, ptr=True):
if ptr:
adjs = [pass_to_ranks(a) for a in adjs]
ase = AdjacencySpectralEmbed(n_components=n_components // len(adjs))
graph_latents = []
for a in adjs:
latent = ase.fit_transform(a)
latent = np.concatenate(latent, axis=-1)
graph_latents.append(latent)
latent = np.concatenate(graph_latents, axis=-1)
return latent
def lse(adj, n_components, regularizer=None, ptr=True):
if ptr:
adj = pass_to_ranks(adj)
lap = to_laplace(adj, form="R-DAD", regularizer=regularizer)
ase = AdjacencySpectralEmbed(n_components=n_components, diag_aug=False)
latent = ase.fit_transform(lap)
# latent = LaplacianSpectralEmbed(
# form="R-DAD", n_components=n_components, regularizer=regularizer
# )
latent = np.concatenate(latent, axis=-1)
return latent
def _embed(self, adj=None):
if adj is None:
adj = self.adj
# TODO look into PTR at this level as well
# lp_inds, rp_inds = get_paired_inds(self.meta)
lp_inds = self.left_pair_inds
rp_inds = self.right_pair_inds
embed_adj = pass_to_ranks(adj)
if self.embed == "ase":
embedder = AdjacencySpectralEmbed(
n_components=self.n_components, n_elbows=self.n_elbows
)
embed = embedder.fit_transform(embed_adj)
elif self.embed == "lse":
embedder = LaplacianSpectralEmbed(
n_components=self.n_components,
n_elbows=self.n_elbows,
regularizer=self.regularizer,
)
embed = embedder.fit_transform(embed_adj)
elif self.embed == "unscaled_ase":
embed_adj = pass_to_ranks(adj)
embed_adj = augment_diagonal(embed_adj)
embed = selectSVD(
embed_adj, n_components=self.n_components, n_elbows=self.n_elbows
)
embed = (embed[0], embed[2].T)
X = np.concatenate(embed, axis=1)
fraction_paired = (len(lp_inds) + len(rp_inds)) / len(self.root_inds)
print(f"Learning transformation with {fraction_paired} neurons paired")
R, _ = orthogonal_procrustes(X[lp_inds], X[rp_inds])
X[self.left_inds] = X[self.left_inds] @ R
if self.normalize:
row_sums = np.sum(X, axis=1)
X /= row_sums[:, None]
return X
def load_HNU1(ptr=None, return_subid=False):
path = Path(MODULE_PATH).parents[1] / "data/raw/HNU1/"
f = sorted(path.glob('*.ssv'))
df = pd.read_csv(path / "HNU1.csv")
subid = [int(fname.stem.split('_')[0].split('-')[1][-5:]) for fname in f]
y = np.array([np.unique(df.SEX[df.SUBID == s])[0] - 1 for s in subid])
g = np.array(import_edgelist(f, 'ssv'))
if ptr is not None:
g = np.array([pass_to_ranks(x) for x in g])
if return_subid:
return g, y, subid
else:
return g, y
def discriminability(self, PTR=True, **kwargs):
"""
Attach discriminability functionality to the object.
See `discr_stat` for full documentation.
Returns
-------
stat : float
Discriminability statistic.
"""
if PTR:
graphs = np.copy(self.graphs)
graphs = np.array([pass_to_ranks(graph) for graph in graphs])
X = self._X(graphs)
return discr_stat(X, self.Y, **kwargs)
return discr_stat(self.X, self.Y, **kwargs)
def reg_lateral_omni(adj, base_adj, lp_inds, rp_inds):
base_ll_adj = pass_to_ranks(base_adj[np.ix_(lp_inds, lp_inds)])
base_rr_adj = pass_to_ranks(base_adj[np.ix_(rp_inds, rp_inds)])
ll_adj = pass_to_ranks(adj[np.ix_(lp_inds, lp_inds)])
rr_adj = pass_to_ranks(adj[np.ix_(rp_inds, rp_inds)])
ipsi_adjs = [base_ll_adj, base_rr_adj, ll_adj, rr_adj]
ipsi_embed = reg_omni(ipsi_adjs)
base_lr_adj = pass_to_ranks(base_adj[np.ix_(lp_inds, rp_inds)])
base_rl_adj = pass_to_ranks(base_adj[np.ix_(rp_inds, lp_inds)])
lr_adj = pass_to_ranks(adj[np.ix_(lp_inds, rp_inds)])
rl_adj = pass_to_ranks(adj[np.ix_(rp_inds, lp_inds)])
contra_adjs = [base_lr_adj, base_rl_adj, lr_adj, rl_adj]
contra_embed = reg_omni(contra_adjs)
embed = np.concatenate((ipsi_embed, contra_embed), axis=1)
return embed
def load_BNU1(ptr=None):
path = Path(MODULE_PATH).parents[1] / "data/raw/BNU1/"
f = sorted(path.glob('*.ssv'))
df = pd.read_csv(path / "BNU1_phenotypic_data.csv")
subid = [int(fname.stem.split('_')[0].split('-')[1][-5:]) for fname in f]
y = np.array([
np.unique(df.SEX[(df.SUBID == s) & (df.SESSION == 'Baseline')])[0]
for s in subid
]).astype(int)
y -= 1
g = np.array(import_edgelist(f, 'ssv'))
if ptr is not None:
g = np.array([pass_to_ranks(x) for x in g])
return g, y
def _embed(self, adj=None):
if adj is None:
adj = self.adj
lp_inds = self.left_pair_inds
rp_inds = self.right_pair_inds
embed_adj = pass_to_ranks(adj) # TODO PTR here?
if self.plus_c:
embed_adj += 1 / adj.size
if self.embed == "ase":
embedder = AdjacencySpectralEmbed(n_components=self.n_components,
n_elbows=self.n_elbows)
embed = embedder.fit_transform(embed_adj)
elif self.embed == "lse":
embedder = LaplacianSpectralEmbed(
n_components=self.n_components,
n_elbows=self.n_elbows,
regularizer=self.regularizer,
)
embed = embedder.fit_transform(embed_adj)
elif self.embed == "unscaled_ase":
embed_adj = augment_diagonal(embed_adj)
embed = selectSVD(embed_adj,
n_components=self.n_components,
n_elbows=self.n_elbows)
embed = (embed[0], embed[2].T)
X = np.concatenate(embed, axis=1)
fraction_paired = (len(lp_inds) + len(rp_inds)) / len(self.root_inds)
print(f"Learning transformation with {fraction_paired} neurons paired")
X = self._procrustes(X)
if self.normalize:
row_norms = np.linalg.norm(X, axis=1)
X /= row_norms[:, None]
return X
def preprocess_adjs(adjs, method="ase"):
"""Preprocessing necessary prior to embedding a graph, opetates on a list
Parameters
----------
adjs : list of adjacency matrices
[description]
method : str, optional
[description], by default "ase"
Returns
-------
[type]
[description]
"""
adjs = [pass_to_ranks(a) for a in adjs]
adjs = [a + 1 / a.size for a in adjs]
if method == "ase":
adjs = [augment_diagonal(a) for a in adjs]
elif method == "lse": # haven't really used much. a few params to look at here
adjs = [to_laplace(a) for a in adjs]
return adjs
def _load_dataset(path, n_nodes, ptr=None):
file = np.load(path)
X = file["X"]
y = file["y"].astype(int)
n_samples = X.shape[0]
y[y == -1] = 0
idx = np.triu_indices(n_nodes, k=1)
X_graphs = np.zeros((n_samples, n_nodes, n_nodes))
for i, x in enumerate(X):
X_graphs[i][idx] = x
X_graphs[i] = symmetrize(X_graphs[i], "triu")
if ptr is not None:
X_graphs = X_graphs - X_graphs.min(axis=(1, 2)).reshape(-1, 1, 1)
for i, x in enumerate(X_graphs):
X_graphs[i] = pass_to_ranks(X_graphs[i])
return X_graphs, y
pair = data["Pair"]
pair_id = data["Pair ID"]
if pair != -1:
if pair not in thresh_g:
thresh_g.node[n]["Pair"] = -1
thresh_g.node[n]["Pair ID"] = -1
n_missing += 1
mg = MetaGraph(thresh_g, weight="max_norm_weight")
meta = mg.meta
adj = mg.adj.copy()
# colsums = np.sum(adj, axis=0)
# colsums[colsums == 0] = 1
# adj = adj / colsums[np.newaxis, :]
adj = pass_to_ranks(adj)
if use_spl:
adj = graph_shortest_path(adj)
if plus_c:
adj += np.min(adj)
if embed == "lse":
latent = lse(adj, None, ptr=False)
elif embed == "ase":
latent = ase(adj, None, ptr=False)
rot_latent, diff = procrustes_match(latent, meta)
rot_latent = latent
n_components = latent.shape[1]
plot_df = pd.DataFrame(data=rot_latent)
stacked_barplot(
cat,
subcat,
ax=ax,
color_dict=CLASS_COLOR_DICT,
plot_names=True,
text_color="dimgrey",
bar_height=0.2,
)
ax.get_legend().remove()
subgraph_inds = temp_meta["inds"].values
subgraph_adj = adj[np.ix_(subgraph_inds, subgraph_inds)]
ax = fig.add_subplot(mid_gs[1:, :])
_, _, top, _ = adjplot(
pass_to_ranks(subgraph_adj),
plot_type="heatmap",
cbar=False,
ax=ax,
meta=temp_meta,
item_order=["merge_class", "sf"],
colors="merge_class",
palette=CLASS_COLOR_DICT,
)
top.set_title("Intra-cluster connectivity", color="dimgrey")
dend_gs = plt.GridSpec(
1,
5,
figure=fig,
left=margin + 5 / n_col + 2 * gap,
def _run_test(self, methodstr, input_graph, expected):
ptr_out = pass_to_ranks(input_graph, method=methodstr)
self.assertTrue(np.allclose(ptr_out, expected, rtol=1e-04))
def test_invalid_inputs(self):
with self.assertRaises(ValueError):
pass_to_ranks(self.loopless_undirected_input,
method="hazelnutcoffe")
with self.assertRaises(TypeError):
pass_to_ranks("hi", "hi")
color=text_color,
)
ax.legend(
ncol=6, # len(category_names),
bbox_to_anchor=(0, 1),
loc="lower left",
fontsize="small",
)
return ax
# Try with my new labels
adj = get_paired_adj("G", nodelist)
adj = adj[:n_per_side, :n_per_side] + adj[n_per_side:, n_per_side:]
embed_adj = pass_to_ranks(adj)
skmeans_kws = dict(n_init=200, n_jobs=-2)
n_clusters = 12
n_components = None
regularizer = 1
lse = LaplacianSpectralEmbed(n_components=n_components,
regularizer=regularizer)
latent = lse.fit_transform(embed_adj)
latent = np.concatenate(latent, axis=-1)
models = []
meta_file = "maggot_models/data/processed/2019-09-18-v2/BP_metadata.csv"
mb_mg = mg.reindex(mb_meta.index, use_ids=True, inplace=False)
mb_mg.calculate_degrees(inplace=True)
mb_mg.meta["Total edgesum"] = -mb_mg.meta["Total edgesum"]
sizes = mb_mg.meta.groupby("class1").size()
mb_mg.meta["class1_sizes"] = -mb_mg.meta["class1"].map(sizes)
# %%
meta = mb_mg.meta
print("n_left")
print(len(meta[meta["left"]]))
print("n_right")
print(len(meta[meta["right"]]))
fig, ax = plt.subplots(1, 1, figsize=(10, 10))
adjplot(
pass_to_ranks(mb_mg.adj),
meta=mb_mg.meta,
sort_class=["hemisphere", "class1"],
class_order=["class1_sizes"],
item_order=["class1", "Total edgesum"],
cbar=False,
row_tick_pad=[0.05, 0.7],
col_tick_pad=[0.2, 0.7],
tick_rot=90,
tick_fontsize=12,
gridline_kws=dict(color="grey", linestyle="--", linewidth=0.5),
ax=ax,
)
left_mb_mg = mb_mg.reindex(meta[meta["left"]].index, use_ids=True, inplace=False)
right_mb_mg = mb_mg.reindex(meta[meta["right"]].index, use_ids=True, inplace=False)
# arrowprops=arrow_args, # ) #%% from graspy.embed import AdjacencySpectralEmbed, OmnibusEmbed from graspy.utils import pass_to_ranks from graspy.plot import pairplot sum_adj = np.sum(np.array(mb_color_graphs), axis=0) n_components = 4 # ptr_adj = pass_to_ranks(sum_adj) ase = AdjacencySpectralEmbed(n_components=n_components) sum_latent = ase.fit_transform(ptr_adj) sum_latent = np.concatenate(sum_latent, axis=-1) pairplot(sum_latent, labels=mb_class_labels) ptr_color_adjs = [pass_to_ranks(a) for a in mb_color_graphs] # graph_sum = [np.sum(a) for a in mb_color_graphs] # ptr_color_adjs = [ptr_color_adjs[i] + (1 / graph_sum[i]) for i in range(4)] omni = OmnibusEmbed(n_components=n_components // 4) color_latent = omni.fit_transform(ptr_color_adjs) color_latent = np.concatenate(color_latent, axis=-1) color_latent = np.concatenate(color_latent, axis=-1) pairplot(color_latent, labels=mb_class_labels) from graspy.embed import MultipleASE
meta["inds"] = range(len(meta))
lp_inds, rp_inds = get_paired_inds(meta)
left_inds = meta[meta["left"]]["inds"]
right_inds = meta[meta["right"]]["inds"]
# %% Load and preprocess all graphs
graph_types = ["Gad", "Gaa", "Gdd", "Gda"]
adjs = []
for g in graph_types:
temp_mg = load_metagraph(g, version="2020-04-01")
temp_mg.reindex(mg.meta.index, use_ids=True)
temp_adj = temp_mg.adj
adjs.append(temp_adj)
embed_adjs = [pass_to_ranks(a) for a in adjs]
embed_adjs = [a + 1 / a.size for a in embed_adjs]
embed_adjs = [augment_diagonal(a) for a in embed_adjs]
# %% [markdown]
# ##
def omni_procrust_svd(embed_adjs):
omni = OmnibusEmbed(n_components=None, check_lcc=False)
joint_embed = omni.fit_transform(embed_adjs)
cat_embed = np.concatenate(joint_embed, axis=-1)
# print(f"Omni concatenated embedding shape: {cat_embed.shape}")
for e in cat_embed:
e[left_inds] = e[left_inds] @ orthogonal_procrustes(
e[lp_inds], e[rp_inds])[0]
def preprocess_adjs(adjs):
adjs = [pass_to_ranks(a) for a in adjs]
adjs = [a + 1 / a.size for a in adjs]
adjs = [augment_diagonal(a) for a in adjs]
return adjs
if "class_size" in meta.columns:
meta = meta.drop("class_size", axis=1)
meta = pd.concat((meta, class_size_mapped), ignore_index=False, axis=1)
meta["idx"] = range(len(meta))
sort_meta = meta.sort_values(
["class_rank", "class_size", sort_class, "median_visit", "dendrite_input"],
inplace=False,
)
perm_inds = sort_meta.idx.values
from graspy.utils import pass_to_ranks
data = mg.adj.copy()
data = data[np.ix_(perm_inds, perm_inds)]
data = pass_to_ranks(data)
sort_meta["idx"] = range(len(sort_meta))
first_df = sort_meta.groupby([sort_class], sort=False).first()
first_inds = list(first_df["idx"].values)
first_inds.append(len(meta) + 1)
# for the tic locs
middle_df = sort_meta.groupby([sort_class], sort=False).mean()
middle_inds = list(middle_df["idx"].values)
middle_labels = list(middle_df.index)
# %% [markdown]
# #
fig = plt.figure(figsize=(20, 20))
gs = plt.GridSpec(
2,
for pg in path_graphs:
path_adj = nx.to_numpy_array(pg, nodelist=meta["idx"].values)
path_adjs.append(path_adj)
from graspy.plot import gridplot
gridplot(path_adjs, labels=from_group_names, transform="simple-all", sizes=(5, 10))
from graspy.utils import get_multigraph_union_lcc
graphs, inds = get_multigraph_union_lcc(path_adjs, return_inds=True)
from graspy.embed import MultipleASE
from graspy.utils import pass_to_ranks
graphs = [pass_to_ranks(g) for g in graphs]
mase = MultipleASE()
embed = mase.fit_transform(graphs)
# %% [markdown]
# #
pairplot(embed[0], labels=meta["Merge Class"].values[inds], palette=CLASS_COLOR_DICT)
# %% [markdown]
# # Sort metadata
# row metadata
ids = pd.Series(index=meta["idx"], data=meta.index, name="id")
]
pal = pal[:path_len]
new_paths = []
for p in paths:
# select paths that got to a stop node
if p[-1] in out_inds:
new_paths.append(p)
paths = new_paths
print(f"Number of paths of length {path_len}: {len(paths)}")
subsample = min(2**13, len(paths))
if subsample != -1:
inds = np.random.choice(len(paths), size=subsample, replace=False)
new_paths = []
for i, p in enumerate(paths):
if i in inds:
new_paths.append(p)
paths = new_paths
print(f"Number of paths after subsampling: {len(paths)}")
# %% [markdown]
# ##
embedder = AdjacencySpectralEmbed(n_components=None, n_elbows=2)
embed = embedder.fit_transform(pass_to_ranks(adj))
embed = np.concatenate(embed, axis=-1)
pairplot(embed, labels=labels)
# remove center neurons # FIXME
idx = mg.meta[mg.meta["hemisphere"].isin(["L", "R"])].index
mg = mg.reindex(idx, use_ids=True)
mg = mg.make_lcc()
print(len(mg))
mg.calculate_degrees(inplace=True)
meta = mg.meta
meta["inds"] = range(len(meta))
adj = mg.adj.copy()
lp_inds, rp_inds = get_paired_inds(meta)
left_inds = meta[meta["left"]]["inds"]
# adj = pass_to_ranks(adj)
left_left_adj = pass_to_ranks(adj[np.ix_(lp_inds, lp_inds)])
right_right_adj = pass_to_ranks(adj[np.ix_(rp_inds, rp_inds)])
omni = OmnibusEmbed(n_components=3, n_elbows=2, check_lcc=False, n_iter=10)
embed = omni.fit_transform([left_left_adj, right_right_adj])
embed = np.concatenate(embed, axis=-1)
embed = np.concatenate(embed, axis=0)
labels = np.concatenate((meta["merge_class"].values[lp_inds],
meta["merge_class"].values[rp_inds]))
plot_pairs(
embed,
labels,
left_pair_inds=np.arange(len(lp_inds)),
right_pair_inds=np.arange(len(lp_inds)) + len(lp_inds),
)
