def visualize(self, i, savedir=""): """ Visualize the ith graph of self.graphs, passed-to-ranks. Parameters ---------- i : int Graph to visualize. savedir : str, optional Directory to save graph into. If left empty, do not save. """ nmax = np.max(self.graphs) if isinstance(i, int): graph = pass_to_ranks(self.graphs[i]) sub = self.subjects[i] sesh = "" # TODO elif isinstance(i, np.ndarray): graph = pass_to_ranks(i) sub = "" sesh = "" else: raise TypeError("Passed value must be integer or np.ndarray.") viz = heatmap( graph, title=f"sub-{sub}_session-{sesh}", xticklabels=True, yticklabels=True, vmin=0, vmax=1, ) # set color of title viz.set_title(viz.get_title(), color="black") # set color of colorbar ticks viz.collections[0].colorbar.ax.yaxis.set_tick_params(color="black") # set font size and color of heatmap ticks for item in viz.get_xticklabels() + viz.get_yticklabels(): item.set_color("black") item.set_fontsize(7) if savedir: p = Path(savedir).resolve() if not p.is_dir(): p.mkdir() plt.savefig( p / f"sub-{sub}_sesh-{sesh}.png", facecolor="white", bbox_inches="tight", dpi=300, ) else: plt.show() plt.cla()
def discriminability(self, PTR=True, **kwargs): """ Attach discriminability functionality to the object. See `discr_stat` for full documentation. Returns ------- stat : float Discriminability statistic. """ if PTR: graphs = np.copy(self.graphs) # No need to pass to ranks and normalize if it's a correlation?? #for graph in graphs: # ranked = np.array([pass_to_ranks(graph)]) # print('oof') graphs = np.array([pass_to_ranks(graph) for graph in graphs]) X = self._X(graphs) z, x, y = graphs.shape spread = np.zeros((x, 0)) maxsort = np.argmax(graphs, axis=2) for qq in range(z): spread = np.insert(spread, 0, maxsort[qq][0:x], axis=1) return spread #return discr_stat(X, self.Y, **kwargs) return discr_stat(self.X, self.Y, **kwargs)
def ase(adj, n_components, ptr=True): if ptr: adj = pass_to_ranks(adj) ase = AdjacencySpectralEmbed(n_components=n_components) latent = ase.fit_transform(adj) latent = np.concatenate(latent, axis=-1) return latent
def omni(adjs, n_components): if PTR: adjs = [pass_to_ranks(a) for a in adjs] omni = OmnibusEmbed(n_components=n_components // len(adjs)) latent = omni.fit_transform(adjs) latent = np.concatenate(latent, axis=-1) # first is for in/out latent = np.concatenate(latent, axis=-1) # second is for concat. each graph return latent
def preprocess_adjs(adjs, method="ase"): adjs = [pass_to_ranks(a) for a in adjs] adjs = [a + 1 / a.size for a in adjs] if method == "ase": adjs = [augment_diagonal(a) for a in adjs] elif method == "lse": adjs = [to_laplace(a) for a in adjs] return adjs
def lse(adj, n_components, regularizer=None): if PTR: adj = pass_to_ranks(adj) lap = to_laplace(adj, form="R-DAD") ase = AdjacencySpectralEmbed(n_components=n_components) latent = ase.fit_transform(lap) latent = np.concatenate(latent, axis=-1) return latent
def lateral_omni(adj, lp_inds, rp_inds): left_left_adj = pass_to_ranks(adj[np.ix_(lp_inds, lp_inds)]) right_right_adj = pass_to_ranks(adj[np.ix_(rp_inds, rp_inds)]) omni = OmnibusEmbed(n_components=3, n_elbows=2, check_lcc=False, n_iter=10) ipsi_embed = omni.fit_transform([left_left_adj, right_right_adj]) ipsi_embed = np.concatenate(ipsi_embed, axis=-1) ipsi_embed = np.concatenate(ipsi_embed, axis=0) left_right_adj = pass_to_ranks(adj[np.ix_(lp_inds, rp_inds)]) right_left_adj = pass_to_ranks(adj[np.ix_(rp_inds, lp_inds)]) omni = OmnibusEmbed(n_components=3, n_elbows=2, check_lcc=False, n_iter=10) contra_embed = omni.fit_transform([left_right_adj, right_left_adj]) contra_embed = np.concatenate(contra_embed, axis=-1) contra_embed = np.concatenate(contra_embed, axis=0) embed = np.concatenate((ipsi_embed, contra_embed), axis=1) return embed
def ase_concatenate(adjs, n_components, ptr=True): if ptr: adjs = [pass_to_ranks(a) for a in adjs] ase = AdjacencySpectralEmbed(n_components=n_components // len(adjs)) graph_latents = [] for a in adjs: latent = ase.fit_transform(a) latent = np.concatenate(latent, axis=-1) graph_latents.append(latent) latent = np.concatenate(graph_latents, axis=-1) return latent
def lse(adj, n_components, regularizer=None, ptr=True): if ptr: adj = pass_to_ranks(adj) lap = to_laplace(adj, form="R-DAD", regularizer=regularizer) ase = AdjacencySpectralEmbed(n_components=n_components, diag_aug=False) latent = ase.fit_transform(lap) # latent = LaplacianSpectralEmbed( # form="R-DAD", n_components=n_components, regularizer=regularizer # ) latent = np.concatenate(latent, axis=-1) return latent
def _embed(self, adj=None): if adj is None: adj = self.adj # TODO look into PTR at this level as well # lp_inds, rp_inds = get_paired_inds(self.meta) lp_inds = self.left_pair_inds rp_inds = self.right_pair_inds embed_adj = pass_to_ranks(adj) if self.embed == "ase": embedder = AdjacencySpectralEmbed( n_components=self.n_components, n_elbows=self.n_elbows ) embed = embedder.fit_transform(embed_adj) elif self.embed == "lse": embedder = LaplacianSpectralEmbed( n_components=self.n_components, n_elbows=self.n_elbows, regularizer=self.regularizer, ) embed = embedder.fit_transform(embed_adj) elif self.embed == "unscaled_ase": embed_adj = pass_to_ranks(adj) embed_adj = augment_diagonal(embed_adj) embed = selectSVD( embed_adj, n_components=self.n_components, n_elbows=self.n_elbows ) embed = (embed[0], embed[2].T) X = np.concatenate(embed, axis=1) fraction_paired = (len(lp_inds) + len(rp_inds)) / len(self.root_inds) print(f"Learning transformation with {fraction_paired} neurons paired") R, _ = orthogonal_procrustes(X[lp_inds], X[rp_inds]) X[self.left_inds] = X[self.left_inds] @ R if self.normalize: row_sums = np.sum(X, axis=1) X /= row_sums[:, None] return X
def load_HNU1(ptr=None, return_subid=False): path = Path(MODULE_PATH).parents[1] / "data/raw/HNU1/" f = sorted(path.glob('*.ssv')) df = pd.read_csv(path / "HNU1.csv") subid = [int(fname.stem.split('_')[0].split('-')[1][-5:]) for fname in f] y = np.array([np.unique(df.SEX[df.SUBID == s])[0] - 1 for s in subid]) g = np.array(import_edgelist(f, 'ssv')) if ptr is not None: g = np.array([pass_to_ranks(x) for x in g]) if return_subid: return g, y, subid else: return g, y
def discriminability(self, PTR=True, **kwargs): """ Attach discriminability functionality to the object. See `discr_stat` for full documentation. Returns ------- stat : float Discriminability statistic. """ if PTR: graphs = np.copy(self.graphs) graphs = np.array([pass_to_ranks(graph) for graph in graphs]) X = self._X(graphs) return discr_stat(X, self.Y, **kwargs) return discr_stat(self.X, self.Y, **kwargs)
def reg_lateral_omni(adj, base_adj, lp_inds, rp_inds): base_ll_adj = pass_to_ranks(base_adj[np.ix_(lp_inds, lp_inds)]) base_rr_adj = pass_to_ranks(base_adj[np.ix_(rp_inds, rp_inds)]) ll_adj = pass_to_ranks(adj[np.ix_(lp_inds, lp_inds)]) rr_adj = pass_to_ranks(adj[np.ix_(rp_inds, rp_inds)]) ipsi_adjs = [base_ll_adj, base_rr_adj, ll_adj, rr_adj] ipsi_embed = reg_omni(ipsi_adjs) base_lr_adj = pass_to_ranks(base_adj[np.ix_(lp_inds, rp_inds)]) base_rl_adj = pass_to_ranks(base_adj[np.ix_(rp_inds, lp_inds)]) lr_adj = pass_to_ranks(adj[np.ix_(lp_inds, rp_inds)]) rl_adj = pass_to_ranks(adj[np.ix_(rp_inds, lp_inds)]) contra_adjs = [base_lr_adj, base_rl_adj, lr_adj, rl_adj] contra_embed = reg_omni(contra_adjs) embed = np.concatenate((ipsi_embed, contra_embed), axis=1) return embed
def load_BNU1(ptr=None): path = Path(MODULE_PATH).parents[1] / "data/raw/BNU1/" f = sorted(path.glob('*.ssv')) df = pd.read_csv(path / "BNU1_phenotypic_data.csv") subid = [int(fname.stem.split('_')[0].split('-')[1][-5:]) for fname in f] y = np.array([ np.unique(df.SEX[(df.SUBID == s) & (df.SESSION == 'Baseline')])[0] for s in subid ]).astype(int) y -= 1 g = np.array(import_edgelist(f, 'ssv')) if ptr is not None: g = np.array([pass_to_ranks(x) for x in g]) return g, y
def _embed(self, adj=None): if adj is None: adj = self.adj lp_inds = self.left_pair_inds rp_inds = self.right_pair_inds embed_adj = pass_to_ranks(adj) # TODO PTR here? if self.plus_c: embed_adj += 1 / adj.size if self.embed == "ase": embedder = AdjacencySpectralEmbed(n_components=self.n_components, n_elbows=self.n_elbows) embed = embedder.fit_transform(embed_adj) elif self.embed == "lse": embedder = LaplacianSpectralEmbed( n_components=self.n_components, n_elbows=self.n_elbows, regularizer=self.regularizer, ) embed = embedder.fit_transform(embed_adj) elif self.embed == "unscaled_ase": embed_adj = augment_diagonal(embed_adj) embed = selectSVD(embed_adj, n_components=self.n_components, n_elbows=self.n_elbows) embed = (embed[0], embed[2].T) X = np.concatenate(embed, axis=1) fraction_paired = (len(lp_inds) + len(rp_inds)) / len(self.root_inds) print(f"Learning transformation with {fraction_paired} neurons paired") X = self._procrustes(X) if self.normalize: row_norms = np.linalg.norm(X, axis=1) X /= row_norms[:, None] return X
def preprocess_adjs(adjs, method="ase"): """Preprocessing necessary prior to embedding a graph, opetates on a list Parameters ---------- adjs : list of adjacency matrices [description] method : str, optional [description], by default "ase" Returns ------- [type] [description] """ adjs = [pass_to_ranks(a) for a in adjs] adjs = [a + 1 / a.size for a in adjs] if method == "ase": adjs = [augment_diagonal(a) for a in adjs] elif method == "lse": # haven't really used much. a few params to look at here adjs = [to_laplace(a) for a in adjs] return adjs
def _load_dataset(path, n_nodes, ptr=None): file = np.load(path) X = file["X"] y = file["y"].astype(int) n_samples = X.shape[0] y[y == -1] = 0 idx = np.triu_indices(n_nodes, k=1) X_graphs = np.zeros((n_samples, n_nodes, n_nodes)) for i, x in enumerate(X): X_graphs[i][idx] = x X_graphs[i] = symmetrize(X_graphs[i], "triu") if ptr is not None: X_graphs = X_graphs - X_graphs.min(axis=(1, 2)).reshape(-1, 1, 1) for i, x in enumerate(X_graphs): X_graphs[i] = pass_to_ranks(X_graphs[i]) return X_graphs, y
pair = data["Pair"] pair_id = data["Pair ID"] if pair != -1: if pair not in thresh_g: thresh_g.node[n]["Pair"] = -1 thresh_g.node[n]["Pair ID"] = -1 n_missing += 1 mg = MetaGraph(thresh_g, weight="max_norm_weight") meta = mg.meta adj = mg.adj.copy() # colsums = np.sum(adj, axis=0) # colsums[colsums == 0] = 1 # adj = adj / colsums[np.newaxis, :] adj = pass_to_ranks(adj) if use_spl: adj = graph_shortest_path(adj) if plus_c: adj += np.min(adj) if embed == "lse": latent = lse(adj, None, ptr=False) elif embed == "ase": latent = ase(adj, None, ptr=False) rot_latent, diff = procrustes_match(latent, meta) rot_latent = latent n_components = latent.shape[1] plot_df = pd.DataFrame(data=rot_latent)
stacked_barplot( cat, subcat, ax=ax, color_dict=CLASS_COLOR_DICT, plot_names=True, text_color="dimgrey", bar_height=0.2, ) ax.get_legend().remove() subgraph_inds = temp_meta["inds"].values subgraph_adj = adj[np.ix_(subgraph_inds, subgraph_inds)] ax = fig.add_subplot(mid_gs[1:, :]) _, _, top, _ = adjplot( pass_to_ranks(subgraph_adj), plot_type="heatmap", cbar=False, ax=ax, meta=temp_meta, item_order=["merge_class", "sf"], colors="merge_class", palette=CLASS_COLOR_DICT, ) top.set_title("Intra-cluster connectivity", color="dimgrey") dend_gs = plt.GridSpec( 1, 5, figure=fig, left=margin + 5 / n_col + 2 * gap,
def _run_test(self, methodstr, input_graph, expected): ptr_out = pass_to_ranks(input_graph, method=methodstr) self.assertTrue(np.allclose(ptr_out, expected, rtol=1e-04))
def test_invalid_inputs(self): with self.assertRaises(ValueError): pass_to_ranks(self.loopless_undirected_input, method="hazelnutcoffe") with self.assertRaises(TypeError): pass_to_ranks("hi", "hi")
color=text_color, ) ax.legend( ncol=6, # len(category_names), bbox_to_anchor=(0, 1), loc="lower left", fontsize="small", ) return ax # Try with my new labels adj = get_paired_adj("G", nodelist) adj = adj[:n_per_side, :n_per_side] + adj[n_per_side:, n_per_side:] embed_adj = pass_to_ranks(adj) skmeans_kws = dict(n_init=200, n_jobs=-2) n_clusters = 12 n_components = None regularizer = 1 lse = LaplacianSpectralEmbed(n_components=n_components, regularizer=regularizer) latent = lse.fit_transform(embed_adj) latent = np.concatenate(latent, axis=-1) models = [] meta_file = "maggot_models/data/processed/2019-09-18-v2/BP_metadata.csv"
mb_mg = mg.reindex(mb_meta.index, use_ids=True, inplace=False) mb_mg.calculate_degrees(inplace=True) mb_mg.meta["Total edgesum"] = -mb_mg.meta["Total edgesum"] sizes = mb_mg.meta.groupby("class1").size() mb_mg.meta["class1_sizes"] = -mb_mg.meta["class1"].map(sizes) # %% meta = mb_mg.meta print("n_left") print(len(meta[meta["left"]])) print("n_right") print(len(meta[meta["right"]])) fig, ax = plt.subplots(1, 1, figsize=(10, 10)) adjplot( pass_to_ranks(mb_mg.adj), meta=mb_mg.meta, sort_class=["hemisphere", "class1"], class_order=["class1_sizes"], item_order=["class1", "Total edgesum"], cbar=False, row_tick_pad=[0.05, 0.7], col_tick_pad=[0.2, 0.7], tick_rot=90, tick_fontsize=12, gridline_kws=dict(color="grey", linestyle="--", linewidth=0.5), ax=ax, ) left_mb_mg = mb_mg.reindex(meta[meta["left"]].index, use_ids=True, inplace=False) right_mb_mg = mb_mg.reindex(meta[meta["right"]].index, use_ids=True, inplace=False)
# arrowprops=arrow_args, # ) #%% from graspy.embed import AdjacencySpectralEmbed, OmnibusEmbed from graspy.utils import pass_to_ranks from graspy.plot import pairplot sum_adj = np.sum(np.array(mb_color_graphs), axis=0) n_components = 4 # ptr_adj = pass_to_ranks(sum_adj) ase = AdjacencySpectralEmbed(n_components=n_components) sum_latent = ase.fit_transform(ptr_adj) sum_latent = np.concatenate(sum_latent, axis=-1) pairplot(sum_latent, labels=mb_class_labels) ptr_color_adjs = [pass_to_ranks(a) for a in mb_color_graphs] # graph_sum = [np.sum(a) for a in mb_color_graphs] # ptr_color_adjs = [ptr_color_adjs[i] + (1 / graph_sum[i]) for i in range(4)] omni = OmnibusEmbed(n_components=n_components // 4) color_latent = omni.fit_transform(ptr_color_adjs) color_latent = np.concatenate(color_latent, axis=-1) color_latent = np.concatenate(color_latent, axis=-1) pairplot(color_latent, labels=mb_class_labels) from graspy.embed import MultipleASE
meta["inds"] = range(len(meta)) lp_inds, rp_inds = get_paired_inds(meta) left_inds = meta[meta["left"]]["inds"] right_inds = meta[meta["right"]]["inds"] # %% Load and preprocess all graphs graph_types = ["Gad", "Gaa", "Gdd", "Gda"] adjs = [] for g in graph_types: temp_mg = load_metagraph(g, version="2020-04-01") temp_mg.reindex(mg.meta.index, use_ids=True) temp_adj = temp_mg.adj adjs.append(temp_adj) embed_adjs = [pass_to_ranks(a) for a in adjs] embed_adjs = [a + 1 / a.size for a in embed_adjs] embed_adjs = [augment_diagonal(a) for a in embed_adjs] # %% [markdown] # ## def omni_procrust_svd(embed_adjs): omni = OmnibusEmbed(n_components=None, check_lcc=False) joint_embed = omni.fit_transform(embed_adjs) cat_embed = np.concatenate(joint_embed, axis=-1) # print(f"Omni concatenated embedding shape: {cat_embed.shape}") for e in cat_embed: e[left_inds] = e[left_inds] @ orthogonal_procrustes( e[lp_inds], e[rp_inds])[0]
def preprocess_adjs(adjs): adjs = [pass_to_ranks(a) for a in adjs] adjs = [a + 1 / a.size for a in adjs] adjs = [augment_diagonal(a) for a in adjs] return adjs
if "class_size" in meta.columns: meta = meta.drop("class_size", axis=1) meta = pd.concat((meta, class_size_mapped), ignore_index=False, axis=1) meta["idx"] = range(len(meta)) sort_meta = meta.sort_values( ["class_rank", "class_size", sort_class, "median_visit", "dendrite_input"], inplace=False, ) perm_inds = sort_meta.idx.values from graspy.utils import pass_to_ranks data = mg.adj.copy() data = data[np.ix_(perm_inds, perm_inds)] data = pass_to_ranks(data) sort_meta["idx"] = range(len(sort_meta)) first_df = sort_meta.groupby([sort_class], sort=False).first() first_inds = list(first_df["idx"].values) first_inds.append(len(meta) + 1) # for the tic locs middle_df = sort_meta.groupby([sort_class], sort=False).mean() middle_inds = list(middle_df["idx"].values) middle_labels = list(middle_df.index) # %% [markdown] # # fig = plt.figure(figsize=(20, 20)) gs = plt.GridSpec( 2,
for pg in path_graphs: path_adj = nx.to_numpy_array(pg, nodelist=meta["idx"].values) path_adjs.append(path_adj) from graspy.plot import gridplot gridplot(path_adjs, labels=from_group_names, transform="simple-all", sizes=(5, 10)) from graspy.utils import get_multigraph_union_lcc graphs, inds = get_multigraph_union_lcc(path_adjs, return_inds=True) from graspy.embed import MultipleASE from graspy.utils import pass_to_ranks graphs = [pass_to_ranks(g) for g in graphs] mase = MultipleASE() embed = mase.fit_transform(graphs) # %% [markdown] # # pairplot(embed[0], labels=meta["Merge Class"].values[inds], palette=CLASS_COLOR_DICT) # %% [markdown] # # Sort metadata # row metadata ids = pd.Series(index=meta["idx"], data=meta.index, name="id")
] pal = pal[:path_len] new_paths = [] for p in paths: # select paths that got to a stop node if p[-1] in out_inds: new_paths.append(p) paths = new_paths print(f"Number of paths of length {path_len}: {len(paths)}") subsample = min(2**13, len(paths)) if subsample != -1: inds = np.random.choice(len(paths), size=subsample, replace=False) new_paths = [] for i, p in enumerate(paths): if i in inds: new_paths.append(p) paths = new_paths print(f"Number of paths after subsampling: {len(paths)}") # %% [markdown] # ## embedder = AdjacencySpectralEmbed(n_components=None, n_elbows=2) embed = embedder.fit_transform(pass_to_ranks(adj)) embed = np.concatenate(embed, axis=-1) pairplot(embed, labels=labels)
# remove center neurons # FIXME idx = mg.meta[mg.meta["hemisphere"].isin(["L", "R"])].index mg = mg.reindex(idx, use_ids=True) mg = mg.make_lcc() print(len(mg)) mg.calculate_degrees(inplace=True) meta = mg.meta meta["inds"] = range(len(meta)) adj = mg.adj.copy() lp_inds, rp_inds = get_paired_inds(meta) left_inds = meta[meta["left"]]["inds"] # adj = pass_to_ranks(adj) left_left_adj = pass_to_ranks(adj[np.ix_(lp_inds, lp_inds)]) right_right_adj = pass_to_ranks(adj[np.ix_(rp_inds, rp_inds)]) omni = OmnibusEmbed(n_components=3, n_elbows=2, check_lcc=False, n_iter=10) embed = omni.fit_transform([left_left_adj, right_right_adj]) embed = np.concatenate(embed, axis=-1) embed = np.concatenate(embed, axis=0) labels = np.concatenate((meta["merge_class"].values[lp_inds], meta["merge_class"].values[rp_inds])) plot_pairs( embed, labels, left_pair_inds=np.arange(len(lp_inds)), right_pair_inds=np.arange(len(lp_inds)) + len(lp_inds), )