def to_minigraph( adj, labels, drop_neg=True, remove_diag=True, size_scaler=1, use_counts=False, use_weights=True, color_map=None, ): # convert the adjacency and a partition to a minigraph based on SBM probs prob_df = get_blockmodel_df( adj, labels, return_counts=use_counts, use_weights=use_weights ) if drop_neg and ("-1" in prob_df.index): prob_df.drop("-1", axis=0, inplace=True) prob_df.drop("-1", axis=1, inplace=True) if remove_diag: adj = prob_df.values adj -= np.diag(np.diag(adj)) prob_df.data = prob_df g = nx.from_pandas_adjacency(prob_df, create_using=nx.DiGraph()) uni_labels, counts = np.unique(labels, return_counts=True) # add size attribute base on number of vertices size_map = dict(zip(uni_labels, size_scaler * counts)) nx.set_node_attributes(g, size_map, name="Size") # add signal flow attribute (for the minigraph itself) mini_adj = nx.to_numpy_array(g, nodelist=uni_labels) node_signal_flow = signal_flow(mini_adj) sf_map = dict(zip(uni_labels, node_signal_flow)) nx.set_node_attributes(g, sf_map, name="Signal Flow") # add spectral properties sym_adj = symmetrize(mini_adj) n_components = 10 latent = AdjacencySpectralEmbed(n_components=n_components).fit_transform(sym_adj) for i in range(n_components): latent_dim = latent[:, i] lap_map = dict(zip(uni_labels, latent_dim)) nx.set_node_attributes(g, lap_map, name=f"AdjEvec-{i}") # add spring layout properties pos = nx.spring_layout(g) spring_x = {} spring_y = {} for key, val in pos.items(): spring_x[key] = val[0] spring_y[key] = val[1] nx.set_node_attributes(g, spring_x, name="Spring-x") nx.set_node_attributes(g, spring_y, name="Spring-y") # add colors if color_map is None: color_map = dict(zip(uni_labels, cc.glasbey_light)) nx.set_node_attributes(g, color_map, name="Color") return g
# %% [markdown] # ## from mpl_toolkits.axes_grid1 import make_axes_locatable from src.utils import get_blockmodel_df labels = full_meta[f"lvl{lowest_level}_labels"].values mid_map = {} for key, val in first_mid_map.items(): new_key = key[:-1] mid_map[new_key] = val blockmodel_df = get_blockmodel_df( full_adj, labels, return_counts=True, use_weights=True ) group_sizes = full_meta.groupby([f"lvl{lowest_level}_labels"]).size() blockmodel_df.index.name = "source" blockmodel_df.columns.name = "target" blockmodel_df.reset_index(inplace=True) blockmodel_df blockmodel_edges = blockmodel_df.melt(id_vars="source", value_name="weight") blockmodel_edges["x"] = blockmodel_edges["target"].map(mid_map) blockmodel_edges["y"] = blockmodel_edges["source"].map(mid_map) blockmodel_edges["source_size"] = blockmodel_edges["source"].map(group_sizes) blockmodel_edges["target_size"] = blockmodel_edges["target"].map(group_sizes) blockmodel_edges["source_n_out"] = blockmodel_edges["source"].map( blockmodel_edges.groupby("source")["weight"].sum() ) blockmodel_edges["out_weight"] = (
partition, class_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=False, figsize=(24, 18), title=title, hatch_dict=None, category_order=category_order, ) stashfig(basename + "barplot-mergeclass-counts") plt.close() # TODO add gridmap counts = False weights = False prob_df = get_blockmodel_df(mg.adj, partition, return_counts=counts, use_weights=weights) prob_df = prob_df.reindex(category_order, axis=0) prob_df = prob_df.reindex(category_order, axis=1) probplot(100 * prob_df, fmt="2.0f", figsize=(20, 20), title=title, font_scale=0.7) stashfig(basename + f"probplot-counts{counts}-weights{weights}") plt.close()
def run_experiment(seed=None, graph_type=None, threshold=None, param_key=None): np.random.seed(seed) if BLIND: temp_param_key = param_key.replace( " ", "") # don't want spaces in filenames savename = f"{temp_param_key}-cell-types-" title = param_key else: savename = f"{graph_type}-t{threshold}-cell-types" title = f"{graph_type}, threshold = {threshold}" mg = load_metagraph(graph_type, version=VERSION) # simple threshold # TODO they will want symmetric threshold... # TODO maybe make that a parameter adj = mg.adj.copy() adj[adj <= threshold] = 0 meta = mg.meta.copy() meta = pd.DataFrame(mg.meta["neuron_name"]) mg = MetaGraph(adj, meta) # run the graphtool code temp_loc = f"maggot_models/data/interim/temp-{param_key}.graphml" block_series = run_minimize_blockmodel(mg, temp_loc) # manage the output mg = load_metagraph(graph_type, version=VERSION) mg.meta = pd.concat((mg.meta, block_series), axis=1) mg.meta["Original index"] = range(len(mg.meta)) keep_inds = mg.meta[~mg.meta["block_label"].isna( )]["Original index"].values mg.reindex(keep_inds) if graph_type != "G": mg.verify(10000, graph_type=graph_type, version=VERSION) # deal with class labels lineage_labels = mg.meta["lineage"].values lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels) class_labels = mg["Merge Class"] skeleton_labels = mg.meta.index.values classlin_labels, color_dict, hatch_dict = augment_classes( skeleton_labels, class_labels, lineage_labels) block_label = mg["block_label"].astype(int) # barplot with unknown class labels merged in, proportions _, _, order = barplot_text( block_label, classlin_labels, norm_bar_width=True, color_dict=color_dict, hatch_dict=hatch_dict, title=title, figsize=(24, 18), return_order=True, ) stashfig(savename + "barplot-mergeclasslin-props") category_order = np.unique(block_label)[order] # barplot with regular class labels barplot_text( block_label, class_labels, norm_bar_width=True, color_dict=color_dict, hatch_dict=hatch_dict, title=title, figsize=(24, 18), category_order=category_order, ) stashfig(savename + "barplot-mergeclass-props") # barplot with unknown class labels merged in, counts barplot_text( block_label, classlin_labels, norm_bar_width=False, color_dict=color_dict, hatch_dict=hatch_dict, title=title, figsize=(24, 18), return_order=True, category_order=category_order, ) stashfig(savename + "barplot-mergeclasslin-counts") # barplot of hemisphere membership fig, ax = plt.subplots(1, 1, figsize=(10, 20)) stacked_barplot( block_label, mg["Hemisphere"], norm_bar_width=True, category_order=category_order, ax=ax, ) remove_spines(ax) stashfig(savename + "barplot-hemisphere") # plot block probability matrix counts = False weights = False prob_df = get_blockmodel_df(mg.adj, block_label, return_counts=counts, use_weights=weights) prob_df = prob_df.reindex(order, axis=0) prob_df = prob_df.reindex(order, axis=1) ax = probplot(100 * prob_df, fmt="2.0f", figsize=(20, 20), title=title, font_scale=0.4) stashfig(savename + "probplot") block_series.name = param_key return block_series
sharex=True, sharey=False, hue="Cluster", hue_order=pred_color_dict.keys(), palette=pred_color_dict.values(), col_order=leaf_names, ) fg.map(sns.distplot, "Total degree", norm_hist=True, kde=False) fg.set(yticks=[], yticklabels=[]) stashfig("total-degree" + name_base) # %% [markdown] # # blockmodel_df = get_blockmodel_df(adj, pred_labels, use_weights=True, return_counts=False) plt.figure(figsize=(20, 20)) sns.heatmap(blockmodel_df, cmap="Reds") g = nx.from_pandas_adjacency(blockmodel_df, create_using=nx.DiGraph()) uni_labels, counts = np.unique(pred_labels, return_counts=True) size_scaler = 5 size_map = dict(zip(uni_labels, size_scaler * counts)) nx.set_node_attributes(g, size_map, name="Size") mini_adj = nx.to_numpy_array(g, nodelist=uni_labels) node_signal_flow = signal_flow(mini_adj) sf_map = dict(zip(uni_labels, node_signal_flow)) nx.set_node_attributes(g, sf_map, name="Signal Flow") sym_adj = symmetrize(mini_adj) node_lap = LaplacianSpectralEmbed(n_components=1).fit_transform(sym_adj)
heatmap( mg.adj, transform="simple-nonzero", figsize=(20, 20), inner_hier_labels=partition, hier_label_fontsize=10, title=title, title_pad=80, ) stashfig(basename + "heatmap") # block probabilities counts = False weights = False prob_df = get_blockmodel_df(mg.adj, partition, return_counts=counts, use_weights=weights) prob_df = prob_df.reindex(sort_partition_sf.index, axis=0) prob_df = prob_df.reindex(sort_partition_sf.index, axis=1) ax = probplot( 100 * prob_df, fmt="2.0f", figsize=(20, 20), title= f"Louvain, res = {r}, counts = {counts}, weights = {weights}", ) ax.set_ylabel(r"Median signal flow $\to$", fontsize=28) stashfig(basename + f"probplot-counts{counts}-weights{weights}")
def run_experiment(graph_type=None, threshold=None, res=None, binarize=None, seed=None, param_key=None): # common names if BLIND: basename = f"{param_key}-" title = param_key else: basename = f"louvain-res{res}-t{threshold}-{graph_type}-" title = f"Louvain, {graph_type}, res = {res}, threshold = {threshold}" np.random.seed(seed) # load and preprocess the data mg = load_metagraph(graph_type, version=BRAIN_VERSION) mg = preprocess( mg, threshold=threshold, sym_threshold=True, remove_pdiff=True, binarize=binarize, ) g_sym = nx.to_undirected(mg.g) skeleton_labels = np.array(list(g_sym.nodes())) partition, modularity = run_louvain(g_sym, res, skeleton_labels) partition_series = pd.Series(partition, index=skeleton_labels) partition_series.name = param_key if SAVEFIGS: # get out some metadata class_label_dict = nx.get_node_attributes(g_sym, "Merge Class") class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict)) lineage_label_dict = nx.get_node_attributes(g_sym, "lineage") lineage_labels = np.array( itemgetter(*skeleton_labels)(lineage_label_dict)) lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels) classlin_labels, color_dict, hatch_dict = augment_classes( class_labels, lineage_labels) # TODO then sort all of them by proportion of sensory/motor # barplot by merge class and lineage _, _, order = barplot_text( partition, classlin_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=True, figsize=(24, 18), title=title, hatch_dict=hatch_dict, return_order=True, ) stashfig(basename + "barplot-mergeclasslin-props") category_order = np.unique(partition)[order] fig, axs = barplot_text( partition, class_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=True, figsize=(24, 18), title=title, hatch_dict=None, category_order=category_order, ) stashfig(basename + "barplot-mergeclass-props") fig, axs = barplot_text( partition, class_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=False, figsize=(24, 18), title=title, hatch_dict=None, category_order=category_order, ) stashfig(basename + "barplot-mergeclass-counts") # TODO add gridmap counts = False weights = False prob_df = get_blockmodel_df(mg.adj, partition, return_counts=counts, use_weights=weights) prob_df = prob_df.reindex(category_order, axis=0) prob_df = prob_df.reindex(category_order, axis=1) probplot(100 * prob_df, fmt="2.0f", figsize=(20, 20), title=title, font_scale=0.7) stashfig(basename + f"probplot-counts{counts}-weights{weights}") return partition_series, modularity
def run_experiment(graph_type=None, thresh=None, res=None): # load and preprocess the data mg = load_metagraph(graph_type, version=BRAIN_VERSION) edgelist = mg.to_edgelist() edgelist = add_max_weight(edgelist) edgelist = edgelist[edgelist["max_weight"] > thresh] mg = edgelist_to_mg(edgelist, mg.meta) mg = mg.make_lcc() mg = mg.remove_pdiff() g_sym = nx.to_undirected(mg.g) skeleton_labels = np.array(list(g_sym.nodes())) partition = run_louvain(g_sym, res, skeleton_labels) # compute signal flow for sorting purposes mg.meta["signal_flow"] = signal_flow(mg.adj) mg.meta["partition"] = partition partition_sf = mg.meta.groupby("partition")["signal_flow"].median() sort_partition_sf = partition_sf.sort_values(ascending=False) # common names basename = f"louvain-res{res}-t{thresh}-{graph_type}-" title = f"Louvain, {graph_type}, res = {res}, thresh = {thresh}" # get out some metadata class_label_dict = nx.get_node_attributes(g_sym, "Merge Class") class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict)) lineage_label_dict = nx.get_node_attributes(g_sym, "lineage") lineage_labels = np.array(itemgetter(*skeleton_labels)(lineage_label_dict)) lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels) classlin_labels, color_dict, hatch_dict = augment_classes( skeleton_labels, class_labels, lineage_labels) # barplot by merge class and lineage fig, axs = barplot_text( partition, classlin_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=True, figsize=(24, 18), title=title, hatch_dict=hatch_dict, ) stashfig(basename + "barplot-mergeclasslin-props") fig, axs = barplot_text( partition, class_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=True, figsize=(24, 18), title=title, hatch_dict=None, ) stashfig(basename + "barplot-mergeclass-props") fig, axs = barplot_text( partition, class_labels, color_dict=color_dict, plot_proportions=False, norm_bar_width=False, figsize=(24, 18), title=title, hatch_dict=None, ) stashfig(basename + "barplot-mergeclass-counts") fig, axs = barplot_text( partition, lineage_labels, color_dict=None, plot_proportions=False, norm_bar_width=True, figsize=(24, 18), title=title, ) stashfig(basename + "barplot-lineage-props") # sorted heatmap heatmap( mg.adj, transform="simple-nonzero", figsize=(20, 20), inner_hier_labels=partition, hier_label_fontsize=10, title=title, title_pad=80, ) stashfig(basename + "heatmap") # block probability matrices counts = False weights = False prob_df = get_blockmodel_df(mg.adj, partition, return_counts=counts, use_weights=weights) prob_df = prob_df.reindex(sort_partition_sf.index, axis=0) prob_df = prob_df.reindex(sort_partition_sf.index, axis=1) ax = probplot( 100 * prob_df, fmt="2.0f", figsize=(20, 20), title=f"Louvain, res = {res}, counts = {counts}, weights = {weights}", ) ax.set_ylabel(r"Median signal flow $\to$", fontsize=28) stashfig(basename + f"probplot-counts{counts}-weights{weights}") # plot minigraph with layout adjusted_partition = adjust_partition(partition, class_labels) minigraph = to_minigraph(mg.adj, adjusted_partition, use_counts=True, size_scaler=10) draw_networkx_nice( minigraph, "Spring-x", "Signal Flow", sizes="Size", colors="Color", cmap="Greys", vmin=100, weight_scale=0.001, ) stashfig(basename + "sbm-drawn-network")