Ejemplo n.º 1
0
def max_symmetrize(mg, n_pairs):
    """ assumes that mg is sorted
    
    Parameters
    ----------
    mg : [type]
        [description]
    n_pairs : [type]
        [description]
    """
    adj = mg.adj
    left_left_adj = adj[:n_pairs, :n_pairs]
    left_right_adj = adj[:n_pairs, n_pairs:2 * n_pairs]
    right_right_adj = adj[n_pairs:2 * n_pairs, n_pairs:2 * n_pairs]
    right_left_adj = adj[n_pairs:2 * n_pairs, :n_pairs]

    # max, average gives similar results
    sym_ipsi_adj = np.maximum(left_left_adj, right_right_adj)
    sym_contra_adj = np.maximum(left_right_adj, right_left_adj)

    sym_adj = adj.copy()
    sym_adj[:n_pairs, :n_pairs] = sym_ipsi_adj
    sym_adj[n_pairs:2 * n_pairs, n_pairs:2 * n_pairs] = sym_ipsi_adj
    sym_adj[:n_pairs, n_pairs:2 * n_pairs] = sym_contra_adj
    sym_adj[n_pairs:2 * n_pairs, :n_pairs] = sym_contra_adj

    sym_mg = MetaGraph(sym_adj,
                       mg.meta)  # did not change indices order so this ok

    return sym_mg
Ejemplo n.º 2
0
def edgelist_to_mg(edgelist, meta):
    g = nx.from_pandas_edgelist(edgelist,
                                edge_attr=True,
                                create_using=nx.DiGraph)
    nx.set_node_attributes(g, meta.to_dict(orient="index"))
    mg = MetaGraph(g)
    return mg
Ejemplo n.º 3
0
def run_minimize_blockmodel(mg, temp_loc=None, weight_model=None):
    meta = mg.meta.copy()
    meta = pd.DataFrame(mg.meta["neuron_name"])
    mg = MetaGraph(mg.adj, meta)
    if temp_loc is None:
        temp_loc = f"maggot_models/data/interim/temp-{np.random.randint(1e8)}.graphml"
    # save to temp
    nx.write_graphml(mg.g, temp_loc)
    # load into graph-tool from temp
    g = load_graph(temp_loc, fmt="graphml")
    os.remove(temp_loc)

    total_degrees = g.get_total_degrees(g.get_vertices())
    remove_verts = np.where(total_degrees == 0)[0]
    g.remove_vertex(remove_verts)

    if weight_model is not None:
        recs = [g.ep.weight]
        rec_types = [weight_model]
    else:
        recs = []
        rec_types = []
    state_args = dict(recs=recs, rec_types=rec_types)
    min_state = minimize_blockmodel_dl(g, verbose=False, state_args=state_args)

    blocks = list(min_state.get_blocks())
    verts = g.get_vertices()

    block_map = {}

    for v, b in zip(verts, blocks):
        cell_id = int(g.vertex_properties["_graphml_vertex_id"][v])
        block_map[cell_id] = int(b)

    block_series = pd.Series(block_map)
    block_series.name = "block_label"
    return block_series
Ejemplo n.º 4
0
n_levels = 10  # max # of splits
metric = "bic"
bic_ratio = 1
d = 10  # embedding dimension
method = "aniso"

basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"

exp = "137.0-BDP-omni-clust"

full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)

# %% [markdown]
# ##
full_meta = full_mg.meta

# parameters
lowest_level = 7

width = 0.5
gap = 10

# this determines the sorting for everybody
level_names = [f"lvl{i}_labels" for i in range(lowest_level + 1)]
sort_class = level_names + ["merge_class"]
class_order = ["sf"]
Ejemplo n.º 5
0
metric = "bic"
bic_ratio = 1
d = 8  # embedding dimension
method = "color_iso"

basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"

exp = "137.2-BDP-omni-clust"

# load data
pair_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
pair_meta["lvl0_labels"] = pair_meta["lvl0_labels"].astype(str)
pair_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
pair_adj = pair_adj.values
mg = MetaGraph(pair_adj, pair_meta)
meta = mg.meta

level_names = [f"lvl{i}_labels" for i in range(level + 1)]


def sort_mg(mg, level_names):
    meta = mg.meta
    sort_class = level_names + ["merge_class"]
    class_order = ["sf"]
    total_sort_by = []
    for sc in sort_class:
        for co in class_order:
            class_value = meta.groupby(sc)[co].mean()
            meta[f"{sc}_{co}_order"] = meta[sc].map(class_value)
            total_sort_by.append(f"{sc}_{co}_order")
Ejemplo n.º 6
0
#     ad_norm_mg.adj, sort_nodes=False, transform="binarize", figsize=(20, 20), cbar=False
# )
# heatmap(
#     ad_norm_mg.adj,
#     sort_nodes=False,
#     transform="binarize",
#     figsize=(20, 20),
#     inner_hier_labels=ad_norm_mg["Merge Class"],
#     outer_hier_labels=ad_norm_mg["Hemisphere"],
#     hier_label_fontsize=5,
# )

threshold = 0.0
adj = ad_norm_mg.adj
adj[adj < threshold] = 0
mod_mg = MetaGraph(adj, ad_norm_mg.meta)

# check pairs

uni_pairs, counts = np.unique(mod_mg["Pair ID"], return_counts=True)
assert (counts[1:] == 2).all()
n_pairs = len(counts) - 1
mod_mg = max_symmetrize(mod_mg, n_pairs)

heatmap(
    mod_mg.adj,
    sort_nodes=False,
    transform="binarize",
    figsize=(20, 20),
    inner_hier_labels=mod_mg["Class 1"],
    outer_hier_labels=mod_mg["Hemisphere"],
Ejemplo n.º 7
0
# ##

# parameters for the experiment
metric = "bic"
bic_ratio = 1
d = 8  # embedding dimension
method = "iso"

# parameters for plotting
lowest_level = 7
width = 0.5
gap = 10

basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"

exp = "137.0-BDP-omni-clust"

# load data
pair_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
pair_meta["lvl0_labels"] = pair_meta["lvl0_labels"].astype(str)
pair_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
pair_mg = MetaGraph(pair_adj.values, pair_meta)
pair_meta = pair_mg.meta

lp_inds, rp_inds = get_paired_inds(pair_meta)

left_adj = pair_adj[np.ix_(lp_inds, lp_inds)]
right_adj = pair_adj[np.ix_(rp_inds, rp_inds)]

calc_blockmodel_df
Ejemplo n.º 8
0
for graph_type in ["Gadn", "Gad", "G"]:
    if graph_type[-1] == "n":
        threshs = [0.01, 0.02]
    else:
        threshs = [1, 2, 3]
    for thresh in threshs:
        for r in [0.3, 0.5, 0.7, 1]:
            mg = load_metagraph(graph_type, version=BRAIN_VERSION)
            edgelist = mg.to_edgelist()
            edgelist = edgelist[edgelist["weight"] > thresh]
            nodelist = list(mg.g.nodes())
            thresh_g = nx.from_pandas_edgelist(edgelist,
                                               edge_attr=True,
                                               create_using=nx.DiGraph)
            nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
            mg = MetaGraph(thresh_g)
            print(len(mg.meta))
            mg = mg.make_lcc()
            print(len(mg.meta))
            not_pdiff = np.where(~mg["is_pdiff"])[0]
            mg = mg.reindex(not_pdiff)
            print(len(mg.meta))
            g_sym = nx.to_undirected(mg.g)
            skeleton_labels = np.array(list(g_sym.nodes()))
            out_dict = cm.best_partition(g_sym, resolution=r)
            partition = np.array(itemgetter(*skeleton_labels)(out_dict))
            adj = nx.to_numpy_array(g_sym, nodelist=skeleton_labels)

            part_unique, part_count = np.unique(partition, return_counts=True)
            for uni, count in zip(part_unique, part_count):
                if count < 3:
Ejemplo n.º 9
0
# %% [markdown]
# #

from src.graph import MetaGraph
import numpy as np
import pandas as pd
import networkx as nx

n_verts = 5
adj = np.arange(0, n_verts**2).reshape(n_verts, n_verts)
adj[0, :] = 0
adj[:, 0] = 0
print(adj)
meta = pd.DataFrame(index=range(n_verts))

meta["title"] = ["zero", "one", "two", "three", "four"]
meta["remove"] = [False, True, False, False, True]
meta.index = ["0", "1", "2", "3", "4"]
print(meta)

mg = MetaGraph(adj, meta)
# mg.make_lcc()
mg.reindex(np.array([2, 3]))
print(mg.meta)
print(mg.adj)

mg2 = MetaGraph(mg.g)
print(mg2.adj)
print(mg2.meta)
Ejemplo n.º 10
0
# %% [markdown]
# # Get rid of unpaired cells for now

mg.meta["is_paired"] = False
pairs = mg["Pair"]
is_paired = pairs != -1
mg.reindex(is_paired)
print(np.unique(mg["Pair"]))
meta = mg.meta
meta["Original index"] = range(meta.shape[0])
meta.sort_values(
    ["Hemisphere", "Pair ID"], inplace=True, kind="mergesort", ascending=False
)
temp_inds = meta["Original index"]
mg = mg.reindex(temp_inds)
mg = MetaGraph(mg.adj, meta)

plot_adj = False
if plot_adj:
    heatmap(
        mg.adj,
        sort_nodes=False,
        inner_hier_labels=mg["Class 1"],
        outer_hier_labels=mg["Hemisphere"],
        transform="binarize",
        cbar=False,
        figsize=(30, 30),
        hier_label_fontsize=10,
    )
    stashfig("heatmap")
Ejemplo n.º 11
0
pairs = mg["Pair"]
is_paired = pairs != -1
mg.reindex(is_paired)
print(np.unique(mg["Pair"]))
meta = mg.meta

# %% [markdown]
# #
meta["Original index"] = range(meta.shape[0])
meta.sort_values(["Hemisphere", "Pair ID"],
                 inplace=True,
                 kind="mergesort",
                 ascending=False)
temp_inds = meta["Original index"]
mg = mg.reindex(temp_inds)
mg = MetaGraph(mg.adj, meta)

plot_adj = True
if plot_adj:
    heatmap(
        mg.adj,
        sort_nodes=False,
        inner_hier_labels=mg["Class 1"],
        outer_hier_labels=mg["Hemisphere"],
        transform="binarize",
        cbar=False,
        figsize=(30, 30),
        hier_label_fontsize=10,
    )
    heatmap(mg.adj,
            sort_nodes=False,
Ejemplo n.º 12
0
metric = "bic"
bic_ratio = 1
d = 8  # embedding dimension
method = "iso"

basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}-Gad"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"

exp = "137.0-BDP-omni-clust"

full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
# full_meta["sf"] = -full_meta["sf"]
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)

full_meta = full_mg.meta

# parameters
lowest_level = 7

width = 0.5
gap = 10

# this determines the sorting for everybody
level_names = [f"lvl{i}_labels" for i in range(lowest_level + 1)]
sort_class = level_names + ["merge_class"]
class_order = ["sf"]
total_sort_by = []
for sc in sort_class:
Ejemplo n.º 13
0
def load_metagraph(graph_type, path=DATA_DIR, version=DATA_VERSION):
    g = load_networkx(graph_type, path=path, version=version)
    mg = MetaGraph(g)
    return mg
Ejemplo n.º 14
0
                                       create_using=nx.DiGraph)
    nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
    thresh_g = get_lcc(thresh_g)
    n_verts = len(thresh_g)

    n_missing = 0
    for n, data in thresh_g.nodes(data=True):
        pair = data["Pair"]
        pair_id = data["Pair ID"]
        if pair != -1:
            if pair not in thresh_g:
                thresh_g.node[n]["Pair"] = -1
                thresh_g.node[n]["Pair ID"] = -1
                n_missing += 1

    mg = MetaGraph(thresh_g)
    meta = mg.meta
    meta["Original index"] = range(len(meta))
    left_paired_df = meta[(meta["Pair"] != -1) & (meta["Hemisphere"] == "L")]
    left_paired_inds = left_paired_df["Original index"].values
    pairs = left_paired_df["Pair"]
    right_paired_inds = meta.loc[pairs, "Original index"].values
    left_inds = meta[meta["Hemisphere"] == "L"]["Original index"].values
    right_inds = meta[meta["Hemisphere"] == "R"]["Original index"].values

    adj = mg.adj.copy()
    colsums = np.sum(adj, axis=0)
    colsums[colsums == 0] = 1
    adj = adj / colsums[np.newaxis, :]
    print(np.sum(adj, axis=0))
    adj = pass_to_ranks(adj)
Ejemplo n.º 15
0
            rows.append(
                dict(
                    train_side="Right",
                    test="Same",
                    test_side="Right",
                    score=score,
                    level=l,
                    model=name,
                    n_params=n_params,
                    norm_score=score / right_adj.sum(),
                )
            )
    return pd.DataFrame(rows)


mg = MetaGraph(new_adj, new_meta)
mg = sort_mg(mg, level_names)
meta = mg.meta
adj = mg.adj

# set up figure
lowest_level = 7
# analysis, bars, colors, graph graph graph...
n_col = 1 + 2 + 1 + lowest_level + 1
n_row = 6
width_ratios = 4 * [1] + (lowest_level + 1) * [1.5]

fig = plt.figure(
    constrained_layout=False, figsize=(5 * 4 + (lowest_level + 1) * 8.5, 20)
)
gs = plt.GridSpec(nrows=n_row, ncols=n_col, figure=fig, width_ratios=width_ratios)
Ejemplo n.º 16
0
# %% [markdown]
# # Imports
from src.graph import MetaGraph
from src.data import load_networkx
from graspy.utils import is_fully_connected

# %% [markdown]
# # Constants
BRAIN_VERSION = "2019-12-18"
GRAPH_TYPE = "Gad"
# %% [markdown]
# # Loads
g = load_networkx(GRAPH_TYPE, BRAIN_VERSION)
mg = MetaGraph(g)

# %% [markdown]
# # Show that getting LCC works
print(is_fully_connected(mg.g))
print(mg.n_verts)
print(mg.meta.shape)
print()
mg = mg.make_lcc()
print(is_fully_connected(mg.g))
print(mg.n_verts)
print(mg.meta.shape)

# %% [markdown]
# #
Ejemplo n.º 17
0
sym_adj[:n_pairs, n_pairs:2 * n_pairs] = sym_contra_adj
sym_adj[n_pairs:2 * n_pairs, :n_pairs] = sym_contra_adj

side_labels = mg["Hemisphere"]
f = gridplot(
    [sym_adj],
    transform="binarize",
    height=20,
    sizes=(10, 20),
    inner_hier_labels=side_labels,
    sort_nodes=False,
    palette="deep",
    legend=False,
)
stashfig("sym-adj")
mg = MetaGraph(sym_adj, mg.meta)

# %% [markdown]
# # Preprocess graph
n_verts = mg.n_verts
mg.make_lcc()
print(f"Removed {n_verts - mg.n_verts} when finding the LCC")
# old_n_verts = sym_adj.shape[0]
# sym_adj, class_labels, side_labels = preprocess_graph(
#     sym_adj, class_labels, side_labels
# )
# n_verts = sym_adj.shape[0]
# print(f"Removed {old_n_verts - n_verts} nodes")
# %% [markdown]
# # Embedding
n_verts = mg.n_verts
Ejemplo n.º 18
0
metric = "bic"
bic_ratio = 1
d = 8  # embedding dimension
method = "color_iso"

basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"

exp = "137.2-BDP-omni-clust"

full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
# full_meta["sf"] = -full_meta["sf"]
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)

full_meta = full_mg.meta

# %% [markdown]
# ##
for i in range(1, 8):
    labels = full_meta[f"lvl{i}_labels"].values
    uni_labels, inv_labels = np.unique(labels, return_inverse=True)
    cmap = dict(zip(np.unique(inv_labels), cc.glasbey))
    colors = np.vectorize(cmap.get)(inv_labels)
    stashskels(
        f"lvl{i}",
        ids=full_meta.index.values.astype(int),
        labels=labels,
        colors=colors,
Ejemplo n.º 19
0
    comm_mg = mg.copy()
    ids = partition[partition == comm].index
    inds = comm_mg.meta.index.isin(ids)
    comm_mg = comm_mg.reindex(inds)
    is_al = comm_mg.meta["Merge Class"].isin(al_classes)
    heatmap(
        comm_mg.adj,
        inner_hier_labels=comm_mg["Merge Class"],
        outer_hier_labels=is_al,
        hier_label_fontsize=7,
        figsize=(20, 20),
        cbar=False,
    )
    adj = comm_mg.adj.copy()
    adj = symmetrize(adj, method="avg")
    sym_mg = MetaGraph(adj, comm_mg.meta)
    g_sym = sym_mg.g
    skeleton_labels = np.array(list(g_sym.nodes()))
    sub_partition, modularity = run_louvain(g_sym, 1, skeleton_labels)
    sub_partition = pd.Series(data=sub_partition, index=skeleton_labels)
    sub_partition.name = "sub-partition"
    sub_partition = sub_partition.reindex(comm_mg.meta.index)
    heatmap(
        comm_mg.adj,
        inner_hier_labels=sub_partition.values,
        hier_label_fontsize=7,
        figsize=(20, 20),
        cbar=False,
        sort_nodes=True,
    )
Ejemplo n.º 20
0
for graph_type in ["Gadn", "Gad", "G"]:
    if graph_type[-1] == "n":
        threshs = [0.01, 0.02]
    else:
        threshs = [1, 2, 3]
    for thresh in threshs:
        for r in [0.3, 0.5, 0.7, 1]:
            mg = load_metagraph(graph_type, version=BRAIN_VERSION)
            edgelist = mg.to_edgelist()
            edgelist = edgelist[edgelist["weight"] > thresh]
            nodelist = list(mg.g.nodes())
            thresh_g = nx.from_pandas_edgelist(edgelist,
                                               edge_attr=True,
                                               create_using=nx.DiGraph)
            nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
            mg = MetaGraph(thresh_g)
            print(len(mg.meta))
            mg = mg.make_lcc()
            print(len(mg.meta))
            not_pdiff = np.where(~mg["is_pdiff"])[0]
            mg = mg.reindex(not_pdiff)
            print(len(mg.meta))
            g_sym = nx.to_undirected(mg.g)
            skeleton_labels = np.array(list(g_sym.nodes()))
            out_dict = cm.best_partition(g_sym, resolution=r)
            partition = np.array(itemgetter(*skeleton_labels)(out_dict))
            adj = nx.to_numpy_array(g_sym, nodelist=skeleton_labels)

            part_unique, part_count = np.unique(partition, return_counts=True)
            for uni, count in zip(part_unique, part_count):
                if count < 3:
Ejemplo n.º 21
0
# %%
metric = "bic"
bic_ratio = 1
d = 8  # embedding dimension
method = "iso"

basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}-G"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"

exp = "137.0-BDP-omni-clust"

# load data
meta = readcsv("meta" + basename, foldername=exp, index_col=0)
meta["lvl0_labels"] = meta["lvl0_labels"].astype(str)
adj = readcsv("adj" + basename, foldername=exp, index_col=0)
mg = MetaGraph(adj.values, meta)

meta = mg.meta
adj = mg.adj
lp_inds, rp_inds = get_paired_inds(meta)


# parameters
lowest_level = 7
n_levels = 10


# %% [markdown]
# ##
rows = []
Ejemplo n.º 22
0
def _process_metagraph(mg, temp_loc):
    adj = mg.adj
    adj = symmetrize(adj, method="avg")
    mg = MetaGraph(adj, mg.meta)
    nx.write_graphml(mg.g, temp_loc)
Ejemplo n.º 23
0
    )

n_pairs = mg.adj.shape[0] // 2

mg.verify()


# %% [markdown]
# # Extract subgraphs


extract_fb = True
if extract_fb:
    from_classes = ["MBON", "FAN", "FBN", "FB2N"]
    to_classes = ["MBIN"]
    sub_mg = MetaGraph(mg.adj, mg.meta)
    from_class_inds = sub_mg.meta["Class 1"].isin(from_classes)
    to_class_inds = sub_mg.meta["Class 1"].isin(to_classes)
    any_inds = np.logical_or(from_class_inds, to_class_inds)
    sub_mg.reindex(any_inds)
    sub_mg.sort_values(["Hemisphere", "Class 1", "Pair ID"])
    # meta = sub_mg.meta
    # meta["Original index"] = range(meta.shape[0])
    # meta.sort_values(
    #     ["Hemisphere", "Class 1", "Pair ID"],
    #     inplace=True,
    #     kind="mergesort",
    #     ascending=False,
    # )
    # temp_inds = meta["Original index"]
    # sub_mg = sub_mg.reindex(temp_inds)
Ejemplo n.º 24
0
def run_experiment(graph_type=None,
                   threshold=None,
                   res=None,
                   binarize=None,
                   seed=None,
                   param_key=None):
    # common names
    if BLIND:
        basename = f"{param_key}-"
        title = param_key
    else:
        basename = f"louvain-res{res}-t{threshold}-{graph_type}-"
        title = f"Louvain, {graph_type}, res = {res}, threshold = {threshold}"

    np.random.seed(seed)

    # load and preprocess the data
    mg = load_metagraph(graph_type, version=BRAIN_VERSION)
    mg = preprocess(
        mg,
        threshold=threshold,
        sym_threshold=True,
        remove_pdiff=True,
        binarize=binarize,
    )
    adj = mg.adj
    adj = symmetrize(adj, method="avg")
    mg = MetaGraph(adj, mg.meta)
    g_sym = mg.g
    skeleton_labels = np.array(list(g_sym.nodes()))
    partition, modularity = run_louvain(g_sym, res, skeleton_labels)

    partition_series = pd.Series(partition, index=skeleton_labels)
    partition_series.name = param_key

    if SAVEFIGS:
        # get out some metadata
        class_label_dict = nx.get_node_attributes(g_sym, "Merge Class")
        class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict))
        lineage_label_dict = nx.get_node_attributes(g_sym, "lineage")
        lineage_labels = np.array(
            itemgetter(*skeleton_labels)(lineage_label_dict))
        lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
        classlin_labels, color_dict, hatch_dict = augment_classes(
            class_labels, lineage_labels)

        # TODO then sort all of them by proportion of sensory/motor
        # barplot by merge class and lineage
        _, _, order = barplot_text(
            partition,
            classlin_labels,
            color_dict=color_dict,
            plot_proportions=False,
            norm_bar_width=True,
            figsize=(24, 18),
            title=title,
            hatch_dict=hatch_dict,
            return_order=True,
        )
        stashfig(basename + "barplot-mergeclasslin-props")
        category_order = np.unique(partition)[order]

        fig, axs = barplot_text(
            partition,
            class_labels,
            color_dict=color_dict,
            plot_proportions=False,
            norm_bar_width=True,
            figsize=(24, 18),
            title=title,
            hatch_dict=None,
            category_order=category_order,
        )
        stashfig(basename + "barplot-mergeclass-props")
        fig, axs = barplot_text(
            partition,
            class_labels,
            color_dict=color_dict,
            plot_proportions=False,
            norm_bar_width=False,
            figsize=(24, 18),
            title=title,
            hatch_dict=None,
            category_order=category_order,
        )
        stashfig(basename + "barplot-mergeclass-counts")

        # TODO add gridmap

        counts = False
        weights = False
        prob_df = get_blockmodel_df(mg.adj,
                                    partition,
                                    return_counts=counts,
                                    use_weights=weights)
        prob_df = prob_df.reindex(category_order, axis=0)
        prob_df = prob_df.reindex(category_order, axis=1)
        probplot(100 * prob_df,
                 fmt="2.0f",
                 figsize=(20, 20),
                 title=title,
                 font_scale=0.7)
        stashfig(basename + f"probplot-counts{counts}-weights{weights}")

    return partition_series, modularity
Ejemplo n.º 25
0
def run_experiment(seed=None, graph_type=None, threshold=None, param_key=None):
    np.random.seed(seed)
    if BLIND:
        temp_param_key = param_key.replace(
            " ", "")  # don't want spaces in filenames
        savename = f"{temp_param_key}-cell-types-"
        title = param_key
    else:
        savename = f"{graph_type}-t{threshold}-cell-types"
        title = f"{graph_type}, threshold = {threshold}"

    mg = load_metagraph(graph_type, version=VERSION)

    # simple threshold
    # TODO they will want symmetric threshold...
    # TODO maybe make that a parameter
    adj = mg.adj.copy()
    adj[adj <= threshold] = 0
    meta = mg.meta.copy()
    meta = pd.DataFrame(mg.meta["neuron_name"])
    mg = MetaGraph(adj, meta)

    # run the graphtool code
    temp_loc = f"maggot_models/data/interim/temp-{param_key}.graphml"
    block_series = run_minimize_blockmodel(mg, temp_loc)

    # manage the output
    mg = load_metagraph(graph_type, version=VERSION)
    mg.meta = pd.concat((mg.meta, block_series), axis=1)
    mg.meta["Original index"] = range(len(mg.meta))
    keep_inds = mg.meta[~mg.meta["block_label"].isna(
    )]["Original index"].values
    mg.reindex(keep_inds)
    if graph_type != "G":
        mg.verify(10000, graph_type=graph_type, version=VERSION)

    # deal with class labels
    lineage_labels = mg.meta["lineage"].values
    lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
    class_labels = mg["Merge Class"]
    skeleton_labels = mg.meta.index.values
    classlin_labels, color_dict, hatch_dict = augment_classes(
        skeleton_labels, class_labels, lineage_labels)
    block_label = mg["block_label"].astype(int)

    # barplot with unknown class labels merged in, proportions
    _, _, order = barplot_text(
        block_label,
        classlin_labels,
        norm_bar_width=True,
        color_dict=color_dict,
        hatch_dict=hatch_dict,
        title=title,
        figsize=(24, 18),
        return_order=True,
    )
    stashfig(savename + "barplot-mergeclasslin-props")
    category_order = np.unique(block_label)[order]

    # barplot with regular class labels
    barplot_text(
        block_label,
        class_labels,
        norm_bar_width=True,
        color_dict=color_dict,
        hatch_dict=hatch_dict,
        title=title,
        figsize=(24, 18),
        category_order=category_order,
    )
    stashfig(savename + "barplot-mergeclass-props")

    # barplot with unknown class labels merged in, counts
    barplot_text(
        block_label,
        classlin_labels,
        norm_bar_width=False,
        color_dict=color_dict,
        hatch_dict=hatch_dict,
        title=title,
        figsize=(24, 18),
        return_order=True,
        category_order=category_order,
    )
    stashfig(savename + "barplot-mergeclasslin-counts")

    # barplot of hemisphere membership
    fig, ax = plt.subplots(1, 1, figsize=(10, 20))
    stacked_barplot(
        block_label,
        mg["Hemisphere"],
        norm_bar_width=True,
        category_order=category_order,
        ax=ax,
    )
    remove_spines(ax)
    stashfig(savename + "barplot-hemisphere")

    # plot block probability matrix
    counts = False
    weights = False
    prob_df = get_blockmodel_df(mg.adj,
                                block_label,
                                return_counts=counts,
                                use_weights=weights)
    prob_df = prob_df.reindex(order, axis=0)
    prob_df = prob_df.reindex(order, axis=1)
    ax = probplot(100 * prob_df,
                  fmt="2.0f",
                  figsize=(20, 20),
                  title=title,
                  font_scale=0.4)
    stashfig(savename + "probplot")
    block_series.name = param_key
    return block_series
Ejemplo n.º 26
0
def stashfig(name, **kws):
    savefig(name, foldername=FNAME, save_on=SAVEFIGS, **kws)


mg = load_metagraph("Gn", BRAIN_VERSION)
meta = mg.meta.copy()
meta["Original index"] = range(meta.shape[0])
degree_df = mg.calculate_degrees()
meta = pd.concat((meta, degree_df), axis=1)
meta.sort_values(["Hemisphere", "Class 1", "Pair ID"],
                 inplace=True,
                 kind="mergesort")
temp_inds = meta["Original index"]
mg = mg.reindex(temp_inds)
mg = MetaGraph(mg.adj, meta)

heatmap(
    mg.adj,
    sort_nodes=False,
    inner_hier_labels=mg["Class 1"],
    outer_hier_labels=mg["Hemisphere"],
    transform="binarize",
    cbar=False,
    figsize=(30, 30),
    hier_label_fontsize=10,
)
stashfig("heatmap")

# %% [markdown]
# #
Ejemplo n.º 27
0
    )
    nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
    thresh_g = get_lcc(thresh_g)
    n_verts = len(thresh_g)

    n_missing = 0
    for n, data in thresh_g.nodes(data=True):
        pair = data["Pair"]
        pair_id = data["Pair ID"]
        if pair != -1:
            if pair not in thresh_g:
                thresh_g.node[n]["Pair"] = -1
                thresh_g.node[n]["Pair ID"] = -1
                n_missing += 1

    mg = MetaGraph(thresh_g, weight="max_norm_weight")
    meta = mg.meta

    adj = mg.adj.copy()
    # colsums = np.sum(adj, axis=0)
    # colsums[colsums == 0] = 1
    # adj = adj / colsums[np.newaxis, :]
    adj = pass_to_ranks(adj)
    if use_spl:
        adj = graph_shortest_path(adj)
    if plus_c:
        adj += np.min(adj)

    if embed == "lse":
        latent = lse(adj, None, ptr=False)
    elif embed == "ase":
Ejemplo n.º 28
0
# take the average of left and right. this could also be max
# average
# sym_ipsi_adj = (left_left_adj + right_right_adj) / 2
# sym_contra_adj = (left_right_adj + right_left_adj) / 2

# max
sym_ipsi_adj = np.maximum(left_left_adj, right_right_adj)
sym_contra_adj = np.maximum(left_right_adj, right_left_adj)

sym_adj = adj.copy()
sym_adj[:n_pairs, :n_pairs] = sym_ipsi_adj
sym_adj[n_pairs : 2 * n_pairs, n_pairs : 2 * n_pairs] = sym_ipsi_adj
sym_adj[:n_pairs, n_pairs : 2 * n_pairs] = sym_contra_adj
sym_adj[n_pairs : 2 * n_pairs, :n_pairs] = sym_contra_adj

full_sym_mg = MetaGraph(sym_adj, mg.meta)

n_verts = full_sym_mg.n_verts
full_sym_mg.make_lcc()
print(f"Removed {n_verts - full_sym_mg.n_verts} when finding the LCC")
adj = full_sym_mg.adj
side_labels = full_sym_mg["Hemisphere"]
class_labels = full_sym_mg["Merge Class"]
latent, laplacian = lse(adj, N_COMPONENTS, regularizer=None, ptr=PTR)
latent_dim = latent.shape[1] // 2
screeplot(
    laplacian, title=f"Laplacian scree plot, R-DAD (ZG2 = {latent_dim} + {latent_dim})"
)

quick_gridmap(adj, side_labels)
stashfig(preface + "adj")
Ejemplo n.º 29
0

rows = []
n_samples = 5
lvls = ["lvl0_labels", "lvl1_labels", "lvl2_labels"]
for lvl in lvls:
    estimator = DCSBMEstimator(degree_directed=True,
                               directed=True,
                               loops=False)
    estimator.fit(adj, meta[lvl].values)
    for i in range(n_samples):
        sample = np.squeeze(estimator.sample())
        sample_meta = meta.copy()
        sf = signal_flow(sample)
        sample_meta["signal_flow"] = -sf
        sample_mg = MetaGraph(sample, sample_meta)
        sample_mg = sample_mg.sort_values("signal_flow", ascending=True)
        prop = upper_triu_prop(sample_mg.adj)
        print(prop)
        row = {"level": lvl.replace("_labels", ""), "prop": prop}
        rows.append(row)
    print()

bin_meta = meta.copy()
bin_adj = binarize(adj)
sf = signal_flow(bin_adj)
bin_meta["signal_flow"] = -sf
bin_mg = MetaGraph(bin_adj, bin_meta)
bin_mb = bin_mg.sort_values("signal_flow", ascending=True)
prop = upper_triu_prop(bin_mg.adj)
print(prop)
Ejemplo n.º 30
0
            meta=full_mg.meta,
            palette=CLASS_COLOR_DICT,
            colors="merge_class",
            ticks=False,
            gridline_kws=dict(linewidth=0.2, color="grey", linestyle="--"),
        )
        top.set_title(f"Level {level} - DCSBM sample")


# %% [markdown]
# ##
# load data
full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)

# TODO add the option for a random permutation here, respecting pairs
if permute_prop > 0:
    meta = full_mg.meta

    label_inds = [meta.columns.get_loc(l) for l in level_names]
    side_label_inds = [meta.columns.get_loc(l + "_side") for l in level_names]
    col_inds = label_inds + side_label_inds

    uni_pairs = meta[meta["pair"].isin(meta.index)]["pair_id"].unique()

    n_pairs = len(uni_pairs)
    n_permute = int(n_pairs * permute_prop)
    perm_pairs = np.random.choice(uni_pairs, size=n_permute, replace=False)