def max_symmetrize(mg, n_pairs):
""" assumes that mg is sorted
Parameters
----------
mg : [type]
[description]
n_pairs : [type]
[description]
"""
adj = mg.adj
left_left_adj = adj[:n_pairs, :n_pairs]
left_right_adj = adj[:n_pairs, n_pairs:2 * n_pairs]
right_right_adj = adj[n_pairs:2 * n_pairs, n_pairs:2 * n_pairs]
right_left_adj = adj[n_pairs:2 * n_pairs, :n_pairs]
# max, average gives similar results
sym_ipsi_adj = np.maximum(left_left_adj, right_right_adj)
sym_contra_adj = np.maximum(left_right_adj, right_left_adj)
sym_adj = adj.copy()
sym_adj[:n_pairs, :n_pairs] = sym_ipsi_adj
sym_adj[n_pairs:2 * n_pairs, n_pairs:2 * n_pairs] = sym_ipsi_adj
sym_adj[:n_pairs, n_pairs:2 * n_pairs] = sym_contra_adj
sym_adj[n_pairs:2 * n_pairs, :n_pairs] = sym_contra_adj
sym_mg = MetaGraph(sym_adj,
mg.meta) # did not change indices order so this ok
return sym_mg
def edgelist_to_mg(edgelist, meta):
g = nx.from_pandas_edgelist(edgelist,
edge_attr=True,
create_using=nx.DiGraph)
nx.set_node_attributes(g, meta.to_dict(orient="index"))
mg = MetaGraph(g)
return mg
def run_minimize_blockmodel(mg, temp_loc=None, weight_model=None):
meta = mg.meta.copy()
meta = pd.DataFrame(mg.meta["neuron_name"])
mg = MetaGraph(mg.adj, meta)
if temp_loc is None:
temp_loc = f"maggot_models/data/interim/temp-{np.random.randint(1e8)}.graphml"
# save to temp
nx.write_graphml(mg.g, temp_loc)
# load into graph-tool from temp
g = load_graph(temp_loc, fmt="graphml")
os.remove(temp_loc)
total_degrees = g.get_total_degrees(g.get_vertices())
remove_verts = np.where(total_degrees == 0)[0]
g.remove_vertex(remove_verts)
if weight_model is not None:
recs = [g.ep.weight]
rec_types = [weight_model]
else:
recs = []
rec_types = []
state_args = dict(recs=recs, rec_types=rec_types)
min_state = minimize_blockmodel_dl(g, verbose=False, state_args=state_args)
blocks = list(min_state.get_blocks())
verts = g.get_vertices()
block_map = {}
for v, b in zip(verts, blocks):
cell_id = int(g.vertex_properties["_graphml_vertex_id"][v])
block_map[cell_id] = int(b)
block_series = pd.Series(block_map)
block_series.name = "block_label"
return block_series
n_levels = 10 # max # of splits
metric = "bic"
bic_ratio = 1
d = 10 # embedding dimension
method = "aniso"
basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"
exp = "137.0-BDP-omni-clust"
full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)
# %% [markdown]
# ##
full_meta = full_mg.meta
# parameters
lowest_level = 7
width = 0.5
gap = 10
# this determines the sorting for everybody
level_names = [f"lvl{i}_labels" for i in range(lowest_level + 1)]
sort_class = level_names + ["merge_class"]
class_order = ["sf"]
metric = "bic"
bic_ratio = 1
d = 8 # embedding dimension
method = "color_iso"
basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"
exp = "137.2-BDP-omni-clust"
# load data
pair_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
pair_meta["lvl0_labels"] = pair_meta["lvl0_labels"].astype(str)
pair_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
pair_adj = pair_adj.values
mg = MetaGraph(pair_adj, pair_meta)
meta = mg.meta
level_names = [f"lvl{i}_labels" for i in range(level + 1)]
def sort_mg(mg, level_names):
meta = mg.meta
sort_class = level_names + ["merge_class"]
class_order = ["sf"]
total_sort_by = []
for sc in sort_class:
for co in class_order:
class_value = meta.groupby(sc)[co].mean()
meta[f"{sc}_{co}_order"] = meta[sc].map(class_value)
total_sort_by.append(f"{sc}_{co}_order")
# ad_norm_mg.adj, sort_nodes=False, transform="binarize", figsize=(20, 20), cbar=False
# )
# heatmap(
# ad_norm_mg.adj,
# sort_nodes=False,
# transform="binarize",
# figsize=(20, 20),
# inner_hier_labels=ad_norm_mg["Merge Class"],
# outer_hier_labels=ad_norm_mg["Hemisphere"],
# hier_label_fontsize=5,
# )
threshold = 0.0
adj = ad_norm_mg.adj
adj[adj < threshold] = 0
mod_mg = MetaGraph(adj, ad_norm_mg.meta)
# check pairs
uni_pairs, counts = np.unique(mod_mg["Pair ID"], return_counts=True)
assert (counts[1:] == 2).all()
n_pairs = len(counts) - 1
mod_mg = max_symmetrize(mod_mg, n_pairs)
heatmap(
mod_mg.adj,
sort_nodes=False,
transform="binarize",
figsize=(20, 20),
inner_hier_labels=mod_mg["Class 1"],
outer_hier_labels=mod_mg["Hemisphere"],
# ##
# parameters for the experiment
metric = "bic"
bic_ratio = 1
d = 8 # embedding dimension
method = "iso"
# parameters for plotting
lowest_level = 7
width = 0.5
gap = 10
basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"
exp = "137.0-BDP-omni-clust"
# load data
pair_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
pair_meta["lvl0_labels"] = pair_meta["lvl0_labels"].astype(str)
pair_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
pair_mg = MetaGraph(pair_adj.values, pair_meta)
pair_meta = pair_mg.meta
lp_inds, rp_inds = get_paired_inds(pair_meta)
left_adj = pair_adj[np.ix_(lp_inds, lp_inds)]
right_adj = pair_adj[np.ix_(rp_inds, rp_inds)]
calc_blockmodel_df
for graph_type in ["Gadn", "Gad", "G"]:
if graph_type[-1] == "n":
threshs = [0.01, 0.02]
else:
threshs = [1, 2, 3]
for thresh in threshs:
for r in [0.3, 0.5, 0.7, 1]:
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
edgelist = mg.to_edgelist()
edgelist = edgelist[edgelist["weight"] > thresh]
nodelist = list(mg.g.nodes())
thresh_g = nx.from_pandas_edgelist(edgelist,
edge_attr=True,
create_using=nx.DiGraph)
nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
mg = MetaGraph(thresh_g)
print(len(mg.meta))
mg = mg.make_lcc()
print(len(mg.meta))
not_pdiff = np.where(~mg["is_pdiff"])[0]
mg = mg.reindex(not_pdiff)
print(len(mg.meta))
g_sym = nx.to_undirected(mg.g)
skeleton_labels = np.array(list(g_sym.nodes()))
out_dict = cm.best_partition(g_sym, resolution=r)
partition = np.array(itemgetter(*skeleton_labels)(out_dict))
adj = nx.to_numpy_array(g_sym, nodelist=skeleton_labels)
part_unique, part_count = np.unique(partition, return_counts=True)
for uni, count in zip(part_unique, part_count):
if count < 3:
# %% [markdown] # # from src.graph import MetaGraph import numpy as np import pandas as pd import networkx as nx n_verts = 5 adj = np.arange(0, n_verts**2).reshape(n_verts, n_verts) adj[0, :] = 0 adj[:, 0] = 0 print(adj) meta = pd.DataFrame(index=range(n_verts)) meta["title"] = ["zero", "one", "two", "three", "four"] meta["remove"] = [False, True, False, False, True] meta.index = ["0", "1", "2", "3", "4"] print(meta) mg = MetaGraph(adj, meta) # mg.make_lcc() mg.reindex(np.array([2, 3])) print(mg.meta) print(mg.adj) mg2 = MetaGraph(mg.g) print(mg2.adj) print(mg2.meta)
# %% [markdown]
# # Get rid of unpaired cells for now
mg.meta["is_paired"] = False
pairs = mg["Pair"]
is_paired = pairs != -1
mg.reindex(is_paired)
print(np.unique(mg["Pair"]))
meta = mg.meta
meta["Original index"] = range(meta.shape[0])
meta.sort_values(
["Hemisphere", "Pair ID"], inplace=True, kind="mergesort", ascending=False
)
temp_inds = meta["Original index"]
mg = mg.reindex(temp_inds)
mg = MetaGraph(mg.adj, meta)
plot_adj = False
if plot_adj:
heatmap(
mg.adj,
sort_nodes=False,
inner_hier_labels=mg["Class 1"],
outer_hier_labels=mg["Hemisphere"],
transform="binarize",
cbar=False,
figsize=(30, 30),
hier_label_fontsize=10,
)
stashfig("heatmap")
pairs = mg["Pair"]
is_paired = pairs != -1
mg.reindex(is_paired)
print(np.unique(mg["Pair"]))
meta = mg.meta
# %% [markdown]
# #
meta["Original index"] = range(meta.shape[0])
meta.sort_values(["Hemisphere", "Pair ID"],
inplace=True,
kind="mergesort",
ascending=False)
temp_inds = meta["Original index"]
mg = mg.reindex(temp_inds)
mg = MetaGraph(mg.adj, meta)
plot_adj = True
if plot_adj:
heatmap(
mg.adj,
sort_nodes=False,
inner_hier_labels=mg["Class 1"],
outer_hier_labels=mg["Hemisphere"],
transform="binarize",
cbar=False,
figsize=(30, 30),
hier_label_fontsize=10,
)
heatmap(mg.adj,
sort_nodes=False,
metric = "bic"
bic_ratio = 1
d = 8 # embedding dimension
method = "iso"
basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}-Gad"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"
exp = "137.0-BDP-omni-clust"
full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
# full_meta["sf"] = -full_meta["sf"]
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)
full_meta = full_mg.meta
# parameters
lowest_level = 7
width = 0.5
gap = 10
# this determines the sorting for everybody
level_names = [f"lvl{i}_labels" for i in range(lowest_level + 1)]
sort_class = level_names + ["merge_class"]
class_order = ["sf"]
total_sort_by = []
for sc in sort_class:
def load_metagraph(graph_type, path=DATA_DIR, version=DATA_VERSION):
g = load_networkx(graph_type, path=path, version=version)
mg = MetaGraph(g)
return mg
create_using=nx.DiGraph)
nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
thresh_g = get_lcc(thresh_g)
n_verts = len(thresh_g)
n_missing = 0
for n, data in thresh_g.nodes(data=True):
pair = data["Pair"]
pair_id = data["Pair ID"]
if pair != -1:
if pair not in thresh_g:
thresh_g.node[n]["Pair"] = -1
thresh_g.node[n]["Pair ID"] = -1
n_missing += 1
mg = MetaGraph(thresh_g)
meta = mg.meta
meta["Original index"] = range(len(meta))
left_paired_df = meta[(meta["Pair"] != -1) & (meta["Hemisphere"] == "L")]
left_paired_inds = left_paired_df["Original index"].values
pairs = left_paired_df["Pair"]
right_paired_inds = meta.loc[pairs, "Original index"].values
left_inds = meta[meta["Hemisphere"] == "L"]["Original index"].values
right_inds = meta[meta["Hemisphere"] == "R"]["Original index"].values
adj = mg.adj.copy()
colsums = np.sum(adj, axis=0)
colsums[colsums == 0] = 1
adj = adj / colsums[np.newaxis, :]
print(np.sum(adj, axis=0))
adj = pass_to_ranks(adj)
rows.append(
dict(
train_side="Right",
test="Same",
test_side="Right",
score=score,
level=l,
model=name,
n_params=n_params,
norm_score=score / right_adj.sum(),
)
)
return pd.DataFrame(rows)
mg = MetaGraph(new_adj, new_meta)
mg = sort_mg(mg, level_names)
meta = mg.meta
adj = mg.adj
# set up figure
lowest_level = 7
# analysis, bars, colors, graph graph graph...
n_col = 1 + 2 + 1 + lowest_level + 1
n_row = 6
width_ratios = 4 * [1] + (lowest_level + 1) * [1.5]
fig = plt.figure(
constrained_layout=False, figsize=(5 * 4 + (lowest_level + 1) * 8.5, 20)
)
gs = plt.GridSpec(nrows=n_row, ncols=n_col, figure=fig, width_ratios=width_ratios)
# %% [markdown] # # Imports from src.graph import MetaGraph from src.data import load_networkx from graspy.utils import is_fully_connected # %% [markdown] # # Constants BRAIN_VERSION = "2019-12-18" GRAPH_TYPE = "Gad" # %% [markdown] # # Loads g = load_networkx(GRAPH_TYPE, BRAIN_VERSION) mg = MetaGraph(g) # %% [markdown] # # Show that getting LCC works print(is_fully_connected(mg.g)) print(mg.n_verts) print(mg.meta.shape) print() mg = mg.make_lcc() print(is_fully_connected(mg.g)) print(mg.n_verts) print(mg.meta.shape) # %% [markdown] # #
sym_adj[:n_pairs, n_pairs:2 * n_pairs] = sym_contra_adj
sym_adj[n_pairs:2 * n_pairs, :n_pairs] = sym_contra_adj
side_labels = mg["Hemisphere"]
f = gridplot(
[sym_adj],
transform="binarize",
height=20,
sizes=(10, 20),
inner_hier_labels=side_labels,
sort_nodes=False,
palette="deep",
legend=False,
)
stashfig("sym-adj")
mg = MetaGraph(sym_adj, mg.meta)
# %% [markdown]
# # Preprocess graph
n_verts = mg.n_verts
mg.make_lcc()
print(f"Removed {n_verts - mg.n_verts} when finding the LCC")
# old_n_verts = sym_adj.shape[0]
# sym_adj, class_labels, side_labels = preprocess_graph(
# sym_adj, class_labels, side_labels
# )
# n_verts = sym_adj.shape[0]
# print(f"Removed {old_n_verts - n_verts} nodes")
# %% [markdown]
# # Embedding
n_verts = mg.n_verts
metric = "bic"
bic_ratio = 1
d = 8 # embedding dimension
method = "color_iso"
basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"
exp = "137.2-BDP-omni-clust"
full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
# full_meta["sf"] = -full_meta["sf"]
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)
full_meta = full_mg.meta
# %% [markdown]
# ##
for i in range(1, 8):
labels = full_meta[f"lvl{i}_labels"].values
uni_labels, inv_labels = np.unique(labels, return_inverse=True)
cmap = dict(zip(np.unique(inv_labels), cc.glasbey))
colors = np.vectorize(cmap.get)(inv_labels)
stashskels(
f"lvl{i}",
ids=full_meta.index.values.astype(int),
labels=labels,
colors=colors,
comm_mg = mg.copy()
ids = partition[partition == comm].index
inds = comm_mg.meta.index.isin(ids)
comm_mg = comm_mg.reindex(inds)
is_al = comm_mg.meta["Merge Class"].isin(al_classes)
heatmap(
comm_mg.adj,
inner_hier_labels=comm_mg["Merge Class"],
outer_hier_labels=is_al,
hier_label_fontsize=7,
figsize=(20, 20),
cbar=False,
)
adj = comm_mg.adj.copy()
adj = symmetrize(adj, method="avg")
sym_mg = MetaGraph(adj, comm_mg.meta)
g_sym = sym_mg.g
skeleton_labels = np.array(list(g_sym.nodes()))
sub_partition, modularity = run_louvain(g_sym, 1, skeleton_labels)
sub_partition = pd.Series(data=sub_partition, index=skeleton_labels)
sub_partition.name = "sub-partition"
sub_partition = sub_partition.reindex(comm_mg.meta.index)
heatmap(
comm_mg.adj,
inner_hier_labels=sub_partition.values,
hier_label_fontsize=7,
figsize=(20, 20),
cbar=False,
sort_nodes=True,
)
for graph_type in ["Gadn", "Gad", "G"]:
if graph_type[-1] == "n":
threshs = [0.01, 0.02]
else:
threshs = [1, 2, 3]
for thresh in threshs:
for r in [0.3, 0.5, 0.7, 1]:
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
edgelist = mg.to_edgelist()
edgelist = edgelist[edgelist["weight"] > thresh]
nodelist = list(mg.g.nodes())
thresh_g = nx.from_pandas_edgelist(edgelist,
edge_attr=True,
create_using=nx.DiGraph)
nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
mg = MetaGraph(thresh_g)
print(len(mg.meta))
mg = mg.make_lcc()
print(len(mg.meta))
not_pdiff = np.where(~mg["is_pdiff"])[0]
mg = mg.reindex(not_pdiff)
print(len(mg.meta))
g_sym = nx.to_undirected(mg.g)
skeleton_labels = np.array(list(g_sym.nodes()))
out_dict = cm.best_partition(g_sym, resolution=r)
partition = np.array(itemgetter(*skeleton_labels)(out_dict))
adj = nx.to_numpy_array(g_sym, nodelist=skeleton_labels)
part_unique, part_count = np.unique(partition, return_counts=True)
for uni, count in zip(part_unique, part_count):
if count < 3:
# %%
metric = "bic"
bic_ratio = 1
d = 8 # embedding dimension
method = "iso"
basename = f"-method={method}-d={d}-bic_ratio={bic_ratio}-G"
title = f"Method={method}, d={d}, BIC ratio={bic_ratio}"
exp = "137.0-BDP-omni-clust"
# load data
meta = readcsv("meta" + basename, foldername=exp, index_col=0)
meta["lvl0_labels"] = meta["lvl0_labels"].astype(str)
adj = readcsv("adj" + basename, foldername=exp, index_col=0)
mg = MetaGraph(adj.values, meta)
meta = mg.meta
adj = mg.adj
lp_inds, rp_inds = get_paired_inds(meta)
# parameters
lowest_level = 7
n_levels = 10
# %% [markdown]
# ##
rows = []
def _process_metagraph(mg, temp_loc):
adj = mg.adj
adj = symmetrize(adj, method="avg")
mg = MetaGraph(adj, mg.meta)
nx.write_graphml(mg.g, temp_loc)
)
n_pairs = mg.adj.shape[0] // 2
mg.verify()
# %% [markdown]
# # Extract subgraphs
extract_fb = True
if extract_fb:
from_classes = ["MBON", "FAN", "FBN", "FB2N"]
to_classes = ["MBIN"]
sub_mg = MetaGraph(mg.adj, mg.meta)
from_class_inds = sub_mg.meta["Class 1"].isin(from_classes)
to_class_inds = sub_mg.meta["Class 1"].isin(to_classes)
any_inds = np.logical_or(from_class_inds, to_class_inds)
sub_mg.reindex(any_inds)
sub_mg.sort_values(["Hemisphere", "Class 1", "Pair ID"])
# meta = sub_mg.meta
# meta["Original index"] = range(meta.shape[0])
# meta.sort_values(
# ["Hemisphere", "Class 1", "Pair ID"],
# inplace=True,
# kind="mergesort",
# ascending=False,
# )
# temp_inds = meta["Original index"]
# sub_mg = sub_mg.reindex(temp_inds)
def run_experiment(graph_type=None,
threshold=None,
res=None,
binarize=None,
seed=None,
param_key=None):
# common names
if BLIND:
basename = f"{param_key}-"
title = param_key
else:
basename = f"louvain-res{res}-t{threshold}-{graph_type}-"
title = f"Louvain, {graph_type}, res = {res}, threshold = {threshold}"
np.random.seed(seed)
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(
mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=True,
binarize=binarize,
)
adj = mg.adj
adj = symmetrize(adj, method="avg")
mg = MetaGraph(adj, mg.meta)
g_sym = mg.g
skeleton_labels = np.array(list(g_sym.nodes()))
partition, modularity = run_louvain(g_sym, res, skeleton_labels)
partition_series = pd.Series(partition, index=skeleton_labels)
partition_series.name = param_key
if SAVEFIGS:
# get out some metadata
class_label_dict = nx.get_node_attributes(g_sym, "Merge Class")
class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict))
lineage_label_dict = nx.get_node_attributes(g_sym, "lineage")
lineage_labels = np.array(
itemgetter(*skeleton_labels)(lineage_label_dict))
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
classlin_labels, color_dict, hatch_dict = augment_classes(
class_labels, lineage_labels)
# TODO then sort all of them by proportion of sensory/motor
# barplot by merge class and lineage
_, _, order = barplot_text(
partition,
classlin_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=hatch_dict,
return_order=True,
)
stashfig(basename + "barplot-mergeclasslin-props")
category_order = np.unique(partition)[order]
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=False,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-counts")
# TODO add gridmap
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(category_order, axis=0)
prob_df = prob_df.reindex(category_order, axis=1)
probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.7)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
return partition_series, modularity
def run_experiment(seed=None, graph_type=None, threshold=None, param_key=None):
np.random.seed(seed)
if BLIND:
temp_param_key = param_key.replace(
" ", "") # don't want spaces in filenames
savename = f"{temp_param_key}-cell-types-"
title = param_key
else:
savename = f"{graph_type}-t{threshold}-cell-types"
title = f"{graph_type}, threshold = {threshold}"
mg = load_metagraph(graph_type, version=VERSION)
# simple threshold
# TODO they will want symmetric threshold...
# TODO maybe make that a parameter
adj = mg.adj.copy()
adj[adj <= threshold] = 0
meta = mg.meta.copy()
meta = pd.DataFrame(mg.meta["neuron_name"])
mg = MetaGraph(adj, meta)
# run the graphtool code
temp_loc = f"maggot_models/data/interim/temp-{param_key}.graphml"
block_series = run_minimize_blockmodel(mg, temp_loc)
# manage the output
mg = load_metagraph(graph_type, version=VERSION)
mg.meta = pd.concat((mg.meta, block_series), axis=1)
mg.meta["Original index"] = range(len(mg.meta))
keep_inds = mg.meta[~mg.meta["block_label"].isna(
)]["Original index"].values
mg.reindex(keep_inds)
if graph_type != "G":
mg.verify(10000, graph_type=graph_type, version=VERSION)
# deal with class labels
lineage_labels = mg.meta["lineage"].values
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
class_labels = mg["Merge Class"]
skeleton_labels = mg.meta.index.values
classlin_labels, color_dict, hatch_dict = augment_classes(
skeleton_labels, class_labels, lineage_labels)
block_label = mg["block_label"].astype(int)
# barplot with unknown class labels merged in, proportions
_, _, order = barplot_text(
block_label,
classlin_labels,
norm_bar_width=True,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
return_order=True,
)
stashfig(savename + "barplot-mergeclasslin-props")
category_order = np.unique(block_label)[order]
# barplot with regular class labels
barplot_text(
block_label,
class_labels,
norm_bar_width=True,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
category_order=category_order,
)
stashfig(savename + "barplot-mergeclass-props")
# barplot with unknown class labels merged in, counts
barplot_text(
block_label,
classlin_labels,
norm_bar_width=False,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
return_order=True,
category_order=category_order,
)
stashfig(savename + "barplot-mergeclasslin-counts")
# barplot of hemisphere membership
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
stacked_barplot(
block_label,
mg["Hemisphere"],
norm_bar_width=True,
category_order=category_order,
ax=ax,
)
remove_spines(ax)
stashfig(savename + "barplot-hemisphere")
# plot block probability matrix
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
block_label,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(order, axis=0)
prob_df = prob_df.reindex(order, axis=1)
ax = probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.4)
stashfig(savename + "probplot")
block_series.name = param_key
return block_series
def stashfig(name, **kws):
savefig(name, foldername=FNAME, save_on=SAVEFIGS, **kws)
mg = load_metagraph("Gn", BRAIN_VERSION)
meta = mg.meta.copy()
meta["Original index"] = range(meta.shape[0])
degree_df = mg.calculate_degrees()
meta = pd.concat((meta, degree_df), axis=1)
meta.sort_values(["Hemisphere", "Class 1", "Pair ID"],
inplace=True,
kind="mergesort")
temp_inds = meta["Original index"]
mg = mg.reindex(temp_inds)
mg = MetaGraph(mg.adj, meta)
heatmap(
mg.adj,
sort_nodes=False,
inner_hier_labels=mg["Class 1"],
outer_hier_labels=mg["Hemisphere"],
transform="binarize",
cbar=False,
figsize=(30, 30),
hier_label_fontsize=10,
)
stashfig("heatmap")
# %% [markdown]
# #
)
nx.set_node_attributes(thresh_g, mg.meta.to_dict(orient="index"))
thresh_g = get_lcc(thresh_g)
n_verts = len(thresh_g)
n_missing = 0
for n, data in thresh_g.nodes(data=True):
pair = data["Pair"]
pair_id = data["Pair ID"]
if pair != -1:
if pair not in thresh_g:
thresh_g.node[n]["Pair"] = -1
thresh_g.node[n]["Pair ID"] = -1
n_missing += 1
mg = MetaGraph(thresh_g, weight="max_norm_weight")
meta = mg.meta
adj = mg.adj.copy()
# colsums = np.sum(adj, axis=0)
# colsums[colsums == 0] = 1
# adj = adj / colsums[np.newaxis, :]
adj = pass_to_ranks(adj)
if use_spl:
adj = graph_shortest_path(adj)
if plus_c:
adj += np.min(adj)
if embed == "lse":
latent = lse(adj, None, ptr=False)
elif embed == "ase":
# take the average of left and right. this could also be max
# average
# sym_ipsi_adj = (left_left_adj + right_right_adj) / 2
# sym_contra_adj = (left_right_adj + right_left_adj) / 2
# max
sym_ipsi_adj = np.maximum(left_left_adj, right_right_adj)
sym_contra_adj = np.maximum(left_right_adj, right_left_adj)
sym_adj = adj.copy()
sym_adj[:n_pairs, :n_pairs] = sym_ipsi_adj
sym_adj[n_pairs : 2 * n_pairs, n_pairs : 2 * n_pairs] = sym_ipsi_adj
sym_adj[:n_pairs, n_pairs : 2 * n_pairs] = sym_contra_adj
sym_adj[n_pairs : 2 * n_pairs, :n_pairs] = sym_contra_adj
full_sym_mg = MetaGraph(sym_adj, mg.meta)
n_verts = full_sym_mg.n_verts
full_sym_mg.make_lcc()
print(f"Removed {n_verts - full_sym_mg.n_verts} when finding the LCC")
adj = full_sym_mg.adj
side_labels = full_sym_mg["Hemisphere"]
class_labels = full_sym_mg["Merge Class"]
latent, laplacian = lse(adj, N_COMPONENTS, regularizer=None, ptr=PTR)
latent_dim = latent.shape[1] // 2
screeplot(
laplacian, title=f"Laplacian scree plot, R-DAD (ZG2 = {latent_dim} + {latent_dim})"
)
quick_gridmap(adj, side_labels)
stashfig(preface + "adj")
rows = []
n_samples = 5
lvls = ["lvl0_labels", "lvl1_labels", "lvl2_labels"]
for lvl in lvls:
estimator = DCSBMEstimator(degree_directed=True,
directed=True,
loops=False)
estimator.fit(adj, meta[lvl].values)
for i in range(n_samples):
sample = np.squeeze(estimator.sample())
sample_meta = meta.copy()
sf = signal_flow(sample)
sample_meta["signal_flow"] = -sf
sample_mg = MetaGraph(sample, sample_meta)
sample_mg = sample_mg.sort_values("signal_flow", ascending=True)
prop = upper_triu_prop(sample_mg.adj)
print(prop)
row = {"level": lvl.replace("_labels", ""), "prop": prop}
rows.append(row)
print()
bin_meta = meta.copy()
bin_adj = binarize(adj)
sf = signal_flow(bin_adj)
bin_meta["signal_flow"] = -sf
bin_mg = MetaGraph(bin_adj, bin_meta)
bin_mb = bin_mg.sort_values("signal_flow", ascending=True)
prop = upper_triu_prop(bin_mg.adj)
print(prop)
meta=full_mg.meta,
palette=CLASS_COLOR_DICT,
colors="merge_class",
ticks=False,
gridline_kws=dict(linewidth=0.2, color="grey", linestyle="--"),
)
top.set_title(f"Level {level} - DCSBM sample")
# %% [markdown]
# ##
# load data
full_meta = readcsv("meta" + basename, foldername=exp, index_col=0)
full_meta["lvl0_labels"] = full_meta["lvl0_labels"].astype(str)
full_adj = readcsv("adj" + basename, foldername=exp, index_col=0)
full_mg = MetaGraph(full_adj.values, full_meta)
# TODO add the option for a random permutation here, respecting pairs
if permute_prop > 0:
meta = full_mg.meta
label_inds = [meta.columns.get_loc(l) for l in level_names]
side_label_inds = [meta.columns.get_loc(l + "_side") for l in level_names]
col_inds = label_inds + side_label_inds
uni_pairs = meta[meta["pair"].isin(meta.index)]["pair_id"].unique()
n_pairs = len(uni_pairs)
n_permute = int(n_pairs * permute_prop)
perm_pairs = np.random.choice(uni_pairs, size=n_permute, replace=False)