def to_minigraph(
adj,
labels,
drop_neg=True,
remove_diag=True,
size_scaler=1,
use_counts=False,
use_weights=True,
color_map=None,
):
# convert the adjacency and a partition to a minigraph based on SBM probs
prob_df = get_blockmodel_df(
adj, labels, return_counts=use_counts, use_weights=use_weights
)
if drop_neg and ("-1" in prob_df.index):
prob_df.drop("-1", axis=0, inplace=True)
prob_df.drop("-1", axis=1, inplace=True)
if remove_diag:
adj = prob_df.values
adj -= np.diag(np.diag(adj))
prob_df.data = prob_df
g = nx.from_pandas_adjacency(prob_df, create_using=nx.DiGraph())
uni_labels, counts = np.unique(labels, return_counts=True)
# add size attribute base on number of vertices
size_map = dict(zip(uni_labels, size_scaler * counts))
nx.set_node_attributes(g, size_map, name="Size")
# add signal flow attribute (for the minigraph itself)
mini_adj = nx.to_numpy_array(g, nodelist=uni_labels)
node_signal_flow = signal_flow(mini_adj)
sf_map = dict(zip(uni_labels, node_signal_flow))
nx.set_node_attributes(g, sf_map, name="Signal Flow")
# add spectral properties
sym_adj = symmetrize(mini_adj)
n_components = 10
latent = AdjacencySpectralEmbed(n_components=n_components).fit_transform(sym_adj)
for i in range(n_components):
latent_dim = latent[:, i]
lap_map = dict(zip(uni_labels, latent_dim))
nx.set_node_attributes(g, lap_map, name=f"AdjEvec-{i}")
# add spring layout properties
pos = nx.spring_layout(g)
spring_x = {}
spring_y = {}
for key, val in pos.items():
spring_x[key] = val[0]
spring_y[key] = val[1]
nx.set_node_attributes(g, spring_x, name="Spring-x")
nx.set_node_attributes(g, spring_y, name="Spring-y")
# add colors
if color_map is None:
color_map = dict(zip(uni_labels, cc.glasbey_light))
nx.set_node_attributes(g, color_map, name="Color")
return g
# %% [markdown]
# ##
from mpl_toolkits.axes_grid1 import make_axes_locatable
from src.utils import get_blockmodel_df
labels = full_meta[f"lvl{lowest_level}_labels"].values
mid_map = {}
for key, val in first_mid_map.items():
new_key = key[:-1]
mid_map[new_key] = val
blockmodel_df = get_blockmodel_df(
full_adj, labels, return_counts=True, use_weights=True
)
group_sizes = full_meta.groupby([f"lvl{lowest_level}_labels"]).size()
blockmodel_df.index.name = "source"
blockmodel_df.columns.name = "target"
blockmodel_df.reset_index(inplace=True)
blockmodel_df
blockmodel_edges = blockmodel_df.melt(id_vars="source", value_name="weight")
blockmodel_edges["x"] = blockmodel_edges["target"].map(mid_map)
blockmodel_edges["y"] = blockmodel_edges["source"].map(mid_map)
blockmodel_edges["source_size"] = blockmodel_edges["source"].map(group_sizes)
blockmodel_edges["target_size"] = blockmodel_edges["target"].map(group_sizes)
blockmodel_edges["source_n_out"] = blockmodel_edges["source"].map(
blockmodel_edges.groupby("source")["weight"].sum()
)
blockmodel_edges["out_weight"] = (
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=False,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-counts")
plt.close()
# TODO add gridmap
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(category_order, axis=0)
prob_df = prob_df.reindex(category_order, axis=1)
probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.7)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
plt.close()
def run_experiment(seed=None, graph_type=None, threshold=None, param_key=None):
np.random.seed(seed)
if BLIND:
temp_param_key = param_key.replace(
" ", "") # don't want spaces in filenames
savename = f"{temp_param_key}-cell-types-"
title = param_key
else:
savename = f"{graph_type}-t{threshold}-cell-types"
title = f"{graph_type}, threshold = {threshold}"
mg = load_metagraph(graph_type, version=VERSION)
# simple threshold
# TODO they will want symmetric threshold...
# TODO maybe make that a parameter
adj = mg.adj.copy()
adj[adj <= threshold] = 0
meta = mg.meta.copy()
meta = pd.DataFrame(mg.meta["neuron_name"])
mg = MetaGraph(adj, meta)
# run the graphtool code
temp_loc = f"maggot_models/data/interim/temp-{param_key}.graphml"
block_series = run_minimize_blockmodel(mg, temp_loc)
# manage the output
mg = load_metagraph(graph_type, version=VERSION)
mg.meta = pd.concat((mg.meta, block_series), axis=1)
mg.meta["Original index"] = range(len(mg.meta))
keep_inds = mg.meta[~mg.meta["block_label"].isna(
)]["Original index"].values
mg.reindex(keep_inds)
if graph_type != "G":
mg.verify(10000, graph_type=graph_type, version=VERSION)
# deal with class labels
lineage_labels = mg.meta["lineage"].values
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
class_labels = mg["Merge Class"]
skeleton_labels = mg.meta.index.values
classlin_labels, color_dict, hatch_dict = augment_classes(
skeleton_labels, class_labels, lineage_labels)
block_label = mg["block_label"].astype(int)
# barplot with unknown class labels merged in, proportions
_, _, order = barplot_text(
block_label,
classlin_labels,
norm_bar_width=True,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
return_order=True,
)
stashfig(savename + "barplot-mergeclasslin-props")
category_order = np.unique(block_label)[order]
# barplot with regular class labels
barplot_text(
block_label,
class_labels,
norm_bar_width=True,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
category_order=category_order,
)
stashfig(savename + "barplot-mergeclass-props")
# barplot with unknown class labels merged in, counts
barplot_text(
block_label,
classlin_labels,
norm_bar_width=False,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
return_order=True,
category_order=category_order,
)
stashfig(savename + "barplot-mergeclasslin-counts")
# barplot of hemisphere membership
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
stacked_barplot(
block_label,
mg["Hemisphere"],
norm_bar_width=True,
category_order=category_order,
ax=ax,
)
remove_spines(ax)
stashfig(savename + "barplot-hemisphere")
# plot block probability matrix
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
block_label,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(order, axis=0)
prob_df = prob_df.reindex(order, axis=1)
ax = probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.4)
stashfig(savename + "probplot")
block_series.name = param_key
return block_series
sharex=True,
sharey=False,
hue="Cluster",
hue_order=pred_color_dict.keys(),
palette=pred_color_dict.values(),
col_order=leaf_names,
)
fg.map(sns.distplot, "Total degree", norm_hist=True, kde=False)
fg.set(yticks=[], yticklabels=[])
stashfig("total-degree" + name_base)
# %% [markdown]
# #
blockmodel_df = get_blockmodel_df(adj,
pred_labels,
use_weights=True,
return_counts=False)
plt.figure(figsize=(20, 20))
sns.heatmap(blockmodel_df, cmap="Reds")
g = nx.from_pandas_adjacency(blockmodel_df, create_using=nx.DiGraph())
uni_labels, counts = np.unique(pred_labels, return_counts=True)
size_scaler = 5
size_map = dict(zip(uni_labels, size_scaler * counts))
nx.set_node_attributes(g, size_map, name="Size")
mini_adj = nx.to_numpy_array(g, nodelist=uni_labels)
node_signal_flow = signal_flow(mini_adj)
sf_map = dict(zip(uni_labels, node_signal_flow))
nx.set_node_attributes(g, sf_map, name="Signal Flow")
sym_adj = symmetrize(mini_adj)
node_lap = LaplacianSpectralEmbed(n_components=1).fit_transform(sym_adj)
heatmap(
mg.adj,
transform="simple-nonzero",
figsize=(20, 20),
inner_hier_labels=partition,
hier_label_fontsize=10,
title=title,
title_pad=80,
)
stashfig(basename + "heatmap")
# block probabilities
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(sort_partition_sf.index, axis=0)
prob_df = prob_df.reindex(sort_partition_sf.index, axis=1)
ax = probplot(
100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=
f"Louvain, res = {r}, counts = {counts}, weights = {weights}",
)
ax.set_ylabel(r"Median signal flow $\to$", fontsize=28)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
def run_experiment(graph_type=None,
threshold=None,
res=None,
binarize=None,
seed=None,
param_key=None):
# common names
if BLIND:
basename = f"{param_key}-"
title = param_key
else:
basename = f"louvain-res{res}-t{threshold}-{graph_type}-"
title = f"Louvain, {graph_type}, res = {res}, threshold = {threshold}"
np.random.seed(seed)
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(
mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=True,
binarize=binarize,
)
g_sym = nx.to_undirected(mg.g)
skeleton_labels = np.array(list(g_sym.nodes()))
partition, modularity = run_louvain(g_sym, res, skeleton_labels)
partition_series = pd.Series(partition, index=skeleton_labels)
partition_series.name = param_key
if SAVEFIGS:
# get out some metadata
class_label_dict = nx.get_node_attributes(g_sym, "Merge Class")
class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict))
lineage_label_dict = nx.get_node_attributes(g_sym, "lineage")
lineage_labels = np.array(
itemgetter(*skeleton_labels)(lineage_label_dict))
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
classlin_labels, color_dict, hatch_dict = augment_classes(
class_labels, lineage_labels)
# TODO then sort all of them by proportion of sensory/motor
# barplot by merge class and lineage
_, _, order = barplot_text(
partition,
classlin_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=hatch_dict,
return_order=True,
)
stashfig(basename + "barplot-mergeclasslin-props")
category_order = np.unique(partition)[order]
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=False,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-counts")
# TODO add gridmap
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(category_order, axis=0)
prob_df = prob_df.reindex(category_order, axis=1)
probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.7)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
return partition_series, modularity
def run_experiment(graph_type=None, thresh=None, res=None):
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
edgelist = mg.to_edgelist()
edgelist = add_max_weight(edgelist)
edgelist = edgelist[edgelist["max_weight"] > thresh]
mg = edgelist_to_mg(edgelist, mg.meta)
mg = mg.make_lcc()
mg = mg.remove_pdiff()
g_sym = nx.to_undirected(mg.g)
skeleton_labels = np.array(list(g_sym.nodes()))
partition = run_louvain(g_sym, res, skeleton_labels)
# compute signal flow for sorting purposes
mg.meta["signal_flow"] = signal_flow(mg.adj)
mg.meta["partition"] = partition
partition_sf = mg.meta.groupby("partition")["signal_flow"].median()
sort_partition_sf = partition_sf.sort_values(ascending=False)
# common names
basename = f"louvain-res{res}-t{thresh}-{graph_type}-"
title = f"Louvain, {graph_type}, res = {res}, thresh = {thresh}"
# get out some metadata
class_label_dict = nx.get_node_attributes(g_sym, "Merge Class")
class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict))
lineage_label_dict = nx.get_node_attributes(g_sym, "lineage")
lineage_labels = np.array(itemgetter(*skeleton_labels)(lineage_label_dict))
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
classlin_labels, color_dict, hatch_dict = augment_classes(
skeleton_labels, class_labels, lineage_labels)
# barplot by merge class and lineage
fig, axs = barplot_text(
partition,
classlin_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=hatch_dict,
)
stashfig(basename + "barplot-mergeclasslin-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=None,
)
stashfig(basename + "barplot-mergeclass-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=False,
figsize=(24, 18),
title=title,
hatch_dict=None,
)
stashfig(basename + "barplot-mergeclass-counts")
fig, axs = barplot_text(
partition,
lineage_labels,
color_dict=None,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
)
stashfig(basename + "barplot-lineage-props")
# sorted heatmap
heatmap(
mg.adj,
transform="simple-nonzero",
figsize=(20, 20),
inner_hier_labels=partition,
hier_label_fontsize=10,
title=title,
title_pad=80,
)
stashfig(basename + "heatmap")
# block probability matrices
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(sort_partition_sf.index, axis=0)
prob_df = prob_df.reindex(sort_partition_sf.index, axis=1)
ax = probplot(
100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=f"Louvain, res = {res}, counts = {counts}, weights = {weights}",
)
ax.set_ylabel(r"Median signal flow $\to$", fontsize=28)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
# plot minigraph with layout
adjusted_partition = adjust_partition(partition, class_labels)
minigraph = to_minigraph(mg.adj,
adjusted_partition,
use_counts=True,
size_scaler=10)
draw_networkx_nice(
minigraph,
"Spring-x",
"Signal Flow",
sizes="Size",
colors="Color",
cmap="Greys",
vmin=100,
weight_scale=0.001,
)
stashfig(basename + "sbm-drawn-network")
