def compute_ari(idx, param_df, classes, class_type="Class 1", remove_non_mb=False):
preprocess_params = dict(param_df.loc[idx, ["binarize", "threshold"]])
graph_type = param_df.loc[idx, "graph_type"]
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(mg, sym_threshold=True, remove_pdiff=True, **preprocess_params)
left_mb_indicator = mg.meta[class_type].isin(classes) & (
mg.meta["Hemisphere"] == "L"
)
right_mb_indicator = mg.meta[class_type].isin(classes) & (
mg.meta["Hemisphere"] == "R"
)
labels = np.zeros(len(mg.meta))
labels[left_mb_indicator.values] = 1
labels[right_mb_indicator.values] = 2
pred_labels = best_block_df[idx]
pred_labels = pred_labels[pred_labels.index.isin(mg.meta.index)]
assert np.array_equal(pred_labels.index, mg.meta.index), print(idx)
if remove_non_mb: # only consider ARI for clusters with some MB mass
uni_pred = np.unique(pred_labels)
keep_mask = np.ones(len(labels), dtype=bool)
for p in uni_pred:
if np.sum(labels[pred_labels == p]) == 0:
keep_mask[pred_labels == p] = False
labels = labels[keep_mask]
pred_labels = pred_labels[keep_mask]
ari = adjusted_rand_score(labels, pred_labels)
return ari
def plot_class_colormap():
from src.visualization import palplot
from src.data import load_metagraph
mg = load_metagraph("G")
uni_class, counts = np.unique(mg["merge_class"], return_counts=True)
count_map = dict(zip(uni_class, counts))
names = []
colors = []
for key, val in CLASS_COLOR_DICT.items():
if key in uni_class:
names.append(f"{key} ({count_map[key]})")
colors.append(val)
fig, ax = plt.subplots(1, 1, figsize=(3, 15))
palplot(len(colors), colors, ax=ax)
ax.yaxis.set_major_formatter(plt.FixedFormatter(names))
plt.savefig("./maggot_models/notebooks/outs/current_cmap.png",
dpi=300,
bbox_inches="tight")
def compute_ari(idx):
preprocess_params = dict(best_param_df.loc[idx, ["binarize", "threshold"]])
graph_type = best_param_df.loc[idx, "graph_type"]
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(mg,
sym_threshold=True,
remove_pdiff=True,
**preprocess_params)
left_mb_indicator = mg.meta["Class 1"].isin(mb_classes) & (
mg.meta["Hemisphere"] == "L")
right_mb_indicator = mg.meta["Class 1"].isin(mb_classes) & (
mg.meta["Hemisphere"] == "R")
labels = np.zeros(len(mg.meta))
labels[left_mb_indicator.values] = 1
labels[right_mb_indicator.values] = 2
pred_labels = best_block_df[idx]
pred_labels = pred_labels[pred_labels.index.isin(mg.meta.index)]
assert np.array_equal(pred_labels.index, mg.meta.index)
ari = adjusted_rand_score(labels, pred_labels)
return ari
def run_experiment(
graph_type=None,
threshold=None,
binarize=None,
seed=None,
param_key=None,
objective_function=None,
implementation="leidenalg",
**kws,
):
np.random.seed(seed)
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(
mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=True,
binarize=binarize,
)
if implementation == "leidenalg":
if objective_function == "CPM":
partition_type = la.CPMVertexPartition
elif objective_function == "modularity":
partition_type = la.ModularityVertexPartition
partition, modularity = run_leiden(
mg,
temp_loc=seed,
implementation=implementation,
partition_type=partition_type,
**kws,
)
elif implementation == "igraph":
partition, modularity = run_leiden(
mg, temp_loc=seed, implementation=implementation, **kws
)
partition.name = param_key
return partition, modularity
tlp_inds = np.arange(len(embed) // 2)
trp_inds = np.arange(len(embed) // 2) + len(embed) // 2
add_connections(
plot_df.iloc[tlp_inds, 0],
plot_df.iloc[trp_inds, 0],
plot_df.iloc[tlp_inds, 1],
plot_df.iloc[trp_inds, 1],
ax=ax,
)
return fig, ax
# %% [markdown]
# ##
graph_type = "G"
master_mg = load_metagraph(graph_type, version="2020-04-01")
mg = preprocess(
master_mg,
threshold=0,
sym_threshold=False,
remove_pdiff=True,
binarize=False,
weight="weight",
)
meta = mg.meta
degrees = mg.calculate_degrees()
quant_val = np.quantile(degrees["Total edgesum"], 0.05)
# remove low degree neurons
idx = meta[degrees["Total edgesum"] > quant_val].index
for co in class_order:
class_value = full_meta.groupby(sc)[co].mean()
full_meta[f"{sc}_{co}_order"] = full_meta[sc].map(class_value)
total_sort_by.append(f"{sc}_{co}_order")
total_sort_by.append(sc)
full_mg = full_mg.sort_values(total_sort_by, ascending=False)
full_meta = full_mg.meta
n_leaf = full_meta[f"lvl{lowest_level}_labels"].nunique()
n_pairs = len(full_meta) // 2
# %% [markdown]
# ## Random walk stuff
ad_mg = load_metagraph("Gad")
ad_mg = preprocess(ad_mg,
sym_threshold=False,
remove_pdiff=True,
binarize=False)
ad_mg.meta["inds"] = range(len(ad_mg))
ad_adj = ad_mg.adj
meta = ad_mg.meta
source_groups = [
("sens-ORN", ),
("sens-MN", ),
("sens-photoRh5", "sens-photoRh6"),
("sens-thermo", ),
("sens-vtd", ),
("sens-AN", ),
def stashcsv(df, name, **kws):
savecsv(df, name)
def invert_permutation(p):
"""The argument p is assumed to be some permutation of 0, 1, ..., len(p)-1.
Returns an array s, where s[i] gives the index of i in p.
"""
p = np.asarray(p)
s = np.empty(p.size, p.dtype)
s[p] = np.arange(p.size)
return s
graph_type = "G"
mg = load_metagraph(graph_type, version="2020-04-01")
mg = preprocess(
mg,
threshold=0,
sym_threshold=False,
remove_pdiff=True,
binarize=False,
weight="weight",
)
meta = mg.meta
# plot where we are cutting out nodes based on degree
degrees = mg.calculate_degrees()
fig, ax = plt.subplots(1, 1, figsize=(5, 2.5))
sns.distplot(np.log10(degrees["Total edgesum"]), ax=ax)
q = np.quantile(degrees["Total edgesum"], 0.05)
def get_best_run(perms, scores, n_opts=None):
if n_opts is None:
n_opts = len(perms)
opt_inds = np.random.choice(len(perms), n_opts, replace=False)
perms = perms[opt_inds]
scores = scores[opt_inds]
max_ind = np.argmax(scores)
return perms[max_ind], scores[max_ind]
# %% [markdown]
# ##
np.random.seed(8888)
graph_type = "G"
master_mg = load_metagraph(graph_type)
mg = master_mg.remove_pdiff()
meta = mg.meta
degrees = mg.calculate_degrees()
quant_val = np.quantile(degrees["Total edgesum"], 0.05)
# remove low degree neurons
idx = meta[degrees["Total edgesum"] > quant_val].index
print(quant_val)
mg = mg.reindex(idx, use_ids=True)
# remove center neurons # FIXME
idx = mg.meta[mg.meta["hemisphere"].isin(["L", "R"])].index
mg = mg.reindex(idx, use_ids=True)
def stashfig(name, **kws):
savefig(name, foldername=FNAME, save_on=True, **kws)
def stashcsv(df, name, **kws):
savecsv(df, name, foldername=FNAME, save_on=True, **kws)
VERSION = "2020-01-29"
print(f"Using version {VERSION}")
graph_type = "Gad"
threshold = 1
weight = "weight"
mg = load_metagraph("Gad", VERSION)
mg = preprocess(
mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=False,
binarize=False,
weight=weight,
)
print(f"Preprocessed graph {graph_type} with threshold={threshold}, weight={weight}")
out_classes = ["O_dVNC"]
sens_classes = ["sens"]
cutoff = 8
print(f"Finding paths from {sens_classes} to {out_classes} of max length {cutoff}")
context = sns.plotting_context(context="talk", font_scale=1, rc=rc_dict)
sns.set_context(context)
np.random.seed(8888)
def stashfig(name, **kws):
savefig(name, foldername=FNAME, save_on=True, **kws)
def stashcsv(df, name, **kws):
savecsv(df, name, foldername=FNAME, **kws)
graph_type = "G"
master_mg = load_metagraph(graph_type, version="2020-04-23")
mg = preprocess(
master_mg,
threshold=0,
sym_threshold=False,
remove_pdiff=True,
binarize=False,
weight="weight",
)
meta = mg.meta
degrees = mg.calculate_degrees()
quant_val = np.quantile(degrees["Total edgesum"], 0.05)
# remove low degree neurons
idx = meta[degrees["Total edgesum"] > quant_val].index
saveskels(
name,
ids,
labels,
colors=colors,
palette=None,
foldername=FNAME,
save_on=SAVESKELS,
**kws,
)
# %% [markdown]
# #
graph_type = "Gad"
mg = load_metagraph(graph_type, BRAIN_VERSION)
# only consider the edges for which we have a mirror edges
edgelist = mg.to_edgelist(remove_unpaired=True)
max_edge = edgelist["weight"].max()
rows = []
for edgeweight in range(1, max_edge + 1):
temp_edgelist = edgelist[edgelist["weight"] == edgeweight]
n_edges = len(temp_edgelist)
n_edges_mirrored = (temp_edgelist["edge pair counts"] == 2).sum()
row = {
"weight": edgeweight,
"n_edges": n_edges,
"n_edges_mirrored": n_edges_mirrored,
"p_edge_mirrored": n_edges_mirrored / n_edges,
return outs
def invert_permutation(p):
"""The argument p is assumed to be some permutation of 0, 1, ..., len(p)-1.
Returns an array s, where s[i] gives the index of i in p.
"""
s = np.empty(p.size, p.dtype)
s[p] = np.arange(p.size)
return s
# %% [markdown]
# # Load data
mg = load_metagraph("Gadn", version=BRAIN_VERSION)
adj = mg.adj
skeleton_labels = mg.meta.index.values
# %% [markdown]
# # Deal with pairs
pair_df = pd.read_csv(
"maggot_models/data/raw/Maggot-Brain-Connectome/pairs/knownpairsatround5.csv"
)
print(pair_df.head())
# extract valid node pairings
left_nodes = pair_df["leftid"].values
right_nodes = pair_df["rightid"].values
left_right_pairs = list(zip(left_nodes, right_nodes))
def get_edges(adj):
adj = adj.copy()
all_edges = []
for i in range(adj.shape[0]):
row = adj[i, :]
col = adj[:, i]
col = np.delete(col, i)
edges = np.concatenate((row, col))
all_edges.append(edges)
return all_edges
# %% [markdown]
# # Play with getting edge df representatino
graph_type = "Gad"
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
g = mg.g
meta = mg.meta
edgelist_df = mg.to_edgelist()
max_pair_edge_df = edgelist_df.groupby("edge pair ID").max()
edge_max_weight_map = dict(
zip(max_pair_edge_df.index.values, max_pair_edge_df["weight"])
)
edgelist_df["max_weight"] = itemgetter(*edgelist_df["edge pair ID"])(
edge_max_weight_map
)
# %% [markdown]
# #
return out_list
def get_edges(adj):
adj = adj.copy()
all_edges = []
for i in range(adj.shape[0]):
row = adj[i, :]
col = adj[:, i]
col = np.delete(col, i)
edges = np.concatenate((row, col))
all_edges.append(edges)
return all_edges
mg = load_metagraph("Gn", version="2019-12-18")
base_path = Path("maggot_models/data/raw/Maggot-Brain-Connectome/")
pair_df = pd.read_csv(base_path / "pairs/bp-pairs-2020-01-13.csv")
all_cells_file = base_path / "neuron-groups/all-neurons-2020-01-13.json"
skeleton_labels = mg.meta.index.values
# extract valid node pairings
left_nodes = pair_df["leftid"].values
right_nodes = pair_df["rightid"].values
left_right_pairs = list(zip(left_nodes, right_nodes))
with open(all_cells_file) as json_file:
temp_dict = json.load(json_file)
all_ids = extract_ids(temp_dict)
trp_inds = np.arange(len(embed) // 2) + len(embed) // 2
add_connections(
plot_df.iloc[tlp_inds, 0],
plot_df.iloc[trp_inds, 0],
plot_df.iloc[tlp_inds, 1],
plot_df.iloc[trp_inds, 1],
ax=ax,
)
return fig, ax
# %% [markdown]
# ## Load and preprocess data
VERSION = "2020-04-23"
graph_type = "G"
master_mg = load_metagraph(graph_type, version="2020-04-23")
mg = preprocess(
master_mg,
threshold=0,
sym_threshold=False,
remove_pdiff=True,
binarize=False,
weight="weight",
)
meta = mg.meta
degrees = mg.calculate_degrees()
quant_val = np.quantile(degrees["Total edgesum"], 0.05)
# remove low degree neurons
idx = meta[degrees["Total edgesum"] > quant_val].index
from src.io import savecsv, savefig, saveskels
import os
FNAME = os.path.basename(__file__)[:-3]
print(FNAME)
def stashfig(name, **kws):
savefig(name, foldername=FNAME, save_on=True, **kws)
VERSION = "2020-03-09"
print(f"Using version {VERSION}")
mg = load_metagraph("G", version=VERSION)
start_instance()
nl = pymaid.get_neurons([mg.meta.index[2]])
# Plot using default settings
fig, ax = nl.plot2d()
# %% [markdown]
# #
mg = mg.sort_values("Pair ID")
nl = pymaid.get_neurons(mg.meta[mg.meta["Merge Class"] == "sens-ORN"].index.values)
fig, ax = nl.plot2d()
# %%
def get_vals_by_k(ks, perm_adj):
ys = []
xs = []
for k in ks:
y = perm_adj[diag_indices(len(perm_adj), k)]
ys.append(y)
x = np.full(len(y), k)
xs.append(x)
return np.concatenate(ys), np.concatenate(xs)
remove_missing = True
mg = load_metagraph("G")
mg = mg.remove_pdiff()
mg = mg.make_lcc()
main_meta = mg.meta.copy()
graph_types = ["G", "Gad", "Gaa", "Gdd", "Gda"]
graph_names = dict(
zip(graph_types,
[r"Sum", r"A $\to$ D", r"A $\to$ A", r"D $\to$ D", r"D $\to$ A"]))
adjs = []
adj_dict = {}
mg_dict = {}
for g in graph_types:
temp_mg = load_metagraph(g)
temp_mg.reindex(mg.meta.index, use_ids=True)
# temp_adj = temp_mg.adj
def run_experiment(graph_type=None, thresh=None, res=None):
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
edgelist = mg.to_edgelist()
edgelist = add_max_weight(edgelist)
edgelist = edgelist[edgelist["max_weight"] > thresh]
mg = edgelist_to_mg(edgelist, mg.meta)
mg = mg.make_lcc()
mg = mg.remove_pdiff()
g_sym = nx.to_undirected(mg.g)
skeleton_labels = np.array(list(g_sym.nodes()))
partition = run_louvain(g_sym, res, skeleton_labels)
# compute signal flow for sorting purposes
mg.meta["signal_flow"] = signal_flow(mg.adj)
mg.meta["partition"] = partition
partition_sf = mg.meta.groupby("partition")["signal_flow"].median()
sort_partition_sf = partition_sf.sort_values(ascending=False)
# common names
basename = f"louvain-res{res}-t{thresh}-{graph_type}-"
title = f"Louvain, {graph_type}, res = {res}, thresh = {thresh}"
# get out some metadata
class_label_dict = nx.get_node_attributes(g_sym, "Merge Class")
class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict))
lineage_label_dict = nx.get_node_attributes(g_sym, "lineage")
lineage_labels = np.array(itemgetter(*skeleton_labels)(lineage_label_dict))
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
classlin_labels, color_dict, hatch_dict = augment_classes(
skeleton_labels, class_labels, lineage_labels)
# barplot by merge class and lineage
fig, axs = barplot_text(
partition,
classlin_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=hatch_dict,
)
stashfig(basename + "barplot-mergeclasslin-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=None,
)
stashfig(basename + "barplot-mergeclass-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=False,
figsize=(24, 18),
title=title,
hatch_dict=None,
)
stashfig(basename + "barplot-mergeclass-counts")
fig, axs = barplot_text(
partition,
lineage_labels,
color_dict=None,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
)
stashfig(basename + "barplot-lineage-props")
# sorted heatmap
heatmap(
mg.adj,
transform="simple-nonzero",
figsize=(20, 20),
inner_hier_labels=partition,
hier_label_fontsize=10,
title=title,
title_pad=80,
)
stashfig(basename + "heatmap")
# block probability matrices
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(sort_partition_sf.index, axis=0)
prob_df = prob_df.reindex(sort_partition_sf.index, axis=1)
ax = probplot(
100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=f"Louvain, res = {res}, counts = {counts}, weights = {weights}",
)
ax.set_ylabel(r"Median signal flow $\to$", fontsize=28)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
# plot minigraph with layout
adjusted_partition = adjust_partition(partition, class_labels)
minigraph = to_minigraph(mg.adj,
adjusted_partition,
use_counts=True,
size_scaler=10)
draw_networkx_nice(
minigraph,
"Spring-x",
"Signal Flow",
sizes="Size",
colors="Color",
cmap="Greys",
vmin=100,
weight_scale=0.001,
)
stashfig(basename + "sbm-drawn-network")
trp_inds = np.arange(len(embed) // 2) + len(embed) // 2
add_connections(
plot_df.iloc[tlp_inds, 0],
plot_df.iloc[trp_inds, 0],
plot_df.iloc[tlp_inds, 1],
plot_df.iloc[trp_inds, 1],
ax=ax,
)
return fig, ax
# %% [markdown]
# ## Load and preprocess data
VERSION = "2020-04-23"
graph_type = "G"
master_mg = load_metagraph(graph_type, version=VERSION)
mg = preprocess(
master_mg,
threshold=0,
sym_threshold=False,
remove_pdiff=True,
binarize=False,
weight="weight",
)
meta = mg.meta
degrees = mg.calculate_degrees()
quant_val = np.quantile(degrees["Total edgesum"], 0.05)
# remove low degree neurons
idx = meta[degrees["Total edgesum"] > quant_val].index
def run_experiment(graph_type=None,
threshold=None,
res=None,
binarize=None,
seed=None,
param_key=None):
# common names
if BLIND:
basename = f"{param_key}-"
title = param_key
else:
basename = f"louvain-res{res}-t{threshold}-{graph_type}-"
title = f"Louvain, {graph_type}, res = {res}, threshold = {threshold}"
np.random.seed(seed)
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(
mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=True,
binarize=binarize,
)
g_sym = nx.to_undirected(mg.g)
skeleton_labels = np.array(list(g_sym.nodes()))
partition, modularity = run_louvain(g_sym, res, skeleton_labels)
partition_series = pd.Series(partition, index=skeleton_labels)
partition_series.name = param_key
if SAVEFIGS:
# get out some metadata
class_label_dict = nx.get_node_attributes(g_sym, "Merge Class")
class_labels = np.array(itemgetter(*skeleton_labels)(class_label_dict))
lineage_label_dict = nx.get_node_attributes(g_sym, "lineage")
lineage_labels = np.array(
itemgetter(*skeleton_labels)(lineage_label_dict))
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
classlin_labels, color_dict, hatch_dict = augment_classes(
class_labels, lineage_labels)
# TODO then sort all of them by proportion of sensory/motor
# barplot by merge class and lineage
_, _, order = barplot_text(
partition,
classlin_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=hatch_dict,
return_order=True,
)
stashfig(basename + "barplot-mergeclasslin-props")
category_order = np.unique(partition)[order]
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=True,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-props")
fig, axs = barplot_text(
partition,
class_labels,
color_dict=color_dict,
plot_proportions=False,
norm_bar_width=False,
figsize=(24, 18),
title=title,
hatch_dict=None,
category_order=category_order,
)
stashfig(basename + "barplot-mergeclass-counts")
# TODO add gridmap
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
partition,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(category_order, axis=0)
prob_df = prob_df.reindex(category_order, axis=1)
probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.7)
stashfig(basename + f"probplot-counts{counts}-weights{weights}")
return partition_series, modularity
def run_experiment(seed=None, graph_type=None, threshold=None, param_key=None):
np.random.seed(seed)
if BLIND:
temp_param_key = param_key.replace(
" ", "") # don't want spaces in filenames
savename = f"{temp_param_key}-cell-types-"
title = param_key
else:
savename = f"{graph_type}-t{threshold}-cell-types"
title = f"{graph_type}, threshold = {threshold}"
mg = load_metagraph(graph_type, version=VERSION)
# simple threshold
# TODO they will want symmetric threshold...
# TODO maybe make that a parameter
adj = mg.adj.copy()
adj[adj <= threshold] = 0
meta = mg.meta.copy()
meta = pd.DataFrame(mg.meta["neuron_name"])
mg = MetaGraph(adj, meta)
# run the graphtool code
temp_loc = f"maggot_models/data/interim/temp-{param_key}.graphml"
block_series = run_minimize_blockmodel(mg, temp_loc)
# manage the output
mg = load_metagraph(graph_type, version=VERSION)
mg.meta = pd.concat((mg.meta, block_series), axis=1)
mg.meta["Original index"] = range(len(mg.meta))
keep_inds = mg.meta[~mg.meta["block_label"].isna(
)]["Original index"].values
mg.reindex(keep_inds)
if graph_type != "G":
mg.verify(10000, graph_type=graph_type, version=VERSION)
# deal with class labels
lineage_labels = mg.meta["lineage"].values
lineage_labels = np.vectorize(lambda x: "~" + x)(lineage_labels)
class_labels = mg["Merge Class"]
skeleton_labels = mg.meta.index.values
classlin_labels, color_dict, hatch_dict = augment_classes(
skeleton_labels, class_labels, lineage_labels)
block_label = mg["block_label"].astype(int)
# barplot with unknown class labels merged in, proportions
_, _, order = barplot_text(
block_label,
classlin_labels,
norm_bar_width=True,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
return_order=True,
)
stashfig(savename + "barplot-mergeclasslin-props")
category_order = np.unique(block_label)[order]
# barplot with regular class labels
barplot_text(
block_label,
class_labels,
norm_bar_width=True,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
category_order=category_order,
)
stashfig(savename + "barplot-mergeclass-props")
# barplot with unknown class labels merged in, counts
barplot_text(
block_label,
classlin_labels,
norm_bar_width=False,
color_dict=color_dict,
hatch_dict=hatch_dict,
title=title,
figsize=(24, 18),
return_order=True,
category_order=category_order,
)
stashfig(savename + "barplot-mergeclasslin-counts")
# barplot of hemisphere membership
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
stacked_barplot(
block_label,
mg["Hemisphere"],
norm_bar_width=True,
category_order=category_order,
ax=ax,
)
remove_spines(ax)
stashfig(savename + "barplot-hemisphere")
# plot block probability matrix
counts = False
weights = False
prob_df = get_blockmodel_df(mg.adj,
block_label,
return_counts=counts,
use_weights=weights)
prob_df = prob_df.reindex(order, axis=0)
prob_df = prob_df.reindex(order, axis=1)
ax = probplot(100 * prob_df,
fmt="2.0f",
figsize=(20, 20),
title=title,
font_scale=0.4)
stashfig(savename + "probplot")
block_series.name = param_key
return block_series
f.plot2d(ax=ax, color=color, method="3d")
title = f"{neuron_class}, {n.neuron_name}, {n.skeleton_id}"
ax.set_title(title, color="grey")
set_axes_equal(ax)
plt.tight_layout()
def get_savename(nl, neuron_class=None):
savename = f"{neuron_class}"
for n in nl:
savename += f"-{n.skeleton_id}"
savename += "-split"
return savename
mg = load_metagraph("G")
meta = mg.meta
start_instance()
skeleton_color_dict = dict(
zip(meta.index,
np.vectorize(CLASS_COLOR_DICT.get)(meta["merge_class"])))
connection_types = ["axon", "dendrite", "unsplittable"]
pal = sns.color_palette("deep", 5)
colors = [pal[1], pal[2], pal[4]]
connection_colors = dict(zip(connection_types, colors))
splits = pymaid.find_treenodes(tags="mw axon split")
splits = splits.set_index("skeleton_id")["treenode_id"].squeeze()
foldername=FNAME,
save_on=SAVESKELS,
**kws,
)
def stashobj(obj, name, **kws):
saveobj(obj, name, foldername=FNAME, save_on=SAVEOBJS, **kws)
graph_type = "G"
threshold = 3
binarize = True
# load and preprocess the data
mg = load_metagraph(graph_type, version=BRAIN_VERSION)
mg = preprocess(mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=True,
binarize=binarize)
#%%
import leidenalg as la
import igraph as ig
def _process_metagraph(mg, temp_loc):
adj = mg.adj
adj = symmetrize(adj, method="avg")
remove_spines(ax)
ax.xaxis.set_major_locator(plt.FixedLocator([0]))
ax.yaxis.set_major_locator(plt.FixedLocator([0]))
ax.set_title(f"Dimension {d}")
def remove_cols(mat, remove_inds):
kept_inds = list(range(mat.shape[1]))
[kept_inds.remove(i) for i in remove_inds]
return mat[:, kept_inds]
# %% [markdown]
# #
thresh = 0.0
ad_norm_mg = load_metagraph("Gadn", BRAIN_VERSION)
ad_norm_mg.adj[ad_norm_mg.adj < thresh] = 0
ad_norm_mg, n_pairs = pair_augment(ad_norm_mg)
left_pair_ids = ad_norm_mg["Pair ID"][:n_pairs]
right_pair_ids = ad_norm_mg["Pair ID"][n_pairs:2 * n_pairs]
print((left_pair_ids == right_pair_ids).all())
sym_mg = max_symmetrize(ad_norm_mg, n_pairs)
left_pair_ids = sym_mg["Pair ID"][:n_pairs]
right_pair_ids = sym_mg["Pair ID"][n_pairs:2 * n_pairs]
print((left_pair_ids == right_pair_ids).all())
sym_mg.make_lcc()
n_pairs = sym_mg.meta["Pair ID"].nunique() - 1
left_pair_ids = sym_mg["Pair ID"][:n_pairs]
stop_reasons[2] += 1
print(stop_reasons / stop_reasons.sum())
print(len(sm_paths))
return sm_paths, visit_orders
#%% Load and preprocess the data
VERSION = "2020-01-29"
print(f"Using version {VERSION}")
graph_type = "G"
threshold = 1
weight = "weight"
mg = load_metagraph(graph_type, VERSION)
mg = preprocess(
mg,
threshold=threshold,
sym_threshold=True,
remove_pdiff=False,
binarize=False,
weight=weight,
)
print(
f"Preprocessed graph {graph_type} with threshold={threshold}, weight={weight}"
)
out_classes = [
"O_dVNC",
"O_dSEZ",
def stashfig(name, **kws):
if SAVEFIGS:
savefig(name, foldername=FNAME, **kws)
brain_version = "2020-02-26"
graph_versions = [
"G", "Gad", "Gaa", "Gdd", "Gda", "Gn", "Gadn", "Gaan", "Gddn", "Gdan"
]
for graph_version in graph_versions:
# sort the graph
mg = load_metagraph(graph_version, brain_version)
paired_inds = np.where(mg.meta["Pair ID"] != -1)[0]
mg = mg.reindex(paired_inds)
mg.sort_values(["Merge Class", "Pair ID", "Hemisphere"], ascending=True)
# if graph_version not in ["G", "Gn"]:
mg.verify(n_checks=10000, graph_type=graph_version, version=brain_version)
# plot the sorted graph
mg.meta["Index"] = range(len(mg))
groups = mg.meta.groupby("Merge Class", as_index=True)
tick_locs = groups["Index"].mean()
border_locs = groups["Index"].first()
fig, ax = plt.subplots(1, 1, figsize=(30, 30))
gridmap(mg.adj, sizes=(3, 5), ax=ax)
for bl in border_locs:
from src.embed import ase, lse, preprocess_graph
from src.graph import MetaGraph
from src.hierarchy import signal_flow
from src.io import savefig, saveobj, saveskels
from src.utils import get_blockmodel_df, get_sbm_prob, invert_permutation
from src.visualization import bartreeplot, get_color_dict, get_colors, sankey, screeplot
FNAME = os.path.basename(__file__)[:-3]
print(FNAME)
print(nx.__version__)
# %% [markdown]
# #
BRAIN_VERSION = "2020-01-16"
mg = load_metagraph("G", BRAIN_VERSION)
mg.make_lcc()
mg.sort_values("Merge Class")
adj = mg.adj
class_labels = mg["Class 1"]
trans_mat = adj.copy()
row_sums = np.sum(adj, axis=1)
trans_mat = trans_mat / row_sums[:, np.newaxis]
trans_mat[np.isnan(trans_mat)] = 0
trans_mat.sum(axis=1)
# %% [markdown]
# #
U, S, V = np.linalg.svd(adj)
#%%
from src.data import load_metagraph
import seaborn as sns
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
mg = load_metagraph("G", "2020-01-21")
is_pdiff = np.where(mg["is_pdiff"])[0]
mg = mg.reindex(is_pdiff)
degree_df = mg.calculate_degrees()
plt.figure()
melt_degree = pd.melt(
degree_df.reset_index(),
id_vars=["ID"],
value_vars=["In degree", "Out degree", "Total degree"],
value_name="Degree",
)
sns.stripplot(y="Degree", data=melt_degree, x="variable", jitter=0.45)
plt.figure()
melt_syns = pd.melt(
degree_df.reset_index(),
id_vars=["ID"],
value_vars=["In edgesum", "Out edgesum", "Total edgesum"],
value_name="Synapses",
)
sns.stripplot(y="Synapses", data=melt_syns, x="variable", jitter=0.45)
while nxt is not None:
last = nxt
nxt = next(level_it, None)
return last
# %% [markdown]
# ## Load data
# In this case we are working with `G`, the directed graph formed by summing the edge
# weights of the 4 different graph types. Preprocessing here includes removing
# partially differentiated cells, and cutting out the lowest 5th percentile of nodes in
# terms of their number of incident synapses. 5th percentile ~= 12 synapses. After this,
# the largest connected component is used.
mg = load_metagraph("G", version="2020-04-01")
mg = preprocess(
mg,
threshold=0,
sym_threshold=False,
remove_pdiff=True,
binarize=False,
weight="weight",
)
meta = mg.meta
# plot where we are cutting out nodes based on degree
degrees = mg.calculate_degrees()
fig, ax = plt.subplots(1, 1, figsize=(5, 2.5))
sns.distplot(np.log10(degrees["Total edgesum"]), ax=ax)
q = np.quantile(degrees["Total edgesum"], 0.05)
import numpy as np
import pandas as pd
from sklearn.metrics import pairwise_distances
from src.data import load_metagraph
from src.io import savecsv
FNAME = os.path.basename(__file__)[:-3]
print(FNAME)
def stashcsv(df, name, **kws):
savecsv(df, name, foldername=FNAME, **kws)
mg = load_metagraph("G", version="2020-05-08")
ids = mg.meta.index.values
# load connectors
connector_path = "maggot_models/data/processed/2020-05-08/connectors.csv"
connectors = pd.read_csv(connector_path)
compartment = "dendrite"
direction = "postsynaptic"
subsample = False
def filter_connectors(connectors, ids, direction, compartment):
label_connectors = connectors[connectors[f"{direction}_to"].isin(ids)]
label_connectors = label_connectors[
label_connectors[f"{direction}_type"] == compartment