def consolidate_parents_structures_cfos(id_table,
ann,
namelist,
verbose=False,
structures=False):
"""Function that generates evenly spaced pixels values based on annotation parents
Removes 0 from list
Inputs:
id_table=path to excel file generated from scripts above
ann = allen annoation file
namelist=list of structues names, typically parent structures*********************
Returns:
-----------
nann = new array of bitdepth
list of value+name combinations
"""
if type(ann) == str: ann = sitk.GetArrayFromImage(sitk.ReadImage(ann))
#remove duplicates and null and root
namelist = list(set(namelist))
namelist = [xx for xx in namelist if xx != "null" and xx != "root"]
namelist.sort()
#make structures to find parents
if not structures:
from tools.analysis.network_analysis import make_structure_objects
structures = make_structure_objects(id_table)
#setup
nann = np.zeros(ann.shape).astype("uint8")
cmap = [xx for xx in np.linspace(1, 255, num=len(namelist))]
#populate
for i in range(len(namelist)):
try:
nm = namelist[i]
s = [xx for xx in structures if xx.name == nm][0]
if verbose:
print("{}, {} of {}, value {}".format(nm, i,
len(namelist) - 1,
cmap[i]))
nann[np.where(ann == int(s.idnum))] = cmap[i]
for ii in s.progeny:
if ii[3] != "null": nann[np.where(ann == int(ii[3]))] = cmap[i]
except Exception, e:
print nm, e
def consolidate_parents_structures_OLD(allen_id_table, ann, namelist, verbose=False):
'''Function that generates evenly spaced pixels values based on annotation parents
Removes 0 from list
Inputs:
allen_id_table=path to excel file generated from scripts above
ann = allen annoation file
namelist=list of structues names, typically parent structures*********************
Returns:
-----------
nann = new array of bitdepth
list of value+name combinations
'''
if type(ann) == str: ann = sitk.GetArrayFromImage(sitk.ReadImage(ann))
#remove duplicates and null and root
namelist = list(set(namelist))
namelist = [xx for xx in namelist if xx != 'null' and xx != 'root']
#make structures to find parents
from tools.analysis.network_analysis import make_structure_objects
structures = make_structure_objects(allen_id_table)
#setup
nann = np.zeros(ann.shape).astype('uint8')
cmap = [int(xx) for xx in np.linspace(0,255, num=len(namelist)+1)]
#populate
for i in range(len(namelist)):
try:
nm=namelist[i]
s = [xx for xx in structures if xx.name==nm][0]
if verbose: print ('{}, {} of {}, value {}'.format(nm, i, len(namelist)-1, cmap[i+1]))
nann[np.where(ann==int(s.idnum))] = cmap[i+1]
for ii in s.progeny:
if ii[3] != 'null': nann[np.where(ann==int(ii[3]))] = cmap[i+1]
except Exception as e:
print(nm, e)
#sitk.Show(sitk.GetImageFromArray(nann))
return nann, zip(cmap[1:], namelist)
"""
Created on Wed Feb 5 14:34:55 2020
@author: wanglab
"""
import pandas as pd, os
os.chdir("/jukebox/wang/zahra/python/lightsheet_py3")
from tools.analysis.network_analysis import make_structure_objects
pth = "/home/wanglab/wang/zahra/registration_error_pma/structures_zeroed_after_80um_erosion_pma_annotation_2018.csv"
df_pth = "/home/wanglab/LightSheetTransfer/atlas/ls_id_table_w_voxelcounts.xlsx"
dst = "/jukebox/wang/zahra/registration_error_pma"
#make structures to traverse hierarchy
structures = make_structure_objects(df_pth)
#read main annotation LUT
anndf = pd.read_csv(pth)
#make new df to only save child structures that don't belong to NC, white matter, or ventricles
sois = ["Isocortex", "ventricular systems", "fiber tracts", "grooves"]
for soi in sois:
soi = [s for s in structures if s.name == soi][0]
anndf = anndf[anndf.name != soi.name]
progeny = [str(xx.name) for xx in soi.progeny]
for progen in progeny:
anndf = anndf[anndf.name != progen]
#format
anndf = anndf.drop(columns=["Unnamed: 0"])
"adultacutePC_lobVI_DREADDs", "adultacutePC_lobVI_DREADDs",
"adultacutePC_lobVI_DREADDs"
]
conditions = {n: c for n, c in zip(nms, cond)}
#set appropriate paths
pth = os.path.join(src, "cell_counts.csv")
df_pth = "/jukebox/LightSheetTransfer/atlas/ls_id_table_w_voxelcounts_16bit.xlsx"
ann_pth = "/jukebox/LightSheetTransfer/atlas/annotation_sagittal_atlas_20um_iso_16bit.tif"
atl_pth = "/jukebox/LightSheetTransfer/atlas/sagittal_atlas_20um_iso.tif"
curated_structures = "/jukebox/wang/Jess/lightsheet_output/201904_ymaze_cfos/structures.csv"
#build structures class
structures = make_structure_objects(
df_pth,
remove_childless_structures_not_repsented_in_ABA=True,
ann_pth=ann_pth)
def generate_data_frame(conditions, lst, pth, flnm):
"""
used to make a pooled csv file of all cell counts in an experiment
inputs:
conditions: zip of file names + condition
lst: list of file names run through analysis
pth: path to save csv output
"""
#generate data frame
bglst = []
for fl in lst:
#extract out info
assert pvol.shape == ann.shape
#anterior is 0 index; 100-250 is good in z
from tools.registration.allen_structure_json_to_pandas import annotation_value_to_structure, isolate_structures_return_list, consolidate_parents_structures, annotation_location_to_structure
allen_id_table = '/home/wanglab/wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx'
#ann = '/home/wanglab/wang/pisano/Python/allenatlas/annotation_25_ccf2015.nrrd'
#find all pixel IDs present in structures
#pix_values=list(np.unique(ann[100:175].ravel().astype('float64')))
#collect names of parents OR ##optionally get sublist
if False:
#make structures to find parents
from tools.analysis.network_analysis import make_structure_objects
structures = make_structure_objects('/home/wanglab/wang/pisano/Python/lightsheet/supp_files/sample_cell_count_output.xlsx')
parent_list = list(set([yy.parent[1] for xx in pix_values for yy in structures if xx == yy.idnum]))
#find counts of highest labeled areas
if False:
olist = annotation_location_to_structure(allen_id_table, args=zip(*np.nonzero(pvol[100:175])), ann=ann[100:175])
srt = sorted(olist, key=lambda x: x[0], reverse=True)
parent_list = [xx[1] for xx in srt]
#select only subset
parent_list=parent_list[0:13]
#manually choose locations
if False: parent_list = ['Anterior cingulate area', 'Hippocampal formation', 'Infralimbic area', 'Prelimbic area', 'Orbital area',
'Agranular insular area', 'Somatosensory areas', 'Striatum']
#generate list of structures present
def make_hiveplot(inputexcel,
outputexcel,
node1='Cerebellum',
node2='Thalamus',
node3=False,
colorlst=['black', 'red', 'white'],
title=False,
annpath=False,
threshold=0.25):
'''leveraging off of: https://github.com/ericmjl/hiveplot
Inputs:
--------------------------
inputexcel = excel file of injection site or starting point
outputexcel = excel file of cell counts or ending points
node1-3: ABA Name of structures. node 1 for inputexcel, node2-3 for outputexcel
threshold = value above to keep. This is a 'score' of connectivity: ((injectionpixels/maxinjectionarea pixels) + (cellcount/maxcellcount)) / 2
'''
#################TESTING:
node1 = 'Cerebellum'
node2 = 'Thalamus'
node3 = 'Cerebrum'
ann_pth = '/home/wanglab/wang/pisano/Python/allenatlas/annotation_25_ccf2015.nrrd'
colorlst = ['black', 'red', 'blue']
title = 'Sample HivePlot'
#
if not ann_pth:
ann_pth = '/home/wanglab/wang/pisano/Python/allenatlas/annotation_25_ccf2015.nrrd'
#generate objects:
innstructures = make_structure_objects(
inputexcel,
remove_childless_structures_not_repsented_in_ABA=True,
ann_pth=ann_pth)
outstructures = make_structure_objects(
outputexcel,
remove_childless_structures_not_repsented_in_ABA=True,
ann_pth=ann_pth)
#make networkx graphs
Ginn = make_network(innstructures,
substructure_name='Basic cell groups and regions',
graphstyle='twopi',
showcellcounts=False,
cutofflevel=10,
font_size=11,
show=False)
Gout = make_network(outstructures,
substructure_name='Basic cell groups and regions',
graphstyle='twopi',
showcellcounts=False,
cutofflevel=10,
font_size=11,
show=False)
#find structure objects for nodes of given input
nn1 = [xx for xx in innstructures if xx.name == node1][0]
nn2 = [xx for xx in outstructures if xx.name == node2][0]
if node3: nn3 = [xx for xx in outstructures if xx.name == node3][0]
#pull out progeny of nodes; might need to make this level based
nodes = dict()
nodes['group1'] = [
(n, d['structure_object'].cellcount_progeny)
for n, d in Ginn.nodes(data=True)
if d.values()[1].name in [str(xx[1]) for xx in nn1.progeny]
] #
nodes['group2'] = [
(n, d['structure_object'].cellcount_progeny)
for n, d in Gout.nodes(data=True)
if d.values()[1].name in [str(xx[1]) for xx in nn2.progeny]
] #
if node3:
nodes['group3'] = [
(n, d['structure_object'].cellcount_progeny)
for n, d in Gout.nodes(data=True)
if d.values()[1].name in [str(xx[1]) for xx in nn3.progeny]
] #
#sort, might need to change sorting measure; currently by count
for group, nodelist in nodes.items():
nodelist.sort(key=lambda x: x[1], reverse=True) #, key=keyfunc())
nodes[group] = [xx[0] for xx in nodelist]
#pull out node ids
#nodes[group] = [n for n, d in nodes[group]]
#lst of [name, counts]:
n1lst = [[i, [xx.cellcount for xx in innstructures if xx.name == i][0]]
for i in nodes['group1']]
n2lst = [[i, [xx.cellcount for xx in outstructures if xx.name == i][0]]
for i in nodes['group2']]
if node3:
n3lst = [[
i, [xx.cellcount for xx in outstructures if xx.name == i][0]
] for i in nodes['group3']]
#create normalization
n1max = max([xx[1] for xx in n1lst])
n2max = max([xx[1] for xx in n2lst])
n3max = max([xx[1] for xx in n3lst])
#FIXME: not yet implemented a true sorting value...you should filter by third number = 0->1
edges = dict()
edges['group1'] = [
(xx[0], yy[0], (((xx[1] / n1max) + (yy[1] / n2max)) / 2))
for xx in n1lst for yy in n2lst
if xx[1] > 0 and yy[1] > 0 and (((xx[1] / n1max) +
(yy[1] / n2max)) / 2) > threshold
]
if node3:
edges['group2'] = [
(xx[0], yy[0], (((xx[1] / n1max) + (yy[1] / n3max)) / 2))
for xx in n1lst for yy in n3lst
if xx[1] > 0 and yy[1] > 0 and (((xx[1] / n1max) +
(yy[1] / n3max)) / 2) > threshold
]
#set colors
nodes_cmap = dict()
nodes_cmap['group1'] = colorlst[0]
nodes_cmap['group2'] = colorlst[1]
if node3: nodes_cmap['group3'] = colorlst[2]
edges_cmap = dict()
edges_cmap['group1'] = (0, 0.5, 0.5)
if node3: edges_cmap['group2'] = (0.2, 0.4, 0.9)
#plot!
f = plt.figure()
ax = f.gca()
ax.set_axis_bgcolor('black')
h = HivePlot(nodes,
edges,
nodes_cmap,
edges_cmap,
is_directed=True,
fig=f,
linewidth=0.3,
ax=ax)
if title: f.suptitle(title, fontsize=14, fontweight='bold')
legend = [
mpatches.Patch(color=colorlst[0], label=node1),
mpatches.Patch(color=colorlst[1], label=node2)
]
if node3: legend.append(mpatches.Patch(color=colorlst[2], label=node3))
f.text(0.05,
0.2,
'{} --> {}'.format(node1, node2),
color=edges_cmap['group1'],
backgroundcolor=(0.6, 0.6, 0.6),
fontsize=14)
if node3:
f.text(0.05,
0.15,
'{} --> {}'.format(node1, node3),
color=edges_cmap['group2'],
backgroundcolor=(0.6, 0.6, 0.6),
fontsize=14)
h.draw()
#%%
if __name__ == '__main__':
##TO FIND NEW STRUCTURE IDs
excelfl = '/home/wanglab/wang/pisano/tracing_output/l7cre_ts/ch00_l7cre_ts01_20150928_005na_z3um_1hfds_488w_647_200msec_5ovlp_stuctures_table.xlsx'
#structure_id= 549 #Thalamus's id NOT atlas_id
ann_pth = '/home/wanglab/wang/pisano/Python/allenatlas/annotation_25_ccf2015.nrrd'
#load
df = pd.read_excel(excelfl)
#generate objects:
structures = make_structure_objects(
excelfl,
remove_childless_structures_not_repsented_in_ABA=True,
ann_pth=ann_pth,
verbose=True)
#optional to find progeny of given level:
cutofflevel = 3
substructures = find_structures_of_given_level(
'Basic cell groups and regions', cutofflevel, structures)
#find structure numbers:
strct = [xx for xx in structures if xx.name == 'Thalamus'][0]
ids = [xx[3] for xx in strct.progeny]
ids.append(strct.idnum)
#%%
primary_as_frac_of_lob = np.array(
[np.argmax(e) for e in expr_all_as_frac_of_lob])
secondary = np.array([np.argsort(e)[-2] for e in expr_all_as_frac_of_inj])
print(expr_all_as_frac_of_lob[15])
#%%
#making dictionary of cells by region
cells_regions = pckl.load(open(cells_regions_pth, "rb"), encoding="latin1")
cells_regions = cells_regions.to_dict(orient="dict")
#make sure brains are the same order
assert brains == list(cells_regions.keys())
#pooling regions
structures = make_structure_objects(
"/jukebox/LightSheetTransfer/atlas/ls_id_table_w_voxelcounts.xlsx",
remove_childless_structures_not_repsented_in_ABA=True,
ann_pth=ann_pth)
#%%
#atlas res
scale_factor = 0.020 #mm/voxel
#GET ONLY THALAMIC POOLS
nuclei = [
"Parafascicular nucleus", "Posterior complex of the thalamus",
"Posterior triangular thalamic nucleus",
"Lateral posterior nucleus of the thalamus", "Lateral habenula",
"Lateral dorsal nucleus of thalamus",
"Central lateral nucleus of the thalamus",
"Paraventricular nucleus of the thalamus", "Nucleus of reuniens",
"Mediodorsal nucleus of thalamus",
def make_2D_overlay_of_heatmaps(src,
atl_pth,
ann_pth,
allen_id_table,
save_dst,
positive=True,
negative=True):
vol = tifffile.imread(src)
atl = tifffile.imread(atl_pth)
ann = tifffile.imread(ann_pth)
pvol = vol[:, :, :, 1]
nvol = vol[:, :, :, 0]
atl = np.rot90(np.transpose(atl, [1, 0, 2]),
axes=(2, 1)) #sagittal to coronal
ann = np.rot90(np.transpose(ann, [1, 0, 2]),
axes=(2, 1)) #sagittal to coronal
assert atl.shape == ann.shape == nvol.shape
pix_values = list(np.unique(ann.ravel().astype("int64")))
#collect names of parents OR ##optionally get sublist
#make structures to find parents
structures = make_structure_objects(allen_id_table)
parent_list = list(
set([
yy.parent[1] for xx in pix_values for yy in structures
if xx == yy.idnum
]))
#find counts of highest labeled areas
olist = annotation_location_to_structure(
allen_id_table, args=zip(*np.nonzero(pvol[100:175])), ann=ann[100:175])
srt = sorted(olist, key=lambda x: x[0], reverse=True)
parent_list = [xx[1] for xx in srt]
#generate list of structures present
#nann, lst = consolidate_parents_structures(allen_id_table, ann, parent_list, verbose=True)
#threshold values
pvol[pvol != 0.0] = 1.0
nvol[nvol != 0.0] = 1.0
no_structures_to_keep = 20
#loop only positives
zstep = 40
colorbar_cutoff = 65 #20,60 #this is related to zstep size...(it"s like apercetnage...)
rngs = range(0, 558, zstep)
for iii in range(len(rngs) - 1):
#range rng = (100,150)
rng = (rngs[iii], rngs[iii + 1])
#positive
if positive:
print(rng, "positive")
#get highest
olist = annotation_location_to_structure(
allen_id_table, zip(*np.nonzero(pvol[rng[0]:rng[1]])),
ann[rng[0]:rng[1]])
srt = sorted(olist, key=lambda x: x[0], reverse=True)
parent_list = [xx[1] for xx in srt]
#select only subset
parent_list = parent_list[:no_structures_to_keep]
nann, lst = consolidate_parents_structures_cfos(
allen_id_table,
ann[rng[0]:rng[1]],
parent_list,
verbose=True,
structures=structures)
#make fig
plt.figure(figsize=(15, 18))
ax = plt.subplot(2, 1, 1)
fig = plt.imshow(np.max(atl[rng[0]:rng[1]], axis=0),
cmap="gray",
alpha=1)
fig.axes.get_xaxis().set_visible(False)
fig.axes.get_yaxis().set_visible(False)
ax.set_title("ABA structures")
mode = scipy.stats.mode(
nann, axis=0, nan_policy="omit") #### THIS IS REALLY IMPORTANT
most = list(np.unique(mode[1][0].ravel()))
most = sorted(most, reverse=True)
ann_mode = mode[0][0]
masked_data = np.ma.masked_where(ann_mode < 0.1, ann_mode)
im = plt.imshow(masked_data,
cmap=tab20cmap_nogray,
alpha=0.8,
vmin=0,
vmax=255)
patches = [
mpatches.Patch(color=im.cmap(im.norm(i[0])),
label="{}".format(i[1])) for i in lst
]
plt.legend(handles=patches,
bbox_to_anchor=(1.05, 1),
loc=2,
borderaxespad=0.)
ax.set_anchor("W")
#pvals
ax = plt.subplot(2, 1, 2)
ax.set_title(
"Number of positively correlated voxels in AP dimension")
#modify colormap
import matplotlib as mpl
my_cmap = plt.cm.viridis(np.arange(plt.cm.RdBu.N))
#my_cmap = plt.cm.RdYlGn(np.arange(plt.cm.RdBu.N))
my_cmap[:colorbar_cutoff, :4] = 0.0
my_cmap = mpl.colors.ListedColormap(my_cmap)
my_cmap.set_under("w")
#plot
fig = plt.imshow(np.max(atl[rng[0]:rng[1]], axis=0),
cmap="gray",
alpha=1)
#plt.imshow(np.max(pvol[rng[0]:rng[1]], axis=0), cmap=my_cmap, alpha=0.9) #old way
plt.imshow(np.sum(pvol[rng[0]:rng[1]], axis=0),
cmap=my_cmap,
alpha=0.95,
vmin=0,
vmax=zstep)
plt.colorbar()
fig.axes.get_xaxis().set_visible(False)
fig.axes.get_yaxis().set_visible(False)
ax.set_anchor("W")
#plt.tight_layout()
pdst = os.path.join(save_dst,
"positive_overlays_zstep{}".format(zstep))
if not os.path.exists(pdst): os.mkdir(pdst)
plt.savefig(os.path.join(pdst,
"cfos_z{}-{}.pdf".format(rng[0], rng[1])),
dpi=300,
transparent=True)
plt.close()
#negative
if negative:
print rng, "negative"
#get highest
olist = annotation_location_to_structure(
allen_id_table, zip(*np.nonzero(nvol[rng[0]:rng[1]])),
ann[rng[0]:rng[1]])
srt = sorted(olist, key=lambda x: x[0], reverse=True)
parent_list = [xx[1] for xx in srt]
#select only subset
parent_list = parent_list[0:no_structures_to_keep]
nann, lst = consolidate_parents_structures_cfos(
allen_id_table,
ann[rng[0]:rng[1]],
parent_list,
verbose=True,
structures=structures)
#make fig
plt.figure(figsize=(15, 18))
ax = plt.subplot(2, 1, 1)
fig = plt.imshow(np.max(atl[rng[0]:rng[1]], axis=0),
cmap="gray",
alpha=1)
fig.axes.get_xaxis().set_visible(False)
fig.axes.get_yaxis().set_visible(False)
ax.set_title("ABA structures")
mode = scipy.stats.mode(
nann, axis=0, nan_policy="omit") #### THIS IS REALLY IMPORTANT
most = list(np.unique(mode[1][0].ravel()))
most = sorted(most, reverse=True)
ann_mode = mode[0][0]
masked_data = np.ma.masked_where(ann_mode < 0.1, ann_mode)
im = plt.imshow(masked_data,
cmap=tab20cmap_nogray,
alpha=0.8,
vmin=0,
vmax=255)
patches = [
mpatches.Patch(color=im.cmap(im.norm(i[0])),
label="{}".format(i[1])) for i in lst
]
plt.legend(handles=patches,
bbox_to_anchor=(1.05, 1),
loc=2,
borderaxespad=0.)
ax.set_anchor("W")
#pvals
ax = plt.subplot(2, 1, 2)
ax.set_title(
"Number of negatively correlated voxels in AP dimension")
#modify colormap
import matplotlib as mpl
my_cmap = plt.cm.plasma(np.arange(plt.cm.RdBu.N))
my_cmap[:colorbar_cutoff, :4] = 0.0
my_cmap = mpl.colors.ListedColormap(my_cmap)
my_cmap.set_under("w")
#plot
fig = plt.imshow(np.max(atl[rng[0]:rng[1]], axis=0),
cmap="gray",
alpha=1)
plt.imshow(np.sum(nvol[rng[0]:rng[1]], axis=0),
cmap=my_cmap,
alpha=0.95,
vmin=0,
vmax=zstep)
plt.colorbar()
fig.axes.get_xaxis().set_visible(False)
fig.axes.get_yaxis().set_visible(False)
ax.set_anchor("W")
#plt.tight_layout()
ndst = os.path.join(save_dst,
"negative_overlays_zstep{}".format(zstep))
if not os.path.exists(ndst): os.mkdir(ndst)
plt.savefig(os.path.join(ndst,
"cfos_z{}-{}.pdf".format(rng[0], rng[1])),
dpi=300,
transparent=True)
plt.close()
return parent_list
ann = np.rot90(np.transpose(ann, [1, 0, 2]), axes=(2, 1)) #sagittal to coronal
df = pd.DataFrame()
for src in p_val_maps:
print(src)
vol = tifffile.imread(src)
#positive correlation
pvol = vol[:, :, :, 1]
pix_values = list(np.unique(ann.ravel().astype("float64")))
#collect names of parents OR ##optionally get sublist
#make structures to find parents
structures = make_structure_objects(allen_id_table)
parent_list = list(
set([
yy.parent[1] for xx in pix_values for yy in structures
if xx == yy.idnum
]))
#find counts of highest labeled areas
olist = annotation_location_to_structure(allen_id_table,
args=zip(*np.nonzero(pvol)),
ann=ann)
srt = sorted(olist, key=lambda x: x[0], reverse=True)
parent_list = [xx[1] for xx in srt]
if "dorsal" in src:
column_name = os.path.basename(src)[8:-4] + "_dorsal_up_positive_corr"
