def align_neuron_obj(neuron_obj,
mesh_center=None,
rotation=None,
align_matrix=None,
in_place=False,
verbose=False,
plot_final_neuron=False,
align_synapses=True,
**kwargs):
"""
Purpose: To rotate all of the meshes
and skeletons of a neuron object
Ex:
neuron_obj_rot = copy.deepcopy(neuron_obj)
mesh_center = neuron_obj["S0"].mesh_center
for i in range(0,10):
neuron_obj_rot = hu.align_neuron_obj(neuron_obj_rot,
mesh_center=mesh_center,
verbose =True)
nviz.visualize_neuron(
neuron_obj_rot,limb_branch_dict = "all")
"""
if not in_place:
neuron_obj = copy.deepcopy(neuron_obj)
if align_matrix is None:
if rotation is None:
if mesh_center is None:
soma_center = neuron_obj["S0"].mesh_center
if verbose:
print(f"soma_center = {soma_center}")
align_matrix = hu.rotation_from_soma_center(
soma_center,
verbose=False,
)
align_matrix = hu.align_matrix_from_rotation(rotation)
neuron_obj.align_matrix = align_matrix
for j, limb_obj in enumerate(neuron_obj):
for branch_obj in limb_obj:
branch_obj.mesh = hu.align_mesh(branch_obj.mesh,
align_matrix=align_matrix,
verbose=False)
branch_obj.mesh_center = tu.mesh_center_vertex_average(
branch_obj.mesh)
branch_obj.skeleton = hu.align_skeleton(branch_obj.skeleton,
align_matrix=align_matrix,
verbose=False)
branch_obj.endpoints = hu.align_array(
branch_obj.endpoints,
align_matrix=align_matrix,
)
if align_synapses:
for syn in branch_obj.synapses:
for att in syu.synapse_coordinate_system_dependent_attributes:
hu.align_attribute(
syn,
att,
align_matrix=align_matrix,
)
#doing the spine alignment
if branch_obj.spines is not None:
branch_obj.spines = [
hu.align_mesh(k, align_matrix=align_matrix)
for k in branch_obj.spines
]
if branch_obj.spines_obj is not None:
for s_obj in branch_obj.spines_obj:
s_obj.mesh = hu.align_mesh(s_obj.mesh,
align_matrix=align_matrix)
#changing the concept network
all_concept_network_data = []
att_to_change = [
"starting_endpoints", "starting_coordinate",
"touching_soma_vertices"
]
for k in limb_obj.all_concept_network_data:
new_data = copy.deepcopy(k)
for att in att_to_change:
new_data[att] = hu.align_array(
k[att],
align_matrix=align_matrix,
)
all_concept_network_data.append(new_data)
for att in att_to_change:
setattr(
limb_obj, f"current_{att}",
hu.align_array(
getattr(limb_obj, f"current_{att}"),
align_matrix=align_matrix,
))
limb_obj.mesh = hu.align_mesh(limb_obj.mesh,
align_matrix=align_matrix,
verbose=False)
limb_obj.all_concept_network_data = copy.deepcopy(
all_concept_network_data)
limb_obj.set_concept_network_directional()
neuron_obj.mesh = hu.align_mesh(neuron_obj.mesh,
align_matrix=align_matrix,
verbose=False)
#finishing soma mesh stuff
for s_name in neuron_obj.get_soma_node_names():
neuron_obj[s_name].mesh = hu.align_mesh(
neuron_obj[s_name].mesh,
align_matrix=align_matrix,
)
if align_synapses:
for syn in neuron_obj[s_name].synapses:
for att in syu.synapse_coordinate_system_dependent_attributes:
hu.align_attribute(
syn,
att,
align_matrix=align_matrix,
)
neuron_obj[s_name].mesh_center = tu.mesh_center_vertex_average(
neuron_obj[s_name].mesh)
#print(f"neuron_obj[s_name].mesh_center = {neuron_obj[s_name].mesh_center}")
#aligning the preprocessing stuff
if plot_final_neuron:
nviz.visualize_neuron(neuron_obj, limb_branch_dict="all")
return neuron_obj
def make(self,key):
"""
Purpose: To decimate a mesh by a perscribed
decimation ratio and algorithm
Pseudocode:
1) Get the current mesh,somas,glia and nuclie faces
2) Get the parameters and run the neuron preprocessing
"""
global_time = time.time()
segment_id = key["segment_id"]
decomposition_hash = key["decomposition_method"]
ver =key["ver"]
if verbose:
print(f"\n\n--Working on {segment_id}: (decomposition_hash = {decomposition_hash})")
# if len(self & dict(
# ver=ver,
# segment_id = segment_id,
# decimation=decomposition_hash)) > 0:
# if verbose:
# print(f"Already processed {segment_id} (decomposition_hash = {decomposition_hash})")
# return
#1)
st = time.time()
mesh = hdju.fetch_segment_id_mesh(segment_id)
somas = hdju.get_soma_mesh_list_filtered(segment_id)
print(f"somas = {somas}")
glia_faces,nuclei_faces = hdju.get_segment_glia_nuclei_faces(segment_id)
if plotting:
soma_mesh_center = hdju.soma_info_center(segment_id,return_nm=True)
# rotating the mesh
nviz.plot_objects(hu.align_mesh_from_soma_coordinate(mesh,
soma_center=soma_mesh_center
))
if verbose:
print(f"Collecting Mesh Info: {time.time() - st}")
#2)
preprocess_args = DecompositionMethod.restrict_one_part_with_hash(
decomposition_method_hash).fetch1()
fill_hole_size = preprocess_args["fill_hole_size"]
current_preprocess_neuron_kwargs = {
k:v for k,v in preprocess_args.items() if k in preprocess_neuron_kwargs.keys()}
current_spines_kwargs = {
k:v for k,v in preprocess_args.items() if k in spines_kwargs.keys()}
print(f"current_preprocess_neuron_kwargs = \n{current_preprocess_neuron_kwargs}")
print(f"current_spines_kwargs = \n{current_spines_kwargs}")
params_to_change = [k for k in current_preprocess_neuron_kwargs if k.split("_")[-1] in ["cgal","map"]]
if verbose:
print(f"params_to_change = {params_to_change}")
current_preprocess_neuron_kwargs["width_threshold_MAP"] = current_preprocess_neuron_kwargs["width_threshold_map"]
current_preprocess_neuron_kwargs["size_threshold_MAP"] = current_preprocess_neuron_kwargs["size_threshold_map"]
current_preprocess_neuron_kwargs["max_stitch_distance_CGAL"] = current_preprocess_neuron_kwargs["max_stitch_distance_cgal"]
del current_preprocess_neuron_kwargs["width_threshold_map"]
del current_preprocess_neuron_kwargs["size_threshold_map"]
del current_preprocess_neuron_kwargs["max_stitch_distance_cgal"]
description = "0_25"
st = time.time()
neuron_obj = neuron.Neuron(
mesh = mesh,
somas = somas,
segment_id=segment_id,
description=description,
suppress_preprocessing_print=False,
suppress_output=False,
calculate_spines=True,
widths_to_calculate=["no_spine_median_mesh_center"],
glia_faces=glia_faces,
nuclei_faces = nuclei_faces,
decomposition_type = "meshafterparty",
preprocess_neuron_kwargs=current_preprocess_neuron_kwargs,
spines_kwargs=spines_kwargs,
fill_hole_size=fill_hole_size
)
neuron_obj_comb = nru.combined_somas_neuron_obj(neuron_obj,
inplace = False,
verbose = verbose,
plot_soma_limb_network = plotting)
if verbose:
print(f"\n\n\n---- Total preprocessing time = {time.time() - st}")
if plotting:
nviz.visualize_neuron(neuron_obj_comb,
limb_branch_dict="all",
mesh_whole_neuron=True)
#3)
st = time.time()
stats_dict = neuron_obj_comb.neuron_stats(stats_to_ignore = [
"n_boutons",
"axon_length",
"axon_area",
"max_soma_volume",
"max_soma_n_faces",],
include_skeletal_stats = True,
include_centroids= True,
voxel_adjustment_vector=voxel_adjustment_vector,
)
if verbose:
print(f"-- Generating Stats: {time.time() - st}")
#4) Save the neuron object in a certain location
file_name = f"{neuron_obj_comb.segment_id}_{decomposition_hash}"
file_name_decomp = f"{file_name}_{dataset}_decomposition"
output_folder=str(target_dir_decomp)
st = time.time()
ret_file_path = neuron_obj_comb.save_compressed_neuron(
output_folder=output_folder,
file_name= file_name_decomp,
return_file_path=True,
export_mesh=False,
suppress_output=True,
)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"-- Neuron Object Save time: {time.time() - st}")
#5) Outputting skeleton object, computing stats and saving
st = time.time()
sk_stats = nst.skeleton_stats_from_neuron_obj(
neuron_obj_comb,
include_centroids=True,
voxel_adjustment_vector=voxel_adjustment_vector,
verbose = True)
skeleton = neuron_obj_comb.skeleton
file_name_decomp_sk = f"{file_name}_{dataset}_decomposition_sk"
ret_sk_filepath = su.compressed_pickle(
skeleton,
filename = file_name_decomp_sk,
folder=str(target_dir_sk),
return_filepath=True)
if verbose:
print(f"ret_sk_filepath = {ret_sk_filepath}")
if verbose:
print(f"-- Skeleton Generation and Save time: {time.time() - st}")
# 6) make the insertions
run_time = run_time=np.round(time.time() - global_time,4)
# -- decomp table --
decomp_dict = dict(key.copy(),
process_version = process_version,
index = 0,
multiplicity=1,
decomposition=ret_file_path_str,
run_time = run_time)
decomp_dict.update(stats_dict)
self.insert1(decomp_dict,
allow_direct_insert = True,
ignore_extra_fields = True,
skip_duplicates=True)
self.Object.insert1(
decomp_dict,
allow_direct_insert = True,
ignore_extra_fields = True,
skip_duplicates=True)
#-- sk table
sk_dict = dict(key.copy() ,
process_version = process_version,
index = 0,
multiplicity=1,
skeleton=ret_sk_filepath,
run_time = run_time)
sk_dict.update(sk_stats)
SkeletonDecomposition.insert1(sk_dict,
allow_direct_insert = True,
ignore_extra_fields = True,
skip_duplicates=True)
SkeletonDecomposition.Object.insert1(
sk_dict,
allow_direct_insert = True,
ignore_extra_fields = True,
skip_duplicates=True)
def make(self, key):
"""
Purpose: To extract the axon/dendrite of a split neuron
1) Pull down the neuron
2) Get the neuron ids and nucleus centers corresponding to
that segent id
Iterate through all the neuron objects
a0) Recompute the width
a) Get the winning nucleus_id
b) Get the cell type info from the central database
c) Add synapses to neuron obj
d) Add spine categories to neuorn object
e) classifiy E/I cell type according to Baylor rules
f) Pick the cell type to use
g) Perfrom complete aon processing
h) Get aon Features
i) Save neurong object
j) Save Axon/Dendrite before proofreading
k) Write to dj table
"""
global_time = time.time()
segment_id = key["segment_id"]
decomposition_cell_type_hash = key["decomposition_cell_type_method"]
decomposition_split_method = hdju.decomposition_split_method_hash_from_segment_id(
segment_id, verbose=True)
if verbose:
print(
f"\n\n--Working on {segment_id}: (decomposition_cell_type_hash = "
f"{decomposition_cell_type_hash}, decomposition_split_method = {decomposition_split_method})"
)
#0) Visualizing the neuron
if plotting:
print(f"Visualizing the intial neuron")
hdju.plot_mesh_with_somas(
segment_id=segment_id,
#split_index=0,
with_skeleton=True,
align_from_soma_center=True)
# ---1) Pulling down the neuron---
st = time.time()
n_objs, sp_indexes = hdju.neuron_objs_from_decomposition_stage(
segment_id, verbose=True, return_one=False)
if verbose:
print(f"---1) Pulling down the neuron---: {time.time() - st}")
st = time.time()
# ---2) Get the nucleus ids and nucleus centers for that segment id---
nucleus_ids, nucleus_centers = hdju.nuclei_from_segment_id(
segment_id, return_centers=True, return_nm=True)
if verbose:
print(f"Number of Corresponding Nuclei = {len(nucleus_ids)}")
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
curr_idx = 0
neuron_obj_pre_filt = n_objs[curr_idx]
split_index = sp_indexes[curr_idx]
if plot_initial_neuron:
neuron_obj_rot = hu.align_neuron_obj(neuron_obj_pre_filt)
nviz.visualize_neuron(neuron_obj_rot, limb_branch_dict="all")
if verbose:
print(f"--> Working on Split Index {split_index} -----")
if verbose:
print(
f"---2) Get the nucleus ids and nucleus centers--- {time.time() - st}"
)
st = time.time()
# -- a0) Prep work: Recompute the Widths --
if filter_low_branch_cluster_dendrite:
neuron_obj, filtering_info_low_branch = pru.apply_proofreading_filters_to_neuron(
input_neuron=neuron_obj_pre_filt,
filter_list=[pru.low_branch_length_clusters_dendrite_filter],
plot_limb_branch_filter_with_disconnect_effect=False,
plot_limb_branch_filter_away=
plot_limb_branch_filter_away_low_branch,
plot_final_neuron=False,
return_error_info=True,
verbose=False,
verbose_outline=verbose)
else:
neuron_obj = neuron_obj_pre_filt
filtering_info_low_branch = {}
neuron_obj = wu.neuron_width_calculation_standard(neuron_obj,
verbose=True)
if verbose:
print(f"a0) Prep work: Recompute the Widths: {time.time() - st}")
st = time.time()
# --- a) Get the winning nucleus_id and nucleus info
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(
neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold=15000,
return_matching_info=True,
verbose=True)
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
if winning_nucleus_id is None:
if verbose:
print(
f"No winning nuclues found so assigning the only nucleus id"
)
winning_nucleus_id = nucleus_ids[0]
if verbose:
print(
f"--- a) Get the winning nucleus_id and nucleus info: {time.time() - st}"
)
st = time.time()
# ---b) Get the cell type info from database
database_cell_type_info = hdju.nuclei_classification_info_from_nucleus_id(
winning_nucleus_id)
database_e_i_class = database_cell_type_info[
f"{data_type}_e_i_cell_type"]
if verbose:
print(f"database_cell_type_info = {database_cell_type_info}")
print(f"database_e_i_class = {database_e_i_class}")
if verbose:
print(
f"---b) Get the cell type info from database: {time.time() - st}"
)
st = time.time()
# ---c/d) Add synapses and spine categories
import synapse_utils as syu
neuron_obj = syu.add_synapses_to_neuron_obj(
neuron_obj,
validation=False,
verbose=verbose,
original_mesh=None,
plot_valid_error_synapses=False,
calculate_synapse_soma_distance=False,
add_valid_synapses=True,
add_error_synapses=False,
)
neuron_obj = spu.add_head_neck_shaft_spine_objs(neuron_obj,
verbose=verbose)
if plot_synapses:
syu.plot_synapses(neuron_obj)
if plot_spines:
spu.plot_spines_head_neck(neuron_obj)
if verbose:
print(
f"---c/d) Add synapses and spine categories: {time.time() - st}"
)
st = time.time()
#---e) classifiy E/I cell type according to Baylor rules
baylor_e_i, baylor_cell_type_info = ctu.e_i_classification_from_neuron_obj(
neuron_obj,
plot_on_model_map=False,
plot_spines_and_sk_filter_for_syn=plot_spines_and_sk_filter_for_syn,
plot_spines_and_sk_filter_for_spine=
plot_spines_and_sk_filter_for_spine,
verbose=True,
return_cell_type_info=True)
baylor_cell_type_info["baylor_e_i"] = baylor_e_i
if verbose:
print(f"baylor_cell_type_info = \n{baylor_cell_type_info}")
if verbose:
print(
f"---e) classifiy E/I cell type according to Baylor rules: {time.time() - st}"
)
st = time.time()
#--- f) Pick the cell type to use
if (inh_exc_class_to_use_for_axon == "h01"
and database_e_i_class in ["excitatory", "inhibitory"]):
e_i_class = database_e_i_class
if verbose:
print(f"Using h01 e/i cell type")
cell_type_used = "h01"
else:
if verbose:
print(f"Using baylor e/i cell type")
e_i_class = baylor_e_i
cell_type_used = "baylor"
if verbose:
print(
f"e_i_class = {e_i_class} with cell_type_used = {cell_type_used}"
)
if verbose:
print(f"---f) Pick the cell type to use: {time.time() - st}")
st = time.time()
#---# g) Perfrom complete aon processing
if plot_aligned_neuron:
print(f"plot_aligned_neuron")
neuron_obj_rot = hu.align_neuron_obj(neuron_obj)
nviz.visualize_neuron(neuron_obj_rot, limb_branch_dict="all")
o_neuron_unalign, filtering_info, axon_angles_dict = au.complete_axon_processing(
neuron_obj,
cell_type=e_i_class,
add_synapses_and_head_neck_shaft_spines=False,
validation=False,
plot_initial_axon=plot_initial_axon,
plot_axon_on_dendrite=plot_axon_on_dendrite,
return_filtering_info=True,
return_axon_angle_info=True,
plot_high_fidelity_axon=plot_high_fidelity_axon,
plot_boutons_web=plot_boutons_web,
add_synapses_after_high_fidelity_axon=True,
verbose=verbose)
#o_neuron_unalign = hu.unalign_neuron_obj(o_neuron)
# if verbose:
# print(f"Readding Synapses to the high fidelity axon after all processing donw")
# o_neuron_unalign = syu.add_synapses_to_neuron_obj(o_neuron_unalign,
# validation = False,
# verbose = verbose,
# original_mesh = None,
# plot_valid_error_synapses = False,
# calculate_synapse_soma_distance = False,
# add_valid_synapses = True,
# add_error_synapses=False,
# limb_branch_dict_to_add_synapses=o_neuron_unalign.axon_limb_branch_dict)
if verbose:
print(
f"After add_synapses_after_high_fidelity_axon: # of neuron_obj.synapses_somas = {len(o_neuron_unalign.synapses_somas)}"
)
if plot_unaligned_synapses:
syu.plot_synapses(o_neuron_unalign, total_synapses=True)
if plot_unaligned_axon:
nviz.plot_axon(o_neuron_unalign)
if verbose:
print(f"---g) Perfrom complete aon processing: {time.time() - st}")
st = time.time()
# --- h) Get the axon and dendrite stats ----
dendrite_stats = nst.skeleton_stats_dendrite(o_neuron_unalign,
include_centroids=False)
axon_stats = nst.skeleton_stats_axon(o_neuron_unalign,
include_centroids=False)
stats_dict = o_neuron_unalign.neuron_stats(
stats_to_ignore=[
"n_not_processed_soma_containing_meshes", "n_error_limbs",
"n_same_soma_multi_touching_limbs",
"n_multi_soma_touching_limbs", "n_somas", "spine_density"
],
include_skeletal_stats=False,
include_centroids=True,
voxel_adjustment_vector=voxel_adjustment_vector,
)
if verbose:
print(
f"--- h) Get the axon and dendrite stats: {time.time() - st}")
st = time.time()
#---- i) Calculating the synapse info ------
syn_dict = syu.n_synapses_analysis_axon_dendrite(o_neuron_unalign,
verbose=True)
# --- j) saving neuron and skeleton ----
#4) Save the neuron object in a certain location
file_name = f"{o_neuron_unalign.segment_id}_{split_index}_{decomposition_cell_type_hash}"
file_name_decomp = f"{file_name}_{dataset}_cell_type_decomp"
output_folder = str(target_dir_decomp)
ret_file_path = o_neuron_unalign.save_compressed_neuron(
output_folder=output_folder,
file_name=file_name_decomp,
return_file_path=True,
export_mesh=False,
suppress_output=True,
)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
axon_skeleton = o_neuron_unalign.axon_skeleton
file_name_decomp_sk_axon = f"{file_name_decomp}_axon_sk"
ret_sk_filepath_ax = su.compressed_pickle(
axon_skeleton,
filename=file_name_decomp_sk_axon,
folder=str(target_dir_sk),
return_filepath=True)
dendrite_skeleton = o_neuron_unalign.dendrite_skeleton
file_name_decomp_sk_dendr = f"{file_name_decomp}_dendr_sk"
ret_sk_filepath_dendr = su.compressed_pickle(
dendrite_skeleton,
filename=file_name_decomp_sk_dendr,
folder=str(target_dir_sk),
return_filepath=True)
if verbose:
print(f"neuron ret_file_path_str = {ret_file_path_str}")
print(f"ret_sk_filepath_ax = {ret_sk_filepath_ax}")
print(f"ret_sk_filepath_dendr = {ret_sk_filepath_dendr}")
if verbose:
print(f"--- i) saving neuron and skeleton ----")
st = time.time()
nucleus_info
h01_e_i_cell_type = database_e_i_class
database_cell_type_info
baylor_e_i, baylor_cell_type_info
e_i_class
cell_type_used
filtering_info, axon_angles_dict
o_neuron_unalign
dendrite_stats
axon_stats
stats_dict
ret_file_path_str
ret_sk_filepath_ax
ret_sk_filepath_dendr
# 7) make the insertions
run_time = run_time = np.round(time.time() - global_time, 4)
# -- decomp table --
n_dict = dict(
key.copy(),
decomposition_split_method=decomposition_split_method,
multiplicity=1,
split_index=split_index,
decomposition=str(ret_file_path_str),
axon_skeleton=str(ret_sk_filepath_ax),
dendrite_skeleton=str(ret_sk_filepath_dendr),
#--- cell types
h01_e_i_cell_type=database_e_i_class,
cell_type=e_i_class,
cell_type_used=cell_type_used,
#----- synapses ---
n_syn_pre=neuron_obj.n_synapses_pre,
n_syn_post=neuron_obj.n_synapses_post,
run_time=run_time,
# statistics for the split
)
dicts_for_update = [
nucleus_info, database_cell_type_info, filtering_info,
axon_angles_dict, dendrite_stats, axon_stats, stats_dict,
baylor_cell_type_info, filtering_info_low_branch, syn_dict
]
for d in dicts_for_update:
n_dict.update(d)
print(f"n_dict = {n_dict}")
for curr_obj in [self, SkeletonAxonDendrite]:
curr_obj.insert1(n_dict,
allow_direct_insert=True,
ignore_extra_fields=True,
skip_duplicates=True)
curr_obj.Object.insert1(n_dict,
allow_direct_insert=True,
ignore_extra_fields=True,
skip_duplicates=True)