def make(self, key):
whole_pass_time = time.time()
# ----------- Doing the v4 Processing ------- #
segment_id = key["segment_id"]
if verbose:
print(f"\n-- Working on neuron {segment_id}---")
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4()
& dict(segment_id=segment_id)).fetch1()
#1) Find the coordinates of the nucleus for that new segment
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
if verbose:
print(f"nuc_center_coords = {nuc_center_coords}")
#2) Make sure that same number of DecompositionAxon objects as in Decomposition
old_segment_id = segment_map_dict["old_segment_id"]
if verbose:
print(f"old_segment_id = {old_segment_id}")
search_key = dict(segment_id=old_segment_id)
n_somas = len(minnie.BaylorSegmentCentroid() & search_key)
n_decomp_axon = len(minnie.DecompositionAxon() & search_key)
if verbose:
print(
f"# of somas = {n_somas} and # of DecompositionAxon = {n_decomp_axon}"
)
if n_somas != n_decomp_axon:
raise Exception(
f"# of somas = {n_somas} NOT MATCH # of DecompositionAxon = {n_decomp_axon}"
)
#3) Pick the neuron object that is closest and within a certain range of the nucleus
neuron_objs, split_idxs = du.decomposition_with_spine_recalculation(
old_segment_id)
if n_somas > 1:
"""
Finding the closest soma:
1) For each neuron object get the mesh center of the soma object
2) Find the distance of each from the nucleus center
3) Find the arg min distance and make sure within threshold
4) Mark the current neuron and the current split index
"""
nuclei_distance_threshold = 15000
soma_center_coords = [k["S0"].mesh_center for k in neuron_objs]
soma_distances = [
np.linalg.norm(k - nuc_center_coords)
for k in soma_center_coords
]
min_dist_arg = np.argmin(soma_distances)
min_dist = soma_distances[min_dist_arg]
if verbose:
print(f"soma_distances = {soma_distances}")
print(
f"min_dist_arg = {min_dist_arg}, with min distance = {min_dist}"
)
if min_dist > nuclei_distance_threshold:
raise Exception(
f"min_dist ({min_dist}) larger than nuclei_distance_threshold ({nuclei_distance_threshold})"
)
neuron_obj = neuron_objs[min_dist_arg]
split_index = split_idxs[min_dist_arg]
if verbose:
print(f"Winning split_index = {split_index}")
else:
split_index = split_idxs[0]
neuron_obj = neuron_objs[0]
(filt_neuron, return_synapse_df_revised, return_synapse_df_errors,
return_validation_df_revised,
return_validation_df_extension) = vu.filtered_neuron_score(
neuron_obj=neuron_obj,
filter_list=pru.v4_exc_filters(),
plot_limb_branch_filter_with_disconnect_effect=False,
verbose=True,
plot_score=False,
nucleus_id=nucleus_id,
return_synapse_df_errors=True,
return_validation_df_extension=True,
split_index=split_index)
print(f"\n\n ----- Done Filtering ----------")
#------- saving off the filtered neuron
save_time = time.time()
file_name = f"{filt_neuron.segment_id}_{filt_neuron.description}_v4_val"
ret_file_path = filt_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
file_name=file_name,
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
print(f"Save time = {time.time() - save_time}")
# ---------- Getting the scores of the proofreading ----- #
presyn_scores_dict = vu.scores_presyn(return_validation_df_revised)
postsyn_scores_dict = vu.scores_postsyn(return_validation_df_revised)
cat = vu.synapse_validation_df_to_category_counts(
return_validation_df_revised,
print_postsyn=True,
print_presyn=False)
run_time = np.round(time.time() - whole_pass_time, 2)
final_dict = dict(
key,
split_index=split_index,
decomposition=ret_file_path_str,
axon_length=filt_neuron.axon_length,
validation_df=return_validation_df_revised.to_numpy(),
validation_df_ext=return_validation_df_extension.to_numpy(),
pre_tp=cat["presyn"]["TP"],
pre_tn=cat["presyn"]["TN"],
pre_fp=cat["presyn"]["FP"],
pre_fn=cat["presyn"]["FN"],
pre_precision=presyn_scores_dict["precision"],
pre_recall=presyn_scores_dict["recall"],
pre_f1=presyn_scores_dict["f1"],
post_tp=cat["postsyn"]["TP"],
post_tn=cat["postsyn"]["TN"],
post_fp=cat["postsyn"]["FP"],
post_fn=cat["postsyn"]["FN"],
post_precision=postsyn_scores_dict["precision"],
post_recall=postsyn_scores_dict["recall"],
post_f1=postsyn_scores_dict["f1"],
run_time=run_time)
self.insert1(final_dict,
skip_duplicates=True,
allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {run_time} ------ ***"
)
def make(self,key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs,neuron_split_idxs = du.decomposition_with_spine_recalculation(segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType = True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
# -------- getting the nuclei info to match
try:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4() & dict(old_segment_id=segment_id)).fetch1()
except:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMapInh() & dict(old_segment_id=segment_id)).fetch1()
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
nucleus_ids = [nucleus_id]
nucleus_centers = [nuc_center_coords]
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index,neuron_obj in zip(neuron_split_idxs,neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold = 15000,
return_matching_info = True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
#4) -------- Running the cell classification and stats--------------
if verbose:
print(f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n")
filter_time = time.time()
#---- adding the synapses and spines data -----#
neuron_obj_exc_syn = syu.add_synapses_to_neuron_obj(neuron_obj,
validation = True,
verbose = True,
original_mesh = None,
plot_valid_error_synapses = False,
calculate_synapse_soma_distance = False,
add_valid_synapses = True,
add_error_synapses=False,)
neuron_obj_exc_syn_sp = spu.add_head_neck_shaft_spine_objs(neuron_obj_exc_syn,
verbose = True
)
limb_branch_dict = ctu.postsyn_branches_near_soma_for_syn_post_density(
neuron_obj = neuron_obj_exc_syn_sp,
verbose = False,)
(syn_density_post,
syn_density_head,
syn_density_neck,
syn_density_shaft,
skeletal_length_processed_syn) = ctu.synapse_density_stats(neuron_obj = neuron_obj_exc_syn_sp,
limb_branch_dict = limb_branch_dict,
verbose = True)
(spine_density,
skeletal_length_processed_spine) = ctu.spine_density_near_soma(neuron_obj = neuron_obj_exc_syn_sp,
verbose = True,
multiplier = 1000)
if verbose:
print(f"Total time for density calculations = {time.time() - filter_time}")
baylor_cell_type_info = dict(
syn_density_post = syn_density_post,
syn_density_head = syn_density_head,
syn_density_neck = syn_density_neck,
syn_density_shaft = syn_density_shaft,
skeletal_length_processed_syn=skeletal_length_processed_syn,
spine_density=spine_density,
skeletal_length_processed_spine = skeletal_length_processed_spine
)
n_dict = dict(key,
split_index = split_index,
run_time = np.round(time.time() - st,2),
)
dicts_for_update = [baylor_cell_type_info,
allen_cell_type_info,
nucleus_info,]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,skip_duplicates=True,allow_direct_insert=True)
print(f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***")
def nucleus_center_from_segment_id(segment_id,split_index=0):
nucleus_id = pv.nucleus_id_from_segment_id(segment_id,split_index)
if nucleus_id == 0:
return None
return du.nuclei_id_to_nucleus_centers(nucleus_id)
def make(self, key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation(
segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType=True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
# -------- getting the nuclei info to match
try:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4()
& dict(old_segment_id=segment_id)).fetch1()
except:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMapInh()
& dict(old_segment_id=segment_id)).fetch1()
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
nucleus_ids = [nucleus_id]
nucleus_centers = [nuc_center_coords]
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(
neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold=15000,
return_matching_info=True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(
winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
#3) ------ Axon Classification (and getting the axon features)------------------
import axon_utils as au
o_neuron, filtering_info, axon_angles_dict = au.complete_axon_processing(
neuron_obj,
add_synapses_and_head_neck_shaft_spines=True,
validation=validation,
plot_initial_axon=False,
plot_axon_on_dendrite=False,
return_filtering_info=True,
return_axon_angle_info=True,
verbose=verbose)
filtering_info = {
k: np.round(v, 2)
for k, v in filtering_info.items()
if "area" in k or "length" in k
}
axon_features = au.axon_features_from_neuron_obj(o_neuron)
#3)------- Running the cell classification and stats--------------
if verbose:
print(
f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n"
)
filter_time = time.time()
#---- adding the synapses and spines data -----#
limb_branch_dict = ctu.postsyn_branches_near_soma_for_syn_post_density(
neuron_obj=o_neuron,
verbose=False,
)
(syn_density_post, syn_density_head, syn_density_neck,
syn_density_shaft,
skeletal_length_processed_syn) = ctu.synapse_density_stats(
neuron_obj=o_neuron,
limb_branch_dict=limb_branch_dict,
verbose=True)
(spine_density,
skeletal_length_processed_spine) = ctu.spine_density_near_soma(
neuron_obj=o_neuron, verbose=True, multiplier=1000)
if verbose:
print(
f"Total time for density calculations = {time.time() - filter_time}"
)
baylor_cell_type_info = dict(
syn_density_post=syn_density_post,
syn_density_head=syn_density_head,
syn_density_neck=syn_density_neck,
syn_density_shaft=syn_density_shaft,
skeletal_length_processed_syn=skeletal_length_processed_syn,
spine_density=spine_density,
skeletal_length_processed_spine=skeletal_length_processed_spine
)
# 4) ------ Predicting the E/I Group Based on the data collected --------
################ NEED TO INSERT CODE TO DO THIS ###########
#5) ----- Deciding on cell type to use for axon
e_i_class = inh_exc_class
if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info[
"e_i"] is not None:
e_i_class = allen_cell_type_info["e_i"]
if verbose:
print(
f"e_i_class = {e_i_class} with inh_exc_class_to_use_for_axon = {inh_exc_class_to_use_for_axon}"
)
# ------- Saving off the neuron object ----------------
save_time = time.time()
ret_file_path = o_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
#output_folder = "./",
file_name=
f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
# ----------------
n_dict = dict(
key,
split_index=split_index,
axon_version=au.axon_version,
decomposition=ret_file_path_str,
run_time=np.round(time.time() - st, 2),
cell_type=e_i_class,
)
dicts_for_update = [
baylor_cell_type_info, allen_cell_type_info, nucleus_info,
filtering_info, axon_features, axon_angles_dict
]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,
skip_duplicates=True,
allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
def make(self,key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs,neuron_split_idxs = du.decomposition_with_spine_recalculation(segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType = True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
# -------- getting the nuclei info to match
# ver = 88
# nucleus_ids,nucleus_centers = du.segment_to_nuclei(segment_id,
# nuclei_version=ver)
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4() & dict(old_segment_id=segment_id)).fetch1()
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
nucleus_ids = [nucleus_id]
nucleus_centers = [nuc_center_coords]
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index,neuron_obj in zip(neuron_split_idxs,neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold = 15000,
return_matching_info = True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
#4) -------- Running the cell classification and stats--------------
if verbose:
print(f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n")
filter_time = time.time()
(inh_exc_class,
spine_category,
axon_angles,
n_axons,
n_apicals,
neuron_spine_density,
n_branches_processed,
skeletal_length_processed,
n_branches_in_search_radius,
skeletal_length_in_search_radius
) = clu.inhibitory_excitatory_classifier(neuron_obj,
return_spine_classification=True,
return_axon_angles=True,
return_n_axons=True,
return_n_apicals=True,
return_spine_statistics=True,
axon_limb_branch_dict_precomputed=None,
axon_angles_precomputed=None,
verbose=verbose)
if verbose:
print(f"Total time for classification = {time.time() - filter_time}")
all_axon_angles = []
for limb_idx,limb_data in axon_angles.items():
for candidate_idx,cand_angle in limb_data.items():
all_axon_angles.append(cand_angle)
if len(axon_angles)>0:
axon_angle_maximum = np.max(all_axon_angles)
else:
axon_angle_maximum = 0
if verbose:
print("\n -- Cell Type Classification Results --")
print(f"inh_exc_class={inh_exc_class}")
print(f"spine_category={spine_category}")
print(f"axon_angles={axon_angles}")
print(f"n_axons={n_axons}")
print(f"n_apicals={n_apicals}")
print(f"neuron_spine_density={neuron_spine_density}")
print(f"n_branches_processed={n_branches_processed}")
print(f"skeletal_length_processed={skeletal_length_processed}")
print(f"n_branches_in_search_radius={n_branches_in_search_radius}")
print(f"skeletal_length_in_search_radius={skeletal_length_in_search_radius}")
baylor_cell_type_info = dict(
cell_type_predicted=inh_exc_class,
spine_category=spine_category,
axon_angle_maximum = axon_angle_maximum,
n_axons=n_axons,
n_apicals=n_apicals,
spine_density_classifier=neuron_spine_density,
n_branches_processed=neuron_spine_density,
skeletal_length_processed=skeletal_length_processed,
n_branches_in_search_radius=n_branches_in_search_radius,
skeletal_length_in_search_radius=skeletal_length_in_search_radius,
)
#5) ----- Deciding on cell type to use for axon
e_i_class = inh_exc_class
if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info["e_i"] is not None:
e_i_class = allen_cell_type_info["e_i"]
if verbose:
print(f"e_i_class = {e_i_class} with inh_exc_class_to_use_for_axon = {inh_exc_class_to_use_for_axon}")
#6) -------- If excitatory running the axon processing--------------
"""
Psuedocode:
If e_i class is excitatory:
1) Filter away the axon on dendrite
2) Do the higher fidelity axon processing
3) Compute the axon features
"""
if e_i_class == "excitatory" and neuron_obj.axon_limb_name is not None:
if verbose:
print(f"Excitatory so performing high fidelity axon and computing axon features")
# 1) Filter away the axon on dendrite
# 2) Do the higher fidelity axon processing
o_neuron, filtering_info = au.complete_axon_processing(neuron_obj,
perform_axon_classification = False,
return_filtering_info = True)
filtering_info = {k:np.round(v,2) for k,v in filtering_info.items() if "area" in k or "length" in k}
#3) Compute the axon features
axon_features = au.axon_features_from_neuron_obj(o_neuron)
else:
nru.clear_all_branch_labels(neuron_obj,labels_to_clear="axon")
o_neuron = neuron_obj
axon_features = dict()
filtering_info = dict()
#3) ------ Adding the Synapses -----------
o_neuron = syu.add_synapses_to_neuron_obj(o_neuron,
validation = validation,
verbose = True,
original_mesh = None,
plot_valid_error_synapses = False,
calculate_synapse_soma_distance = False,
add_valid_synapses = True,
add_error_synapses=False)
# ------- Saving off the neuron object ----------------
save_time = time.time()
ret_file_path = o_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
#output_folder = "./",
file_name=f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
n_dict = dict(key,
split_index = split_index,
axon_version = au.axon_version,
decomposition=ret_file_path_str,
run_time = np.round(time.time() - st,2),
cell_type_for_axon = e_i_class,
)
dicts_for_update = [baylor_cell_type_info,
allen_cell_type_info,
nucleus_info,
filtering_info,
axon_features]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,skip_duplicates=True,allow_direct_insert=True)
#dict_to_write.append(n_dict)
#write the
#self.insert(dict_to_write,skip_duplicates=True,allow_direct_insert=True)
print(f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***")