def make(self, key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation(
segment_id)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
#Run the Axon Decomposition
neuron_obj_with_web = au.complete_axon_processing(neuron_obj,
verbose=True)
save_time = time.time()
ret_file_path = neuron_obj_with_web.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
#output_folder = "./",
file_name=
f"{neuron_obj_with_web.segment_id}_{split_index}_split_axon_v{au.axon_version}",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
n_dict = dict(key,
split_index=split_index,
axon_version=au.axon_version,
decomposition=ret_file_path_str,
axon_length=neuron_obj_with_web.axon_length,
run_time=np.round(time.time() - st, 2))
dict_to_write.append(n_dict)
#write the
self.insert(dict_to_write,
skip_duplicates=True,
allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
def make(self, key):
whole_pass_time = time.time()
# ----------- Doing the v4 Processing ------- #
segment_id = key["segment_id"]
if verbose:
print(f"\n-- Working on neuron {segment_id}---")
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4()
& dict(segment_id=segment_id)).fetch1()
#1) Find the coordinates of the nucleus for that new segment
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
if verbose:
print(f"nuc_center_coords = {nuc_center_coords}")
#2) Make sure that same number of DecompositionAxon objects as in Decomposition
old_segment_id = segment_map_dict["old_segment_id"]
if verbose:
print(f"old_segment_id = {old_segment_id}")
search_key = dict(segment_id=old_segment_id)
n_somas = len(minnie.BaylorSegmentCentroid() & search_key)
n_decomp_axon = len(minnie.DecompositionAxon() & search_key)
if verbose:
print(
f"# of somas = {n_somas} and # of DecompositionAxon = {n_decomp_axon}"
)
if n_somas != n_decomp_axon:
raise Exception(
f"# of somas = {n_somas} NOT MATCH # of DecompositionAxon = {n_decomp_axon}"
)
#3) Pick the neuron object that is closest and within a certain range of the nucleus
neuron_objs, split_idxs = du.decomposition_with_spine_recalculation(
old_segment_id)
if n_somas > 1:
"""
Finding the closest soma:
1) For each neuron object get the mesh center of the soma object
2) Find the distance of each from the nucleus center
3) Find the arg min distance and make sure within threshold
4) Mark the current neuron and the current split index
"""
nuclei_distance_threshold = 15000
soma_center_coords = [k["S0"].mesh_center for k in neuron_objs]
soma_distances = [
np.linalg.norm(k - nuc_center_coords)
for k in soma_center_coords
]
min_dist_arg = np.argmin(soma_distances)
min_dist = soma_distances[min_dist_arg]
if verbose:
print(f"soma_distances = {soma_distances}")
print(
f"min_dist_arg = {min_dist_arg}, with min distance = {min_dist}"
)
if min_dist > nuclei_distance_threshold:
raise Exception(
f"min_dist ({min_dist}) larger than nuclei_distance_threshold ({nuclei_distance_threshold})"
)
neuron_obj = neuron_objs[min_dist_arg]
split_index = split_idxs[min_dist_arg]
if verbose:
print(f"Winning split_index = {split_index}")
else:
split_index = split_idxs[0]
neuron_obj = neuron_objs[0]
(filt_neuron, return_synapse_df_revised, return_synapse_df_errors,
return_validation_df_revised,
return_validation_df_extension) = vu.filtered_neuron_score(
neuron_obj=neuron_obj,
filter_list=pru.v4_exc_filters(),
plot_limb_branch_filter_with_disconnect_effect=False,
verbose=True,
plot_score=False,
nucleus_id=nucleus_id,
return_synapse_df_errors=True,
return_validation_df_extension=True,
split_index=split_index)
print(f"\n\n ----- Done Filtering ----------")
#------- saving off the filtered neuron
save_time = time.time()
file_name = f"{filt_neuron.segment_id}_{filt_neuron.description}_v4_val"
ret_file_path = filt_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
file_name=file_name,
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
print(f"Save time = {time.time() - save_time}")
# ---------- Getting the scores of the proofreading ----- #
presyn_scores_dict = vu.scores_presyn(return_validation_df_revised)
postsyn_scores_dict = vu.scores_postsyn(return_validation_df_revised)
cat = vu.synapse_validation_df_to_category_counts(
return_validation_df_revised,
print_postsyn=True,
print_presyn=False)
run_time = np.round(time.time() - whole_pass_time, 2)
final_dict = dict(
key,
split_index=split_index,
decomposition=ret_file_path_str,
axon_length=filt_neuron.axon_length,
validation_df=return_validation_df_revised.to_numpy(),
validation_df_ext=return_validation_df_extension.to_numpy(),
pre_tp=cat["presyn"]["TP"],
pre_tn=cat["presyn"]["TN"],
pre_fp=cat["presyn"]["FP"],
pre_fn=cat["presyn"]["FN"],
pre_precision=presyn_scores_dict["precision"],
pre_recall=presyn_scores_dict["recall"],
pre_f1=presyn_scores_dict["f1"],
post_tp=cat["postsyn"]["TP"],
post_tn=cat["postsyn"]["TN"],
post_fp=cat["postsyn"]["FP"],
post_fn=cat["postsyn"]["FN"],
post_precision=postsyn_scores_dict["precision"],
post_recall=postsyn_scores_dict["recall"],
post_f1=postsyn_scores_dict["f1"],
run_time=run_time)
self.insert1(final_dict,
skip_duplicates=True,
allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {run_time} ------ ***"
)
def make(self, key):
"""
Pseudocode:
1) Pull down the neuron object
2) Run the complete axon preprocessing on the neuron
3) Run the borders attributes dictionary
4) Save off the neuron object
5) Write the Attribute records
"""
print(f"\n\n\n---- Working on Neuron {key['segment_id']} ----")
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
whole_pass_time = time.time()
neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation(
segment_id)
neuron_obj = neuron_objs[0]
#2) Run the complete axon preprocessing on the neuron
neuron_obj_with_web = au.complete_axon_processing(neuron_obj,
verbose=True)
branch_attr = vu.neuron_to_border_branching_attributes(
neuron_obj_with_web,
plot_valid_border_branches=False,
plot_invalid_border_branches=False,
verbose=False)
#3) Run the borders attributes dictionary
branch_attr_keys = []
for k in branch_attr:
new_dict = dict(key)
new_dict.update(k)
new_dict["axon_version"] = axon_version
branch_attr_keys.append(new_dict)
if verbose:
print(f"\n\nlen(branch_attr_keys) = {len(branch_attr_keys)}")
#4) Save the file in a certain location
save_time = time.time()
ret_file_path = neuron_obj_with_web.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
file_name=f"{neuron_obj_with_web.segment_id}_validation_full_axon",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
n_dict = dict(key, decomposition=ret_file_path_str)
AutoProofreadValidationBorderNeurons.insert1(n_dict,
skip_duplicates=True)
#5) Write the Attribute records
if len(branch_attr_keys) > 0:
AutoProofreadValidationBorder.insert(branch_attr_keys,
skip_duplicates=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
def make(self, key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation(
segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType=True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
# -------- getting the nuclei info to match
# ver = 88
# nucleus_ids,nucleus_centers = du.segment_to_nuclei(segment_id,
# nuclei_version=ver)
nucleus_ids, nucleus_centers = du.segment_to_nuclei(segment_id,
nuclei_version=ver)
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(
neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold=15000,
return_matching_info=True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(
winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
#4) -------- Running the cell classification and stats--------------
if verbose:
print(
f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n"
)
filter_time = time.time()
(inh_exc_class, spine_category, axon_angles, n_axons, n_apicals,
neuron_spine_density, n_branches_processed,
skeletal_length_processed, n_branches_in_search_radius,
skeletal_length_in_search_radius
) = clu.inhibitory_excitatory_classifier(
neuron_obj,
return_spine_classification=True,
return_axon_angles=True,
return_n_axons=True,
return_n_apicals=True,
return_spine_statistics=True,
axon_limb_branch_dict_precomputed=None,
axon_angles_precomputed=None,
verbose=verbose)
if verbose:
print(
f"Total time for classification = {time.time() - filter_time}"
)
all_axon_angles = []
for limb_idx, limb_data in axon_angles.items():
for candidate_idx, cand_angle in limb_data.items():
all_axon_angles.append(cand_angle)
if len(axon_angles) > 0:
axon_angle_maximum = np.max(all_axon_angles)
else:
axon_angle_maximum = 0
if verbose:
print("\n -- Cell Type Classification Results --")
print(f"inh_exc_class={inh_exc_class}")
print(f"spine_category={spine_category}")
print(f"axon_angles={axon_angles}")
print(f"n_axons={n_axons}")
print(f"n_apicals={n_apicals}")
print(f"neuron_spine_density={neuron_spine_density}")
print(f"n_branches_processed={n_branches_processed}")
print(f"skeletal_length_processed={skeletal_length_processed}")
print(
f"n_branches_in_search_radius={n_branches_in_search_radius}"
)
print(
f"skeletal_length_in_search_radius={skeletal_length_in_search_radius}"
)
baylor_cell_type_info = dict(
cell_type_predicted=inh_exc_class,
spine_category=spine_category,
axon_angle_maximum=axon_angle_maximum,
n_axons=n_axons,
n_apicals=n_apicals,
spine_density_classifier=neuron_spine_density,
n_branches_processed=neuron_spine_density,
skeletal_length_processed=skeletal_length_processed,
n_branches_in_search_radius=n_branches_in_search_radius,
skeletal_length_in_search_radius=
skeletal_length_in_search_radius,
)
#5) ----- Deciding on cell type to use for axon
e_i_class = inh_exc_class
if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info[
"e_i"] is not None:
e_i_class = allen_cell_type_info["e_i"]
if verbose:
print(
f"e_i_class = {e_i_class} with inh_exc_class_to_use_for_axon = {inh_exc_class_to_use_for_axon}"
)
#6) -------- If excitatory running the axon processing--------------
"""
Psuedocode:
If e_i class is excitatory:
1) Filter away the axon on dendrite
2) Do the higher fidelity axon processing
3) Compute the axon features
"""
if e_i_class == "excitatory" and neuron_obj.axon_limb_name is not None:
if verbose:
print(
f"Excitatory so performing high fidelity axon and computing axon features"
)
# 1) Filter away the axon on dendrite
# 2) Do the higher fidelity axon processing
o_neuron, filtering_info = au.complete_axon_processing(
neuron_obj,
perform_axon_classification=False,
return_filtering_info=True)
filtering_info = {
k: np.round(v, 2)
for k, v in filtering_info.items()
if "area" in k or "length" in k
}
#3) Compute the axon features
axon_features = au.axon_features_from_neuron_obj(o_neuron)
else:
nru.clear_all_branch_labels(neuron_obj, labels_to_clear="axon")
o_neuron = neuron_obj
axon_features = dict()
filtering_info = dict()
#3) ------ Adding the Synapses -----------
o_neuron = syu.add_synapses_to_neuron_obj(
o_neuron,
validation=validation,
verbose=True,
original_mesh=None,
plot_valid_error_synapses=False,
calculate_synapse_soma_distance=False,
add_valid_synapses=True,
add_error_synapses=False)
# ------- Saving off the neuron object ----------------
save_time = time.time()
ret_file_path = o_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
#output_folder = "./",
file_name=
f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
n_dict = dict(
key,
split_index=split_index,
axon_version=au.axon_version,
decomposition=ret_file_path_str,
run_time=np.round(time.time() - st, 2),
cell_type_for_axon=e_i_class,
)
dicts_for_update = [
baylor_cell_type_info, allen_cell_type_info, nucleus_info,
filtering_info, axon_features
]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,
skip_duplicates=True,
allow_direct_insert=True)
#dict_to_write.append(n_dict)
#write the
#self.insert(dict_to_write,skip_duplicates=True,allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
def make(self,key):
"""
Pseudocode:
1) Pull Down All of the Neurons
2) Get the nucleus centers and the original mesh
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
print(f"\n\n------- AutoProofreadNeuron {segment_id} ----------")
neuron_objs,neuron_split_idxs = du.decomposition_with_spine_recalculation(segment_id)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
# 2) ----- Pre-work ------
nucleus_ids,nucleus_centers = du.segment_to_nuclei(segment_id)
if verbose:
print(f"Number of Corresponding Nuclei = {len(nucleus_ids)}")
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
original_mesh = du.fetch_segment_id_mesh(segment_id)
original_mesh_kdtree = KDTree(original_mesh.triangles_center)
# 3) ----- Iterate through all of the Neurons and Proofread --------
# lists to help save stats until write to ProofreadStats Table
filtering_info_list = []
synapse_stats_list = []
total_error_synapse_ids_list = []
for split_index,neuron_obj_pre_split in zip(neuron_split_idxs,neuron_objs):
whole_pass_time = time.time()
if verbose:
print(f"\n-----Working on Neuron Split {split_index}-----")
neuron_obj = neuron_obj_pre_split
# if neuron_obj_pre_split.n_error_limbs > 0:
# if verbose:
# print(f" ---> Pre-work: Splitting Neuron Limbs Because still error limbs exist--- ")
# neuron_objs_split = pru.split_neuron(neuron_obj_pre_split,
# verbose=False)
# if len(neuron_objs_split) > 1:
# raise Exception(f"After splitting the neuron there were more than 1: {neuron_objs_split}")
# neuron_obj= neuron_objs_split[0]
# else:
# neuron_obj = neuron_obj_pre_split
# Part A: Proofreading the Neuron
if verbose:
print(f"\n --> Part A: Proofreading the Neuron ----")
# nviz.visualize_neuron(neuron_obj,
# limb_branch_dict="all")
output_dict= pru.proofread_neuron(neuron_obj,
plot_limb_branch_filter_with_disconnect_effect=False,
plot_final_filtered_neuron=False,
verbose=True)
filtered_neuron = output_dict["filtered_neuron"]
cell_type_info = output_dict["cell_type_info"]
filtering_info = output_dict["filtering_info"]
# Part B: Getting Soma Centers and Matching To Nuclei
if verbose:
print(f"\n\n --> Part B: Getting Soma Centers and Matching To Nuclei ----")
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold = 15000,
return_matching_info = True,
verbose=True)
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
# Part C: Getting the Faces of the Original Mesh
if verbose:
print(f"\n\n --> Part C: Getting the Faces of the Original Mesh ----")
original_mesh_faces = tu.original_mesh_faces_map(original_mesh,
filtered_neuron.mesh,
exact_match=True,
original_mesh_kdtree=original_mesh_kdtree)
original_mesh_faces_file = du.save_proofread_faces(original_mesh_faces,
segment_id=segment_id,
split_index=split_index)
# nviz.plot_objects(recovered_mesh)
# Part D: Getting the Synapse Information
if verbose:
print(f"\n\n --> Part D: Getting the Synapse Information ----")
(keys_to_write,
synapse_stats,
total_error_synapse_ids) = pru.synapse_filtering(filtered_neuron,
split_index,
nucleus_id=winning_nucleus_id,
segment_id=None,
return_synapse_filter_info = True,
return_synapse_center_data = False,
return_error_synapse_ids = True,
mapping_threshold = 500,
plot_synapses=False,
verbose = True,
original_mesh_method = True,
original_mesh = original_mesh,
original_mesh_kdtree = original_mesh_kdtree,
valid_faces_on_original_mesh=original_mesh_faces,
)
soma_x,soma_y,soma_z = nru.soma_centers(filtered_neuron,
soma_name="S0",
voxel_adjustment=True)
#7) Creating the dictionary to insert into the AutoProofreadNeuron
new_key = dict(key,
split_index = split_index,
proof_version = proof_version,
multiplicity = len(neuron_objs),
# -------- Important Excitatory Inhibitory Classfication ------- #
cell_type_predicted = cell_type_info["inh_exc_class"],
spine_category=cell_type_info["spine_category"],
n_axons=cell_type_info["n_axons"],
n_apicals=cell_type_info["n_axons"],
# ----- Soma Information ----#
nucleus_id = nucleus_info["nuclei_id"],
nuclei_distance = np.round(nucleus_info["nuclei_distance"],2),
n_nuclei_in_radius = nucleus_info["n_nuclei_in_radius"],
n_nuclei_in_bbox = nucleus_info["n_nuclei_in_bbox"],
soma_x = soma_x,
soma_y =soma_y,
soma_z =soma_z,
# ---------- Mesh Faces ------ #
mesh_faces = original_mesh_faces_file,
# ------------- The Regular Neuron Information (will be computed in the stats dict) ----------------- #
# ------ Information Used For Excitatory Inhibitory Classification --------
axon_angle_maximum=cell_type_info["axon_angle_maximum"],
spine_density_classifier=cell_type_info["neuron_spine_density"],
n_branches_processed=cell_type_info["n_branches_processed"],
skeletal_length_processed=cell_type_info["skeletal_length_processed"],
n_branches_in_search_radius=cell_type_info["n_branches_in_search_radius"],
skeletal_length_in_search_radius=cell_type_info["skeletal_length_in_search_radius"],
run_time=np.round(time.time() - whole_pass_time,4)
)
stats_dict = filtered_neuron.neuron_stats()
new_key.update(stats_dict)
# ------ Writing the Data To the Tables ----- #
SynapseProofread.insert(keys_to_write,skip_duplicates=True)
self.insert1(new_key,skip_duplicates=True,allow_direct_insert=True)
#saving following information for later processing:
filtering_info_list.append(filtering_info)
synapse_stats_list.append(synapse_stats)
total_error_synapse_ids_list.append(total_error_synapse_ids)
# Once have inserted all the new neurons need to compute the stats
if verbose:
print("Computing the overall stats")
overall_syn_error_rates = pru.calculate_error_rate(total_error_synapse_ids_list,
synapse_stats_list,
verbose=True)
# Final Part: Create the stats table entries and insert
proofread_stats_entries = []
stats_to_make_sure_in_proofread_stats = [
'axon_on_dendrite_merges_error_area',
'axon_on_dendrite_merges_error_length',
'low_branch_clusters_error_area',
'low_branch_clusters_error_length',
'dendrite_on_axon_merges_error_area',
'dendrite_on_axon_merges_error_length',
'double_back_and_width_change_error_area',
'double_back_and_width_change_error_length',
'crossovers_error_area',
'crossovers_error_length',
'high_degree_coordinates_error_area',
'high_degree_coordinates_error_length',
]
for sp_idx,split_index in enumerate(neuron_split_idxs):
synapse_stats = synapse_stats_list[sp_idx]
filtering_info = filtering_info_list[sp_idx]
curr_key = dict(key,
split_index = split_index,
proof_version = proof_version,
# ------------ For local valid synapses to that split_index
n_valid_syn_presyn_for_split=synapse_stats["n_valid_syn_presyn"],
n_valid_syn_postsyn_for_split=synapse_stats["n_valid_syn_postsyn"],
)
for s in stats_to_make_sure_in_proofread_stats:
if s not in filtering_info.keys():
curr_key[s] = None
filter_key = {k:np.round(v,2) for k,v in filtering_info.items() if "area" in k or "length" in k}
curr_key.update(filter_key)
curr_key.update(overall_syn_error_rates)
proofread_stats_entries.append(curr_key)
ProofreadStats.insert(proofread_stats_entries,skip_duplicates=True)
print(f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***")
def make(self, key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if len(key_source_inh & dict(segment_id=segment_id)) > 0:
manual_e_i = "inhibitory"
elif len(key_source_exc & dict(segment_id=segment_id)) > 0:
manual_e_i = "excitatory"
else:
raise Exception("Not in exc or inh table")
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation(
segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType=True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
''' ------ Old way of getting the nucleus info for the manual proofread data -------
# -------- getting the nuclei info to match
try:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4() & dict(old_segment_id=segment_id)).fetch1()
except:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMapInh() & dict(old_segment_id=segment_id)).fetch1()
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
nucleus_ids = [nucleus_id]
nucleus_centers = [nuc_center_coords]
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")'''
nucleus_ids, nucleus_centers = du.segment_to_nuclei(
segment_id,
#nuclei_version=ver
)
if verbose:
print(f"Number of Corresponding Nuclei = {len(nucleus_ids)}")
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(
neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold=15000,
return_matching_info=True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(
winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
# 3) ---- Doing Baylor Cell Type Classification ---------
# 3a) --- Adding the synapses and spine labels
if verbose:
print(
f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n"
)
st = time.time()
if verbose:
print(f"Adding the synapses and the head_neck_shaft")
neuron_obj = syu.add_synapses_to_neuron_obj(
neuron_obj,
validation=validation,
verbose=verbose,
original_mesh=None,
plot_valid_error_synapses=False,
calculate_synapse_soma_distance=False,
add_valid_synapses=True,
add_error_synapses=False,
)
neuron_obj = spu.add_head_neck_shaft_spine_objs(neuron_obj,
verbose=verbose)
if verbose:
print(
f"Done adding synapses and head_neck_shaft: {time.time() - st}"
)
# 3b) --- Running the stats for Baylor Classification
filter_time = time.time()
'''
limb_branch_dict = ctu.postsyn_branches_near_soma_for_syn_post_density(
neuron_obj = neuron_obj,
verbose = False,)
(syn_density_post,
syn_density_head,
syn_density_neck,
syn_density_shaft,
skeletal_length_processed_syn) = ctu.synapse_density_stats(neuron_obj = neuron_obj,
limb_branch_dict = limb_branch_dict,
verbose = True)
(spine_density,
skeletal_length_processed_spine) = ctu.spine_density_near_soma(neuron_obj = neuron_obj,
verbose = True,
multiplier = 1000)
if verbose:
print(f"Total time for density calculations = {time.time() - filter_time}")
# 4) ------ Predicting the E/I Group Based on the data collected --------
baylor_cell_type_info = dict(
syn_density_post = syn_density_post,
syn_density_head = syn_density_head,
syn_density_neck = syn_density_neck,
syn_density_shaft = syn_density_shaft,
skeletal_length_processed_syn=skeletal_length_processed_syn,
spine_density=spine_density,
skeletal_length_processed_spine = skeletal_length_processed_spine
)
baylor_e_i = ctu.e_i_classification_single(data=[syn_density_shaft,spine_density],
features=["syn_density_shaft","spine_density"],
verbose = True,
return_label_name = True
)
'''
baylor_e_i, baylor_cell_type_info = ctu.e_i_classification_from_neuron_obj(
neuron_obj,
verbose=True,
return_cell_type_info=True,
return_dendrite_branch_stats=True)
baylor_cell_type_info["baylor_e_i"] = baylor_e_i
#5) ----- Deciding on cell type to use for axon
if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info[
"allen_e_i"] is not None:
e_i_class = allen_cell_type_info["allen_e_i"]
cell_type_used = "allen"
else:
e_i_class = baylor_e_i
cell_type_used = "baylor"
if verbose:
print(
f"e_i_class = {e_i_class} with cell_type_used = {cell_type_used}"
)
#3) ------ Axon Classification (and getting the axon features)------------------
o_neuron, filtering_info, axon_angles_dict = au.complete_axon_processing(
neuron_obj,
cell_type=e_i_class,
add_synapses_and_head_neck_shaft_spines=False,
validation=validation,
plot_initial_axon=False,
plot_axon_on_dendrite=False,
return_filtering_info=True,
return_axon_angle_info=True,
verbose=verbose)
filtering_info = {
k: np.round(v, 2)
for k, v in filtering_info.items()
if "area" in k or "length" in k
}
axon_features = au.axon_features_from_neuron_obj(o_neuron)
# ------- Saving off the neuron object ----------------
save_time = time.time()
ret_file_path = o_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
#output_folder = "./",
file_name=
f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}_pipe_v6_e_i_val_3",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
# ----------------
# ---- 8/10 Addition ----------
if save_axon_skeleton:
axon_skeleton_file = du.save_proofread_skeleton(
o_neuron.axon_skeleton,
segment_id=o_neuron.segment_id,
split_index=split_index,
file_name_ending=f"decomp_cell_type_axon_skeleton_e_i_val_3"
)
else:
axon_skeleton_file = None
#---- 8/29 Addition: Will compute the soma center of the mesh in nm ---
soma_x_nm, soma_y_nm, soma_z_nm = neuron_obj["S0"].mesh_center
if verbose:
print(
f"soma_x_nm, soma_y_nm, soma_z_nm = {soma_x_nm, soma_y_nm, soma_z_nm}"
)
n_dict = dict(
key,
split_index=split_index,
axon_version=au.axon_version,
decomposition=ret_file_path_str,
run_time=np.round(time.time() - st, 2),
manual_e_i=manual_e_i,
cell_type=e_i_class,
cell_type_used=cell_type_used,
axon_skeleton=str(axon_skeleton_file),
soma_x_nm=soma_x_nm,
soma_y_nm=soma_y_nm,
soma_z_nm=soma_z_nm,
n_syn_pre=neuron_obj.n_synapses_pre,
n_syn_post=neuron_obj.n_synapses_post,
)
soma_stats_dict = ctu.soma_stats_for_cell_type(neuron_obj)
dicts_for_update = [
baylor_cell_type_info, allen_cell_type_info, nucleus_info,
filtering_info, axon_features, axon_angles_dict,
soma_stats_dict
]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,
skip_duplicates=True,
allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
def make(self,key):
"""
Pseudocode:
1) Pull Down All of the Neurons
2) Get the nucleus centers and the original mesh
"""
whole_pass_time = time.time()
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
split_index = key["split_index"]
if verbose:
print(f"\n\n------- Working on Neuron {segment_id}_{split_index} -----")
cell_type,nucleus_id = (minnie.DecompositionCellTypeV7() & key).fetch1(f"{cell_type_used}_e_i","nucleus_id")
if verbose:
print(f"---- Working on Neuron {segment_id}:{split_index}")
print(f"nucleus_id = {nucleus_id},cell_type = {cell_type}")
neuron_obj = du.decomposition_with_spine_recalculation(segment_id,split_index = split_index)
import axon_utils as au
au.compute_axon_on_dendrite_limb_branch_dict(neuron_obj = neuron_obj,
plot_axon_on_dendrite=False,
verbose = True,
clean_prior_labels = True)
neuron_obj = syu.add_synapses_to_neuron_obj(neuron_obj,
validation = False,
verbose = verbose,
original_mesh = None,
plot_valid_error_synapses = False,
calculate_synapse_soma_distance = False,
add_valid_synapses = True,
add_error_synapses=False,)
if plot_data:
nviz.plot_axon(neuron_obj)
#2) Running the neuron proofreading
# neuron_obj_proof,filtering_info = pru.proofread_neuron_full(
# neuron_obj,
# filter_list = [pru.exc_axon_on_dendrite_merges_filter()],
# # arguments for processing down in DecompositionCellTypeV7
# cell_type=cell_type,
# add_valid_synapses = False,
# validation = validation,
# add_spines = False,
# perform_axon_processing = False,
# return_after_axon_processing = False,
# #arguments for processing after DecompositionCellTypeV7 to Proofread Neuron
# plot_head_neck_shaft_synapses = plot_data,
# plot_soma_synapses = plot_data,
# proofread_verbose = proofread_verbose,
# verbose_outline = verbose,
# plot_limb_branch_filter_with_disconnect_effect = plot_data,
# plot_final_filtered_neuron = False,
# plot_synapses_after_proofread = False,
# plot_compartments = plot_data,
# plot_valid_synapses = plot_data,
# plot_error_synapses = plot_data,
# verbose = verbose,
# debug_time = verbose,
# )
o_neuron,filtering_info = pru.proofread_neuron_class_predetermined(
neuron_obj=neuron_obj,
inh_exc_class = cell_type,
plot_limb_branch_filter_with_disconnect_effect = plot_data,
verbose = verbose,
verbose_outline = verbose,
high_fidelity_axon_on_excitatory = False,
plot_final_filtered_neuron = plot_data,
filter_list=[pru.exc_axon_on_dendrite_merges_filter()],
)
#4) Collect and Write Neuron Stats
limb_branch_to_cancel = pru.extract_from_filter_info(filtering_info,
name_to_extract="limb_branch_dict_to_cancel")
red_blue_suggestions = pru.extract_from_filter_info(filtering_info,
name_to_extract = "red_blue_suggestions")
filter_key = {k:np.round(v,2) for k,v in filtering_info.items() if "area" in k or "length" in k}
neuron_stats_dict = dict(key,
proof_version = proof_version,
limb_branch_to_cancel=limb_branch_to_cancel,
red_blue_suggestions=red_blue_suggestions,
cell_type=cell_type,
nucleus_id=nucleus_id)
neuron_stats_dict.update(filter_key)
AutoProofreadAxonOnDendrite.insert1(neuron_stats_dict,
skip_duplicates = True,
ignore_extra_fields = True,)
print(f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***")
def make(self,key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs,neuron_split_idxs = du.decomposition_with_spine_recalculation(segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType = True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
# -------- getting the nuclei info to match
try:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4() & dict(old_segment_id=segment_id)).fetch1()
except:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMapInh() & dict(old_segment_id=segment_id)).fetch1()
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
nucleus_ids = [nucleus_id]
nucleus_centers = [nuc_center_coords]
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index,neuron_obj in zip(neuron_split_idxs,neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold = 15000,
return_matching_info = True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
#4) -------- Running the cell classification and stats--------------
if verbose:
print(f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n")
filter_time = time.time()
#---- adding the synapses and spines data -----#
neuron_obj_exc_syn = syu.add_synapses_to_neuron_obj(neuron_obj,
validation = True,
verbose = True,
original_mesh = None,
plot_valid_error_synapses = False,
calculate_synapse_soma_distance = False,
add_valid_synapses = True,
add_error_synapses=False,)
neuron_obj_exc_syn_sp = spu.add_head_neck_shaft_spine_objs(neuron_obj_exc_syn,
verbose = True
)
limb_branch_dict = ctu.postsyn_branches_near_soma_for_syn_post_density(
neuron_obj = neuron_obj_exc_syn_sp,
verbose = False,)
(syn_density_post,
syn_density_head,
syn_density_neck,
syn_density_shaft,
skeletal_length_processed_syn) = ctu.synapse_density_stats(neuron_obj = neuron_obj_exc_syn_sp,
limb_branch_dict = limb_branch_dict,
verbose = True)
(spine_density,
skeletal_length_processed_spine) = ctu.spine_density_near_soma(neuron_obj = neuron_obj_exc_syn_sp,
verbose = True,
multiplier = 1000)
if verbose:
print(f"Total time for density calculations = {time.time() - filter_time}")
baylor_cell_type_info = dict(
syn_density_post = syn_density_post,
syn_density_head = syn_density_head,
syn_density_neck = syn_density_neck,
syn_density_shaft = syn_density_shaft,
skeletal_length_processed_syn=skeletal_length_processed_syn,
spine_density=spine_density,
skeletal_length_processed_spine = skeletal_length_processed_spine
)
n_dict = dict(key,
split_index = split_index,
run_time = np.round(time.time() - st,2),
)
dicts_for_update = [baylor_cell_type_info,
allen_cell_type_info,
nucleus_info,]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,skip_duplicates=True,allow_direct_insert=True)
print(f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***")
def make(self, key):
"""
Pseudocode:
1) Pull Down all the Neuron Objects associated with a segment_id
For each neuron:
2) Run the full axon preprocessing
3) Save off the neuron
4) Save dict entry to list
5) Write the new entry to the table
"""
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
if verbose:
print(f"------- Working on Neuron {segment_id} -----")
whole_pass_time = time.time()
#1) Pull Down all the Neuron Objects associated with a segment_id
neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation(
segment_id,
ignore_DecompositionAxon=True,
ignore_DecompositionCellType=True)
if verbose:
print(f"Number of Neurons found ={len(neuron_objs)}")
#For each neuron:
dict_to_write = []
# -------- getting the nuclei info to match
try:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4()
& dict(old_segment_id=segment_id)).fetch1()
except:
segment_map_dict = (minnie.AutoProofreadValidationSegmentMapInh()
& dict(old_segment_id=segment_id)).fetch1()
nucleus_id = segment_map_dict["nucleus_id"]
nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id)
nucleus_ids = [nucleus_id]
nucleus_centers = [nuc_center_coords]
print(f"nucleus_ids = {nucleus_ids}")
print(f"nucleus_centers = {nucleus_centers}")
for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs):
if verbose:
print(f"--> Working on Split Index {split_index} -----")
st = time.time()
# ------------- Does all of the processing -------------------
#1) ------ Getting the paired nuclei ------
winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei(
neuron_obj,
"S0",
nucleus_ids,
nucleus_centers,
nuclei_distance_threshold=15000,
return_matching_info=True,
verbose=True)
# else:
# winning_nucleus_id = 12345
# nucleus_info = dict()
# nucleus_info["nucleus_id"] = winning_nucleus_id
# nucleus_info["nuclei_distance"] = 0
# nucleus_info["n_nuclei_in_radius"] = 1
# nucleus_info["n_nuclei_in_bbox"] = 1
if verbose:
print(f"nucleus_info = {nucleus_info}")
print(f"winning_nucleus_id = {winning_nucleus_id}")
#2) ------- Finding the Allen Cell Types -------
allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id(
winning_nucleus_id)
if verbose:
print(f"allen_cell_type_info = {allen_cell_type_info}")
#3) ------ Axon Classification (and getting the axon features)------------------
import axon_utils as au
o_neuron, filtering_info, axon_angles_dict = au.complete_axon_processing(
neuron_obj,
add_synapses_and_head_neck_shaft_spines=True,
validation=validation,
plot_initial_axon=False,
plot_axon_on_dendrite=False,
return_filtering_info=True,
return_axon_angle_info=True,
verbose=verbose)
filtering_info = {
k: np.round(v, 2)
for k, v in filtering_info.items()
if "area" in k or "length" in k
}
axon_features = au.axon_features_from_neuron_obj(o_neuron)
#3)------- Running the cell classification and stats--------------
if verbose:
print(
f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n"
)
filter_time = time.time()
#---- adding the synapses and spines data -----#
limb_branch_dict = ctu.postsyn_branches_near_soma_for_syn_post_density(
neuron_obj=o_neuron,
verbose=False,
)
(syn_density_post, syn_density_head, syn_density_neck,
syn_density_shaft,
skeletal_length_processed_syn) = ctu.synapse_density_stats(
neuron_obj=o_neuron,
limb_branch_dict=limb_branch_dict,
verbose=True)
(spine_density,
skeletal_length_processed_spine) = ctu.spine_density_near_soma(
neuron_obj=o_neuron, verbose=True, multiplier=1000)
if verbose:
print(
f"Total time for density calculations = {time.time() - filter_time}"
)
baylor_cell_type_info = dict(
syn_density_post=syn_density_post,
syn_density_head=syn_density_head,
syn_density_neck=syn_density_neck,
syn_density_shaft=syn_density_shaft,
skeletal_length_processed_syn=skeletal_length_processed_syn,
spine_density=spine_density,
skeletal_length_processed_spine=skeletal_length_processed_spine
)
# 4) ------ Predicting the E/I Group Based on the data collected --------
################ NEED TO INSERT CODE TO DO THIS ###########
#5) ----- Deciding on cell type to use for axon
e_i_class = inh_exc_class
if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info[
"e_i"] is not None:
e_i_class = allen_cell_type_info["e_i"]
if verbose:
print(
f"e_i_class = {e_i_class} with inh_exc_class_to_use_for_axon = {inh_exc_class_to_use_for_axon}"
)
# ------- Saving off the neuron object ----------------
save_time = time.time()
ret_file_path = o_neuron.save_compressed_neuron(
output_folder=str(du.get_decomposition_path()),
#output_folder = "./",
file_name=
f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}",
return_file_path=True,
export_mesh=False,
suppress_output=True)
ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2"
if verbose:
print(f"ret_file_path_str = {ret_file_path_str}")
print(f"Save time = {time.time() - save_time}")
# ----------------
n_dict = dict(
key,
split_index=split_index,
axon_version=au.axon_version,
decomposition=ret_file_path_str,
run_time=np.round(time.time() - st, 2),
cell_type=e_i_class,
)
dicts_for_update = [
baylor_cell_type_info, allen_cell_type_info, nucleus_info,
filtering_info, axon_features, axon_angles_dict
]
for d in dicts_for_update:
n_dict.update(d)
self.insert1(n_dict,
skip_duplicates=True,
allow_direct_insert=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
def make(self, key):
"""
Pseudocode:
1) Pull Down All of the Neurons
2) Get the nucleus centers and the original mesh
"""
whole_pass_time = time.time()
# 1) Pull Down All of the Neurons
segment_id = key["segment_id"]
split_index = key["split_index"]
if verbose:
print(
f"\n\n------- Working on Neuron {segment_id}_{split_index} -----"
)
cell_type, nucleus_id = (minnie.DecompositionCellTypeV7()
& key).fetch1(f"{cell_type_used}_e_i",
"nucleus_id")
if verbose:
print(f"---- Working on Neuron {segment_id}:{split_index}")
print(f"nucleus_id = {nucleus_id},cell_type = {cell_type}")
neuron_obj = du.decomposition_with_spine_recalculation(
segment_id, split_index=split_index)
if plot_data:
nviz.plot_axon(neuron_obj)
#2) Running the neuron proofreading
neuron_obj_proof, filtering_info = pru.proofread_neuron_full(
neuron_obj,
# arguments for processing down in DecompositionCellTypeV7
cell_type=cell_type,
add_valid_synapses=False,
validation=validation,
add_spines=False,
perform_axon_processing=False,
return_after_axon_processing=False,
#arguments for processing after DecompositionCellTypeV7 to Proofread Neuron
plot_head_neck_shaft_synapses=plot_data,
plot_soma_synapses=plot_data,
proofread_verbose=proofread_verbose,
verbose_outline=verbose,
plot_limb_branch_filter_with_disconnect_effect=plot_data,
plot_final_filtered_neuron=False,
plot_synapses_after_proofread=False,
plot_compartments=plot_data,
plot_valid_synapses=plot_data,
plot_error_synapses=plot_data,
verbose=verbose,
debug_time=verbose,
)
#3) Collect and Write Data to Synapse Table
dj_keys_valid = syu.synapses_to_dj_keys(neuron_obj_proof,
valid_synapses=True,
verbose=verbose,
nucleus_id=nucleus_id,
split_index=split_index)
dj_keys_error = syu.synapses_to_dj_keys(neuron_obj_proof,
valid_synapses=False,
verbose=verbose,
nucleus_id=nucleus_id,
split_index=split_index)
if verbose:
print(f"n_synapses_total = {neuron_obj_proof.n_synapses_total}")
AutoProofreadSynapse7.insert(dj_keys_valid, skip_duplicates=True)
AutoProofreadSynapseErrors7.insert(dj_keys_error, skip_duplicates=True)
#4) Collect and Write Neuron Stats
limb_branch_to_cancel = pru.extract_from_filter_info(
filtering_info, name_to_extract="limb_branch_dict_to_cancel")
red_blue_suggestions = pru.extract_from_filter_info(
filtering_info, name_to_extract="red_blue_suggestions")
filter_key = {
k: np.round(v, 2)
for k, v in filtering_info.items() if "area" in k or "length" in k
}
mesh_skeleton_file_paths = pru.save_off_meshes_skeletons(
neuron_obj_proof,
verbose=False,
split_index=key["split_index"],
file_name_ending=f"proofv{proof_version}")
neuron_stats_dict = dict(
key,
proof_version=proof_version,
limb_branch_to_cancel=limb_branch_to_cancel,
red_blue_suggestions=red_blue_suggestions,
)
neuron_stats_dict.update(mesh_skeleton_file_paths)
neuron_stats_dict.update(filter_key)
#--------- 12/8: Adding the neuron_graph object that will be retrieved later ----
if save_G_with_attrs:
G = ctcu.G_with_attrs_from_neuron_obj(neuron_obj_proof,
plot_G=False)
G_path = ctcu.save_G_with_attrs(G,
segment_id=segment_id,
split_index=split_index)
if verbose:
print(f"Saved G_path = {G_path}")
neuron_stats_dict["neuron_graph"] = G_path
AutoProofreadStats7.insert1(neuron_stats_dict, skip_duplicates=True)
# 5) Collecting Stats for the AutoProofreadNeurons6 table
#a) Neuron basics
if verbose:
print(f"\n--5a) Neuron basics")
dicts_to_update = []
multiplicity = du.multiplicity_from_segment_id(segment_id)
soma_x, soma_y, soma_z = nru.soma_centers(neuron_obj,
soma_name="S0",
voxel_adjustment=True)
basic_cell_dict = dict(multiplicity=multiplicity,
soma_x=soma_x,
soma_y=soma_y,
soma_z=soma_z,
cell_type=cell_type,
cell_type_used=cell_type_used)
dicts_to_update.append(basic_cell_dict)
#b) Neuron Overall Statistics
if verbose:
print(f"\n--5b) Neuron Overall Statistics")
neuron_stats_dict = neuron_obj_proof.neuron_stats(
stats_to_ignore=["axon_length", "axon_area"])
dicts_to_update.append(neuron_stats_dict)
#c) compartment Stats
if verbose:
print(f"\n--5c) compartment Stats")
comp_stats = apu.compartments_stats(neuron_obj_proof,
compartment_labels=None,
verbose=False)
dicts_to_update.append(comp_stats)
#d) Synapse Stats
if verbose:
print(f"\n--5d) Synapse Stats")
syn_stats = syu.complete_n_synapses_analysis(neuron_obj_proof)
dicts_to_update.append(syn_stats)
#e) Cell Typing Info after proofreading
if verbose:
print(f"\n--5e) Cell Typing Info after proofreading")
baylor_e_i, baylor_cell_type_info = ctu.e_i_classification_from_neuron_obj(
neuron_obj_proof, verbose=False, return_cell_type_info=True)
baylor_cell_type_info["baylor_e_i"] = baylor_e_i
baylor_cell_type_info = {
f"{k}_after_proof": v
for k, v in baylor_cell_type_info.items()
}
dicts_to_update.append(baylor_cell_type_info)
#c
if verbose:
print(f"\n--5e) Cell Typing Info after proofreading")
axon_feature_dict = au.axon_features_from_neuron_obj(
neuron_obj_proof, features_to_exclude=("length", "n_branches"))
apical_feature_dict = apu.compartment_features_from_skeleton_and_soma_center(
neuron_obj_proof,
compartment_label="apical_total",
name_prefix="apical",
features_to_exclude=("length", "n_branches"),
)
basal_feature_dict = apu.compartment_features_from_skeleton_and_soma_center(
neuron_obj_proof,
compartment_label="basal",
name_prefix="basal",
features_to_exclude=("length", "n_branches"),
)
dendrite_feature_dict = apu.compartment_features_from_skeleton_and_soma_center(
neuron_obj_proof,
compartment_label="dendrite",
name_prefix="dendrite",
features_to_exclude=("length", "n_branches"),
)
dicts_to_update += [
axon_feature_dict, apical_feature_dict, basal_feature_dict,
dendrite_feature_dict
]
#g) Repeating old features from DecompositionCellType table
if verbose:
print(
f"\n--5g) Repeating old features from DecompositionCellTypeV7 table"
)
decomp_cell_type_features = [
"nucleus_id",
"nuclei_distance",
"n_nuclei_in_radius",
"n_nuclei_in_bbox",
"soma_x_nm",
"soma_y_nm",
"soma_z_nm",
"baylor_e_i",
"allen_e_i",
"cell_type_used",
"cell_type",
"axon_angle_max",
"axon_angle_min",
"n_axon_angles",
"allen_e_i_n_nuc",
"allen_cell_type",
"allen_cell_type_n_nuc",
"allen_cell_type_e_i",
]
decomp_dict = (minnie.DecompositionCellTypeV7() & key).fetch(
*decomp_cell_type_features, as_dict=True)[0]
decomp_dict["cell_type_used_for_axon"] = decomp_dict["cell_type_used"]
decomp_dict["cell_type_for_axon"] = decomp_dict["cell_type"]
del decomp_dict["cell_type_used"]
del decomp_dict["cell_type"]
dicts_to_update.append(decomp_dict)
if plot_data:
nviz.plot_compartments(neuron_obj_proof)
#h) Writing the Data
if verbose:
print(f"\n--5h) Writing the Data")
neuron_proof_dict = dict(key,
proof_version=proof_version,
run_time=np.round(
time.time() - whole_pass_time, 2))
for d_u in dicts_to_update:
neuron_proof_dict.update(d_u)
AutoProofreadNeurons7.insert1(neuron_proof_dict, skip_duplicates=True)
print(
f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***"
)
