class DecompositionSplit(dj.Computed): definition = """ -> minnie.Decomposition() split_index: tinyint unsigned #the index of the neuron object that resulted AFTER THE SPLITTING ALGORITHM split_version: tinyint unsigned #the version of the splitting algorithm used --- multiplicity=null : tinyint unsigned # the number of somas found for this base segment n_splits : int unsigned # the number of cuts required to help split the neuron split_success : tinyint unsigned # the successfulness of the splitting n_error_limbs_cancelled : tinyint unsigned # number of limbs that couldn't be resolved and cancelled out n_same_soma_limbs_cancelled : tinyint unsigned # number of same soma touching limbs that couldn't be resolved and cancelled out n_multi_soma_limbs_cancelled : tinyint unsigned # number of multi soma touching limbs that couldn't be resolved and cancelled out error_imbs_cancelled_area=NULL : double # the total area (in microns^2) of the limbs that was cancelled out because touching the same soma multiple times or multiple somas error_imbs_cancelled_skeletal_length = NULL : double #the total skeletal length (in microns) of the limbs that were called out because could not be resolved split_results: longblob #will store the results of how to split the limbs of neuron objects from original neuron decomposition: <decomposition> n_vertices : int unsigned # number of vertices n_faces : int unsigned # number of faces n_not_processed_soma_containing_meshes : int unsigned #the number of meshes with somas that were not processed n_error_limbs: int #the number of limbs that are touching multiple somas or 1 soma in multiple places n_same_soma_multi_touching_limbs: int # number of limbs that touch the same soma multiple times n_multi_soma_touching_limbs: int # number of limbs that touch multiple somas n_somas: int #number of soma meshes detected max_soma_n_faces: int unsigned # The largest number of faces of the somas max_soma_volume: int unsigned # The largest volume of the somas the (volume in billions (10*9 nm^3)) n_limbs: int n_branches: int max_limb_n_branches=NULL:int skeletal_length=NULL: double max_limb_skeletal_length=NULL:double median_branch_length=NULL:double #gives information on average skeletal length to next branch point width_median=NULL: double #median width from mesh center without spines removed width_no_spine_median=NULL: double #median width from mesh center with spines removed width_90_perc=NULL: double # 90th percentile for width without spines removed width_no_spine_90_perc=NULL: double # 90th percentile for width with spines removed n_spines: bigint spine_density=NULL: double # n_spines/ skeletal_length spines_per_branch=NULL: double skeletal_length_eligible=NULL: double # the skeletal length for all branches searched for spines n_spine_eligible_branches=NULL: int # the number of branches that were checked for spines because passed width threshold spine_density_eligible=NULL:double # n_spines/skeletal_length_eligible spines_per_branch_eligible=NULL:double # n_spines/n_spine_eligible_branches total_spine_volume=NULL: double # the sum of all spine volume spine_volume_median=NULL: double # median of the spine volume for those spines with able to calculate volume spine_volume_density=NULL: double #total_spine_volume/skeletal_length spine_volume_density_eligible=NULL: double #total_spine_volume/skeletal_length_eligible spine_volume_per_branch_eligible=NULL: double #total_spine_volume/n_spine_eligible_branches run_time=NULL : double # the amount of time to run (seconds) """ ''' Old keysource used for inhibitory excitatory check classified_table = (minnie.BaylorManualCellType() & 'nucleus_version=3' & "(cell_type = 'excitatory') or (cell_type = 'inhibitory')") key_source = ((minnie.Decomposition & (minnie.NeuronSplitSuggestions.proj()) & (classified_table.proj()) & f"n_somas<{max_n_somas}" & "n_error_limbs>0"))''' # This keysource acounts that you could have more than 1 possible soma but not a significant limb connecting them (no error limbs) key_source = minnie.Decomposition( ) & "n_somas>1 OR n_error_limbs>0" & du.proofreading_segment_id_restriction( ) def make(self, key): """ Pseudocode for process: 1) Get the segment id from the key 2) Get the decomposed neurong object from Decomposition table 3) Run the multi_soma split suggestions algorithm 4) Get the number of splits required for this neuron 5) Split the neuron into a list of neuron objects 6) For each neuron object in the list: - get the number of errored limbs (to indicate the success type) - Change the description to include the multiplicity - Compute the information on the largest soma faces and volume - Save the neuron object to the external - Add the new write key to a list to commit 7) Write all of the keys """ whole_pass_time = time.time() # 1) Get the segment id from the key segment_id = key["segment_id"] print(f"\n\n\n---- Working on Neuron {key['segment_id']} ----") # 2) Get the decomposed neuron object from Decomposition table and the split suggestions neuron_obj_path = (minnie.Decomposition & key).fetch1("decomposition") neuron_obj = du.filepath_to_neuron_obj(neuron_obj_path) """ Old way that downloaded from another table # 3) Retrieve the multi soma suggestions split_results = (minnie.NeuronSplitSuggestions & key).fetch1("split_results") """ #3) Calculated the split results split_results = pru.multi_soma_split_suggestions( neuron_obj, plot_intermediates=False) # 4) Get the number of splits required for this neuron n_paths_cut = pru.get_n_paths_cut(split_results) if verbose: print(f"n_paths_cut = {n_paths_cut}") # 5) Split the neuron into a list of neuron objects (neuron_list, neuron_list_errored_limbs_area, neuron_list_errored_limbs_skeletal_length, neuron_list_n_multi_soma_errors, neuron_list_n_same_soma_errors) = pru.split_neuron( neuron_obj, limb_results=split_results, verbose=verbose, return_error_info=True) print(f"neuron_list = {neuron_list}") print( f"neuron_list_errored_limbs_area = {neuron_list_errored_limbs_area}" ) print( f"neuron_list_n_multi_soma_errors = {neuron_list_n_multi_soma_errors}" ) print( f"neuron_list_n_same_soma_errors = {neuron_list_n_same_soma_errors}" ) if verbose: print(f"Number of neurons: {len(neuron_list)}") neuron_entries = [] for neuron_idx in range(len(neuron_list)): """ # 6) For each neuron object in the list: # - get the number of errored limbs (to indicate the success type) # - Compute the information on the largest soma faces and volume # - Save the neuron object to the external # - Add the new write key to a list to commit """ n = neuron_list[neuron_idx] error_imbs_cancelled_area = neuron_list_errored_limbs_area[ neuron_idx] error_imbs_cancelled_skeletal_length = neuron_list_errored_limbs_skeletal_length[ neuron_idx] n_multi_soma_limbs_cancelled = neuron_list_n_multi_soma_errors[ neuron_idx] n_same_soma_limbs_cancelled = neuron_list_n_same_soma_errors[ neuron_idx] #for n in neuron_list: # nviz.visualize_neuron(n, # limb_branch_dict="all") # - get the number of errored limbs (to indicate the success type) if n.n_error_limbs == 0: split_success = 0 elif n.multi_soma_touching_limbs == 0: split_successs = 1 elif n.same_soma_multi_touching_limbs == 0: split_success = 2 else: split_success = 3 if verbose: print(f"split_success = {split_success}") # - Compute the information on the largest soma faces and volume soma_volumes = [ n[k].volume / 1000000000 for k in n.get_soma_node_names() ] soma_n_faces = [ len(n[k].mesh.faces) for k in n.get_soma_node_names() ] largest_n_faces = np.max(soma_n_faces) largest_volume = np.max(soma_volumes) if verbose: print(f"largest_n_faces = {largest_n_faces}") print(f"largest_volume = {largest_volume}") if "split" not in n.description: n.description += "_soma_0_split" #6) Save the file in a certain location if True: save_time = time.time() ret_file_path = n.save_compressed_neuron(output_folder=str( du.get_decomposition_path()), return_file_path=True, export_mesh=False, suppress_output=True) ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2" print(f"Save time = {time.time() - save_time}") else: print("Storing a dummy value for neuron") ret_file_path_str = "dummy" #7) Pass stats and file location to insert new_key = dict( key, split_index=neuron_idx, split_version=split_version, multiplicity=len(neuron_list), n_splits=n_paths_cut, split_success=split_success, n_error_limbs_cancelled=len(error_imbs_cancelled_area), n_multi_soma_limbs_cancelled=n_multi_soma_limbs_cancelled, n_same_soma_limbs_cancelled=n_same_soma_limbs_cancelled, error_imbs_cancelled_area=np.round( np.sum(error_imbs_cancelled_area), 4), error_imbs_cancelled_skeletal_length=np.round( np.sum(error_imbs_cancelled_skeletal_length) / 1000, 4), split_results=split_results, max_soma_n_faces=largest_n_faces, max_soma_volume=largest_volume, decomposition=ret_file_path_str, n_vertices=len(n.mesh.vertices), n_faces=len(n.mesh.faces), run_time=np.round(time.time() - whole_pass_time, 4)) stats_dict = n.neuron_stats() new_key.update(stats_dict) attributes_to_remove = ["axon_length", "axon_area", "n_boutons"] for k in attributes_to_remove: del new_key[k] neuron_entries.append(new_key) self.insert(neuron_entries, allow_direct_insert=True, skip_duplicates=True) print( f"\n\n ------ Total time for {segment_id} = {time.time() - whole_pass_time} ------" )
import meshlab meshlab.set_meshlab_port(current_port=None) # In[8]: decimation_version = 0 decimation_ratio = 0.25 # key_source = (minnie.Decimation().proj(decimation_version='version') & # dict(decimation_version=decimation_version,decimation_ratio=decimation_ratio) # & minnie.MultiSomaProofread2() & (dj.U("segment_id") & (minnie.BaylorSegmentCentroid()).proj())) key_source = ((minnie.Decimation).proj(decimation_version='version') & "decimation_version=" + str(decimation_version) & f"decimation_ratio={decimation_ratio}" & (minnie.BaylorSegmentCentroid() & "multiplicity>0") & du.proofreading_segment_id_restriction()) key_source # In[11]: # minnie.DecompositionVersions.insert1(dict(process_version=9, # description="uses meshafterparty for decomposition type and stitch distance 2000"), # skip_duplicates=True) # minnie.DecompositionVersions() # In[12]: import numpy as np import time decimation_version = 0
class Decomposition(dj.Computed): definition = """ -> minnie.Decimation.proj(decimation_version='version') ver : decimal(6,2) #the version number of the materializaiton process_version : int unsigned #the version of the preprocessing pipeline run index : tinyint unsigned #the index of the neuron object that resulted from that mesh (indexed starting at 0) --- multiplicity=null : tinyint unsigned # the number of somas found for this base segment decomposition: <decomposition> n_vertices : int unsigned # number of vertices n_faces : int unsigned # number of faces n_not_processed_soma_containing_meshes : int unsigned #the number of meshes with somas that were not processed n_error_limbs: int #the number of limbs that are touching multiple somas or 1 soma in multiple places n_same_soma_multi_touching_limbs: int # number of limbs that touch the same soma multiple times n_multi_soma_touching_limbs: int # number of limbs that touch multiple somas n_somas: int #number of soma meshes detected n_limbs: int n_branches: int max_limb_n_branches=NULL:int skeletal_length=NULL: double max_limb_skeletal_length=NULL:double median_branch_length=NULL:double #gives information on average skeletal length to next branch point width_median=NULL: double #median width from mesh center without spines removed width_no_spine_median=NULL: double #median width from mesh center with spines removed width_90_perc=NULL: double # 90th percentile for width without spines removed width_no_spine_90_perc=NULL: double # 90th percentile for width with spines removed n_spines: bigint spine_density=NULL: double # n_spines/ skeletal_length spines_per_branch=NULL: double skeletal_length_eligible=NULL: double # the skeletal length for all branches searched for spines n_spine_eligible_branches=NULL: int # the number of branches that were checked for spines because passed width threshold spine_density_eligible=NULL:double # n_spines/skeletal_length_eligible spines_per_branch_eligible=NULL:double # n_spines/n_spine_eligible_branches total_spine_volume=NULL: double # the sum of all spine volume spine_volume_median=NULL: double # median of the spine volume for those spines with able to calculate volume spine_volume_density=NULL: double #total_spine_volume/skeletal_length spine_volume_density_eligible=NULL: double #total_spine_volume/skeletal_length_eligible spine_volume_per_branch_eligible=NULL: double #total_spine_volume/n_spine_eligible_branches run_time=NULL : double # the amount of time to run (seconds) """ key_source = ((minnie.Decimation).proj(decimation_version='version') & "decimation_version=" + str(decimation_version) & f"decimation_ratio={decimation_ratio}" & (minnie.BaylorSegmentCentroid() & "multiplicity>0") & du.proofreading_segment_id_restriction()) def make(self, key): """ Pseudocode for process: 1) Get the segment id from the key 2) Get the decimated mesh 3) Get the somas info 4) Run the preprocessing 5) Calculate all starter stats 6) Save the file in a certain location 7) Pass stats and file location to insert """ whole_pass_time = time.time() #1) Get the segment id from the key segment_id = key["segment_id"] description = str(key['decimation_version']) + "_25" print(f"\n\n\n---- Working on Neuron {key['segment_id']} ----") global_start = time.time() #2) Get the decimated mesh current_neuron_mesh = du.fetch_segment_id_mesh(segment_id) #3) Get the somas info somas = du.get_soma_mesh_list(segment_id) soma_ver = du.get_soma_mesh_list_ver(segment_id) print(f"somas = {somas}") #3b) Get the glia and nuclei information glia_faces, nuclei_faces = du.get_segment_glia_nuclei_faces( segment_id, return_empty_list=True) #4) Run the preprocessing total_neuron_process_time = time.time() print(f"\n--- Beginning preprocessing of {segment_id}---") recovered_neuron = neuron.Neuron( mesh=current_neuron_mesh, somas=somas, segment_id=segment_id, description=description, suppress_preprocessing_print=False, suppress_output=False, calculate_spines=True, widths_to_calculate=["no_spine_median_mesh_center"], glia_faces=glia_faces, nuclei_faces=nuclei_faces, decomposition_type="meshafterparty", ) print( f"\n\n\n---- Total preprocessing time = {time.time() - total_neuron_process_time}" ) #5) Don't have to do any of the processing anymore because will do in the neuron object stats_dict = recovered_neuron.neuron_stats(stats_to_ignore=[ "n_boutons", "axon_length", "axon_area", "max_soma_volume", "max_soma_n_faces", ]) #6) Save the file in a certain location save_time = time.time() ret_file_path = recovered_neuron.save_compressed_neuron( output_folder=str(du.get_decomposition_path()), return_file_path=True, export_mesh=False, suppress_output=True) ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2" print(f"Save time = {time.time() - save_time}") #7) Pass stats and file location to insert new_key = dict(key, ver=soma_ver, process_version=process_version, index=0, multiplicity=1, decomposition=ret_file_path_str, n_vertices=len(current_neuron_mesh.vertices), n_faces=len(current_neuron_mesh.faces), run_time=np.round(time.time() - whole_pass_time, 4)) new_key.update(stats_dict) self.insert1(new_key, allow_direct_insert=True, skip_duplicates=True) print( f"\n\n ------ Total time for {segment_id} = {time.time() - global_start} ------" )
class DecompositionCellType(dj.Computed): definition = """ -> minnie.Decomposition() split_index : tinyint unsigned # the index of the neuron object that resulted AFTER THE SPLITTING ALGORITHM -> minnie.DecompositonAxonVersion() # the version of code used for this cell typing classification --- # -- attributes for the cell type classification --- decomposition : <decomposition> # saved neuron object with high fidelity axon # ----- Nucleus Information ----# nucleus_id : int unsigned # id of nucleus from the flat segmentation Equivalent to Allen: 'id'. nuclei_distance : double # the distance to the closest nuclei (even if no matching nuclei found) n_nuclei_in_radius : tinyint unsigned # the number of nuclei within the search radius of 15000 belonging to that segment n_nuclei_in_bbox : tinyint unsigned # the number of nuclei within the bounding box of that soma # ------ Information Used For Excitatory Inhibitory Classification (Baylor Cell) -------- cell_type_predicted: enum('excitatory','inhibitory','other','unknown') # morphology predicted by classifier spine_category: enum('no_spined','sparsely_spined','densely_spined') n_axons: tinyint unsigned # Number of axon candidates identified n_apicals: tinyint unsigned # Number of apicals identified axon_angle_maximum=NULL:double #the anlge of an identified axon spine_density_classifier:double # the number of spines divided by skeletal length for branches analyzed in classification n_branches_processed: int unsigned # the number branches used for the spine density analysis skeletal_length_processed: double # The total skeletal length of the viable branches used for the spine density analysis n_branches_in_search_radius: int unsigned # the number branches existing in the search radius used for spine density skeletal_length_in_search_radius : double # The total skeletal length of the branches existing in the search radius used for spine density #---- allen classification info ----- allen_e_i=NULL: enum('excitatory','inhibitory','other','unknown') allen_e_i_n_nuc=NULL: tinyint unsigned allen_cell_type=NULL:varchar(256) allen_cell_type_n_nuc=NULL:tinyint unsigned allen_cell_type_e_i=NULL:enum('excitatory','inhibitory','other','unknown') # ----- for the dendrite on axon filtering away -------- dendrite_on_axon_merges_error_area=NULL : double #the area (in um ^ 2) of the faces canceled out by filter dendrite_on_axon_merges_error_length =NULL: double #the length (in um) of skeleton distance canceled out by filter # ----- attributes from the axon_features --- cell_type_for_axon : enum('excitatory','inhibitory','other','unknown') axon_volume=NULL: double #volume of the oriented bounding box of axon (divided by 10^14) axon_length=NULL: double # length (in um) of the classified axon skeleton axon_branch_length_median=NULL: double # length (in um) of the classified axon skeleton axon_branch_length_mean=NULL: double # length (in um) of the classified axon skeleton # number of branches in the axon axon_n_branches=NULL: int unsigned axon_n_short_branches=NULL: int unsigned axon_n_long_branches=NULL: int unsigned axon_n_medium_branches=NULL: int unsigned #bounding box features axon_bbox_x_min=NULL: double axon_bbox_y_min=NULL: double axon_bbox_z_min=NULL: double axon_bbox_x_max=NULL: double axon_bbox_y_max=NULL: double axon_bbox_z_max=NULL: double axon_bbox_x_min_soma_relative=NULL: double axon_bbox_y_min_soma_relative=NULL: double axon_bbox_z_min_soma_relative=NULL: double axon_bbox_x_max_soma_relative=NULL: double axon_bbox_y_max_soma_relative=NULL: double axon_bbox_z_max_soma_relative=NULL: double run_time=NULL : double # the amount of time to run (seconds) """ key_source = minnie.Decomposition( ) & du.proofreading_segment_id_restriction() def make(self, key): """ Pseudocode: 1) Pull Down all the Neuron Objects associated with a segment_id For each neuron: 2) Run the full axon preprocessing 3) Save off the neuron 4) Save dict entry to list 5) Write the new entry to the table """ # 1) Pull Down All of the Neurons segment_id = key["segment_id"] if verbose: print(f"------- Working on Neuron {segment_id} -----") whole_pass_time = time.time() #1) Pull Down all the Neuron Objects associated with a segment_id neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation( segment_id, ignore_DecompositionAxon=True, ignore_DecompositionCellType=True) if verbose: print(f"Number of Neurons found ={len(neuron_objs)}") #For each neuron: dict_to_write = [] # -------- getting the nuclei info to match # ver = 88 # nucleus_ids,nucleus_centers = du.segment_to_nuclei(segment_id, # nuclei_version=ver) nucleus_ids, nucleus_centers = du.segment_to_nuclei(segment_id, nuclei_version=ver) print(f"nucleus_ids = {nucleus_ids}") print(f"nucleus_centers = {nucleus_centers}") for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs): if verbose: print(f"--> Working on Split Index {split_index} -----") st = time.time() # ------------- Does all of the processing ------------------- #1) ------ Getting the paired nuclei ------ winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei( neuron_obj, "S0", nucleus_ids, nucleus_centers, nuclei_distance_threshold=15000, return_matching_info=True, verbose=True) # else: # winning_nucleus_id = 12345 # nucleus_info = dict() # nucleus_info["nucleus_id"] = winning_nucleus_id # nucleus_info["nuclei_distance"] = 0 # nucleus_info["n_nuclei_in_radius"] = 1 # nucleus_info["n_nuclei_in_bbox"] = 1 if verbose: print(f"nucleus_info = {nucleus_info}") print(f"winning_nucleus_id = {winning_nucleus_id}") #2) ------- Finding the Allen Cell Types ------- allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id( winning_nucleus_id) if verbose: print(f"allen_cell_type_info = {allen_cell_type_info}") #4) -------- Running the cell classification and stats-------------- if verbose: print( f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n" ) filter_time = time.time() (inh_exc_class, spine_category, axon_angles, n_axons, n_apicals, neuron_spine_density, n_branches_processed, skeletal_length_processed, n_branches_in_search_radius, skeletal_length_in_search_radius ) = clu.inhibitory_excitatory_classifier( neuron_obj, return_spine_classification=True, return_axon_angles=True, return_n_axons=True, return_n_apicals=True, return_spine_statistics=True, axon_limb_branch_dict_precomputed=None, axon_angles_precomputed=None, verbose=verbose) if verbose: print( f"Total time for classification = {time.time() - filter_time}" ) all_axon_angles = [] for limb_idx, limb_data in axon_angles.items(): for candidate_idx, cand_angle in limb_data.items(): all_axon_angles.append(cand_angle) if len(axon_angles) > 0: axon_angle_maximum = np.max(all_axon_angles) else: axon_angle_maximum = 0 if verbose: print("\n -- Cell Type Classification Results --") print(f"inh_exc_class={inh_exc_class}") print(f"spine_category={spine_category}") print(f"axon_angles={axon_angles}") print(f"n_axons={n_axons}") print(f"n_apicals={n_apicals}") print(f"neuron_spine_density={neuron_spine_density}") print(f"n_branches_processed={n_branches_processed}") print(f"skeletal_length_processed={skeletal_length_processed}") print( f"n_branches_in_search_radius={n_branches_in_search_radius}" ) print( f"skeletal_length_in_search_radius={skeletal_length_in_search_radius}" ) baylor_cell_type_info = dict( cell_type_predicted=inh_exc_class, spine_category=spine_category, axon_angle_maximum=axon_angle_maximum, n_axons=n_axons, n_apicals=n_apicals, spine_density_classifier=neuron_spine_density, n_branches_processed=neuron_spine_density, skeletal_length_processed=skeletal_length_processed, n_branches_in_search_radius=n_branches_in_search_radius, skeletal_length_in_search_radius= skeletal_length_in_search_radius, ) #5) ----- Deciding on cell type to use for axon e_i_class = inh_exc_class if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info[ "e_i"] is not None: e_i_class = allen_cell_type_info["e_i"] if verbose: print( f"e_i_class = {e_i_class} with inh_exc_class_to_use_for_axon = {inh_exc_class_to_use_for_axon}" ) #6) -------- If excitatory running the axon processing-------------- """ Psuedocode: If e_i class is excitatory: 1) Filter away the axon on dendrite 2) Do the higher fidelity axon processing 3) Compute the axon features """ if e_i_class == "excitatory" and neuron_obj.axon_limb_name is not None: if verbose: print( f"Excitatory so performing high fidelity axon and computing axon features" ) # 1) Filter away the axon on dendrite # 2) Do the higher fidelity axon processing o_neuron, filtering_info = au.complete_axon_processing( neuron_obj, perform_axon_classification=False, return_filtering_info=True) filtering_info = { k: np.round(v, 2) for k, v in filtering_info.items() if "area" in k or "length" in k } #3) Compute the axon features axon_features = au.axon_features_from_neuron_obj(o_neuron) else: nru.clear_all_branch_labels(neuron_obj, labels_to_clear="axon") o_neuron = neuron_obj axon_features = dict() filtering_info = dict() #3) ------ Adding the Synapses ----------- o_neuron = syu.add_synapses_to_neuron_obj( o_neuron, validation=validation, verbose=True, original_mesh=None, plot_valid_error_synapses=False, calculate_synapse_soma_distance=False, add_valid_synapses=True, add_error_synapses=False) # ------- Saving off the neuron object ---------------- save_time = time.time() ret_file_path = o_neuron.save_compressed_neuron( output_folder=str(du.get_decomposition_path()), #output_folder = "./", file_name= f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}", return_file_path=True, export_mesh=False, suppress_output=True) ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2" if verbose: print(f"ret_file_path_str = {ret_file_path_str}") print(f"Save time = {time.time() - save_time}") n_dict = dict( key, split_index=split_index, axon_version=au.axon_version, decomposition=ret_file_path_str, run_time=np.round(time.time() - st, 2), cell_type_for_axon=e_i_class, ) dicts_for_update = [ baylor_cell_type_info, allen_cell_type_info, nucleus_info, filtering_info, axon_features ] for d in dicts_for_update: n_dict.update(d) self.insert1(n_dict, skip_duplicates=True, allow_direct_insert=True) #dict_to_write.append(n_dict) #write the #self.insert(dict_to_write,skip_duplicates=True,allow_direct_insert=True) print( f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***" )
class DecompositionSplitMultiSoma(dj.Computed): definition = """ -> minnie.Decomposition() split_version: tinyint unsigned #the version of the splitting algorithm used --- n_splits : int unsigned # the number of cuts required to help split the neuron split_results: longblob #will store the results of how to split the limbs of neuron objects from original neuron red_blue_split_results: longblob run_time=NULL : double # the amount of time to run (seconds) """ ''' Old keysource used for inhibitory excitatory check classified_table = (minnie.BaylorManualCellType() & 'nucleus_version=3' & "(cell_type = 'excitatory') or (cell_type = 'inhibitory')") key_source = ((minnie.Decomposition & (minnie.NeuronSplitSuggestions.proj()) & (classified_table.proj()) & f"n_somas<{max_n_somas}" & "n_error_limbs>0"))''' # This keysource acounts that you could have more than 1 possible soma but not a significant limb connecting them (no error limbs) key_source = minnie.Decomposition( ) & "n_somas>1" & du.proofreading_segment_id_restriction( ) #& dict(segment_id=864691136756471662) def make(self, key): """ Pseudocode for process: 1) Get the segment id from the key 2) Get the decomposed neurong object from Decomposition table 3) Run the multi_soma split suggestions algorithm 4) Get the number of splits required for this neuron 5) Split the neuron into a list of neuron objects 6) For each neuron object in the list: - get the number of errored limbs (to indicate the success type) - Change the description to include the multiplicity - Compute the information on the largest soma faces and volume - Save the neuron object to the external - Add the new write key to a list to commit 7) Write all of the keys """ whole_pass_time = time.time() # 1) Get the segment id from the key segment_id = key["segment_id"] print(f"\n\n\n---- Working on Neuron {key['segment_id']} ----") # 2) Get the decomposed neuron object from Decomposition table and the split suggestions neuron_obj_path = (minnie.Decomposition & key).fetch1("decomposition") neuron_obj = du.filepath_to_neuron_obj(neuron_obj_path) """ Old way that downloaded from another table # 3) Retrieve the multi soma suggestions split_results = (minnie.NeuronSplitSuggestions & key).fetch1("split_results") """ #3) Calculated the split results split_results, red_blue_split_results = pru.multi_soma_split_suggestions( neuron_obj, plot_intermediates=False, only_multi_soma_paths=True, default_cut_edge="last", verbose=True, debug=False, output_red_blue_suggestions=True, split_red_blue_by_common_upstream=True, apply_valid_upstream_branches_restriction=True, debug_red_blue=False, ) # 4) Get the number of splits required for this neuron n_paths_cut = pru.get_n_paths_cut(split_results) #7) Pass stats and file location to insert new_key = dict(key, split_version=split_version, n_splits=n_paths_cut, split_results=split_results, red_blue_split_results=red_blue_split_results, run_time=np.round(time.time() - whole_pass_time, 4)) self.insert1(new_key, allow_direct_insert=True, skip_duplicates=True) print( f"\n\n ------ Total time for {segment_id} = {time.time() - whole_pass_time} ------" )
class DecompositionAxon(dj.Computed): definition=""" -> minnie.Decomposition() split_index : tinyint unsigned # the index of the neuron object that resulted AFTER THE SPLITTING ALGORITHM -> minnie.DecompositonAxonVersion() # the version of code used for this cell typing classification --- decomposition : <decomposition> # saved neuron object with high fidelity axon axon_length: double # length (in um) of the classified axon skeleton run_time=NULL : double # the amount of time to run (seconds) """ #key_source = minnie.Decomposition() & minnie.NucleiSegmentsRun2() & "segment_id=864691136540183458" key_source = minnie.Decomposition() & du.proofreading_segment_id_restriction() def make(self,key): """ Pseudocode: 1) Pull Down all the Neuron Objects associated with a segment_id For each neuron: 2) Run the full axon preprocessing 3) Save off the neuron 4) Save dict entry to list 5) Write the new entry to the table """ # 1) Pull Down All of the Neurons segment_id = key["segment_id"] if verbose: print(f"------- Working on Neuron {segment_id} -----") whole_pass_time = time.time() #1) Pull Down all the Neuron Objects associated with a segment_id neuron_objs,neuron_split_idxs = du.decomposition_with_spine_recalculation(segment_id) if verbose: print(f"Number of Neurons found ={len(neuron_objs)}") #For each neuron: dict_to_write = [] for split_index,neuron_obj in zip(neuron_split_idxs,neuron_objs): if verbose: print(f"--> Working on Split Index {split_index} -----") st = time.time() #Run the Axon Decomposition neuron_obj_with_web = au.complete_axon_processing(neuron_obj, verbose=True) save_time = time.time() ret_file_path = neuron_obj_with_web.save_compressed_neuron( output_folder=str(du.get_decomposition_path()), #output_folder = "./", file_name=f"{neuron_obj_with_web.segment_id}_{split_index}_split_axon_v{au.axon_version}", return_file_path=True, export_mesh=False, suppress_output=True) ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2" if verbose: print(f"ret_file_path_str = {ret_file_path_str}") print(f"Save time = {time.time() - save_time}") n_dict = dict(key, split_index = split_index, axon_version = au.axon_version, decomposition=ret_file_path_str, axon_length=neuron_obj_with_web.axon_length, run_time = np.round(time.time() - st,2) ) dict_to_write.append(n_dict) #write the self.insert(dict_to_write,skip_duplicates=True,allow_direct_insert=True) print(f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***")
class DecompositionCellTypeV7(dj.Computed): definition = """ -> minnie.Decomposition() split_index : tinyint unsigned # the index of the neuron object that resulted AFTER THE SPLITTING ALGORITHM -> minnie.DecompositonAxonVersion() # the version of code used for this cell typing classification --- # -- attributes for the cell type classification --- decomposition : <decomposition> # saved neuron object with high fidelity axon # ----- Nucleus Information ----# nucleus_id : int unsigned # id of nucleus from the flat segmentation Equivalent to Allen: 'id'. nuclei_distance : double # the distance to the closest nuclei (even if no matching nuclei found) n_nuclei_in_radius : tinyint unsigned # the number of nuclei within the search radius of 15000 belonging to that segment n_nuclei_in_bbox : tinyint unsigned # the number of nuclei within the bounding box of that soma soma_x_nm : double # x coordinate of soma mesh in nm soma_y_nm : double # y coordinate of soma mesh in nm soma_z_nm : double # z coordinate of soma mesh in nm # ---- chosen cell type for proofreading rules baylor_e_i=NULL: enum('excitatory','inhibitory','other','unknown') allen_e_i=NULL: enum('excitatory','inhibitory','other','unknown') cell_type_used=NULL: enum('allen','baylor') cell_type=NULL: enum('excitatory','inhibitory','other','unknown') # ------ Information Used For Excitatory Inhibitory Classification (Baylor Cell) -------- syn_density_post: double syn_density_head: double syn_density_neck: double syn_density_shaft: double skeletal_length_processed_syn: double spine_density: double skeletal_length_processed_spine: double n_syn_pre: int unsigned n_syn_post: int unsigned # -- regarding the axon that was found --- axon_angle_max=NULL : double axon_angle_min=NULL : double n_axon_angles : tinyint unsigned #---- allen classification info ----- allen_e_i_n_nuc=NULL: tinyint unsigned allen_cell_type=NULL:varchar(256) allen_cell_type_n_nuc=NULL:tinyint unsigned allen_cell_type_e_i=NULL:enum('excitatory','inhibitory','other','unknown') # ----- for the dendrite on axon filtering away -------- dendrite_on_axon_merges_error_area=NULL : double #the area (in um ^ 2) of the faces canceled out by filter dendrite_on_axon_merges_error_length =NULL: double #the length (in um) of skeleton distance canceled out by filter # ----- attributes from the axon_features--- axon_skeleton: <skeleton> # the skeleton of the axon of the final proofread neuorn axon_volume=NULL: double #volume of the oriented bounding box of axon (divided by 10^14) axon_length=NULL: double # length (in um) of the classified axon skeleton axon_branch_length_median=NULL: double # length (in um) of the classified axon skeleton axon_branch_length_mean=NULL: double # length (in um) of the classified axon skeleton # number of branches in the axon axon_n_branches=NULL: int unsigned axon_n_short_branches=NULL: int unsigned axon_n_long_branches=NULL: int unsigned axon_n_medium_branches=NULL: int unsigned #bounding box features axon_bbox_volume=NULL: double axon_bbox_x_min=NULL: double axon_bbox_y_min=NULL: double axon_bbox_z_min=NULL: double axon_bbox_x_max=NULL: double axon_bbox_y_max=NULL: double axon_bbox_z_max=NULL: double axon_bbox_x_min_soma_relative=NULL: double axon_bbox_y_min_soma_relative=NULL: double axon_bbox_z_min_soma_relative=NULL: double axon_bbox_x_max_soma_relative=NULL: double axon_bbox_y_max_soma_relative=NULL: double axon_bbox_z_max_soma_relative=NULL: double run_time=NULL : double # the amount of time to run (seconds) """ key_source = minnie.Decomposition( ) & du.proofreading_segment_id_restriction() #key_source = minnie.Decomposition & dict(n_error_limbs = 0,n_same_soma_multi_touching_limbs=0,n_multi_soma_touching_limbs=0) & "n_limbs > 5" def make(self, key): """ Pseudocode: 1) Pull Down all the Neuron Objects associated with a segment_id For each neuron: 2) Run the full axon preprocessing 3) Save off the neuron 4) Save dict entry to list 5) Write the new entry to the table """ # 1) Pull Down All of the Neurons segment_id = key["segment_id"] if verbose: print(f"------- Working on Neuron {segment_id} -----") whole_pass_time = time.time() #1) Pull Down all the Neuron Objects associated with a segment_id neuron_objs, neuron_split_idxs = du.decomposition_with_spine_recalculation( segment_id, ignore_DecompositionAxon=True, ignore_DecompositionCellType=True) if verbose: print(f"Number of Neurons found ={len(neuron_objs)}") #For each neuron: dict_to_write = [] ''' ------ Old way of getting the nucleus info for the manual proofread data ------- # -------- getting the nuclei info to match try: segment_map_dict = (minnie.AutoProofreadValidationSegmentMap4() & dict(old_segment_id=segment_id)).fetch1() except: segment_map_dict = (minnie.AutoProofreadValidationSegmentMapInh() & dict(old_segment_id=segment_id)).fetch1() nucleus_id = segment_map_dict["nucleus_id"] nuc_center_coords = du.nuclei_id_to_nucleus_centers(nucleus_id) nucleus_ids = [nucleus_id] nucleus_centers = [nuc_center_coords] print(f"nucleus_ids = {nucleus_ids}") print(f"nucleus_centers = {nucleus_centers}")''' nucleus_ids, nucleus_centers = du.segment_to_nuclei( segment_id, #nuclei_version=ver ) if verbose: print(f"Number of Corresponding Nuclei = {len(nucleus_ids)}") print(f"nucleus_ids = {nucleus_ids}") print(f"nucleus_centers = {nucleus_centers}") for split_index, neuron_obj in zip(neuron_split_idxs, neuron_objs): if verbose: print(f"--> Working on Split Index {split_index} -----") st = time.time() # ------------- Does all of the processing ------------------- #1) ------ Getting the paired nuclei ------ winning_nucleus_id, nucleus_info = nru.pair_neuron_obj_to_nuclei( neuron_obj, "S0", nucleus_ids, nucleus_centers, nuclei_distance_threshold=15000, return_matching_info=True, verbose=True) # else: # winning_nucleus_id = 12345 # nucleus_info = dict() # nucleus_info["nucleus_id"] = winning_nucleus_id # nucleus_info["nuclei_distance"] = 0 # nucleus_info["n_nuclei_in_radius"] = 1 # nucleus_info["n_nuclei_in_bbox"] = 1 if verbose: print(f"nucleus_info = {nucleus_info}") print(f"winning_nucleus_id = {winning_nucleus_id}") #2) ------- Finding the Allen Cell Types ------- allen_cell_type_info = ctu.allen_nuclei_classification_info_from_nucleus_id( winning_nucleus_id) if verbose: print(f"allen_cell_type_info = {allen_cell_type_info}") # 3) ---- Doing Baylor Cell Type Classification --------- # 3a) --- Adding the synapses and spine labels if verbose: print( f"\n\n ------ Part C: Inhibitory Excitatory Classification ---- \n\n" ) st = time.time() if verbose: print(f"Adding the synapses and the head_neck_shaft") neuron_obj = syu.add_synapses_to_neuron_obj( neuron_obj, validation=validation, verbose=verbose, original_mesh=None, plot_valid_error_synapses=False, calculate_synapse_soma_distance=False, add_valid_synapses=True, add_error_synapses=False, ) neuron_obj = spu.add_head_neck_shaft_spine_objs(neuron_obj, verbose=verbose) if verbose: print( f"Done adding synapses and head_neck_shaft: {time.time() - st}" ) # 3b) --- Running the stats for Baylor Classification filter_time = time.time() ''' limb_branch_dict = ctu.postsyn_branches_near_soma_for_syn_post_density( neuron_obj = neuron_obj, verbose = False,) (syn_density_post, syn_density_head, syn_density_neck, syn_density_shaft, skeletal_length_processed_syn) = ctu.synapse_density_stats(neuron_obj = neuron_obj, limb_branch_dict = limb_branch_dict, verbose = True) (spine_density, skeletal_length_processed_spine) = ctu.spine_density_near_soma(neuron_obj = neuron_obj, verbose = True, multiplier = 1000) if verbose: print(f"Total time for density calculations = {time.time() - filter_time}") # 4) ------ Predicting the E/I Group Based on the data collected -------- baylor_cell_type_info = dict( syn_density_post = syn_density_post, syn_density_head = syn_density_head, syn_density_neck = syn_density_neck, syn_density_shaft = syn_density_shaft, skeletal_length_processed_syn=skeletal_length_processed_syn, spine_density=spine_density, skeletal_length_processed_spine = skeletal_length_processed_spine ) baylor_e_i = ctu.e_i_classification_single(data=[syn_density_shaft,spine_density], features=["syn_density_shaft","spine_density"], verbose = True, return_label_name = True ) ''' baylor_e_i, baylor_cell_type_info = ctu.e_i_classification_from_neuron_obj( neuron_obj, verbose=True, return_cell_type_info=True) baylor_cell_type_info["baylor_e_i"] = baylor_e_i #5) ----- Deciding on cell type to use for axon if inh_exc_class_to_use_for_axon == "Allen" and allen_cell_type_info[ "allen_e_i"] is not None: e_i_class = allen_cell_type_info["allen_e_i"] cell_type_used = "allen" else: e_i_class = baylor_e_i cell_type_used = "baylor" if verbose: print( f"e_i_class = {e_i_class} with cell_type_used = {cell_type_used}" ) #3) ------ Axon Classification (and getting the axon features)------------------ o_neuron, filtering_info, axon_angles_dict = au.complete_axon_processing( neuron_obj, cell_type=e_i_class, add_synapses_and_head_neck_shaft_spines=False, validation=validation, plot_initial_axon=False, plot_axon_on_dendrite=False, return_filtering_info=True, return_axon_angle_info=True, verbose=verbose) filtering_info = { k: np.round(v, 2) for k, v in filtering_info.items() if "area" in k or "length" in k } axon_features = au.axon_features_from_neuron_obj(o_neuron) # ------- Saving off the neuron object ---------------- save_time = time.time() rand_numb = np.random.randint(0, 3000) ret_file_path = o_neuron.save_compressed_neuron( output_folder=str(du.get_decomposition_path()), #output_folder = "./", file_name= f"{o_neuron.segment_id}_{split_index}_split_axon_v{au.axon_version}_pipe_v{pipe_version}_bcm_{rand_numb}", return_file_path=True, export_mesh=False, suppress_output=True) ret_file_path_str = str(ret_file_path.absolute()) + ".pbz2" if verbose: print(f"ret_file_path_str = {ret_file_path_str}") print(f"Save time = {time.time() - save_time}") # ---------------- # ---- 8/10 Addition ---------- if save_axon_skeleton: axon_skeleton_file = du.save_proofread_skeleton( o_neuron.axon_skeleton, segment_id=o_neuron.segment_id, split_index=split_index, file_name_ending= f"decomp_cell_type_axon_skeleton_v{pipe_version}_bcm_{rand_numb}" ) else: axon_skeleton_file = None #---- 8/29 Addition: Will compute the soma center of the mesh in nm --- soma_x_nm, soma_y_nm, soma_z_nm = neuron_obj["S0"].mesh_center if verbose: print( f"soma_x_nm, soma_y_nm, soma_z_nm = {soma_x_nm, soma_y_nm, soma_z_nm}" ) n_dict = dict( key, split_index=split_index, axon_version=au.axon_version, decomposition=ret_file_path_str, run_time=np.round(time.time() - st, 2), cell_type=e_i_class, cell_type_used=cell_type_used, axon_skeleton=str(axon_skeleton_file), soma_x_nm=soma_x_nm, soma_y_nm=soma_y_nm, soma_z_nm=soma_z_nm, n_syn_pre=neuron_obj.n_synapses_pre, n_syn_post=neuron_obj.n_synapses_post, ) dicts_for_update = [ baylor_cell_type_info, allen_cell_type_info, nucleus_info, filtering_info, axon_features, axon_angles_dict ] for d in dicts_for_update: n_dict.update(d) self.insert1(n_dict, skip_duplicates=True, allow_direct_insert=True) print( f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***" )
class Decimation(dj.Computed): # definition = minnie.Decimation.describe(printout=False) #key_source = minnie.Mesh.proj() * (minnie.DecimationConfig & 'decimation_ratio=0.25') & minnie.MultiSomaProofread2.proj() #key_source = (du.version_to_segment_id_table() - (minnie.Decimation() & dict(decimation_ratio=decimation_ratio_global)).proj()) key_source = (du.proofreading_segment_id_restriction() - (minnie.Decimation() & dict(decimation_ratio=decimation_ratio_global)).proj() ) & minnie.Mesh.proj() #key_source = minnie.Mesh() & dict(segment_id=864691136008592558) # Creates hf file at the proper location, returns the filepath of the newly created file @classmethod def make_file(cls, segment_id, version, decimation_ratio, vertices, faces): """Creates hf file at the proper location, returns the filepath of the newly created file""" assert vertices.ndim == 2 and vertices.shape[1] == 3 assert faces.ndim == 2 and faces.shape[1] == 3 filename = f'{segment_id}_{version}_{int(decimation_ratio*100):02}.h5' filepath = os.path.join(external_decimated_mesh_path, filename) with h5py.File(filepath, 'w') as hf: hf.create_dataset('segment_id', data=segment_id) hf.create_dataset('version', data=version) hf.create_dataset('decimation_ratio', data=float(decimation_ratio)) hf.create_dataset('vertices', data=vertices) hf.create_dataset('faces', data=faces) return filepath @classmethod def make_entry(cls, segment_id, version, decimation_ratio, vertices, faces): key = dict(segment_id=segment_id, version=version, decimation_ratio=decimation_ratio, n_vertices=len(vertices), n_faces=len(faces)) filepath = cls.make_file(segment_id, version, decimation_ratio, vertices, faces) cls.insert1(dict(key, mesh=filepath), allow_direct_insert=True) def make(self, key): print(key) segment_id = key['segment_id'] # -------- 1/27: Inserting the data into Mesh and Segment first ----- # if perform_mesh_segment_insert: #2) Manually add segmnet ids to segment tables minnie.Segment.insert1(dict(segment_id=segment_id), skip_duplicates=True) #3) Fill in the Mesh Table du.fill_from_ids(segment_ids=[segment_id]) if verbose: print("Done Inserting Mesh") if "version" not in key.keys(): version = decimation_version if "decimation_ratio" not in key.keys(): decimation_ratio = decimation_ratio_global mesh = (minnie.Mesh & key).fetch1('mesh') print( f"Mesh size: n_vertices = {len(mesh.vertices)}, n_faces = {len(mesh.faces)}" ) if decimation_ratio not in meshlab_scripts: meshlab_scripts[decimation_ratio] = meshlab.Decimator( decimation_ratio, temporary_folder, overwrite=False) mls_func = meshlab_scripts[decimation_ratio] try: expected_filepath = os.path.join(external_decimated_mesh_path, f'{segment_id}_{version}.h5') if not os.path.isfile(expected_filepath): new_mesh, _path = mls_func(mesh.vertices, mesh.faces, segment_id) new_vertices, new_faces = new_mesh.vertices, new_mesh.faces self.make_entry( segment_id=segment_id, version=version, decimation_ratio=decimation_ratio, vertices=new_vertices, faces=new_faces, ) else: print('File already exists.') with h5py.File(expected_filepath, 'r') as hf: vertices = hf['vertices'][()].astype(np.float64) faces = hf['faces'][()].reshape(-1, 3).astype(np.uint32) self.insert1(dict(key, n_vertices=len(vertices), n_faces=len(faces), mesh=expected_filepath), allow_direct_insert=True) except Exception as e: minnie.DecimationError.insert1(dict(key, log=str(e))) print(e) raise e
class AutoProofreadNeurons7(dj.Computed): definition = djt.AutoProofreadNeurons7 key_source = ( minnie.DecompositionCellTypeV7() & "n_syn_pre + n_syn_post > 0" & du.proofreading_segment_id_restriction() #& dict(segment_id=864691134884836602) #worked for excitatory #& dict(segment_id=864691135012928150) ) & (minnie.ManualCellTypesAllen() & "table_name != 'allen_v1_column_types_slanted'").proj() def make(self, key): """ Pseudocode: 1) Pull Down All of the Neurons 2) Get the nucleus centers and the original mesh """ whole_pass_time = time.time() # 1) Pull Down All of the Neurons segment_id = key["segment_id"] split_index = key["split_index"] if verbose: print( f"\n\n------- Working on Neuron {segment_id}_{split_index} -----" ) cell_type, nucleus_id = (minnie.DecompositionCellTypeV7() & key).fetch1(f"{cell_type_used}_e_i", "nucleus_id") if verbose: print(f"---- Working on Neuron {segment_id}:{split_index}") print(f"nucleus_id = {nucleus_id},cell_type = {cell_type}") neuron_obj = du.decomposition_with_spine_recalculation( segment_id, split_index=split_index) if plot_data: nviz.plot_axon(neuron_obj) #2) Running the neuron proofreading neuron_obj_proof, filtering_info = pru.proofread_neuron_full( neuron_obj, # arguments for processing down in DecompositionCellTypeV7 cell_type=cell_type, add_valid_synapses=False, validation=validation, add_spines=False, perform_axon_processing=False, return_after_axon_processing=False, #arguments for processing after DecompositionCellTypeV7 to Proofread Neuron plot_head_neck_shaft_synapses=plot_data, plot_soma_synapses=plot_data, proofread_verbose=proofread_verbose, verbose_outline=verbose, plot_limb_branch_filter_with_disconnect_effect=plot_data, plot_final_filtered_neuron=False, plot_synapses_after_proofread=False, plot_compartments=plot_data, plot_valid_synapses=plot_data, plot_error_synapses=plot_data, verbose=verbose, debug_time=verbose, ) #3) Collect and Write Data to Synapse Table dj_keys_valid = syu.synapses_to_dj_keys(neuron_obj_proof, valid_synapses=True, verbose=verbose, nucleus_id=nucleus_id, split_index=split_index) dj_keys_error = syu.synapses_to_dj_keys(neuron_obj_proof, valid_synapses=False, verbose=verbose, nucleus_id=nucleus_id, split_index=split_index) if verbose: print(f"n_synapses_total = {neuron_obj_proof.n_synapses_total}") AutoProofreadSynapse7.insert(dj_keys_valid, skip_duplicates=True) AutoProofreadSynapseErrors7.insert(dj_keys_error, skip_duplicates=True) #4) Collect and Write Neuron Stats limb_branch_to_cancel = pru.extract_from_filter_info( filtering_info, name_to_extract="limb_branch_dict_to_cancel") red_blue_suggestions = pru.extract_from_filter_info( filtering_info, name_to_extract="red_blue_suggestions") filter_key = { k: np.round(v, 2) for k, v in filtering_info.items() if "area" in k or "length" in k } mesh_skeleton_file_paths = pru.save_off_meshes_skeletons( neuron_obj_proof, verbose=False, split_index=key["split_index"], file_name_ending=f"proofv{proof_version}") neuron_stats_dict = dict( key, proof_version=proof_version, limb_branch_to_cancel=limb_branch_to_cancel, red_blue_suggestions=red_blue_suggestions, ) neuron_stats_dict.update(mesh_skeleton_file_paths) neuron_stats_dict.update(filter_key) #--------- 12/8: Adding the neuron_graph object that will be retrieved later ---- if save_G_with_attrs: G = ctcu.G_with_attrs_from_neuron_obj(neuron_obj_proof, plot_G=False) G_path = ctcu.save_G_with_attrs(G, segment_id=segment_id, split_index=split_index) if verbose: print(f"Saved G_path = {G_path}") neuron_stats_dict["neuron_graph"] = G_path AutoProofreadStats7.insert1(neuron_stats_dict, skip_duplicates=True) # 5) Collecting Stats for the AutoProofreadNeurons6 table #a) Neuron basics if verbose: print(f"\n--5a) Neuron basics") dicts_to_update = [] multiplicity = du.multiplicity_from_segment_id(segment_id) soma_x, soma_y, soma_z = nru.soma_centers(neuron_obj, soma_name="S0", voxel_adjustment=True) basic_cell_dict = dict(multiplicity=multiplicity, soma_x=soma_x, soma_y=soma_y, soma_z=soma_z, cell_type=cell_type, cell_type_used=cell_type_used) dicts_to_update.append(basic_cell_dict) #b) Neuron Overall Statistics if verbose: print(f"\n--5b) Neuron Overall Statistics") neuron_stats_dict = neuron_obj_proof.neuron_stats( stats_to_ignore=["axon_length", "axon_area"]) dicts_to_update.append(neuron_stats_dict) #c) compartment Stats if verbose: print(f"\n--5c) compartment Stats") comp_stats = apu.compartments_stats(neuron_obj_proof, compartment_labels=None, verbose=False) dicts_to_update.append(comp_stats) #d) Synapse Stats if verbose: print(f"\n--5d) Synapse Stats") syn_stats = syu.complete_n_synapses_analysis(neuron_obj_proof) dicts_to_update.append(syn_stats) #e) Cell Typing Info after proofreading if verbose: print(f"\n--5e) Cell Typing Info after proofreading") baylor_e_i, baylor_cell_type_info = ctu.e_i_classification_from_neuron_obj( neuron_obj_proof, verbose=False, return_cell_type_info=True) baylor_cell_type_info["baylor_e_i"] = baylor_e_i baylor_cell_type_info = { f"{k}_after_proof": v for k, v in baylor_cell_type_info.items() } dicts_to_update.append(baylor_cell_type_info) #c if verbose: print(f"\n--5e) Cell Typing Info after proofreading") axon_feature_dict = au.axon_features_from_neuron_obj( neuron_obj_proof, features_to_exclude=("length", "n_branches")) apical_feature_dict = apu.compartment_features_from_skeleton_and_soma_center( neuron_obj_proof, compartment_label="apical_total", name_prefix="apical", features_to_exclude=("length", "n_branches"), ) basal_feature_dict = apu.compartment_features_from_skeleton_and_soma_center( neuron_obj_proof, compartment_label="basal", name_prefix="basal", features_to_exclude=("length", "n_branches"), ) dendrite_feature_dict = apu.compartment_features_from_skeleton_and_soma_center( neuron_obj_proof, compartment_label="dendrite", name_prefix="dendrite", features_to_exclude=("length", "n_branches"), ) dicts_to_update += [ axon_feature_dict, apical_feature_dict, basal_feature_dict, dendrite_feature_dict ] #g) Repeating old features from DecompositionCellType table if verbose: print( f"\n--5g) Repeating old features from DecompositionCellTypeV7 table" ) decomp_cell_type_features = [ "nucleus_id", "nuclei_distance", "n_nuclei_in_radius", "n_nuclei_in_bbox", "soma_x_nm", "soma_y_nm", "soma_z_nm", "baylor_e_i", "allen_e_i", "cell_type_used", "cell_type", "axon_angle_max", "axon_angle_min", "n_axon_angles", "allen_e_i_n_nuc", "allen_cell_type", "allen_cell_type_n_nuc", "allen_cell_type_e_i", ] decomp_dict = (minnie.DecompositionCellTypeV7() & key).fetch( *decomp_cell_type_features, as_dict=True)[0] decomp_dict["cell_type_used_for_axon"] = decomp_dict["cell_type_used"] decomp_dict["cell_type_for_axon"] = decomp_dict["cell_type"] del decomp_dict["cell_type_used"] del decomp_dict["cell_type"] dicts_to_update.append(decomp_dict) if plot_data: nviz.plot_compartments(neuron_obj_proof) #h) Writing the Data if verbose: print(f"\n--5h) Writing the Data") neuron_proof_dict = dict(key, proof_version=proof_version, run_time=np.round( time.time() - whole_pass_time, 2)) for d_u in dicts_to_update: neuron_proof_dict.update(d_u) AutoProofreadNeurons7.insert1(neuron_proof_dict, skip_duplicates=True) print( f"\n\n ***------ Total time for {key['segment_id']} = {time.time() - whole_pass_time} ------ ***" )
class BaylorSegmentCentroid(dj.Computed): definition=""" -> minnie.Decimation.proj(decimation_version='version') soma_index : tinyint unsigned #index given to this soma to account for multiple somas in one base semgnet ver : decimal(6,2) #the version number of the materializaiton --- centroid_x=NULL : int unsigned # (EM voxels) centroid_y=NULL : int unsigned # (EM voxels) centroid_z=NULL : int unsigned # (EM voxels) n_vertices=NULL : bigint #number of vertices n_faces=NULL : bigint #number of faces mesh: <somas> #datajoint adapter to get the somas mesh objects multiplicity=NULL : tinyint unsigned # the number of somas found for this base segment sdf=NULL : double # sdf width value for the soma volume=NULL : double # the volume in billions (10*9 nm^3) of the convex hull max_side_ratio=NULL : double # the maximum of the side length ratios used for check if soma bbox_volume_ratio=NULL : double # ratio of bbox (axis aligned) volume to mesh volume to use for check if soma max_hole_length=NULL : double #euclidean distance of the maximum hole size run_time=NULL : double # the amount of time to run (seconds) """ key_source = (((minnie.Decimation & f"n_vertices > {verts_min}").proj(decimation_version='version') & "decimation_version=" + str(decimation_version) & f"decimation_ratio={decimation_ratio}") & ( du.proofreading_segment_id_restriction() ) ) def make(self,key): """ Pseudocode: 1) Compute all of the 2) Save the mesh as an h5 py file 3) Store the saved path as the decomposition part of the dictionary and erase the vertices and faces 4) Insert """ #get the mesh data print(f"\n\n\n---- Working on Neuron {key['segment_id']} ----") print(key) new_mesh = (minnie.Decimation() & key).fetch1("mesh") current_mesh_verts,current_mesh_faces = new_mesh.vertices,new_mesh.faces segment_id = key["segment_id"] (total_soma_list, run_time, total_soma_list_sdf, glia_pieces, nuclei_pieces) = sm.extract_soma_center( segment_id, current_mesh_verts, current_mesh_faces, return_glia_nuclei_pieces=True, ) # -------- 1/9 Addition: Going to save off the glia and nuclei pieces ----------- # """ Psuedocode: For both glia and nuclie pieces 1) If the length of array is greater than 0 --> combine the mesh and map the indices to original mesh 2) If not then just put None """ orig_mesh = trimesh.Trimesh(vertices=current_mesh_verts, faces=current_mesh_faces) if len(glia_pieces)>0: glia_faces = tu.original_mesh_faces_map(orig_mesh,tu.combine_meshes(glia_pieces)) n_glia_faces = len(glia_faces) else: glia_faces = None n_glia_faces = 0 if len(nuclei_pieces)>0: nuclei_faces = tu.original_mesh_faces_map(orig_mesh,tu.combine_meshes(nuclei_pieces)) n_nuclei_faces = len(nuclei_faces) else: nuclei_faces = None n_nuclei_faces = 0 # --------- saving the nuclei and glia saves glia_path,nuclei_path = du.save_glia_nuclei_files(glia_faces=glia_faces, nuclei_faces=nuclei_faces, segment_id=segment_id) print(f" glia_path = {glia_path} \n nuclei_path = {nuclei_path}") glia_nuclei_key = dict(key, ver=current_version, n_glia_faces=n_glia_faces, #glia_faces = glia_faces, glia_faces = glia_path, n_nuclei_faces = n_nuclei_faces, #nuclei_faces = nuclei_faces nuclei_faces = nuclei_path, ) NeuronGliaNuclei.insert1(glia_nuclei_key,replace=True) print(f"Finished saving off glia and nuclei information : {glia_nuclei_key}") # ---------------- End of 1/9 Addition --------------------------------- # print(f"Run time was {run_time} \n total_soma_list = {total_soma_list}" f"\n with sdf values = {total_soma_list_sdf}") #check if soma list is empty and did not find soma if len(total_soma_list) <= 0: print("There were no somas found for this mesh so just writing empty data") returned_file_path = tu.write_h5_file( vertices=np.array([]), faces=np.array([]), segment_id=segment_id, filename = f'{segment_id}_0.h5', filepath=str(du.get_somas_path()) ) insert_dict = dict(key, soma_index=0, ver=current_version, centroid_x=None, centroid_y=None, centroid_z=None, #distance_from_prediction=None, #prediction_matching_index = None, n_vertices=0, n_faces=0, mesh=returned_file_path, multiplicity=0, sdf = None, volume = None, max_side_ratio = None, bbox_volume_ratio = None, max_hole_length=None, run_time=run_time ) #raise Exception("to prevent writing because none were found") self.insert1(insert_dict,skip_duplicates=True) return #if there is one or more soma found, get the volume and side length checks max_side_ratio = [np.max(sm.side_length_ratios(m)) for m in total_soma_list] bbox_volume_ratio = [sm.soma_volume_ratio(m) for m in total_soma_list] dicts_to_insert = [] for i,(current_soma,soma_sdf,sz_ratio,vol_ratio) in enumerate(zip(total_soma_list,total_soma_list_sdf,max_side_ratio,bbox_volume_ratio)): print("Trying to write off file") """ Currently don't need to export the meshes current_soma.export(f"{key['segment_id']}/{key['segment_id']}_soma_{i}.off") """ auto_prediction_center = np.mean(current_soma.vertices,axis=0) / np.array([4,4,40]) auto_prediction_center = auto_prediction_center.astype("int") print(f"Predicted Coordinates are {auto_prediction_center}") max_hole_length = tu.largest_hole_length(current_soma) returned_file_path = tu.write_h5_file( vertices=current_soma.vertices, faces=current_soma.faces, segment_id=segment_id, filename = f'{segment_id}_{i}.h5', filepath=str(du.get_somas_path()) ) insert_dict = dict(key, soma_index=i+1, ver=current_version, centroid_x=auto_prediction_center[0], centroid_y=auto_prediction_center[1], centroid_z=auto_prediction_center[2], n_vertices = len(current_soma.vertices), n_faces = len(current_soma.faces), mesh=returned_file_path, multiplicity=len(total_soma_list), sdf = np.round(soma_sdf,3), volume = current_soma.convex_hull.volume/1000000000, max_side_ratio = np.round(sz_ratio,3), bbox_volume_ratio = np.round(vol_ratio,3), max_hole_length = np.round(max_hole_length,3), run_time=np.round(run_time,4) ) dicts_to_insert.append(insert_dict) self.insert(dicts_to_insert,skip_duplicates=True)