def main(args): """Main""" stage_jobconfig = args['stage_jobconfig'] highlevel_job_props = args['highlevel_jobconfig'] seed = args.get('seed', 1) logging.basicConfig(level=highlevel_job_props['log_level']) opt = create_optimizer(args) cp_file = os.path.join(stage_jobconfig['cp_dir'], 'seed%s.pkl' % seed) utility.create_filepath(cp_file) if stage_jobconfig.get('cp_backup_dir'): cp_backup_file = os.path.join(stage_jobconfig['cp_backup_dir'], 'seed%s.pkl' % seed) utility.create_filepath(cp_backup_file) else: cp_backup_file = None cp_backup_frequency = stage_jobconfig['cp_backup_frequency'] max_ngen = stage_jobconfig['max_ngen'] offspring_size = stage_jobconfig['offspring_size'] continue_cp = os.path.exists(cp_file) logger.debug('Doing start or continue') if os.path.exists(cp_file): try: _ = utility.load_pickle(cp_file) except: logger.debug('Checkpoint file is corrupt! Looking for backup') if cp_backup_file and os.path.exists(cp_backup_file): shutil.copyfile(cp_backup_file, cp_file) opt.run(max_ngen=max_ngen, offspring_size=offspring_size, continue_cp=continue_cp, cp_filename=cp_file, cp_backup=cp_backup_file, cp_backup_frequency=cp_backup_frequency)
def Main(): args = parser.parse_args() cty_type = args.cty cty_list = args.cty_list data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir, 'assets', 'aggregated_data') mouse_data_filename = os.path.join(data_path, 'Mouse_class_data.csv') mouse_datatype_filename = os.path.join(data_path, 'Mouse_class_datatype.csv') me_ttype_map_path = os.path.join(data_path, 'me_ttype.pkl') sdk_data_filename = os.path.join(data_path, 'sdk.csv') sdk_datatype_filename = os.path.join(data_path, 'sdk_datatype.csv') sdk_data = man_utils.read_csv_with_dtype(sdk_data_filename, sdk_datatype_filename) if cty_type == 'ttype': mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename, mouse_datatype_filename) me_ttype_map = utility.load_pickle(me_ttype_map_path) metype_cluster = mouse_data.loc[ mouse_data.hof_index == 0, ['Cell_id', 'Dendrite_type', 'me_type']] sdk_me = pd.merge(sdk_data, metype_cluster, how='left', on='Cell_id').dropna(how='any', subset=['me_type']) sdk_me['ttype'] = sdk_me['me_type'].apply(lambda x: me_ttype_map[x]) cell_df = sdk_me.loc[sdk_me.ttype.isin(cty_list), ] elif cty_type == 'Cre_line': cell_df = sdk_data.loc[sdk_data.line_name.isin(cty_list), ] cell_ids = cell_df.Cell_id.unique().tolist() rc = Client(profile=os.getenv('IPYTHON_PROFILE')) logger.debug('Using ipyparallel with %d engines', len(rc)) lview = rc.load_balanced_view() lview.map_sync(get_efeatures, cell_ids)
import seaborn as sns import numpy as np from ateam.data import lims from ateamopt.utils import utility from allensdk.core.cell_types_cache import CellTypesCache import pandas as pd import os def get_morph_path(cell_id): lr = lims.LimsReader() morph_path = lr.get_swc_path_from_lims(int(cell_id)) return morph_path data_path = os.path.join(os.getcwd(),os.pardir,os.pardir,'assets','aggregated_data') cre_color_tasic16_filename = os.path.join(data_path,'cre_color_tasic16.pkl') cre_color_dict = utility.load_pickle(cre_color_tasic16_filename) cre_color_dict['Other'] = (0,0,0) depth_data_filename = os.path.join(data_path,'mouse_me_and_met_avg_layer_depths.json') # Average layerwise depths for mouse depth_data = utility.load_json(depth_data_filename) total_depth = depth_data['wm'] # Cells are chosen to sample from diverse types within each layer cell_id_dict = { #'1':['574734127','564349611','475585413','555341581','536951541'], '1':['574734127','475585413','536951541'], # '2/3':['485184849','475515168','485468180','476087653','571306690'], '2/3':['485184849','475515168','485468180'], # '4':['483101699','602822298','490205998','569723367','324257146'], '4':['483101699','602822298','569723367'], '5':['479225052','607124114','515249852'],
import os from matplotlib.cm import ScalarMappable from matplotlib.colors import ListedColormap import seaborn as sns import matplotlib.pyplot as plt import numpy as np from ateamopt.utils import utility import matplotlib matplotlib.use('Agg') cell_id = '483101699' model_data_path = ('/allen/aibs/mat/anin/hpc_trials/BluePyOpt_speedup/' '%s/Stage2' % cell_id) hof_responses_path = os.path.join(model_data_path, 'analysis_params/hof_response_all.pkl') hof_responses = utility.load_pickle(hof_responses_path) hof_num = 40 color_palette = sns.color_palette('Blues_r', hof_num) select_stim_names = ['LongDC_55', 'Short_Square_Triple_101', 'Ramp_5', 'Noise_1_63'] sns.set_style('whitegrid') fig, ax = plt.subplots(1, len(select_stim_names), sharey=True, squeeze=False, gridspec_kw={'width_ratios': [1, 1, 1.5, 1.5]}, figsize=(12, 3.5)) hof_array_r = np.arange(hof_num - 1, -1, -1) hof_responses_r = list(reversed(hof_responses))
def main(args): stage_jobconfig = args['stage_jobconfig'] highlevel_job_props = args['highlevel_jobconfig'] logging.basicConfig(level=highlevel_job_props['log_level']) parent_dir = os.path.abspath(os.path.join('.', os.pardir)) path_to_cell_metadata = glob.glob(parent_dir + '/cell_metadata*.json')[0] cell_metadata = utility.load_json(path_to_cell_metadata) cell_id = cell_metadata['cell_id'] peri_model_id = cell_metadata.get('peri_model_id') released_aa_model_id = cell_metadata.get('released_aa_model_id') all_protocols_path = highlevel_job_props['all_protocols_path'] all_features_path = highlevel_job_props['all_features_path'] morph_path = highlevel_job_props['swc_path'] axon_type = highlevel_job_props['axon_type'] ephys_dir = highlevel_job_props['ephys_dir'] train_features_path = args['train_features'] test_features_path = args['test_features'] param_write_path = args['parameters'] mech_write_path = args['mechanism'] release_param_write_path = args['released_aa_model'] mech_release_write_path = args['released_aa_mechanism'] analysis_parallel = (stage_jobconfig['analysis_config'].get('ipyparallel') and stage_jobconfig['run_hof_analysis']) props = dict(axon_type=axon_type, ephys_dir=ephys_dir) map_function = analyzer_map(analysis_parallel) opt_train = get_opt_obj(all_protocols_path, train_features_path, morph_path, param_write_path, mech_write_path, map_function, **props) opt_all = get_opt_obj(all_protocols_path, all_features_path, morph_path, param_write_path, mech_write_path, map_function, **props) opt_test = get_opt_obj(all_protocols_path, test_features_path, morph_path, param_write_path, mech_write_path, map_function, **props) analysis_handler = Optim_Analyzer(args, opt_all) best_model = analysis_handler.get_best_model( ) # Model with least training error aibs_params_modelname = 'fitted_params/optim_param_%s.json' % cell_id analysis_handler.save_params_aibs_format(aibs_params_modelname, best_model[0], expand_params=True) aibs_params_compact_modelname = 'fitted_params/optim_param_%s_compact.json' % cell_id analysis_handler.save_params_aibs_format(aibs_params_compact_modelname, best_model[0]) bpopt_params_modelname = 'fitted_params/optim_param_%s_bpopt.json' % cell_id analysis_handler.save_params_bpopt_format(bpopt_params_modelname, best_model[0]) # Export hoc model if stage_jobconfig['hoc_export']: hoc_export_path = 'fitted_params/model_template_%s.hoc' % cell_id utility.create_filepath(hoc_export_path) best_param_dict = {key:best_model[0][i] for i,key in \ enumerate(opt_train.evaluator.param_names)} model_string = opt_train.evaluator.cell_model.create_hoc( best_param_dict) with open(hoc_export_path, "w") as hoc_template: hoc_template.write(model_string) hof_model_params, seed_indices = analysis_handler.get_all_models() if not stage_jobconfig.get('run_hof_analysis'): hof_model_params, seed_indices = best_model, [seed_indices[0]] hof_params_filename = 'analysis_params/hof_model_params.pkl' hof_responses_filename = 'analysis_params/hof_response_all.pkl' obj_list_train_filename = 'analysis_params/hof_obj_train.pkl' obj_list_all_filename = 'analysis_params/hof_obj_all.pkl' feat_list_all_filename = 'analysis_params/hof_features_all.pkl' obj_list_test_filename = 'analysis_params/hof_obj_test.pkl' seed_indices_filename = 'analysis_params/seed_indices.pkl' score_list_train_filename = 'analysis_params/score_list_train.pkl' # Response for the entire hall of fame not arranged hof_response_list = analysis_handler.get_model_responses( hof_model_params, hof_responses_filename) analysis_handler._opt = opt_train obj_list_train = analysis_handler.get_response_scores(hof_response_list) # Sort everything with respect to training error if not os.path.exists(score_list_train_filename): score_list_train = [ np.sum(list(obj_dict_train.values())) for obj_dict_train in obj_list_train ] utility.create_filepath(score_list_train_filename) utility.save_pickle(score_list_train_filename, score_list_train) else: score_list_train = utility.load_pickle(score_list_train_filename) if not os.path.exists(seed_indices_filename): seed_indices_sorted = analysis_handler.organize_models( seed_indices, score_list_train) utility.save_pickle(seed_indices_filename, seed_indices_sorted) if not os.path.exists(obj_list_train_filename): obj_list_train_sorted = analysis_handler.organize_models( obj_list_train, score_list_train) utility.save_pickle(obj_list_train_filename, obj_list_train_sorted) if not os.path.exists(obj_list_all_filename): analysis_handler._opt = opt_all obj_list_gen = analysis_handler.get_response_scores(hof_response_list) obj_list_gen_sorted = analysis_handler.organize_models( obj_list_gen, score_list_train) utility.save_pickle(obj_list_all_filename, obj_list_gen_sorted) if not os.path.exists(feat_list_all_filename): analysis_handler._opt = opt_all feat_list_gen = analysis_handler.get_response_features( hof_response_list) feat_list_gen_sorted = analysis_handler.organize_models( feat_list_gen, score_list_train) utility.save_pickle(feat_list_all_filename, feat_list_gen_sorted) if not os.path.exists(obj_list_test_filename): analysis_handler._opt = opt_test obj_list_test = analysis_handler.get_response_scores(hof_response_list) obj_list_test_sorted = analysis_handler.organize_models( obj_list_test, score_list_train) utility.save_pickle(obj_list_test_filename, obj_list_test_sorted) analysis_handler._opt = opt_train # Save the sorted hof responses at the end hof_response_sorted = analysis_handler.organize_models( hof_response_list, score_list_train) utility.save_pickle(hof_responses_filename, hof_response_sorted) # Save the sorted hall of fame output in .pkl hof_model_params_sorted = analysis_handler.save_hof_output_params( hof_model_params, hof_params_filename, score_list_train) # Save the entire hall of fame parameters for i, hof_param in enumerate(hof_model_params_sorted): aibs_params_modelname = os.path.join( 'fitted_params', 'hof_param_%s_%s.json' % (cell_id, i)) analysis_handler.save_params_aibs_format(aibs_params_modelname, hof_param, expand_params=True) # Now save the sorted score utility.save_pickle(score_list_train_filename, sorted(score_list_train)) GA_evol_path = os.path.join('analysis_params', 'GA_evolution_params.pkl') analysis_handler.save_GA_evolultion_info(GA_evol_path) resp_filename = os.path.join(os.getcwd(), 'resp_opt.txt') analysis_handler.save_best_response(hof_response_sorted[0], resp_filename) if release_param_write_path: eval_handler_release = Bpopt_Evaluator(all_protocols_path, all_features_path, morph_path, release_param_write_path, mech_release_write_path, stub_axon=False, do_replace_axon=True, ephys_dir=ephys_dir) evaluator_release = eval_handler_release.create_evaluator() opt_release = bpopt.optimisations.DEAPOptimisation( evaluator=evaluator_release) resp_release_filename = os.path.join(os.getcwd(), 'resp_release.txt') analysis_handler.get_release_responses(opt_release, resp_release_filename) resp_release_aa = utility.load_pickle(resp_release_filename)[0] features_release_aa = opt_release.evaluator.fitness_calculator.\ calculate_features(resp_release_aa) features_aa_filename = os.path.join( 'Validation_Responses', 'Features_released_aa_%s.pkl' % cell_id) utility.create_filepath(features_aa_filename) utility.save_pickle(features_aa_filename, features_release_aa) else: resp_release_filename = None stim_mapfile = highlevel_job_props['stimmap_file'] analysis_write_path = '%s_%s.pdf' % (cell_id, stage_jobconfig['stage_name']) pdf_pages = PdfPages(analysis_write_path) model_type = 'All-active' pdf_pages = analysis_handler.plot_grid_Response(resp_filename, resp_release_filename, stim_mapfile, pdf_pages) pdf_pages = analysis_handler.plot_feature_comp(resp_filename, resp_release_filename, pdf_pages) pdf_pages = analysis_handler.plot_GA_evol(GA_evol_path, pdf_pages) pdf_pages = analysis_handler.plot_param_diversity(hof_params_filename, pdf_pages) if stage_jobconfig['model_postprocess']: exp_fi_path = os.path.join('Validation_Responses', 'fI_exp_%s.pkl' % cell_id) model_fi_path = os.path.join('Validation_Responses', 'fI_aa_%s.pkl' % cell_id) exp_AP_shape_path = os.path.join('Validation_Responses', 'AP_shape_exp_%s.pkl' % cell_id) model_AP_shape_path = os.path.join('Validation_Responses', 'AP_shape_aa_%s.pkl' % cell_id) pdf_pages = analysis_handler.postprocess( stim_mapfile, resp_filename, pdf_pages, exp_fi_path, model_fi_path, exp_AP_shape_path, model_AP_shape_path, model_type) # Perisomatic model if peri_model_id and stage_jobconfig['run_peri_comparison']: resp_peri_filename = os.path.join(os.getcwd(), 'resp_peri.txt') peri_param_path = args['released_peri_model'] peri_mech_path = args['released_peri_mechanism'] props_peri = props.copy() props_peri['axon_type'] = 'stub_axon' eval_handler_peri = Bpopt_Evaluator(all_protocols_path, all_features_path, morph_path, peri_param_path, peri_mech_path, **props_peri) evaluator_peri = eval_handler_peri.create_evaluator() opt_peri = bpopt.optimisations.DEAPOptimisation( evaluator=evaluator_peri) model_type = 'Perisomatic' analysis_handler.get_release_responses(opt_peri, resp_peri_filename) resp_peri = utility.load_pickle(resp_peri_filename)[0] features_peri = opt_peri.evaluator.fitness_calculator.calculate_features( resp_peri) features_peri_filename = os.path.join('Validation_Responses', 'Features_peri_%s.pkl' % cell_id) utility.create_filepath(features_peri_filename) utility.save_pickle(features_peri_filename, features_peri) pdf_pages = analysis_handler.plot_grid_Response( resp_filename, resp_peri_filename, stim_mapfile, pdf_pages, resp_comparison=model_type) if stage_jobconfig['model_postprocess']: model_fi_path = os.path.join('Validation_Responses', 'fI_peri_%s.pkl' % cell_id) model_AP_shape_path = os.path.join( 'Validation_Responses', 'AP_shape_peri_%s.pkl' % cell_id) pdf_pages = analysis_handler.postprocess( stim_mapfile, resp_peri_filename, pdf_pages, exp_fi_path, model_fi_path, exp_AP_shape_path, model_AP_shape_path, model_type) if stage_jobconfig.get('calc_model_perf'): spiketimes_exp_path = os.path.join('Validation_Responses', 'spiketimes_exp_noise.pkl') all_features = utility.load_json(all_features_path) spiketimes_noise_exp = {} for stim_, feat in all_features.items(): if 'Noise' in stim_: if 'peak_time' in feat['soma'].keys(): spiketimes_noise_exp[stim_] = feat['soma']['peak_time'][2] if bool(spiketimes_noise_exp): utility.create_filepath(spiketimes_exp_path) utility.save_pickle(spiketimes_exp_path, spiketimes_noise_exp) spiketimes_hof_path = os.path.join('Validation_Responses', 'spiketimes_model_noise.pkl') exp_variance_hof_path = os.path.join('Validation_Responses', 'exp_variance_hof.pkl') model_perf_filename = os.path.join('Validation_Responses', 'fitness_metrics_%s.csv' % cell_id) pdf_pages = analysis_handler.hof_statistics( stim_mapfile, pdf_pages, obj_list_all_filename, hof_responses_filename, obj_list_train_filename, obj_list_test_filename, seed_indices_filename, spiketimes_exp_path, spiketimes_hof_path, exp_variance_hof_path, cell_metadata, model_perf_filename) pdf_pages.close() if stage_jobconfig.get('calc_time_statistics'): # time_by_gen_filename = 'time_info.txt' # if os.path.exists(time_by_gen_filename): # time_metrics_filename = 'time_metrics_%s.csv' % cell_id # analysis_module.save_optimization_time(time_by_gen_filename, # time_metrics_filename, cell_metadata) compute_statistics_filename = 'compute_metrics_%s.csv' % cell_id opt_logbook = 'logbook_info.txt' analysis_module.save_compute_statistics(opt_logbook, compute_statistics_filename)
from statsmodels.stats.multitest import fdrcorrection from itertools import combinations from scipy.stats import mannwhitneyu import matplotlib as mpl # %% Data paths data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir, 'assets', 'aggregated_data') annotation_datapath = os.path.join(data_path, 'anno.feather') cre_coloring_filename = os.path.join(data_path, 'cre_color_tasic16.pkl') exc_expression_profile_path = os.path.join(data_path, 'exc_expression_all.csv') # %% Read the data annotation_df = feather.read_dataframe(annotation_datapath) cre_color_dict = utility.load_pickle(cre_coloring_filename) # Only for better contrast cre_color_dict["Rbp4-Cre_KL100"] = mpl.colors.to_rgb('#008033') exc_lines = ["Nr5a1-Cre", "Rbp4-Cre_KL100"] exc_palette = {exc_line_: cre_color_dict[exc_line_] for exc_line_ in exc_lines} exc_expression_data = pd.read_csv(exc_expression_profile_path, index_col=None) exc_expression_data = exc_expression_data.rename( columns={'cre_label': 'Cre_line'}) # %% Filtering # Get rid of genes expressed in less than 10% of the cells exc_expression_data = exc_expression_data.loc[:, ( exc_expression_data != 0).sum(axis=0) >= .1 * exc_expression_data.shape[0]]
hof_num = 1 inh_lines = ["Htr3a-Cre_NO152","Sst-IRES-Cre","Pvalb-IRES-Cre"] data_path = os.path.join(os.path.dirname(man_opt.__file__),os.pardir,'assets','aggregated_data') mouse_data_filename = os.path.join(data_path,'Mouse_class_data.csv') mouse_datatype_filename = os.path.join(data_path,'Mouse_class_datatype.csv') sdk_data_filename = os.path.join(data_path,'sdk.csv') sdk_datatype_filename = os.path.join(data_path,'sdk_datatype.csv') cre_coloring_filename = os.path.join(data_path,'cre_color_tasic16.pkl') filtered_me_inh_cells_filename = os.path.join(data_path,'filtered_me_inh_cells.pkl') mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename,mouse_datatype_filename) mouse_data = mouse_data.loc[mouse_data.hof_index<hof_num,] mouse_data = mouse_data.loc[mouse_data.Cre_line.isin(inh_lines),['Cell_id','Cre_line','hof_index']] cre_color_dict = utility.load_pickle(cre_coloring_filename) filtered_me_inh_cells = utility.load_pickle(filtered_me_inh_cells_filename) sdk_data= man_utils.read_csv_with_dtype(sdk_data_filename,sdk_datatype_filename) sdk_data= sdk_data.rename(columns={'line_name':'Cre_line','ef__threshold_i_long_square':'rheobase'}) sdk_data = sdk_data.loc[sdk_data.Cre_line.isin(inh_lines),['Cell_id','rheobase']] mouse_data = pd.merge(mouse_data,sdk_data,on='Cell_id') cell_ids = list(set(mouse_data.Cell_id.tolist())) efel_feature_path = 'eFEL_features' spiking_features_df_list = [] spiking_features = ['AP_amplitude_from_voltagebase','AP1_amp','APlast_amp', "AP_width",'AHP_depth','mean_frequency', 'Spikecount','inv_first_ISI','voltage_base']
import numpy as np from ateamopt.utils import utility import seaborn as sns import matplotlib.pyplot as plt from ateamopt.animation import animation_module from ateamopt.bpopt_evaluator import Bpopt_Evaluator import glob,os import pandas as pd from ipyparallel import Client model_path = os.getcwd() checkpoint_path = os.path.join(model_path,'checkpoints_final/seed4.pkl') checkpoint = utility.load_pickle(checkpoint_path) hof_inds = checkpoint['halloffame'] config_dir= os.path.join(model_path,'config/483101699') protocol_path = os.path.join(config_dir,'protocols.json') feature_path = os.path.join(config_dir,'features.json') parameter_path = os.path.join(config_dir,'parameters.json') mechanism_path = os.path.join(config_dir,'mechanism.json') morph_path = os.path.join(model_path,'reconstruction.swc') eval_handler = Bpopt_Evaluator(protocol_path,feature_path,morph_path, parameter_path,mechanism_path,timeout=60) evaluator = eval_handler.create_evaluator() param_names = evaluator.param_names offspring_size = 512 max_ngen = (len(checkpoint['history'].genealogy_history.values())- offspring_size)/(2*offspring_size)+1 max_ngen = int(max_ngen)
inh_expression_profile_path = os.path.join(data_path, 'inh_expression_all.csv') mouse_data_filename = os.path.join(data_path, 'Mouse_class_data.csv') mouse_datatype_filename = os.path.join(data_path, 'Mouse_class_datatype.csv') param_data_filename = os.path.join(data_path, 'allactive_params.csv') param_datatype_filename = os.path.join(data_path, 'allactive_params_datatype.csv') me_ttype_map_path = os.path.join(data_path, 'me_ttype.pkl') annotation_datapath = os.path.join(data_path, 'anno.feather') expression_profile_path = os.path.join(data_path, 'exc_ttype_expression.csv') # %% Read the data mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename, mouse_datatype_filename) me_ttype_map = utility.load_pickle(me_ttype_map_path) metype_cluster = mouse_data.loc[mouse_data.hof_index == 0, ['Cell_id', 'Dendrite_type', 'me_type']] hof_param_data = man_utils.read_csv_with_dtype(param_data_filename, param_datatype_filename) hof_param_data = hof_param_data.loc[hof_param_data.hof_index == 0, ] hof_param_data_ttype = pd.merge(hof_param_data, metype_cluster, how='left', on='Cell_id').dropna(how='any', subset=['me_type']) hof_param_data_ttype['ttype'] = hof_param_data_ttype['me_type'].apply( lambda x: me_ttype_map[x])
param_name_ = param_name_.split('_', 1)[-1] param_name_ = param_name_.rsplit('_', 1)[0] if bool(re.search('NaT', param_name_)): param_name_ = 'NaT' elif bool(re.search('Nap', param_name_)): param_name_ = 'NaP' elif bool(re.search('K_P', param_name_)): param_name_ = 'KP' elif bool(re.search('K_T', param_name_)): param_name_ = 'KT' elif bool(re.search('Kv3_1', param_name_)): param_name_ = 'Kv31' return param_name_ filtered_me_inh_cells = utility.load_pickle('filtered_me_inh_cells.pkl') cre_color_dict = utility.load_pickle('rnaseq_sorted_cre.pkl') mouse_data_path = os.path.join(os.getcwd(), os.pardir, 'Mouse_class_data.csv') mouse_datatype_path = os.path.join(os.getcwd(), os.pardir, 'Mouse_class_datatype.csv') datatypes = pd.read_csv(mouse_datatype_path)['types'] mouse_data = pd.read_csv(mouse_data_path, dtype=datatypes.to_dict()) mouse_data = mouse_data.loc[mouse_data.hof_index == 0, ] cre_cluster = mouse_data.loc[mouse_data.hof_index == 0, ['Cell_id', 'Cre_line']] sens_analysis_datapath = '/allen/aibs/mat/anin/Sobol_analysis/Inh_cells/*/'\ 'sensitivity_data/*.csv' sens_analysis_pathlist = glob.glob(sens_analysis_datapath) cell_id_list = [path_.split('/')[-3] for path_ in sens_analysis_pathlist]
mouse_data_df = man_utils.read_csv_with_dtype( mouse_data_filename, mouse_datatype_filename) morph_data = man_utils.read_csv_with_dtype( morph_data_filename, morph_datatype_filename) # morph_data = morph_data.loc[:,[morph_feature for morph_feature in morph_data.columns # if not any(sec in morph_feature for sec in['apical','axon'])]] morph_fields = man_utils.get_data_fields(morph_data) ephys_data = man_utils.read_csv_with_dtype(train_ephys_max_amp_fname, train_ephys_max_amp_dtype_fname) ephys_fields = utility.load_json(train_ephys_max_amp_fields_fname) hof_param_data = man_utils.read_csv_with_dtype( param_data_filename, param_datatype_filename) cre_color_dict = utility.load_pickle(cre_coloring_filename) bcre_color_dict = utility.load_pickle(bcre_coloring_filename) bcre_index_order = list(bcre_color_dict.keys()) filtered_me_inh_cells = utility.load_pickle(filtered_me_inh_cells_filename) filtered_me_exc_cells = utility.load_pickle(filtered_me_exc_cells_filename) cre_cluster_color_dict = OrderedDict() cre_type_cluster = utility.load_pickle(cre_ttype_filename) for cre, color in cre_color_dict.items(): if cre_type_cluster[cre] not in cre_cluster_color_dict.keys(): cre_cluster_color_dict[cre_type_cluster[cre]] = color # %% Filtering
'Cell_id'].tolist() filtered_pyr_cells = sup_pyr + deep_pyr param_pyr = param_pyr.loc[param_pyr.Cell_id.isin(filtered_pyr_cells), ] param_pyr['depth'] = param_pyr['Layer'].apply(lambda x: 'L2/3 PC' if x == '2/3' else 'L5 PC') depth_cty = param_pyr.depth.unique().tolist() all_cre_lines = mouse_data.loc[mouse_data.Cell_id.isin(filtered_pyr_cells), 'Cre_line'].unique().tolist() # Save cells list cell_list_path = os.path.join(data_path, 'pyr_cells_list.csv') pyr_cell_list = param_pyr.loc[:, ['Cell_id', 'Layer', 'depth']] pyr_cell_list.to_csv(cell_list_path, index=False) layer_cty_colors = utility.load_pickle(layer_cty_colors_filename) # %% Statistical comparison conductance_params = [ dens_ for dens_ in list(param_pyr) if dens_.startswith('gbar') ] param_pyr = param_pyr.dropna(axis=1, how='all') param_pyr = param_pyr.dropna( subset=[cond for cond in list(param_pyr) if cond in conductance_params]) select_conds = [ 'gbar_Ih.apical', 'gbar_Ih.somatic', 'gbar_NaTa_t.axonal', 'gbar_Kv3_1.axonal', 'gbar_Kv3_1.somatic', 'gbar_K_Tst.axonal', 'gbar_K_Tst.somatic', 'gbar_Ca_LVA.axonal', 'gbar_Ca_LVA.somatic' ]
def read_pickle_file(file): pickle_data = utility.load_pickle(file) return pickle_data
mouse_data_df = man_utils.read_csv_with_dtype(mouse_data_filename, mouse_datatype_filename) mouse_data_df = man_utils.add_transcriptomic_subclass(mouse_data_df, me_ttype_map_path) mouse_data_df = man_utils.add_broad_subclass(mouse_data_df) hof_param_data = man_utils.read_csv_with_dtype(param_data_filename, param_datatype_filename) subclass_cluster = mouse_data_df.loc[mouse_data_df.hof_index == 0, ['Cell_id', 'Broad_subclass']] hof_param_subclass = pd.merge(hof_param_data, subclass_cluster, how='left', on='Cell_id') broad_subclass_color_dict = utility.load_pickle(broad_subclass_colors_filename) broad_subclass = [ subclass for subclass in broad_subclass_color_dict.keys() if subclass != 'Other' ] hof_param_subclass = hof_param_subclass.loc[ hof_param_subclass.Broad_subclass.isin(broad_subclass), ] hof_param_subclass.Broad_subclass = pd.Categorical( hof_param_subclass.Broad_subclass, categories=broad_subclass) hof_param_subclass = hof_param_subclass.sort_values( ['Broad_subclass', 'Cell_id', 'hof_index']) hof_param_subclass.reset_index(drop=True, inplace=True) hof_num = 40 intra_dist, inter_dist = {}, {}
# Data paths data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir, 'assets', 'aggregated_data') cre_coloring_filename = os.path.join(data_path, 'cre_color_tasic16.pkl') sag_features_exp_filename = os.path.join(data_path, 'sag_data_exp.csv') sag_features_model_filename = os.path.join(data_path, 'sag_data_model.csv') filtered_me_exc_cell_list_path = os.path.join(data_path, 'filtered_me_exc_cells.pkl') perturbed_Nr5like_feature_file = os.path.join( data_path, 'sag_data_perturbed_Nr5like.csv') perturbed_Rbp4like_feature_file = os.path.join( data_path, 'sag_data_perturbed_Rbp4like.csv') # Read the data exc_lines = ["Nr5a1-Cre", "Rbp4-Cre_KL100"] cre_color_dict = utility.load_pickle(cre_coloring_filename) cre_color_dict["Rbp4-Cre_KL100"] = mpl.colors.to_rgb( '#008033') # Only for better contrast filtered_me_exc_cells = utility.load_pickle(filtered_me_exc_cell_list_path) # All experiment sag_features_exp = pd.read_csv(sag_features_exp_filename, index_col=0, dtype={'Cell_id': str}) sag_features_exp = sag_features_exp.loc[ sag_features_exp.Cell_id.isin(filtered_me_exc_cells), ] sag_features_exp['type'] = 'Exp' # Models sag_features_model = pd.read_csv(sag_features_model_filename, index_col=0,
def add_transcriptomic_subclass(data, me_ttype_map_path): me_ttype_map = utility.load_pickle(me_ttype_map_path) me_ttype_map['Other'] = 'Other' data['me_type'] = data['me_type'].fillna('Other') data['ttype'] = data['me_type'].apply(lambda x: me_ttype_map[x]) return data
HoF_obj_train_filelist = glob.glob(HoF_obj_train_path) #%% Create heatmap of the feature errors obj_dict_train_list = [] feat_reject_list = ['Spikecount', 'depol_block', 'check_AISInitiation'] max_feat_count = {} feature_mat_list = [] feature_mat_dict_all = defaultdict(list) feature_heatmap_filename = os.path.join('figures', 'feature_score_comparison.pdf') utility.create_filepath(feature_heatmap_filename) for obj_file in HoF_obj_train_filelist: obj_dict_train = utility.load_pickle(obj_file)[0] cell_id = obj_file.split('/')[-2] temp_dict = defaultdict(list) for obj_key, obj_val in obj_dict_train.items(): feat = obj_key.split('.')[-1] feat = feat if feat != 'AP_amplitude_from_voltagebase' else 'AP_amplitude' if feat not in feat_reject_list: temp_dict[feat].append(obj_val) feature_mat_dict_all[feat].append(obj_val) feature_mat_dict_all['Cell_id'].append(cell_id) temp_dict = {key: np.mean(val) for key, val in temp_dict.items()} max_feat = max(temp_dict, key=temp_dict.get) temp_dict['max_feature'] = max_feat temp_dict['max_feature_val'] = temp_dict[max_feat]