def main(args):
"""Main"""
stage_jobconfig = args['stage_jobconfig']
highlevel_job_props = args['highlevel_jobconfig']
seed = args.get('seed', 1)
logging.basicConfig(level=highlevel_job_props['log_level'])
opt = create_optimizer(args)
cp_file = os.path.join(stage_jobconfig['cp_dir'], 'seed%s.pkl' % seed)
utility.create_filepath(cp_file)
if stage_jobconfig.get('cp_backup_dir'):
cp_backup_file = os.path.join(stage_jobconfig['cp_backup_dir'],
'seed%s.pkl' % seed)
utility.create_filepath(cp_backup_file)
else:
cp_backup_file = None
cp_backup_frequency = stage_jobconfig['cp_backup_frequency']
max_ngen = stage_jobconfig['max_ngen']
offspring_size = stage_jobconfig['offspring_size']
continue_cp = os.path.exists(cp_file)
logger.debug('Doing start or continue')
if os.path.exists(cp_file):
try:
_ = utility.load_pickle(cp_file)
except:
logger.debug('Checkpoint file is corrupt! Looking for backup')
if cp_backup_file and os.path.exists(cp_backup_file):
shutil.copyfile(cp_backup_file, cp_file)
opt.run(max_ngen=max_ngen,
offspring_size=offspring_size,
continue_cp=continue_cp,
cp_filename=cp_file,
cp_backup=cp_backup_file,
cp_backup_frequency=cp_backup_frequency)
def Main():
args = parser.parse_args()
cty_type = args.cty
cty_list = args.cty_list
data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir,
'assets', 'aggregated_data')
mouse_data_filename = os.path.join(data_path, 'Mouse_class_data.csv')
mouse_datatype_filename = os.path.join(data_path,
'Mouse_class_datatype.csv')
me_ttype_map_path = os.path.join(data_path, 'me_ttype.pkl')
sdk_data_filename = os.path.join(data_path, 'sdk.csv')
sdk_datatype_filename = os.path.join(data_path, 'sdk_datatype.csv')
sdk_data = man_utils.read_csv_with_dtype(sdk_data_filename,
sdk_datatype_filename)
if cty_type == 'ttype':
mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename,
mouse_datatype_filename)
me_ttype_map = utility.load_pickle(me_ttype_map_path)
metype_cluster = mouse_data.loc[
mouse_data.hof_index == 0, ['Cell_id', 'Dendrite_type', 'me_type']]
sdk_me = pd.merge(sdk_data, metype_cluster, how='left',
on='Cell_id').dropna(how='any', subset=['me_type'])
sdk_me['ttype'] = sdk_me['me_type'].apply(lambda x: me_ttype_map[x])
cell_df = sdk_me.loc[sdk_me.ttype.isin(cty_list), ]
elif cty_type == 'Cre_line':
cell_df = sdk_data.loc[sdk_data.line_name.isin(cty_list), ]
cell_ids = cell_df.Cell_id.unique().tolist()
rc = Client(profile=os.getenv('IPYTHON_PROFILE'))
logger.debug('Using ipyparallel with %d engines', len(rc))
lview = rc.load_balanced_view()
lview.map_sync(get_efeatures, cell_ids)
import seaborn as sns
import numpy as np
from ateam.data import lims
from ateamopt.utils import utility
from allensdk.core.cell_types_cache import CellTypesCache
import pandas as pd
import os
def get_morph_path(cell_id):
lr = lims.LimsReader()
morph_path = lr.get_swc_path_from_lims(int(cell_id))
return morph_path
data_path = os.path.join(os.getcwd(),os.pardir,os.pardir,'assets','aggregated_data')
cre_color_tasic16_filename = os.path.join(data_path,'cre_color_tasic16.pkl')
cre_color_dict = utility.load_pickle(cre_color_tasic16_filename)
cre_color_dict['Other'] = (0,0,0)
depth_data_filename = os.path.join(data_path,'mouse_me_and_met_avg_layer_depths.json') # Average layerwise depths for mouse
depth_data = utility.load_json(depth_data_filename)
total_depth = depth_data['wm']
# Cells are chosen to sample from diverse types within each layer
cell_id_dict = {
#'1':['574734127','564349611','475585413','555341581','536951541'],
'1':['574734127','475585413','536951541'],
# '2/3':['485184849','475515168','485468180','476087653','571306690'],
'2/3':['485184849','475515168','485468180'],
# '4':['483101699','602822298','490205998','569723367','324257146'],
'4':['483101699','602822298','569723367'],
'5':['479225052','607124114','515249852'],
import os
from matplotlib.cm import ScalarMappable
from matplotlib.colors import ListedColormap
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
from ateamopt.utils import utility
import matplotlib
matplotlib.use('Agg')
cell_id = '483101699'
model_data_path = ('/allen/aibs/mat/anin/hpc_trials/BluePyOpt_speedup/'
'%s/Stage2' % cell_id)
hof_responses_path = os.path.join(model_data_path, 'analysis_params/hof_response_all.pkl')
hof_responses = utility.load_pickle(hof_responses_path)
hof_num = 40
color_palette = sns.color_palette('Blues_r', hof_num)
select_stim_names = ['LongDC_55', 'Short_Square_Triple_101',
'Ramp_5', 'Noise_1_63']
sns.set_style('whitegrid')
fig, ax = plt.subplots(1, len(select_stim_names), sharey=True, squeeze=False,
gridspec_kw={'width_ratios': [1, 1, 1.5, 1.5]},
figsize=(12, 3.5))
hof_array_r = np.arange(hof_num - 1, -1, -1)
hof_responses_r = list(reversed(hof_responses))
def main(args):
stage_jobconfig = args['stage_jobconfig']
highlevel_job_props = args['highlevel_jobconfig']
logging.basicConfig(level=highlevel_job_props['log_level'])
parent_dir = os.path.abspath(os.path.join('.', os.pardir))
path_to_cell_metadata = glob.glob(parent_dir + '/cell_metadata*.json')[0]
cell_metadata = utility.load_json(path_to_cell_metadata)
cell_id = cell_metadata['cell_id']
peri_model_id = cell_metadata.get('peri_model_id')
released_aa_model_id = cell_metadata.get('released_aa_model_id')
all_protocols_path = highlevel_job_props['all_protocols_path']
all_features_path = highlevel_job_props['all_features_path']
morph_path = highlevel_job_props['swc_path']
axon_type = highlevel_job_props['axon_type']
ephys_dir = highlevel_job_props['ephys_dir']
train_features_path = args['train_features']
test_features_path = args['test_features']
param_write_path = args['parameters']
mech_write_path = args['mechanism']
release_param_write_path = args['released_aa_model']
mech_release_write_path = args['released_aa_mechanism']
analysis_parallel = (stage_jobconfig['analysis_config'].get('ipyparallel')
and stage_jobconfig['run_hof_analysis'])
props = dict(axon_type=axon_type, ephys_dir=ephys_dir)
map_function = analyzer_map(analysis_parallel)
opt_train = get_opt_obj(all_protocols_path, train_features_path,
morph_path, param_write_path, mech_write_path,
map_function, **props)
opt_all = get_opt_obj(all_protocols_path, all_features_path, morph_path,
param_write_path, mech_write_path, map_function,
**props)
opt_test = get_opt_obj(all_protocols_path, test_features_path, morph_path,
param_write_path, mech_write_path, map_function,
**props)
analysis_handler = Optim_Analyzer(args, opt_all)
best_model = analysis_handler.get_best_model(
) # Model with least training error
aibs_params_modelname = 'fitted_params/optim_param_%s.json' % cell_id
analysis_handler.save_params_aibs_format(aibs_params_modelname,
best_model[0],
expand_params=True)
aibs_params_compact_modelname = 'fitted_params/optim_param_%s_compact.json' % cell_id
analysis_handler.save_params_aibs_format(aibs_params_compact_modelname,
best_model[0])
bpopt_params_modelname = 'fitted_params/optim_param_%s_bpopt.json' % cell_id
analysis_handler.save_params_bpopt_format(bpopt_params_modelname,
best_model[0])
# Export hoc model
if stage_jobconfig['hoc_export']:
hoc_export_path = 'fitted_params/model_template_%s.hoc' % cell_id
utility.create_filepath(hoc_export_path)
best_param_dict = {key:best_model[0][i] for i,key in \
enumerate(opt_train.evaluator.param_names)}
model_string = opt_train.evaluator.cell_model.create_hoc(
best_param_dict)
with open(hoc_export_path, "w") as hoc_template:
hoc_template.write(model_string)
hof_model_params, seed_indices = analysis_handler.get_all_models()
if not stage_jobconfig.get('run_hof_analysis'):
hof_model_params, seed_indices = best_model, [seed_indices[0]]
hof_params_filename = 'analysis_params/hof_model_params.pkl'
hof_responses_filename = 'analysis_params/hof_response_all.pkl'
obj_list_train_filename = 'analysis_params/hof_obj_train.pkl'
obj_list_all_filename = 'analysis_params/hof_obj_all.pkl'
feat_list_all_filename = 'analysis_params/hof_features_all.pkl'
obj_list_test_filename = 'analysis_params/hof_obj_test.pkl'
seed_indices_filename = 'analysis_params/seed_indices.pkl'
score_list_train_filename = 'analysis_params/score_list_train.pkl'
# Response for the entire hall of fame not arranged
hof_response_list = analysis_handler.get_model_responses(
hof_model_params, hof_responses_filename)
analysis_handler._opt = opt_train
obj_list_train = analysis_handler.get_response_scores(hof_response_list)
# Sort everything with respect to training error
if not os.path.exists(score_list_train_filename):
score_list_train = [
np.sum(list(obj_dict_train.values()))
for obj_dict_train in obj_list_train
]
utility.create_filepath(score_list_train_filename)
utility.save_pickle(score_list_train_filename, score_list_train)
else:
score_list_train = utility.load_pickle(score_list_train_filename)
if not os.path.exists(seed_indices_filename):
seed_indices_sorted = analysis_handler.organize_models(
seed_indices, score_list_train)
utility.save_pickle(seed_indices_filename, seed_indices_sorted)
if not os.path.exists(obj_list_train_filename):
obj_list_train_sorted = analysis_handler.organize_models(
obj_list_train, score_list_train)
utility.save_pickle(obj_list_train_filename, obj_list_train_sorted)
if not os.path.exists(obj_list_all_filename):
analysis_handler._opt = opt_all
obj_list_gen = analysis_handler.get_response_scores(hof_response_list)
obj_list_gen_sorted = analysis_handler.organize_models(
obj_list_gen, score_list_train)
utility.save_pickle(obj_list_all_filename, obj_list_gen_sorted)
if not os.path.exists(feat_list_all_filename):
analysis_handler._opt = opt_all
feat_list_gen = analysis_handler.get_response_features(
hof_response_list)
feat_list_gen_sorted = analysis_handler.organize_models(
feat_list_gen, score_list_train)
utility.save_pickle(feat_list_all_filename, feat_list_gen_sorted)
if not os.path.exists(obj_list_test_filename):
analysis_handler._opt = opt_test
obj_list_test = analysis_handler.get_response_scores(hof_response_list)
obj_list_test_sorted = analysis_handler.organize_models(
obj_list_test, score_list_train)
utility.save_pickle(obj_list_test_filename, obj_list_test_sorted)
analysis_handler._opt = opt_train
# Save the sorted hof responses at the end
hof_response_sorted = analysis_handler.organize_models(
hof_response_list, score_list_train)
utility.save_pickle(hof_responses_filename, hof_response_sorted)
# Save the sorted hall of fame output in .pkl
hof_model_params_sorted = analysis_handler.save_hof_output_params(
hof_model_params, hof_params_filename, score_list_train)
# Save the entire hall of fame parameters
for i, hof_param in enumerate(hof_model_params_sorted):
aibs_params_modelname = os.path.join(
'fitted_params', 'hof_param_%s_%s.json' % (cell_id, i))
analysis_handler.save_params_aibs_format(aibs_params_modelname,
hof_param,
expand_params=True)
# Now save the sorted score
utility.save_pickle(score_list_train_filename, sorted(score_list_train))
GA_evol_path = os.path.join('analysis_params', 'GA_evolution_params.pkl')
analysis_handler.save_GA_evolultion_info(GA_evol_path)
resp_filename = os.path.join(os.getcwd(), 'resp_opt.txt')
analysis_handler.save_best_response(hof_response_sorted[0], resp_filename)
if release_param_write_path:
eval_handler_release = Bpopt_Evaluator(all_protocols_path,
all_features_path,
morph_path,
release_param_write_path,
mech_release_write_path,
stub_axon=False,
do_replace_axon=True,
ephys_dir=ephys_dir)
evaluator_release = eval_handler_release.create_evaluator()
opt_release = bpopt.optimisations.DEAPOptimisation(
evaluator=evaluator_release)
resp_release_filename = os.path.join(os.getcwd(), 'resp_release.txt')
analysis_handler.get_release_responses(opt_release,
resp_release_filename)
resp_release_aa = utility.load_pickle(resp_release_filename)[0]
features_release_aa = opt_release.evaluator.fitness_calculator.\
calculate_features(resp_release_aa)
features_aa_filename = os.path.join(
'Validation_Responses', 'Features_released_aa_%s.pkl' % cell_id)
utility.create_filepath(features_aa_filename)
utility.save_pickle(features_aa_filename, features_release_aa)
else:
resp_release_filename = None
stim_mapfile = highlevel_job_props['stimmap_file']
analysis_write_path = '%s_%s.pdf' % (cell_id,
stage_jobconfig['stage_name'])
pdf_pages = PdfPages(analysis_write_path)
model_type = 'All-active'
pdf_pages = analysis_handler.plot_grid_Response(resp_filename,
resp_release_filename,
stim_mapfile, pdf_pages)
pdf_pages = analysis_handler.plot_feature_comp(resp_filename,
resp_release_filename,
pdf_pages)
pdf_pages = analysis_handler.plot_GA_evol(GA_evol_path, pdf_pages)
pdf_pages = analysis_handler.plot_param_diversity(hof_params_filename,
pdf_pages)
if stage_jobconfig['model_postprocess']:
exp_fi_path = os.path.join('Validation_Responses',
'fI_exp_%s.pkl' % cell_id)
model_fi_path = os.path.join('Validation_Responses',
'fI_aa_%s.pkl' % cell_id)
exp_AP_shape_path = os.path.join('Validation_Responses',
'AP_shape_exp_%s.pkl' % cell_id)
model_AP_shape_path = os.path.join('Validation_Responses',
'AP_shape_aa_%s.pkl' % cell_id)
pdf_pages = analysis_handler.postprocess(
stim_mapfile, resp_filename, pdf_pages, exp_fi_path, model_fi_path,
exp_AP_shape_path, model_AP_shape_path, model_type)
# Perisomatic model
if peri_model_id and stage_jobconfig['run_peri_comparison']:
resp_peri_filename = os.path.join(os.getcwd(), 'resp_peri.txt')
peri_param_path = args['released_peri_model']
peri_mech_path = args['released_peri_mechanism']
props_peri = props.copy()
props_peri['axon_type'] = 'stub_axon'
eval_handler_peri = Bpopt_Evaluator(all_protocols_path,
all_features_path, morph_path,
peri_param_path, peri_mech_path,
**props_peri)
evaluator_peri = eval_handler_peri.create_evaluator()
opt_peri = bpopt.optimisations.DEAPOptimisation(
evaluator=evaluator_peri)
model_type = 'Perisomatic'
analysis_handler.get_release_responses(opt_peri, resp_peri_filename)
resp_peri = utility.load_pickle(resp_peri_filename)[0]
features_peri = opt_peri.evaluator.fitness_calculator.calculate_features(
resp_peri)
features_peri_filename = os.path.join('Validation_Responses',
'Features_peri_%s.pkl' % cell_id)
utility.create_filepath(features_peri_filename)
utility.save_pickle(features_peri_filename, features_peri)
pdf_pages = analysis_handler.plot_grid_Response(
resp_filename,
resp_peri_filename,
stim_mapfile,
pdf_pages,
resp_comparison=model_type)
if stage_jobconfig['model_postprocess']:
model_fi_path = os.path.join('Validation_Responses',
'fI_peri_%s.pkl' % cell_id)
model_AP_shape_path = os.path.join(
'Validation_Responses', 'AP_shape_peri_%s.pkl' % cell_id)
pdf_pages = analysis_handler.postprocess(
stim_mapfile, resp_peri_filename, pdf_pages, exp_fi_path,
model_fi_path, exp_AP_shape_path, model_AP_shape_path,
model_type)
if stage_jobconfig.get('calc_model_perf'):
spiketimes_exp_path = os.path.join('Validation_Responses',
'spiketimes_exp_noise.pkl')
all_features = utility.load_json(all_features_path)
spiketimes_noise_exp = {}
for stim_, feat in all_features.items():
if 'Noise' in stim_:
if 'peak_time' in feat['soma'].keys():
spiketimes_noise_exp[stim_] = feat['soma']['peak_time'][2]
if bool(spiketimes_noise_exp):
utility.create_filepath(spiketimes_exp_path)
utility.save_pickle(spiketimes_exp_path, spiketimes_noise_exp)
spiketimes_hof_path = os.path.join('Validation_Responses',
'spiketimes_model_noise.pkl')
exp_variance_hof_path = os.path.join('Validation_Responses',
'exp_variance_hof.pkl')
model_perf_filename = os.path.join('Validation_Responses',
'fitness_metrics_%s.csv' % cell_id)
pdf_pages = analysis_handler.hof_statistics(
stim_mapfile, pdf_pages, obj_list_all_filename,
hof_responses_filename, obj_list_train_filename,
obj_list_test_filename, seed_indices_filename, spiketimes_exp_path,
spiketimes_hof_path, exp_variance_hof_path, cell_metadata,
model_perf_filename)
pdf_pages.close()
if stage_jobconfig.get('calc_time_statistics'):
# time_by_gen_filename = 'time_info.txt'
# if os.path.exists(time_by_gen_filename):
# time_metrics_filename = 'time_metrics_%s.csv' % cell_id
# analysis_module.save_optimization_time(time_by_gen_filename,
# time_metrics_filename, cell_metadata)
compute_statistics_filename = 'compute_metrics_%s.csv' % cell_id
opt_logbook = 'logbook_info.txt'
analysis_module.save_compute_statistics(opt_logbook,
compute_statistics_filename)
from statsmodels.stats.multitest import fdrcorrection
from itertools import combinations
from scipy.stats import mannwhitneyu
import matplotlib as mpl
# %% Data paths
data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir,
'assets', 'aggregated_data')
annotation_datapath = os.path.join(data_path, 'anno.feather')
cre_coloring_filename = os.path.join(data_path, 'cre_color_tasic16.pkl')
exc_expression_profile_path = os.path.join(data_path, 'exc_expression_all.csv')
# %% Read the data
annotation_df = feather.read_dataframe(annotation_datapath)
cre_color_dict = utility.load_pickle(cre_coloring_filename)
# Only for better contrast
cre_color_dict["Rbp4-Cre_KL100"] = mpl.colors.to_rgb('#008033')
exc_lines = ["Nr5a1-Cre", "Rbp4-Cre_KL100"]
exc_palette = {exc_line_: cre_color_dict[exc_line_] for exc_line_ in exc_lines}
exc_expression_data = pd.read_csv(exc_expression_profile_path, index_col=None)
exc_expression_data = exc_expression_data.rename(
columns={'cre_label': 'Cre_line'})
# %% Filtering
# Get rid of genes expressed in less than 10% of the cells
exc_expression_data = exc_expression_data.loc[:, (
exc_expression_data != 0).sum(axis=0) >= .1 * exc_expression_data.shape[0]]
hof_num = 1
inh_lines = ["Htr3a-Cre_NO152","Sst-IRES-Cre","Pvalb-IRES-Cre"]
data_path = os.path.join(os.path.dirname(man_opt.__file__),os.pardir,'assets','aggregated_data')
mouse_data_filename = os.path.join(data_path,'Mouse_class_data.csv')
mouse_datatype_filename = os.path.join(data_path,'Mouse_class_datatype.csv')
sdk_data_filename = os.path.join(data_path,'sdk.csv')
sdk_datatype_filename = os.path.join(data_path,'sdk_datatype.csv')
cre_coloring_filename = os.path.join(data_path,'cre_color_tasic16.pkl')
filtered_me_inh_cells_filename = os.path.join(data_path,'filtered_me_inh_cells.pkl')
mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename,mouse_datatype_filename)
mouse_data = mouse_data.loc[mouse_data.hof_index<hof_num,]
mouse_data = mouse_data.loc[mouse_data.Cre_line.isin(inh_lines),['Cell_id','Cre_line','hof_index']]
cre_color_dict = utility.load_pickle(cre_coloring_filename)
filtered_me_inh_cells = utility.load_pickle(filtered_me_inh_cells_filename)
sdk_data= man_utils.read_csv_with_dtype(sdk_data_filename,sdk_datatype_filename)
sdk_data= sdk_data.rename(columns={'line_name':'Cre_line','ef__threshold_i_long_square':'rheobase'})
sdk_data = sdk_data.loc[sdk_data.Cre_line.isin(inh_lines),['Cell_id','rheobase']]
mouse_data = pd.merge(mouse_data,sdk_data,on='Cell_id')
cell_ids = list(set(mouse_data.Cell_id.tolist()))
efel_feature_path = 'eFEL_features'
spiking_features_df_list = []
spiking_features = ['AP_amplitude_from_voltagebase','AP1_amp','APlast_amp',
"AP_width",'AHP_depth','mean_frequency',
'Spikecount','inv_first_ISI','voltage_base']
import numpy as np
from ateamopt.utils import utility
import seaborn as sns
import matplotlib.pyplot as plt
from ateamopt.animation import animation_module
from ateamopt.bpopt_evaluator import Bpopt_Evaluator
import glob,os
import pandas as pd
from ipyparallel import Client
model_path = os.getcwd()
checkpoint_path = os.path.join(model_path,'checkpoints_final/seed4.pkl')
checkpoint = utility.load_pickle(checkpoint_path)
hof_inds = checkpoint['halloffame']
config_dir= os.path.join(model_path,'config/483101699')
protocol_path = os.path.join(config_dir,'protocols.json')
feature_path = os.path.join(config_dir,'features.json')
parameter_path = os.path.join(config_dir,'parameters.json')
mechanism_path = os.path.join(config_dir,'mechanism.json')
morph_path = os.path.join(model_path,'reconstruction.swc')
eval_handler = Bpopt_Evaluator(protocol_path,feature_path,morph_path,
parameter_path,mechanism_path,timeout=60)
evaluator = eval_handler.create_evaluator()
param_names = evaluator.param_names
offspring_size = 512
max_ngen = (len(checkpoint['history'].genealogy_history.values())-
offspring_size)/(2*offspring_size)+1
max_ngen = int(max_ngen)
inh_expression_profile_path = os.path.join(data_path, 'inh_expression_all.csv')
mouse_data_filename = os.path.join(data_path, 'Mouse_class_data.csv')
mouse_datatype_filename = os.path.join(data_path, 'Mouse_class_datatype.csv')
param_data_filename = os.path.join(data_path, 'allactive_params.csv')
param_datatype_filename = os.path.join(data_path,
'allactive_params_datatype.csv')
me_ttype_map_path = os.path.join(data_path, 'me_ttype.pkl')
annotation_datapath = os.path.join(data_path, 'anno.feather')
expression_profile_path = os.path.join(data_path, 'exc_ttype_expression.csv')
# %% Read the data
mouse_data = man_utils.read_csv_with_dtype(mouse_data_filename,
mouse_datatype_filename)
me_ttype_map = utility.load_pickle(me_ttype_map_path)
metype_cluster = mouse_data.loc[mouse_data.hof_index == 0,
['Cell_id', 'Dendrite_type', 'me_type']]
hof_param_data = man_utils.read_csv_with_dtype(param_data_filename,
param_datatype_filename)
hof_param_data = hof_param_data.loc[hof_param_data.hof_index == 0, ]
hof_param_data_ttype = pd.merge(hof_param_data,
metype_cluster,
how='left',
on='Cell_id').dropna(how='any',
subset=['me_type'])
hof_param_data_ttype['ttype'] = hof_param_data_ttype['me_type'].apply(
lambda x: me_ttype_map[x])
param_name_ = param_name_.split('_', 1)[-1]
param_name_ = param_name_.rsplit('_', 1)[0]
if bool(re.search('NaT', param_name_)):
param_name_ = 'NaT'
elif bool(re.search('Nap', param_name_)):
param_name_ = 'NaP'
elif bool(re.search('K_P', param_name_)):
param_name_ = 'KP'
elif bool(re.search('K_T', param_name_)):
param_name_ = 'KT'
elif bool(re.search('Kv3_1', param_name_)):
param_name_ = 'Kv31'
return param_name_
filtered_me_inh_cells = utility.load_pickle('filtered_me_inh_cells.pkl')
cre_color_dict = utility.load_pickle('rnaseq_sorted_cre.pkl')
mouse_data_path = os.path.join(os.getcwd(), os.pardir, 'Mouse_class_data.csv')
mouse_datatype_path = os.path.join(os.getcwd(), os.pardir,
'Mouse_class_datatype.csv')
datatypes = pd.read_csv(mouse_datatype_path)['types']
mouse_data = pd.read_csv(mouse_data_path, dtype=datatypes.to_dict())
mouse_data = mouse_data.loc[mouse_data.hof_index == 0, ]
cre_cluster = mouse_data.loc[mouse_data.hof_index == 0,
['Cell_id', 'Cre_line']]
sens_analysis_datapath = '/allen/aibs/mat/anin/Sobol_analysis/Inh_cells/*/'\
'sensitivity_data/*.csv'
sens_analysis_pathlist = glob.glob(sens_analysis_datapath)
cell_id_list = [path_.split('/')[-3] for path_ in sens_analysis_pathlist]
mouse_data_df = man_utils.read_csv_with_dtype(
mouse_data_filename, mouse_datatype_filename)
morph_data = man_utils.read_csv_with_dtype(
morph_data_filename, morph_datatype_filename)
# morph_data = morph_data.loc[:,[morph_feature for morph_feature in morph_data.columns
# if not any(sec in morph_feature for sec in['apical','axon'])]]
morph_fields = man_utils.get_data_fields(morph_data)
ephys_data = man_utils.read_csv_with_dtype(train_ephys_max_amp_fname,
train_ephys_max_amp_dtype_fname)
ephys_fields = utility.load_json(train_ephys_max_amp_fields_fname)
hof_param_data = man_utils.read_csv_with_dtype(
param_data_filename, param_datatype_filename)
cre_color_dict = utility.load_pickle(cre_coloring_filename)
bcre_color_dict = utility.load_pickle(bcre_coloring_filename)
bcre_index_order = list(bcre_color_dict.keys())
filtered_me_inh_cells = utility.load_pickle(filtered_me_inh_cells_filename)
filtered_me_exc_cells = utility.load_pickle(filtered_me_exc_cells_filename)
cre_cluster_color_dict = OrderedDict()
cre_type_cluster = utility.load_pickle(cre_ttype_filename)
for cre, color in cre_color_dict.items():
if cre_type_cluster[cre] not in cre_cluster_color_dict.keys():
cre_cluster_color_dict[cre_type_cluster[cre]] = color
# %% Filtering
'Cell_id'].tolist()
filtered_pyr_cells = sup_pyr + deep_pyr
param_pyr = param_pyr.loc[param_pyr.Cell_id.isin(filtered_pyr_cells), ]
param_pyr['depth'] = param_pyr['Layer'].apply(lambda x: 'L2/3 PC'
if x == '2/3' else 'L5 PC')
depth_cty = param_pyr.depth.unique().tolist()
all_cre_lines = mouse_data.loc[mouse_data.Cell_id.isin(filtered_pyr_cells),
'Cre_line'].unique().tolist()
# Save cells list
cell_list_path = os.path.join(data_path, 'pyr_cells_list.csv')
pyr_cell_list = param_pyr.loc[:, ['Cell_id', 'Layer', 'depth']]
pyr_cell_list.to_csv(cell_list_path, index=False)
layer_cty_colors = utility.load_pickle(layer_cty_colors_filename)
# %% Statistical comparison
conductance_params = [
dens_ for dens_ in list(param_pyr) if dens_.startswith('gbar')
]
param_pyr = param_pyr.dropna(axis=1, how='all')
param_pyr = param_pyr.dropna(
subset=[cond for cond in list(param_pyr) if cond in conductance_params])
select_conds = [
'gbar_Ih.apical', 'gbar_Ih.somatic', 'gbar_NaTa_t.axonal',
'gbar_Kv3_1.axonal', 'gbar_Kv3_1.somatic', 'gbar_K_Tst.axonal',
'gbar_K_Tst.somatic', 'gbar_Ca_LVA.axonal', 'gbar_Ca_LVA.somatic'
]
def read_pickle_file(file):
pickle_data = utility.load_pickle(file)
return pickle_data
mouse_data_df = man_utils.read_csv_with_dtype(mouse_data_filename,
mouse_datatype_filename)
mouse_data_df = man_utils.add_transcriptomic_subclass(mouse_data_df,
me_ttype_map_path)
mouse_data_df = man_utils.add_broad_subclass(mouse_data_df)
hof_param_data = man_utils.read_csv_with_dtype(param_data_filename,
param_datatype_filename)
subclass_cluster = mouse_data_df.loc[mouse_data_df.hof_index == 0,
['Cell_id', 'Broad_subclass']]
hof_param_subclass = pd.merge(hof_param_data,
subclass_cluster,
how='left',
on='Cell_id')
broad_subclass_color_dict = utility.load_pickle(broad_subclass_colors_filename)
broad_subclass = [
subclass for subclass in broad_subclass_color_dict.keys()
if subclass != 'Other'
]
hof_param_subclass = hof_param_subclass.loc[
hof_param_subclass.Broad_subclass.isin(broad_subclass), ]
hof_param_subclass.Broad_subclass = pd.Categorical(
hof_param_subclass.Broad_subclass, categories=broad_subclass)
hof_param_subclass = hof_param_subclass.sort_values(
['Broad_subclass', 'Cell_id', 'hof_index'])
hof_param_subclass.reset_index(drop=True, inplace=True)
hof_num = 40
intra_dist, inter_dist = {}, {}
# Data paths
data_path = os.path.join(os.path.dirname(man_opt.__file__), os.pardir,
'assets', 'aggregated_data')
cre_coloring_filename = os.path.join(data_path, 'cre_color_tasic16.pkl')
sag_features_exp_filename = os.path.join(data_path, 'sag_data_exp.csv')
sag_features_model_filename = os.path.join(data_path, 'sag_data_model.csv')
filtered_me_exc_cell_list_path = os.path.join(data_path,
'filtered_me_exc_cells.pkl')
perturbed_Nr5like_feature_file = os.path.join(
data_path, 'sag_data_perturbed_Nr5like.csv')
perturbed_Rbp4like_feature_file = os.path.join(
data_path, 'sag_data_perturbed_Rbp4like.csv')
# Read the data
exc_lines = ["Nr5a1-Cre", "Rbp4-Cre_KL100"]
cre_color_dict = utility.load_pickle(cre_coloring_filename)
cre_color_dict["Rbp4-Cre_KL100"] = mpl.colors.to_rgb(
'#008033') # Only for better contrast
filtered_me_exc_cells = utility.load_pickle(filtered_me_exc_cell_list_path)
# All experiment
sag_features_exp = pd.read_csv(sag_features_exp_filename,
index_col=0,
dtype={'Cell_id': str})
sag_features_exp = sag_features_exp.loc[
sag_features_exp.Cell_id.isin(filtered_me_exc_cells), ]
sag_features_exp['type'] = 'Exp'
# Models
sag_features_model = pd.read_csv(sag_features_model_filename,
index_col=0,
def add_transcriptomic_subclass(data, me_ttype_map_path):
me_ttype_map = utility.load_pickle(me_ttype_map_path)
me_ttype_map['Other'] = 'Other'
data['me_type'] = data['me_type'].fillna('Other')
data['ttype'] = data['me_type'].apply(lambda x: me_ttype_map[x])
return data
HoF_obj_train_filelist = glob.glob(HoF_obj_train_path)
#%% Create heatmap of the feature errors
obj_dict_train_list = []
feat_reject_list = ['Spikecount', 'depol_block', 'check_AISInitiation']
max_feat_count = {}
feature_mat_list = []
feature_mat_dict_all = defaultdict(list)
feature_heatmap_filename = os.path.join('figures',
'feature_score_comparison.pdf')
utility.create_filepath(feature_heatmap_filename)
for obj_file in HoF_obj_train_filelist:
obj_dict_train = utility.load_pickle(obj_file)[0]
cell_id = obj_file.split('/')[-2]
temp_dict = defaultdict(list)
for obj_key, obj_val in obj_dict_train.items():
feat = obj_key.split('.')[-1]
feat = feat if feat != 'AP_amplitude_from_voltagebase' else 'AP_amplitude'
if feat not in feat_reject_list:
temp_dict[feat].append(obj_val)
feature_mat_dict_all[feat].append(obj_val)
feature_mat_dict_all['Cell_id'].append(cell_id)
temp_dict = {key: np.mean(val) for key, val in temp_dict.items()}
max_feat = max(temp_dict, key=temp_dict.get)
temp_dict['max_feature'] = max_feat
temp_dict['max_feature_val'] = temp_dict[max_feat]
