fig_name = 'Figure 1_a2a_chr2_str' newbase = savepath.joinpath(fig_name).joinpath('data').joinpath('raw') # %% # inc = [x + ['AG3233_5'] for x in inc] ethovision_tools.raw_csv_to_preprocessed_csv(newbase, inc, exc, force_replace=True, win=10) # %% pns = dataloc.raw_csv(newbase, inc[0], exc[0]) if not isinstance(pns, list): pns = [pns] saveit = True closeit = True for pn in pns: df, meta = ethovision_tools.csv_load(pn, columns='All', method='preproc') plots.plot_openloop_day(df, meta, save=saveit, close=closeit, save_dir=pn.parent) # %% analysis = 'Str_A2a_ChR2_1mw'
inc = [ ['AG', 'GPe', 'FoxP2', 'ChR2', '10x10_20mW'], ] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' ethovision_tools.unify_raw_to_csv(basepath, inc, exc, force_replace=False, win=10) ethovision_tools.raw_csv_to_preprocessed_csv(basepath, inc, exc, force_replace=False, win=10) for ii, ee in zip(inc, exc): pns = dataloc.raw_csv(basepath, ii, ee) for pn in pns: temp = {} raw, meta = ethovision_tools.csv_load(pn, method='preproc') plots.plot_openloop_day(raw, meta) # %% # def test_batch_analyze(inc,exc): data = pd.DataFrame([], columns=[ 'anid', 'proto', 'cell_area_opsin', 'amb_vel', 'amb_meander', 'amb_bouts', 'amb_directed' ]) temp = data min_bout = 1
from statistics import mode # %% ex0 = [ 'exclude', 'and_GPe', 'and_Str', 'Left', 'Right', 'Other XLS', 'Exclude', 'zone_1_d1r', '_gpe_muscimol', '_gpe_pbs', 'mW', 'mw' ] inc = [['AG', 'GPe', 'CAG', 'Arch', 'zone_1'], ['AG', 'Str', 'A2A', 'Ai32', 'zone_1'], ['AG', 'Str', 'A2A', 'ChR2', 'zone_1']] exc = [ex0, ex0, ex0] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' # ethovision_tools.add_dlc_to_csv(basepath,inc,exc,save=True) pns = dataloc.raw_csv(basepath, inc[1], ex0) raw, meta = ethovision_tools.csv_load(pns[1], method='preproc') r, _ = ethovision_tools.csv_load(pns[1]) # raw,meta=ethovision_tools.csv_load(pns[0]) # %% ID Crossings: ac_on, ac_off = signals.thresh(raw['iz1'].astype(int), 0.5, 'Pos') min = 0 #meta['fs'][0] * 4 #4 seconds all_cross = [] cross_t = [] fs = meta['fs'][0] for on, off in zip(ac_on, ac_off): if (off - on) > min: all_cross.append([on, off]) cross_t.append(on / fs) durs = np.diff(np.array(all_cross), axis=1) / fs
'Other XLS' ] exc = [ex0] save = False plt.close('all') y_col = 'vel' load_method = 'preproc' stim_dur = 2 percentage = lambda x: (np.nansum(x) / len(x)) * 100 rate = lambda x: len(signals.thresh(x, 0.5)[0]) / stim_dur sum_fun = np.mean # inc=[['AG','GPe','A2A','ChR2','light',''50x2',]] inc = ['GPe', 'A2a', 'ChR2', '50x2'] pns = dataloc.raw_csv(basepath.parent, inc, ex0) if not isinstance(pns, list): pns = [pns] button_dict = { 'AG7128_4': button_base.joinpath('gpe_buttonB6_0-255_11mW_max_cal_cleaned_model.pkl'), 'AG7128_5': button_base.joinpath('gpe_buttonB3_0-255_11mW_max_cal_cleaned_model.pkl'), 'AG7192_2': button_base.joinpath('gpe_buttonB11_0-255_11mW_max_cal_cleaned_model.pkl'), 'AG7192_3': button_base.joinpath('gpe_buttonB13_0-255_11mW_max_cal_cleaned_model.pkl'), 'AG7192_4': button_base.joinpath('gpe_buttonB8_0-255_11mW_max_cal_cleaned_model.pkl') } #For better precision, load each mouse's calibration file separately.
import numpy as np from pathlib import Path from gittislab import dataloc, ethovision_tools, signals, plots import matplotlib.pyplot as plt basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/Str/Naive/A2A/Ai32/Bilateral/10x10/' inc = [['AG5477_4']] # basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/GPe/Naive/CAG/Arch/Right/5x30/' # inc=[['AG4486_1']] exc = [['exclude']] ethovision_tools.unify_raw_to_csv(basepath, inc, exc) ethovision_tools.raw_csv_to_preprocessed_csv(basepath, inc, exc, force_replace=True) pns = dataloc.raw_csv(basepath, inc[0], exc[0]) raw, meta = ethovision_tools.csv_load(pns, method='preproc') ethovision_tools.boris_prep(basepath, inc, exc, plot_cols=['time', 'mouse_height', 'vel'], event_col='rear', event_thresh=0.5, method='preproc') dlc = ethovision_tools.add_dlc_helper(raw, meta, pns.parent, force_replace=True) dlc = dlc[0] mouse_height = (dlc['dlc_rear_centroid_y'] - dlc['dlc_front_centroid_y'])
win=10) summary = ethovision_tools.meta_sum_csv(basepath, inc, exc) # %% Plot comparing im & im2 (improved) inc = [ 'AG', 'Str', 'A2A', 'Ai32', '10x10', ] exc = ['exclude', 'gpe_', 'mw'] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' pns = dataloc.raw_csv(basepath, inc, exc) for pn in pns: raw, meta = ethovision_tools.csv_load(pn, method='preproc') plt.figure() plt.plot(raw['time'], raw['im']) plt.plot(raw['time'], raw['im2'], '--') percent_match = sum(raw['im'] & raw['im2']) / sum(raw['im']) * 100 plt.title('%s r= %1.3f, %2.1f %% Hit' % (meta['anid'][0], meta['im_im2_pearson'][0], percent_match)) basepath = pn.parent # boris,raw,meta=ethovision_tools.boris_prep(basepath,[inc],[exc],plot_cols=['time','im','im2'], # event_col=['im','im2'],event_thresh=0.5, method='preproc') # %% Compare ethovision immobility measurement to using DLC side camera stats
'video_resolution', 'human_scored_rear', 'side_length_px', 'head_hind_5hz_pw', 'snout_hind_px_height', 'snout_hind_px_height_detrend', 'front_hind_px_height_detrend', 'side_length_px_detrend', ] #'dlc_front_over_rear_length', valid_video_pn = test_video_pn test_video_boris_obs = test_video_boris_obs # %% IF needed: pn = dataloc.raw_csv(test_video_pn[0]) raw, meta = ethovision_tools.csv_load(pn, method='preproc') raw, meta = ethovision_tools.add_dlc_helper(raw, meta, pn.parent) # %% Create training and test data from rear_scored videos: train = model.combine_raw_csv_for_modeling(train_video_pn, train_video_boris_obs, use_cols, rescale=True, avg_model_detrend=True, z_score_x_y=True, flip_y=True) test = model.combine_raw_csv_for_modeling(test_video_pn, test_video_boris_obs, use_cols,
def rear_nn_auroc_perf( ffn, boris_obs, prob_thresh=0.5, low_pass_freq=None, weights_fn='/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/DLC Examples/train_rear_model/bi_rearing_nn_weightsv2', tab_fn='/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/DLC Examples/train_rear_model/to_nnv2.pkl', rf_model_fn=None): ''' Specify an experiment folder (ffn) and the observation name to use in the Rearing observations.boris file with human scored rearing data in that folder. Assumes many other files are in that folder (raw csv, metatdata, etc,) and that videos have already been both: 1) scored by a human observer and 2) pre-processed with deeplabcut. ''' use_cols = [ 'vel', 'area', 'delta_area', # 'dlc_front_over_rear_length' 'dlc_side_head_x', 'dlc_side_head_y', 'dlc_front_centroid_x', 'dlc_front_centroid_y', 'dlc_rear_centroid_x', 'dlc_rear_centroid_y', 'dlc_snout_x', 'dlc_snout_y', 'dlc_side_left_fore_x', 'dlc_side_left_fore_y', 'dlc_side_right_fore_x', 'dlc_side_right_fore_y', 'dlc_side_left_hind_x', 'dlc_side_left_hind_y', 'dlc_side_right_hind_x', 'dlc_side_right_hind_y', 'dlc_top_head_x', 'dlc_top_head_y', 'dlc_top_body_center_x', 'dlc_top_body_center_y', 'dlc_top_tail_base_x', 'dlc_top_tail_base_y', 'video_resolution', 'human_scored_rear', 'side_length_px', 'head_hind_5hz_pw', 'snout_hind_px_height', 'snout_hind_px_height_detrend', 'front_hind_px_height_detrend', 'side_length_px_detrend', ] dep_var = 'human_scored_rear' raw, meta = ethovision_tools.csv_load(dataloc.raw_csv(ffn), columns=['time'], method='raw') boris_fn = Path(ffn).joinpath('Rearing Observations.boris') f = open(boris_fn, "r") boris = json.loads(f.read()) f.close() dat = combine_raw_csv_for_modeling([ffn], [boris_obs], use_cols, rescale=True, avg_model_detrend=True, z_score_x_y=True, flip_y=True) if 'Unnamed: 0' in dat.columns: dat.drop('Unnamed: 0', axis=1, inplace=True) dat.fillna(method='ffill', inplace=True) dat.fillna(method='bfill', inplace=True) pred = tabular_predict_from_nn(tab_fn, weights_fn, xs=dat) if isinstance(rf_model_fn, str): ensembling = True xs = dat.drop(dep_var, axis=1) rf_pred = tabular_predict_from_rf(rf_model_fn, xs) raw['rf_pred'] = rf_pred else: ensembling = False # pdb.set_trace() # human_rear_score = behavior.boris_to_logical_vector( raw, boris, boris_obs, 'a', 'd') raw['human_scored'] = human_rear_score raw['nn_pred'] = pred[:, 1] if ensembling == True: raw['final_pred'] = (raw['nn_pred'] + raw['rf_pred']) / 2 else: raw['final_pred'] = raw['nn_pred'] #Lowpass filter prediction: if not (low_pass_freq == None): raw['final_pred'] = signals.pad_lowpass_unpad(raw['final_pred'], low_pass_freq, meta['fs'][0]) b, r, m = ethovision_tools.boris_prep_from_df( raw, meta, plot_cols=['time', 'final_pred', 'human_scored'], event_col=['final_pred'], event_thresh=prob_thresh, ) auroc = metrics.roc_auc_score(raw['human_scored'].values.astype(np.int16), raw['final_pred'].values) print('%1.5f AUROC' % auroc) return auroc, raw['final_pred'], human_rear_score
def combine_raw_csv_for_modeling(raw_pns, boris_obs_names, use_cols, uniform_boris_fn='Rearing Observations.boris', rescale=False, avg_model_detrend=False, z_score_x_y=False, flip_y=False, meta_to_raw=[]): combined = pd.DataFrame([], columns=use_cols) for pn, obs in zip(raw_pns, boris_obs_names): p = Path(pn) inc = [['AG']] exc = [['exclude']] ethovision_tools.unify_raw_to_csv(p, inc, exc) ethovision_tools.raw_csv_to_preprocessed_csv(p, inc, exc, force_replace=False) pns = dataloc.raw_csv(p, inc[0], exc[0]) raw, meta = ethovision_tools.csv_load(pns, method='raw') fn_ka = p.joinpath(uniform_boris_fn) f = open(fn_ka, 'r') boris = json.loads(f.read()) # Get human scored rearing events and add as vector! human_scored_rearing = behavior.boris_to_logical_vector( raw, boris, obs, 'a', 'd') dlc = ethovision_tools.add_dlc_helper(raw, meta, p, force_replace=True) dlc = dlc[0] # pdb.set_trace() if meta['exp_room_number'][0] == 228: x_scale = 512 / 480 y_scale = 1.455 #Scale pixels #Video takes up only half of screen in these recordings vid_res = np.ones((dlc.shape[0], 1)) * 704 #Video width 704 x 480 # pdb.set_trace() else: x_scale = 1 y_scale = 1 vid_res = np.ones( (dlc.shape[0], 1)) * 1280 #Video width height = 512 if rescale == True: for col in dlc.columns: if ('dlc' in col) and ('x' in col): dlc[col] = dlc[col].values * x_scale if ('dlc' in col) and ('y' in col): dlc[col] = dlc[col].values * y_scale dlc['video_resolution'] = vid_res dlc['human_scored_rear'] = human_scored_rearing # dlc['head_hind_px_height']= dlc ['dlc_rear_centroid_y'] - dlc['dlc_side_head_y'] dlc['front_hind_px_height'] = dlc['dlc_rear_centroid_y'] - dlc[ 'dlc_front_centroid_y'] dlc['head_hind_5hz_pw'] = signals.get_spectral_band_power( dlc['front_hind_px_height'], meta['fs'][0], 4.5, 6.5) dlc['snout_hind_px_height'] = dlc['dlc_rear_centroid_y'] - dlc[ 'dlc_snout_y'] dlc['side_length_px'] = signals.calculateDistance( dlc['dlc_front_centroid_x'].values, dlc['dlc_front_centroid_y'].values, dlc['dlc_rear_centroid_x'].values, dlc['dlc_rear_centroid_y'].values) # dlc['top_length_px']=signals.calculateDistance(dlc['dlc_top_head_x'].values, # dlc['dlc_top_head_y'].values, # dlc ['dlc_top_tail_base_x'].values, # dlc ['dlc_top_tail_base_y'].values) #Detrend effect of mouse distance from camera using an average pre-fitted #z-score approach: if avg_model_detrend == True: detrend_cols = [ 'snout_hind_px_height', 'front_hind_px_height', 'side_length_px' ] for col in detrend_cols: y = dlc[col] x = dlc['x'] dlc[col + '_detrend'] = average_z_score(x, y) if z_score_x_y == True: for col in dlc.columns: if ('dlc' in col) and (('x' in col) or ('y' in col)): temp = dlc[col].values temp = temp - np.nanmean(temp) temp = temp / np.nanstd(temp) dlc[col] = temp if flip_y == True: for col in dlc.columns: if ('dlc' in col) and ('y' in col): dlc[col] = -1 * dlc[col] # dlc[col] = dlc[col] - np.nanmin(dlc[col]) for col in meta_to_raw: dlc[col] = meta[col][0] temp = dlc[use_cols] combined = pd.concat([combined, temp]) #Add time lags (?) combined.reset_index(drop=True, inplace=True) return combined
def batch_analyze(basepath, inc, exc): ''' With comments ''' data = pd.DataFrame([], columns=[ 'anid', 'proto', 'cell_area_opsin', 'amb_vel', 'amb_meander', 'amb_bouts', 'amb_directed' ]) temp = data min_bout = 1 use_dlc = False use_cols = ['time', 'vel', 'im', 'dir', 'ambulation', 'meander'] for ii, ee in zip(inc, exc): pns = dataloc.raw_csv(basepath, ii, ee) for pn in pns: temp = {} raw, meta = ethovision_tools.csv_load(pn, columns=use_cols, method='preproc') temp['anid'] = meta['anid'][0] temp['cell_area_opsin'] = '%s_%s_%s' % (meta['cell_type'][0], meta['stim_area'][0], meta['opsin_type'][0]) temp['proto'] = meta['protocol'][0] stim_dur = round(np.mean(meta['stim_dur'])) vel_clip = behavior.stim_clip_grab(raw, meta, y_col='vel', stim_dur=stim_dur) clip_ave = behavior.stim_clip_average(vel_clip) #### Calculate stim-triggered %time mobile: percentage = lambda x: (np.nansum(x) / len(x)) * 100 raw['m'] = ~raw['im'] m_clip = behavior.stim_clip_grab(raw, meta, y_col='m', stim_dur=stim_dur, summarization_fun=percentage) #### Calculate ambulation bout properties: raw['run'] = (raw['ambulation'] == True) & (raw['vel'] > 5) # raw['flight']=(raw['vel'] > (4* np.mean(raw['vel']))) #Flight, Yilmaz & Meister 2013 raw['flight'] = (raw['vel'] > (3 * np.mean(raw['vel']))) if any(raw['run']): amb_bouts = behavior.bout_analyze(raw, meta, 'flight', stim_dur=stim_dur, min_bout_dur_s=min_bout, use_dlc=use_dlc) temp['amb_meander'] = np.nanmean(amb_bouts['meander'], axis=0) temp['amb_directed'] = np.nanmean(amb_bouts['directed'], axis=0) temp['amb_bouts'] = np.nanmean(amb_bouts['rate'], axis=0) else: temp['amb_meander'] = [np.nan, np.nan, np.nan] temp['amb_directed'] = [np.nan, np.nan, np.nan] temp['amb_bouts'] = [0, 0, 0] #### Calculate immobile bout properties: im_bouts = behavior.bout_analyze(raw, meta, 'im', stim_dur=stim_dur, min_bout_dur_s=min_bout, use_dlc=use_dlc) data = data.append(temp, ignore_index=True)