# %% ex0 = [ 'exclude', 'and_GPe', 'and_Str', 'Left', 'Right', 'Other XLS', 'Exclude', 'zone_1_d1r', '_gpe_muscimol', '_gpe_pbs', 'mW', 'mw' ] inc = [['AG', 'GPe', 'CAG', 'Arch', 'zone_1'], ['AG', 'Str', 'A2A', 'Ai32', 'zone_1'], ['AG', 'Str', 'A2A', 'ChR2', 'zone_1']] exc = [ex0, ex0, ex0] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' # ethovision_tools.add_dlc_to_csv(basepath,inc,exc,save=True) pns = dataloc.raw_csv(basepath, inc[1], ex0) raw, meta = ethovision_tools.csv_load(pns[1], method='preproc') r, _ = ethovision_tools.csv_load(pns[1]) # raw,meta=ethovision_tools.csv_load(pns[0]) # %% ID Crossings: ac_on, ac_off = signals.thresh(raw['iz1'].astype(int), 0.5, 'Pos') min = 0 #meta['fs'][0] * 4 #4 seconds all_cross = [] cross_t = [] fs = meta['fs'][0] for on, off in zip(ac_on, ac_off): if (off - on) > min: all_cross.append([on, off]) cross_t.append(on / fs) durs = np.diff(np.array(all_cross), axis=1) / fs print('%d crossings detected. Median dur: %1.2fs' % \
# inc = [x + ['AG3233_5'] for x in inc] ethovision_tools.raw_csv_to_preprocessed_csv(newbase, inc, exc, force_replace=True, win=10) # %% pns = dataloc.raw_csv(newbase, inc[0], exc[0]) if not isinstance(pns, list): pns = [pns] saveit = True closeit = True for pn in pns: df, meta = ethovision_tools.csv_load(pn, columns='All', method='preproc') plots.plot_openloop_day(df, meta, save=saveit, close=closeit, save_dir=pn.parent) # %% analysis = 'Str_A2a_ChR2_1mw' behavior_str = '10x' fn = '%s_openloop_data' % analysis qans = [ 'AG3233_5', 'AG3233_4',
import numpy as np import pandas as pd from matplotlib import pyplot as plt import matplotlib.gridspec as gridspec from scipy.stats import sem, t import pdb import math import time # %% Plot one 10x30 experiment day inc = [['AG', 'Str', 'A2A', 'Ai32', 'hm4di_cno', '10x10']] exc = [['exclude', '_and_Str', 'Left', 'Right', 'Other XLS', 'Exclude']] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' pns = dataloc.raw_csv(basepath, inc[0], exc[0]) # a=time.time() df, meta = ethovision_tools.csv_load(pns[2], columns='All', method='raw') df, meta = behavior.preproc_raw(df, meta) # b=time.time() print('%2.2f seconds to load' % (b - a)) # raw=ethovision_tools.add_amb_to_raw(raw,meta) #The magic: plots.plot_openloop_day(df, meta, save=True) # %% Debug new metric: meander #DLC Measure: dir_smooth = behavior.smooth_direction(raw, meta, use_dlc=True) dir_smooth_etho = behavior.smooth_direction(raw, meta) diff_angle = signal.angle_vector_delta(dir_smooth[0:-1], dir_smooth[1:],
from pathlib import Path from gittislab import dataloc, ethovision_tools, signals, plots import matplotlib.pyplot as plt basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/Str/Naive/A2A/Ai32/Bilateral/10x10/' inc = [['AG5477_4']] # basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/GPe/Naive/CAG/Arch/Right/5x30/' # inc=[['AG4486_1']] exc = [['exclude']] ethovision_tools.unify_raw_to_csv(basepath, inc, exc) ethovision_tools.raw_csv_to_preprocessed_csv(basepath, inc, exc, force_replace=True) pns = dataloc.raw_csv(basepath, inc[0], exc[0]) raw, meta = ethovision_tools.csv_load(pns, method='preproc') ethovision_tools.boris_prep(basepath, inc, exc, plot_cols=['time', 'mouse_height', 'vel'], event_col='rear', event_thresh=0.5, method='preproc') dlc = ethovision_tools.add_dlc_helper(raw, meta, pns.parent, force_replace=True) dlc = dlc[0] mouse_height = (dlc['dlc_rear_centroid_y'] - dlc['dlc_front_centroid_y']) head_xy = np.array(dlc.loc[:,
) mean_power = np.mean(dat, axis=0) f = np.polyfit(x, mean_power, deg=2) laser_cal_fit = np.poly1d(f) plt.sca(ax) plt.plot(x, laser_cal_fit(x), 'r') ax.set_ylabel('Blue laser output (mW)') ax.set_xlabel('Arduino PWM level') # %% Load in 10x10 day for comparison: # pnfn=Path('/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/Str/Naive/A2A/Ai32/Bilateral/10x10_hm4di_saline/AG6846_5_BI040121/Raw_AG6846_5_BI040121.csv') pnfn = Path( '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/Str/Naive/A2A/Ai32/Bilateral/10x10_hm4di_cno/AG6846_5_BI033021/Raw_AG6846_5_BI033021.csv' ) raw, meta = ethovision_tools.csv_load(pnfn, columns='All', method='raw') meta['stim_dur'] = 2 meta['stim_off'] = meta['stim_on'] + 2 meta['stim_mean_dur'] = 2 percentage = lambda x: (np.nansum(x) / len(x)) * 100 m_clip = behavior.stim_clip_grab(raw, meta, y_col='im', stim_dur=2, baseline=2, summarization_fun=percentage) print(np.mean(m_clip['disc'][:, 1])) # %%
summary = ethovision_tools.meta_sum_csv(basepath, inc, exc) # %% Plot comparing im & im2 (improved) inc = [ 'AG', 'Str', 'A2A', 'Ai32', '10x10', ] exc = ['exclude', 'gpe_', 'mw'] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' pns = dataloc.raw_csv(basepath, inc, exc) for pn in pns: raw, meta = ethovision_tools.csv_load(pn, method='preproc') plt.figure() plt.plot(raw['time'], raw['im']) plt.plot(raw['time'], raw['im2'], '--') percent_match = sum(raw['im'] & raw['im2']) / sum(raw['im']) * 100 plt.title('%s r= %1.3f, %2.1f %% Hit' % (meta['anid'][0], meta['im_im2_pearson'][0], percent_match)) basepath = pn.parent # boris,raw,meta=ethovision_tools.boris_prep(basepath,[inc],[exc],plot_cols=['time','im','im2'], # event_col=['im','im2'],event_thresh=0.5, method='preproc') # %% Compare ethovision immobility measurement to using DLC side camera stats # and or velocity threshold.
'human_scored_rear', 'side_length_px', 'head_hind_5hz_pw', 'snout_hind_px_height', 'snout_hind_px_height_detrend', 'front_hind_px_height_detrend', 'side_length_px_detrend', ] #'dlc_front_over_rear_length', valid_video_pn = test_video_pn test_video_boris_obs = test_video_boris_obs # %% IF needed: pn = dataloc.raw_csv(test_video_pn[0]) raw, meta = ethovision_tools.csv_load(pn, method='preproc') raw, meta = ethovision_tools.add_dlc_helper(raw, meta, pn.parent) # %% Create training and test data from rear_scored videos: train = model.combine_raw_csv_for_modeling(train_video_pn, train_video_boris_obs, use_cols, rescale=True, avg_model_detrend=True, z_score_x_y=True, flip_y=True) test = model.combine_raw_csv_for_modeling(test_video_pn, test_video_boris_obs, use_cols, rescale=True,
inc = [ [ 'AG', '10x10_gpe_pbs', ], ] exc = [ex0] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' for ii, ee in zip(inc, exc): pns = dataloc.raw_csv(basepath, ii, ee) for pn in pns: temp = {} raw, meta = ethovision_tools.csv_load(pn, method='preproc') plots.plot_openloop_day(raw, meta, save=True, close=False) # %% Combined openloop day summary: inc = [ [ 'AG', '10x10_gpe_pbs', ], ] exc = [[ 'exclude', 'and_GPe', 'and_Str', 'Left', 'Right', 'Other XLS', 'Exclude', 'mW', 'mw', 'AG6343' ]] data = behavior.open_loop_summary_collect(basepath, [inc[0]], [exc[0]]) # %%
data = pd.DataFrame([], columns=[ 'anid', 'proto', 'cell_area_opsin', 'amb_vel', 'amb_meander', 'amb_bouts', 'amb_directed' ]) temp = data min_bout = 1 use_dlc = False use_cols = ['time', 'vel', 'im', 'dir', 'ambulation', 'meander'] for ii, ee in zip(inc, exc): pns = dataloc.raw_csv(basepath, ii, ee) for pn in pns: temp = {} raw, meta = ethovision_tools.csv_load(pn, columns=use_cols, method='preproc') temp['anid'] = meta['anid'][0] temp['cell_area_opsin'] = '%s_%s_%s' % ( meta['cell_type'][0], meta['stim_area'][0], meta['opsin_type'][0]) temp['proto'] = meta['protocol'][0] stim_dur = round(np.mean(meta['stim_dur'])) vel_clip = behavior.stim_clip_grab(raw, meta, y_col='vel', stim_dur=stim_dur) clip_ave = behavior.stim_clip_average(vel_clip) #### Calculate stim-triggered %time mobile: percentage = lambda x: (np.nansum(x) / len(x)) * 100 raw['m'] = ~raw['im']
# %% Print summary of metadata retrieved by query: # inc = [['AG','A2a','ChR2','Str']] # 'zone_1_30mW' # exc = [['exclude','_and_SNr','_and_Str','20min_10Hz', # 'grooming','20min_4Hz','Exclude']] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' summary = ethovision_tools.meta_sum_csv(basepath, inc, exc) print(summary) plt.hist(summary.stim_n) # %% Load example data inc = [['AG', 'GPe', 'CAG', 'Arch', '10x30']] exc = [['exclude', '_and_Str', 'Left', 'Right', 'Other XLS', 'Exclude']] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' pns = dataloc.raw_csv(basepath, inc[0], exc[0]) raw, meta = ethovision_tools.csv_load(pns[1]) # %% Attempt to chunk velocity by stim start times: useday = 'stim' if useday == 'nostim': #Conditioning days with no stim: inc = [['AG', 'GPe', 'CAG', 'Arch', 'pp30_cond_dish']] # exc = [['exclude', '_and_Str', 'Left', 'Right', 'pp30_cond_dish_fc_stim']] else: #Days with stim: inc = [['AG', 'GPe', 'CAG', 'Arch', 'pp30_cond_dish_fc_stim']] # exc = [['exclude', '_and_Str', 'Left', 'Right']] basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/' pns = dataloc.raw_csv(basepath, inc[0], exc[0]) summary = ethovision_tools.meta_sum_csv(basepath, inc, exc)
def rear_nn_auroc_perf( ffn, boris_obs, prob_thresh=0.5, low_pass_freq=None, weights_fn='/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/DLC Examples/train_rear_model/bi_rearing_nn_weightsv2', tab_fn='/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/DLC Examples/train_rear_model/to_nnv2.pkl', rf_model_fn=None): ''' Specify an experiment folder (ffn) and the observation name to use in the Rearing observations.boris file with human scored rearing data in that folder. Assumes many other files are in that folder (raw csv, metatdata, etc,) and that videos have already been both: 1) scored by a human observer and 2) pre-processed with deeplabcut. ''' use_cols = [ 'vel', 'area', 'delta_area', # 'dlc_front_over_rear_length' 'dlc_side_head_x', 'dlc_side_head_y', 'dlc_front_centroid_x', 'dlc_front_centroid_y', 'dlc_rear_centroid_x', 'dlc_rear_centroid_y', 'dlc_snout_x', 'dlc_snout_y', 'dlc_side_left_fore_x', 'dlc_side_left_fore_y', 'dlc_side_right_fore_x', 'dlc_side_right_fore_y', 'dlc_side_left_hind_x', 'dlc_side_left_hind_y', 'dlc_side_right_hind_x', 'dlc_side_right_hind_y', 'dlc_top_head_x', 'dlc_top_head_y', 'dlc_top_body_center_x', 'dlc_top_body_center_y', 'dlc_top_tail_base_x', 'dlc_top_tail_base_y', 'video_resolution', 'human_scored_rear', 'side_length_px', 'head_hind_5hz_pw', 'snout_hind_px_height', 'snout_hind_px_height_detrend', 'front_hind_px_height_detrend', 'side_length_px_detrend', ] dep_var = 'human_scored_rear' raw, meta = ethovision_tools.csv_load(dataloc.raw_csv(ffn), columns=['time'], method='raw') boris_fn = Path(ffn).joinpath('Rearing Observations.boris') f = open(boris_fn, "r") boris = json.loads(f.read()) f.close() dat = combine_raw_csv_for_modeling([ffn], [boris_obs], use_cols, rescale=True, avg_model_detrend=True, z_score_x_y=True, flip_y=True) if 'Unnamed: 0' in dat.columns: dat.drop('Unnamed: 0', axis=1, inplace=True) dat.fillna(method='ffill', inplace=True) dat.fillna(method='bfill', inplace=True) pred = tabular_predict_from_nn(tab_fn, weights_fn, xs=dat) if isinstance(rf_model_fn, str): ensembling = True xs = dat.drop(dep_var, axis=1) rf_pred = tabular_predict_from_rf(rf_model_fn, xs) raw['rf_pred'] = rf_pred else: ensembling = False # pdb.set_trace() # human_rear_score = behavior.boris_to_logical_vector( raw, boris, boris_obs, 'a', 'd') raw['human_scored'] = human_rear_score raw['nn_pred'] = pred[:, 1] if ensembling == True: raw['final_pred'] = (raw['nn_pred'] + raw['rf_pred']) / 2 else: raw['final_pred'] = raw['nn_pred'] #Lowpass filter prediction: if not (low_pass_freq == None): raw['final_pred'] = signals.pad_lowpass_unpad(raw['final_pred'], low_pass_freq, meta['fs'][0]) b, r, m = ethovision_tools.boris_prep_from_df( raw, meta, plot_cols=['time', 'final_pred', 'human_scored'], event_col=['final_pred'], event_thresh=prob_thresh, ) auroc = metrics.roc_auc_score(raw['human_scored'].values.astype(np.int16), raw['final_pred'].values) print('%1.5f AUROC' % auroc) return auroc, raw['final_pred'], human_rear_score
def combine_raw_csv_for_modeling(raw_pns, boris_obs_names, use_cols, uniform_boris_fn='Rearing Observations.boris', rescale=False, avg_model_detrend=False, z_score_x_y=False, flip_y=False, meta_to_raw=[]): combined = pd.DataFrame([], columns=use_cols) for pn, obs in zip(raw_pns, boris_obs_names): p = Path(pn) inc = [['AG']] exc = [['exclude']] ethovision_tools.unify_raw_to_csv(p, inc, exc) ethovision_tools.raw_csv_to_preprocessed_csv(p, inc, exc, force_replace=False) pns = dataloc.raw_csv(p, inc[0], exc[0]) raw, meta = ethovision_tools.csv_load(pns, method='raw') fn_ka = p.joinpath(uniform_boris_fn) f = open(fn_ka, 'r') boris = json.loads(f.read()) # Get human scored rearing events and add as vector! human_scored_rearing = behavior.boris_to_logical_vector( raw, boris, obs, 'a', 'd') dlc = ethovision_tools.add_dlc_helper(raw, meta, p, force_replace=True) dlc = dlc[0] # pdb.set_trace() if meta['exp_room_number'][0] == 228: x_scale = 512 / 480 y_scale = 1.455 #Scale pixels #Video takes up only half of screen in these recordings vid_res = np.ones((dlc.shape[0], 1)) * 704 #Video width 704 x 480 # pdb.set_trace() else: x_scale = 1 y_scale = 1 vid_res = np.ones( (dlc.shape[0], 1)) * 1280 #Video width height = 512 if rescale == True: for col in dlc.columns: if ('dlc' in col) and ('x' in col): dlc[col] = dlc[col].values * x_scale if ('dlc' in col) and ('y' in col): dlc[col] = dlc[col].values * y_scale dlc['video_resolution'] = vid_res dlc['human_scored_rear'] = human_scored_rearing # dlc['head_hind_px_height']= dlc ['dlc_rear_centroid_y'] - dlc['dlc_side_head_y'] dlc['front_hind_px_height'] = dlc['dlc_rear_centroid_y'] - dlc[ 'dlc_front_centroid_y'] dlc['head_hind_5hz_pw'] = signals.get_spectral_band_power( dlc['front_hind_px_height'], meta['fs'][0], 4.5, 6.5) dlc['snout_hind_px_height'] = dlc['dlc_rear_centroid_y'] - dlc[ 'dlc_snout_y'] dlc['side_length_px'] = signals.calculateDistance( dlc['dlc_front_centroid_x'].values, dlc['dlc_front_centroid_y'].values, dlc['dlc_rear_centroid_x'].values, dlc['dlc_rear_centroid_y'].values) # dlc['top_length_px']=signals.calculateDistance(dlc['dlc_top_head_x'].values, # dlc['dlc_top_head_y'].values, # dlc ['dlc_top_tail_base_x'].values, # dlc ['dlc_top_tail_base_y'].values) #Detrend effect of mouse distance from camera using an average pre-fitted #z-score approach: if avg_model_detrend == True: detrend_cols = [ 'snout_hind_px_height', 'front_hind_px_height', 'side_length_px' ] for col in detrend_cols: y = dlc[col] x = dlc['x'] dlc[col + '_detrend'] = average_z_score(x, y) if z_score_x_y == True: for col in dlc.columns: if ('dlc' in col) and (('x' in col) or ('y' in col)): temp = dlc[col].values temp = temp - np.nanmean(temp) temp = temp / np.nanstd(temp) dlc[col] = temp if flip_y == True: for col in dlc.columns: if ('dlc' in col) and ('y' in col): dlc[col] = -1 * dlc[col] # dlc[col] = dlc[col] - np.nanmin(dlc[col]) for col in meta_to_raw: dlc[col] = meta[col][0] temp = dlc[use_cols] combined = pd.concat([combined, temp]) #Add time lags (?) combined.reset_index(drop=True, inplace=True) return combined
def batch_analyze(basepath, inc, exc): ''' With comments ''' data = pd.DataFrame([], columns=[ 'anid', 'proto', 'cell_area_opsin', 'amb_vel', 'amb_meander', 'amb_bouts', 'amb_directed' ]) temp = data min_bout = 1 use_dlc = False use_cols = ['time', 'vel', 'im', 'dir', 'ambulation', 'meander'] for ii, ee in zip(inc, exc): pns = dataloc.raw_csv(basepath, ii, ee) for pn in pns: temp = {} raw, meta = ethovision_tools.csv_load(pn, columns=use_cols, method='preproc') temp['anid'] = meta['anid'][0] temp['cell_area_opsin'] = '%s_%s_%s' % (meta['cell_type'][0], meta['stim_area'][0], meta['opsin_type'][0]) temp['proto'] = meta['protocol'][0] stim_dur = round(np.mean(meta['stim_dur'])) vel_clip = behavior.stim_clip_grab(raw, meta, y_col='vel', stim_dur=stim_dur) clip_ave = behavior.stim_clip_average(vel_clip) #### Calculate stim-triggered %time mobile: percentage = lambda x: (np.nansum(x) / len(x)) * 100 raw['m'] = ~raw['im'] m_clip = behavior.stim_clip_grab(raw, meta, y_col='m', stim_dur=stim_dur, summarization_fun=percentage) #### Calculate ambulation bout properties: raw['run'] = (raw['ambulation'] == True) & (raw['vel'] > 5) # raw['flight']=(raw['vel'] > (4* np.mean(raw['vel']))) #Flight, Yilmaz & Meister 2013 raw['flight'] = (raw['vel'] > (3 * np.mean(raw['vel']))) if any(raw['run']): amb_bouts = behavior.bout_analyze(raw, meta, 'flight', stim_dur=stim_dur, min_bout_dur_s=min_bout, use_dlc=use_dlc) temp['amb_meander'] = np.nanmean(amb_bouts['meander'], axis=0) temp['amb_directed'] = np.nanmean(amb_bouts['directed'], axis=0) temp['amb_bouts'] = np.nanmean(amb_bouts['rate'], axis=0) else: temp['amb_meander'] = [np.nan, np.nan, np.nan] temp['amb_directed'] = [np.nan, np.nan, np.nan] temp['amb_bouts'] = [0, 0, 0] #### Calculate immobile bout properties: im_bouts = behavior.bout_analyze(raw, meta, 'im', stim_dur=stim_dur, min_bout_dur_s=min_bout, use_dlc=use_dlc) data = data.append(temp, ignore_index=True)