fig_name = 'Figure 1_a2a_chr2_str'
newbase = savepath.joinpath(fig_name).joinpath('data').joinpath('raw')
# %%
# inc = [x + ['AG3233_5'] for x in inc]
ethovision_tools.raw_csv_to_preprocessed_csv(newbase,
inc,
exc,
force_replace=True,
win=10)
# %%
pns = dataloc.raw_csv(newbase, inc[0], exc[0])
if not isinstance(pns, list):
pns = [pns]
saveit = True
closeit = True
for pn in pns:
df, meta = ethovision_tools.csv_load(pn, columns='All', method='preproc')
plots.plot_openloop_day(df,
meta,
save=saveit,
close=closeit,
save_dir=pn.parent)
# %%
analysis = 'Str_A2a_ChR2_1mw'
inc = [
['AG', 'GPe', 'FoxP2', 'ChR2', '10x10_20mW'],
]
basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/'
ethovision_tools.unify_raw_to_csv(basepath,
inc,
exc,
force_replace=False,
win=10)
ethovision_tools.raw_csv_to_preprocessed_csv(basepath,
inc,
exc,
force_replace=False,
win=10)
for ii, ee in zip(inc, exc):
pns = dataloc.raw_csv(basepath, ii, ee)
for pn in pns:
temp = {}
raw, meta = ethovision_tools.csv_load(pn, method='preproc')
plots.plot_openloop_day(raw, meta)
# %%
# def test_batch_analyze(inc,exc):
data = pd.DataFrame([],
columns=[
'anid', 'proto', 'cell_area_opsin', 'amb_vel',
'amb_meander', 'amb_bouts', 'amb_directed'
])
temp = data
min_bout = 1
from statistics import mode
# %%
ex0 = [
'exclude', 'and_GPe', 'and_Str', 'Left', 'Right', 'Other XLS', 'Exclude',
'zone_1_d1r', '_gpe_muscimol', '_gpe_pbs', 'mW', 'mw'
]
inc = [['AG', 'GPe', 'CAG', 'Arch', 'zone_1'],
['AG', 'Str', 'A2A', 'Ai32', 'zone_1'],
['AG', 'Str', 'A2A', 'ChR2', 'zone_1']]
exc = [ex0, ex0, ex0]
basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/'
# ethovision_tools.add_dlc_to_csv(basepath,inc,exc,save=True)
pns = dataloc.raw_csv(basepath, inc[1], ex0)
raw, meta = ethovision_tools.csv_load(pns[1], method='preproc')
r, _ = ethovision_tools.csv_load(pns[1])
# raw,meta=ethovision_tools.csv_load(pns[0])
# %% ID Crossings:
ac_on, ac_off = signals.thresh(raw['iz1'].astype(int), 0.5, 'Pos')
min = 0 #meta['fs'][0] * 4 #4 seconds
all_cross = []
cross_t = []
fs = meta['fs'][0]
for on, off in zip(ac_on, ac_off):
if (off - on) > min:
all_cross.append([on, off])
cross_t.append(on / fs)
durs = np.diff(np.array(all_cross), axis=1) / fs
'Other XLS'
]
exc = [ex0]
save = False
plt.close('all')
y_col = 'vel'
load_method = 'preproc'
stim_dur = 2
percentage = lambda x: (np.nansum(x) / len(x)) * 100
rate = lambda x: len(signals.thresh(x, 0.5)[0]) / stim_dur
sum_fun = np.mean
# inc=[['AG','GPe','A2A','ChR2','light',''50x2',]]
inc = ['GPe', 'A2a', 'ChR2', '50x2']
pns = dataloc.raw_csv(basepath.parent, inc, ex0)
if not isinstance(pns, list):
pns = [pns]
button_dict = {
'AG7128_4':
button_base.joinpath('gpe_buttonB6_0-255_11mW_max_cal_cleaned_model.pkl'),
'AG7128_5':
button_base.joinpath('gpe_buttonB3_0-255_11mW_max_cal_cleaned_model.pkl'),
'AG7192_2':
button_base.joinpath('gpe_buttonB11_0-255_11mW_max_cal_cleaned_model.pkl'),
'AG7192_3':
button_base.joinpath('gpe_buttonB13_0-255_11mW_max_cal_cleaned_model.pkl'),
'AG7192_4':
button_base.joinpath('gpe_buttonB8_0-255_11mW_max_cal_cleaned_model.pkl')
}
#For better precision, load each mouse's calibration file separately.
import numpy as np
from pathlib import Path
from gittislab import dataloc, ethovision_tools, signals, plots
import matplotlib.pyplot as plt
basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/Str/Naive/A2A/Ai32/Bilateral/10x10/'
inc = [['AG5477_4']]
# basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/GPe/Naive/CAG/Arch/Right/5x30/'
# inc=[['AG4486_1']]
exc = [['exclude']]
ethovision_tools.unify_raw_to_csv(basepath, inc, exc)
ethovision_tools.raw_csv_to_preprocessed_csv(basepath,
inc,
exc,
force_replace=True)
pns = dataloc.raw_csv(basepath, inc[0], exc[0])
raw, meta = ethovision_tools.csv_load(pns, method='preproc')
ethovision_tools.boris_prep(basepath,
inc,
exc,
plot_cols=['time', 'mouse_height', 'vel'],
event_col='rear',
event_thresh=0.5,
method='preproc')
dlc = ethovision_tools.add_dlc_helper(raw,
meta,
pns.parent,
force_replace=True)
dlc = dlc[0]
mouse_height = (dlc['dlc_rear_centroid_y'] - dlc['dlc_front_centroid_y'])
win=10)
summary = ethovision_tools.meta_sum_csv(basepath, inc, exc)
# %% Plot comparing im & im2 (improved)
inc = [
'AG',
'Str',
'A2A',
'Ai32',
'10x10',
]
exc = ['exclude', 'gpe_', 'mw']
basepath = '/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/'
pns = dataloc.raw_csv(basepath, inc, exc)
for pn in pns:
raw, meta = ethovision_tools.csv_load(pn, method='preproc')
plt.figure()
plt.plot(raw['time'], raw['im'])
plt.plot(raw['time'], raw['im2'], '--')
percent_match = sum(raw['im'] & raw['im2']) / sum(raw['im']) * 100
plt.title('%s r= %1.3f, %2.1f %% Hit' %
(meta['anid'][0], meta['im_im2_pearson'][0], percent_match))
basepath = pn.parent
# boris,raw,meta=ethovision_tools.boris_prep(basepath,[inc],[exc],plot_cols=['time','im','im2'],
# event_col=['im','im2'],event_thresh=0.5, method='preproc')
# %% Compare ethovision immobility measurement to using DLC side camera stats
'video_resolution',
'human_scored_rear',
'side_length_px',
'head_hind_5hz_pw',
'snout_hind_px_height',
'snout_hind_px_height_detrend',
'front_hind_px_height_detrend',
'side_length_px_detrend',
] #'dlc_front_over_rear_length',
valid_video_pn = test_video_pn
test_video_boris_obs = test_video_boris_obs
# %% IF needed:
pn = dataloc.raw_csv(test_video_pn[0])
raw, meta = ethovision_tools.csv_load(pn, method='preproc')
raw, meta = ethovision_tools.add_dlc_helper(raw, meta, pn.parent)
# %% Create training and test data from rear_scored videos:
train = model.combine_raw_csv_for_modeling(train_video_pn,
train_video_boris_obs,
use_cols,
rescale=True,
avg_model_detrend=True,
z_score_x_y=True,
flip_y=True)
test = model.combine_raw_csv_for_modeling(test_video_pn,
test_video_boris_obs,
use_cols,
def rear_nn_auroc_perf(
ffn,
boris_obs,
prob_thresh=0.5,
low_pass_freq=None,
weights_fn='/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/DLC Examples/train_rear_model/bi_rearing_nn_weightsv2',
tab_fn='/home/brian/Dropbox/Gittis Lab Data/OptoBehavior/DLC Examples/train_rear_model/to_nnv2.pkl',
rf_model_fn=None):
'''
Specify an experiment folder (ffn) and the observation name to use in the
Rearing observations.boris file with human scored rearing data in that folder.
Assumes many other files are in that folder (raw csv, metatdata, etc,)
and that videos have already been both: 1) scored by a human observer and 2)
pre-processed with deeplabcut.
'''
use_cols = [
'vel',
'area',
'delta_area', # 'dlc_front_over_rear_length'
'dlc_side_head_x',
'dlc_side_head_y',
'dlc_front_centroid_x',
'dlc_front_centroid_y',
'dlc_rear_centroid_x',
'dlc_rear_centroid_y',
'dlc_snout_x',
'dlc_snout_y',
'dlc_side_left_fore_x',
'dlc_side_left_fore_y',
'dlc_side_right_fore_x',
'dlc_side_right_fore_y',
'dlc_side_left_hind_x',
'dlc_side_left_hind_y',
'dlc_side_right_hind_x',
'dlc_side_right_hind_y',
'dlc_top_head_x',
'dlc_top_head_y',
'dlc_top_body_center_x',
'dlc_top_body_center_y',
'dlc_top_tail_base_x',
'dlc_top_tail_base_y',
'video_resolution',
'human_scored_rear',
'side_length_px',
'head_hind_5hz_pw',
'snout_hind_px_height',
'snout_hind_px_height_detrend',
'front_hind_px_height_detrend',
'side_length_px_detrend',
]
dep_var = 'human_scored_rear'
raw, meta = ethovision_tools.csv_load(dataloc.raw_csv(ffn),
columns=['time'],
method='raw')
boris_fn = Path(ffn).joinpath('Rearing Observations.boris')
f = open(boris_fn, "r")
boris = json.loads(f.read())
f.close()
dat = combine_raw_csv_for_modeling([ffn], [boris_obs],
use_cols,
rescale=True,
avg_model_detrend=True,
z_score_x_y=True,
flip_y=True)
if 'Unnamed: 0' in dat.columns:
dat.drop('Unnamed: 0', axis=1, inplace=True)
dat.fillna(method='ffill', inplace=True)
dat.fillna(method='bfill', inplace=True)
pred = tabular_predict_from_nn(tab_fn, weights_fn, xs=dat)
if isinstance(rf_model_fn, str):
ensembling = True
xs = dat.drop(dep_var, axis=1)
rf_pred = tabular_predict_from_rf(rf_model_fn, xs)
raw['rf_pred'] = rf_pred
else:
ensembling = False
# pdb.set_trace()
#
human_rear_score = behavior.boris_to_logical_vector(
raw, boris, boris_obs, 'a', 'd')
raw['human_scored'] = human_rear_score
raw['nn_pred'] = pred[:, 1]
if ensembling == True:
raw['final_pred'] = (raw['nn_pred'] + raw['rf_pred']) / 2
else:
raw['final_pred'] = raw['nn_pred']
#Lowpass filter prediction:
if not (low_pass_freq == None):
raw['final_pred'] = signals.pad_lowpass_unpad(raw['final_pred'],
low_pass_freq,
meta['fs'][0])
b, r, m = ethovision_tools.boris_prep_from_df(
raw,
meta,
plot_cols=['time', 'final_pred', 'human_scored'],
event_col=['final_pred'],
event_thresh=prob_thresh,
)
auroc = metrics.roc_auc_score(raw['human_scored'].values.astype(np.int16),
raw['final_pred'].values)
print('%1.5f AUROC' % auroc)
return auroc, raw['final_pred'], human_rear_score
def combine_raw_csv_for_modeling(raw_pns,
boris_obs_names,
use_cols,
uniform_boris_fn='Rearing Observations.boris',
rescale=False,
avg_model_detrend=False,
z_score_x_y=False,
flip_y=False,
meta_to_raw=[]):
combined = pd.DataFrame([], columns=use_cols)
for pn, obs in zip(raw_pns, boris_obs_names):
p = Path(pn)
inc = [['AG']]
exc = [['exclude']]
ethovision_tools.unify_raw_to_csv(p, inc, exc)
ethovision_tools.raw_csv_to_preprocessed_csv(p,
inc,
exc,
force_replace=False)
pns = dataloc.raw_csv(p, inc[0], exc[0])
raw, meta = ethovision_tools.csv_load(pns, method='raw')
fn_ka = p.joinpath(uniform_boris_fn)
f = open(fn_ka, 'r')
boris = json.loads(f.read())
# Get human scored rearing events and add as vector!
human_scored_rearing = behavior.boris_to_logical_vector(
raw, boris, obs, 'a', 'd')
dlc = ethovision_tools.add_dlc_helper(raw, meta, p, force_replace=True)
dlc = dlc[0]
# pdb.set_trace()
if meta['exp_room_number'][0] == 228:
x_scale = 512 / 480
y_scale = 1.455 #Scale pixels #Video takes up only half of screen in these recordings
vid_res = np.ones((dlc.shape[0], 1)) * 704 #Video width 704 x 480
# pdb.set_trace()
else:
x_scale = 1
y_scale = 1
vid_res = np.ones(
(dlc.shape[0], 1)) * 1280 #Video width height = 512
if rescale == True:
for col in dlc.columns:
if ('dlc' in col) and ('x' in col):
dlc[col] = dlc[col].values * x_scale
if ('dlc' in col) and ('y' in col):
dlc[col] = dlc[col].values * y_scale
dlc['video_resolution'] = vid_res
dlc['human_scored_rear'] = human_scored_rearing
# dlc['head_hind_px_height']= dlc ['dlc_rear_centroid_y'] - dlc['dlc_side_head_y']
dlc['front_hind_px_height'] = dlc['dlc_rear_centroid_y'] - dlc[
'dlc_front_centroid_y']
dlc['head_hind_5hz_pw'] = signals.get_spectral_band_power(
dlc['front_hind_px_height'], meta['fs'][0], 4.5, 6.5)
dlc['snout_hind_px_height'] = dlc['dlc_rear_centroid_y'] - dlc[
'dlc_snout_y']
dlc['side_length_px'] = signals.calculateDistance(
dlc['dlc_front_centroid_x'].values,
dlc['dlc_front_centroid_y'].values,
dlc['dlc_rear_centroid_x'].values,
dlc['dlc_rear_centroid_y'].values)
# dlc['top_length_px']=signals.calculateDistance(dlc['dlc_top_head_x'].values,
# dlc['dlc_top_head_y'].values,
# dlc ['dlc_top_tail_base_x'].values,
# dlc ['dlc_top_tail_base_y'].values)
#Detrend effect of mouse distance from camera using an average pre-fitted
#z-score approach:
if avg_model_detrend == True:
detrend_cols = [
'snout_hind_px_height', 'front_hind_px_height',
'side_length_px'
]
for col in detrend_cols:
y = dlc[col]
x = dlc['x']
dlc[col + '_detrend'] = average_z_score(x, y)
if z_score_x_y == True:
for col in dlc.columns:
if ('dlc' in col) and (('x' in col) or ('y' in col)):
temp = dlc[col].values
temp = temp - np.nanmean(temp)
temp = temp / np.nanstd(temp)
dlc[col] = temp
if flip_y == True:
for col in dlc.columns:
if ('dlc' in col) and ('y' in col):
dlc[col] = -1 * dlc[col]
# dlc[col] = dlc[col] - np.nanmin(dlc[col])
for col in meta_to_raw:
dlc[col] = meta[col][0]
temp = dlc[use_cols]
combined = pd.concat([combined, temp])
#Add time lags (?)
combined.reset_index(drop=True, inplace=True)
return combined
def batch_analyze(basepath, inc, exc):
'''
With comments
'''
data = pd.DataFrame([],
columns=[
'anid', 'proto', 'cell_area_opsin', 'amb_vel',
'amb_meander', 'amb_bouts', 'amb_directed'
])
temp = data
min_bout = 1
use_dlc = False
use_cols = ['time', 'vel', 'im', 'dir', 'ambulation', 'meander']
for ii, ee in zip(inc, exc):
pns = dataloc.raw_csv(basepath, ii, ee)
for pn in pns:
temp = {}
raw, meta = ethovision_tools.csv_load(pn,
columns=use_cols,
method='preproc')
temp['anid'] = meta['anid'][0]
temp['cell_area_opsin'] = '%s_%s_%s' % (meta['cell_type'][0],
meta['stim_area'][0],
meta['opsin_type'][0])
temp['proto'] = meta['protocol'][0]
stim_dur = round(np.mean(meta['stim_dur']))
vel_clip = behavior.stim_clip_grab(raw,
meta,
y_col='vel',
stim_dur=stim_dur)
clip_ave = behavior.stim_clip_average(vel_clip)
#### Calculate stim-triggered %time mobile:
percentage = lambda x: (np.nansum(x) / len(x)) * 100
raw['m'] = ~raw['im']
m_clip = behavior.stim_clip_grab(raw,
meta,
y_col='m',
stim_dur=stim_dur,
summarization_fun=percentage)
#### Calculate ambulation bout properties:
raw['run'] = (raw['ambulation'] == True) & (raw['vel'] > 5)
# raw['flight']=(raw['vel'] > (4* np.mean(raw['vel']))) #Flight, Yilmaz & Meister 2013
raw['flight'] = (raw['vel'] > (3 * np.mean(raw['vel'])))
if any(raw['run']):
amb_bouts = behavior.bout_analyze(raw,
meta,
'flight',
stim_dur=stim_dur,
min_bout_dur_s=min_bout,
use_dlc=use_dlc)
temp['amb_meander'] = np.nanmean(amb_bouts['meander'], axis=0)
temp['amb_directed'] = np.nanmean(amb_bouts['directed'],
axis=0)
temp['amb_bouts'] = np.nanmean(amb_bouts['rate'], axis=0)
else:
temp['amb_meander'] = [np.nan, np.nan, np.nan]
temp['amb_directed'] = [np.nan, np.nan, np.nan]
temp['amb_bouts'] = [0, 0, 0]
#### Calculate immobile bout properties:
im_bouts = behavior.bout_analyze(raw,
meta,
'im',
stim_dur=stim_dur,
min_bout_dur_s=min_bout,
use_dlc=use_dlc)
data = data.append(temp, ignore_index=True)
