def get_PV1(mouse, arena, day, speed_thresh=1.5, pf_file='placefields_cm1_manlims_1000shuf.pkl'):
"""
Gets PV for each session with no spatial bins
:param mouse:
:param arena:
:param day:
:param speed_thresh: exclude data points where mouse's smoothed speed is below this (1.5cm/s default)
:param pf_file: default = 'placefields_cm1_manlims_1000shuf.pkl'_
:return: PV1: nneurons long 1-d np array of event rates for each neuron across the whole session
"""
try:
PF = load_pf(mouse, arena, day, pf_file=pf_file)
# Speed threshold PSAbool
PFthresh = PF.PSAbool_align[:, PF.speed_sm > speed_thresh]
sr_image = PF.sr_image
except FileNotFoundError:
print('No placefields file found - creating PV1 from neural data only - NO SPEED THRESHOLDING')
dir_use = get_dir(mouse, arena, day)
im_data_file = path.join(dir_use, 'FinalOutput.mat')
im_data = sio.loadmat(im_data_file)
PSAbool = im_data['PSAbool']
PFthresh = PSAbool
sr_image = im_data['SampleRate'][0]
# Calculate PV
nframes = PFthresh.shape[1]
try:
PV1d = PFthresh.sum(axis=1)/nframes * sr_image[0]
except TypeError: # Catch a few errors for mice where sr_image is not properly formatted
PV1d = PFthresh.sum(axis=1) / nframes * sr_image
return PV1d
def get_running_bool(mouse, arena, day, speed_thresh=1.5, lims_method='auto', list_dir=master_directory,
align_from_end=False):
"""Generate a boolean of times the animal is running to use with placefields. Can modify and then feed into
placefields function to generate place field maps from arbitrary times throughout the session
:param all same as in placefields function above
:return isrunning: boolean when the mouse is running. """
make_session_list(list_dir)
# Get position and time information for .csv file (later need to align to imaging)
dir_use = get_dir(mouse, arena, day)
speed, pos, t_track, sr = get_speed(dir_use)
t_track = t_track[0:-1] # chop last time data point to match t_track match speed/pos length
# Display warning if "aligntoend" is in folder name but you are running with align_from_end=False
if str(dir_use).upper().find('ALIGNTOEND') != -1 and align_from_end is False:
print('Folder structure for ' + mouse + ' ' + arena + ': Day ' + str(
day) + ' suggests you should align data from end of recording')
print('RE-RUN WITH align_from_end=False!!!')
# Import imaging data
# im_data_file = path.join(dir_use + '\imaging', 'FinalOutput.mat')
im_data_file = path.join(dir_use, 'FinalOutput.mat')
im_data = sio.loadmat(im_data_file)
PSAbool = im_data['PSAbool']
nneurons, _ = np.shape(PSAbool)
try:
sr_image = im_data['SampleRate'].squeeze()
except KeyError:
sr_image = 20
# Convert position to cm
pix2cm = er.get_conv_factors(arena) # get conversion to cm for the arena
pos_cm = pos * pix2cm
speed_cm = speed * pix2cm
# Align imaging and position data
pos_align, speed_align, PSAbool_align, time_interp = \
align_imaging_to_tracking(pos_cm, speed_cm, t_track, PSAbool, sr_image, align_from_end=align_from_end)
# Smooth speed data for legitimate thresholding, get limits of data
speed_sm = np.convolve(speed_align, np.ones(2 * int(sr)) / (2 * sr), mode='same') # smooth speed
# Get data limits
if lims_method == 'auto': # automatic by default
lims = [np.min(pos_cm, axis=1), np.max(pos_cm, axis=1)]
elif lims_method == 'file': # file not yet enabled
print('lims_method=''file'' not enabled yet')
else: # grab array!
lims = lims_method
good = np.ones(len(speed_sm)) == 1
isrunning = good
isrunning[speed_sm < speed_thresh] = False
return isrunning
def plot_trajectory_overlay(mouse,
day_des=[-2, -1, 4, 1, 2, 7],
arenas=['Open', 'Shock'],
xmin=False,
ymin=False,
xlim=False,
ylim=False):
"""
Plot mouse trajectory for each session
:param
mouse: name of mouse
day_des: days to plot (day 0 = shock day, day -2 = 2 days before shock, 4 = 4 hr after shock (special case))
arenas: 'Open' and/or 'Shock'
xmin, ymin: #arenas by #days arrays or lists of minimum x/y values. False = don't adjust
xlim, ylim: limits of x and y data you want to set as plotted. False = don't adjust.
:return: h: figure handle
"""
nsesh = len(day_des)
narena = len(arenas)
# Allocate xmin and ymin to zero if not designated
if not xmin:
xmin = np.zeros((narena, nsesh))
if not ymin:
ymin = np.zeros((narena, nsesh))
# Cast xmin and ymin as 2d ndarrays for later
xmin = np.reshape(xmin, (1, -1))
ymin = np.reshape(ymin, (1, -1))
fig, ax = plt.subplots(1,
narena,
figsize=(2.3, narena * 2.3),
squeeze=False)
# Iterate through all sessions and plot stuff
for ida, arena in enumerate(arenas):
for idd, day in enumerate(day_des):
dir_use = get_dir(mouse, arena, day)
plot_frame_and_traj(ax[0, ida],
dir_use,
plot_frame=False,
xcorr=xmin[ida, idd],
ycorr=ymin[ida, idd])
# set x and y limits
if xlim is not False:
ax[0, ida].set_xlim(xlim)
if ylim is not False:
ax[0, ida].set_ylim(ylim)
# resize figure
fig.set_size_inches(6.2, 5.7)
return fig, ax
def get_im_sample_rate(mouse, arena, day):
"""Gets sample rate for imaging data"""
dir_use = get_dir(mouse, arena, day)
im_data_file = path.join(dir_use, 'FinalOutput.mat')
im_data = sio.loadmat(im_data_file, variable_names='SampleRate')
try:
sr_image = im_data['SampleRate'].squeeze()
except KeyError:
sr_image = 20
return sr_image
def get_all_freezing(mouse,
day_des=[-2, -1, 4, 1, 2, 7],
arenas=['Open', 'Shock'],
velocity_threshold=1.044,
min_freeze_duration=10):
"""
Gets freezing ratio for all experimental sessions for a given mouse.
:param
mouse: Mouse name (string), e.g. 'DVHPC_5' or 'Marble7'
arenas: 'Open' (denotes square) or 'Shock' or 'Circle' (denotes open field circle arena)
day_des: array of session days -2,-1,0,1,2,7 and 4 = 4hr session on day 0
list_dir: alternate location of SessionDirectories
:return:
fratios: narena x nsession array of fratios
"""
nsesh = len(day_des)
narena = len(arenas)
# Iterate through all sessions and get fratio
fratios = np.ones(
(narena, nsesh)) * float('NaN') # pre-allocate fratio as nan
for idd, day in enumerate(day_des):
for ida, arena in enumerate(arenas):
# print(mouse + " " + str(day) + " " + arena)
try:
pix2cm = get_conv_factors(arena)
dir_use = get_dir(mouse, arena, day)
freezing, _ = detect_freezing(
dir_use,
velocity_threshold=velocity_threshold,
min_freeze_duration=min_freeze_duration,
arena=arena,
pix2cm=pix2cm)
fratios[ida, idd] = freezing.sum() / freezing.__len__()
except (IOError, IndexError, TypeError
): # FileNotFoundError is IOError in earlier versions
# print(['Unknown error processing ' + mouse + ' ' + arena + ' ' + str(day)])
print([
'Unknown file missing and/or IndexError for ' + mouse +
' ' + arena + ' ' + str(day)
])
print('Freezing left as NaN for this session')
return fratios
def load_pf(mouse, arena, day, session_index=None, pf_file='placefields_cm1_manlims_1000shuf.pkl'):
if session_index is None:
dir_use = get_dir(mouse, arena, day)
elif isinstance(session_index, int) and session_index >= 0:
dir_use = session_list[session_index]["Location"]
elif session_index == 'cwd':
dir_use = os.getcwd()
elif os.path.exists(session_index):
dir_use = session_index
position_path = path.join(dir_use, pf_file)
with open(position_path, 'rb') as file:
PF = load(file)
if type(PF.sr_image) is not int:
PF.sr_image = PF.sr_image.squeeze() # Backwards compatibility fix
return PF
def get_ROIs(mouse: str, arena: str, day: int in [-2, -1, 0, 1, 4, 2, 7], **kwargs):
"""
Get neuron ROI array.
:param mouse: str of the format 'Marble##'
:param arena: str in ['Open', Shock']
:param day: int in [-2, -1, 0, 4, 1, 2, 7]
:param **kwargs: input to session_directory.find_eraser_directory
:return: rois an ncells x npixx x npixy ndarray of neuron ROIs
"""
dir_use = Path(get_dir(mouse, arena, day, **kwargs))
neural_data = loadmat(dir_use / 'FinalOutput.mat')
neuron_image = neural_data['NeuronImage']
# Convert from a series of xpix x ypix arrays to a 3d ndarray
assert neuron_image.shape[0] == 1, 'Inappropriate format for NeuronImage in FinalOutput.mat - write code!'
rois = np.asarray([roi for roi in neuron_image[0]])
return rois
def get_bad_epochs(mouse, arena, day):
"""
Identifies bad epochs where mouse is at 0,0 for manual correction
:param mouse:
:param arena:
:param day:
:return:
"""
# Commment here
dir_use = get_dir(mouse, arena, day)
# Comment here
pos = get_pos(dir_use)
# Comment here
bad_bool = np.logical_and(pos[0, :] == 0, pos[1, :] == 0)
# Comment here
bad_epochs = get_freezing_epochs(bad_bool)
# Insert code here to print bad epochs to screen if you wish. Might be easier in the long run
#print(bad_epochs)
return bad_epochs
def plot_experiment_traj(mouse,
day_des=[-2, -1, 4, 1, 2, 7],
arenas=['Open', 'Shock'],
disp_fratio=False,
plot_frame=False):
"""
Plot mouse trajectory for each session
:param
mouse: name of mouse
day_des: days to plot (day 0 = shock day, day -2 = 2 days before shock, 4 = 4 hr after shock (special case))
arenas: 'Open' and/or 'Shock'
disp_fratio: true = display freezing ratio on plot
:return: h: figure handle
"""
nsesh = len(day_des)
narena = len(arenas)
fig, ax = plt.subplots(narena, nsesh, figsize=(12.7, 4.8), squeeze=False)
# Iterate through all sessions and plot stuff
for idd, day in enumerate(day_des):
for ida, arena in enumerate(arenas):
# try:
dir_use = get_dir(mouse, arena, day)
# Label stuff
ax[ida, idd].set_xlabel(str(day))
if idd == 0:
ax[ida, idd].set_ylabel(arena)
if ida == 0 and idd == 0:
ax[ida, idd].set_title(mouse)
axis_off(ax[ida, idd])
plot_frame_and_traj(ax[ida, idd], dir_use, plot_frame=plot_frame)
if disp_fratio:
velocity_threshold, min_freeze_duration, pix2cm = get_conv_factors(
arena)
# if arena == 'Open':
# # NK Note - velocity threshold is just a guess at this point
# # Also need to ignore positions at 0,0 somehow and/or interpolate
# velocity_threshold = 15
# min_freeze_duration = 75
# elif arena == 'Shock':
# velocity_threshold = 15
# min_freeze_duration = 10
freezing, velocity = detect_freezing(
dir_use,
velocity_threshold=velocity_threshold,
min_freeze_duration=min_freeze_duration,
arena=arena,
pix2cm=pix2cm)
fratio = freezing.sum() / freezing.__len__()
fratio_str = '%0.2f' % fratio # make it a string
# Label stuff - hack here to make sure things get labeled if try statement fails during plotting
ax[ida, idd].set_xlabel(str(day))
if idd == 0:
ax[ida, idd].set_ylabel(arena)
if ida == 0 and idd == 0:
ax[ida, idd].set_title(mouse)
if idd == 0 and ida == 0 and disp_fratio:
ax[ida, idd].set_ylabel(fratio_str)
elif disp_fratio:
ax[ida, idd].set_title(fratio_str)
# except:
# print(['Error processing ' + mouse + ' ' + arena + ' ' + str(day)])
return fig, ax
shock1,
nan_policy='omit')
# Plot stats in second subplot if it is there (only for 1st two groups though).
label1 = xticklabels[0]
label2 = xticklabels[1]
if len(ax) == 2:
# Plot across group stats in each arena
ax[1].text(
0.1, 0.9, label1 + ': ' + group_names[0] + ' v ' + group_names[1] +
' pval=' + "{0:.3g}".format(pval[0, 1, 0]) + ' tstat=' +
"{0:.3g}".format(tstat[0, 1, 0]))
ax[1].text(
0.1, 0.75, label2 + ': ' + group_names[0] + ' v ' +
group_names[1] + ' pval=' + "{0:.3g}".format(pval[0, 1, 1]) +
' tstat=' + "{0:.3g}".format(tstat[0, 1, 1]))
# Plot within group stats between arenas...
for idg, group_name in enumerate(group_names):
ax[1].text(
0.1, 0.50 - 0.15 * idg, group_name + ': ' + label1 + 'v ' +
label2 + ' pval=' + "{0:.3g}".format(pval[idg, idg, 2]) +
' tstat=' + "{0:.3g}".format(tstat[idg, idg, 2]))
return fig, ax, pval, tstat
if __name__ == '__main__':
dir_use = get_dir('Marble07', 'Open', -2)
detect_freezing(dir_use, arena='Open')
pass
plt.close(fig)
## Workhorse code below - run before doing much of the above
## For mice with fixed registrations move all files to "archive" folders
arenas = ['Shock', 'Open']
days = [-2, -1, 0, 4, 1, 2, 7]
name_append = '_bad4' # super ocd tracking of # times you've had to redo stuff _2 = 2nd, _87 = 87th, etc.
# IMPORTANT - comment out files you don't want to move in code below!
for mouse in err.all_mice_good:
for arena1 in arenas:
for arena2 in arenas:
for id1, day1 in enumerate(days):
for id2, day2 in enumerate(days):
if id1 <= id2 and arena1 != arena2 or id1 < id2 and arena1 == arena2:
dir_use = get_dir(mouse, arena1, day1)
archive_dir = os.path.join(dir_use, 'rot_archive')
if not os.path.isdir(archive_dir):
try:
os.mkdir(archive_dir)
except FileNotFoundError:
print('Error for ' + mouse + ' ' + arena1 +
' day ' + str(day1) + ' to ' + arena2 +
' day ' + str(day2))
files_move = glob(
os.path.join(dir_use,
'shuffle_map_mean*nshuf1000.pkl'))
# files_move = glob(os.path.join(dir_use, 'best_rot*.pkl'))
# files_move.extend(glob(os.path.join(dir_use, 'PV1shuf*nshuf_1000.pkl')))
# files_move.extend(glob(os.path.join(dir_use, 'shuffle_map_mean*nshuf100.pkl')))
for file in files_move:
def save_data(self, filename='placefields_cm1.pkl'):
dir_use = get_dir(self.mouse, self.arena, self.day, self.list_dir)
save_file = path.join(dir_use, filename)
with open(save_file, 'wb') as output:
dump(self, output)
def placefields(mouse, arena, day, cmperbin=1, nshuf=1000, speed_thresh=1.5, half=None,
lims_method='auto', save_file='placefields_cm1.pkl', list_dir=master_directory,
align_from_end=False, keep_shuffled=False, isrunning_custom=None):
"""
Make placefields of each neuron. Ported over from Will Mau's/Dave Sullivan's MATLAB
function
:param mouse: mouse name to analyze
:param arena: arena to analyze
:param day: day to analyze
:param cmperbin: 4 default
:param lims_method: 'auto' (default) takes limits of data, 'file' looks for arena_lims.csv
file in the session directory which supplies [[xmin, ymin],[xmax, ymax]], or you
you can enter in [[xmin, ymin], [xmax, ymax]] manually
:param nshuf: number of shuffles to perform for determining significance
:param speed_thresh: speed threshold in cm/s
:param save_file: default = 'placefields_cm1.pkl'. None = do not save
:param align_from_end: False (default) align data assuming start of neural/behavioral data acquisition was
synchronized, True = use end time-points to align (in case of bad triggering at beginning but good at end).
:param half: None (default) = run whole session, 1 = run 1st half only, 2 = run 2nd half only, (odd/even not yet
implemented as of 2021_02_01).
:param keep_shuffled: True = keep shuffled smoothed tmaps (saved as .tmap_sm_shuf). False = default.
:param isrunning_custom: use your own frames, specified in a boolean created by get_running_bool and modified by you
as you please, to calculate the place field map for an arbitrary period of the session.
:return:
"""
make_session_list(list_dir)
# Get position and time information for .csv file (later need to align to imaging)
dir_use = get_dir(mouse, arena, day)
speed, pos, t_track, sr = get_speed(dir_use)
t_track = t_track[0:-1] # chop last time data point to match t_track match speed/pos length
# Display warning if "aligntoend" is in folder name but you are running with align_from_end=False
if str(dir_use).upper().find('ALIGNTOEND') != -1 and align_from_end is False:
print('Folder structure for ' + mouse + ' ' + arena + ': Day ' + str(day) + ' suggests you should align data from end of recording')
print('RE-RUN WITH align_from_end=False!!!')
# Import imaging data
# im_data_file = path.join(dir_use + '\imaging', 'FinalOutput.mat')
im_data_file = path.join(dir_use, 'FinalOutput.mat')
im_data = sio.loadmat(im_data_file)
PSAbool = im_data['PSAbool']
nneurons, _ = np.shape(PSAbool)
try:
sr_image = im_data['SampleRate'].squeeze()
except KeyError:
sr_image = 20
# Convert position to cm
pix2cm = er.get_conv_factors(arena) # get conversion to cm for the arena
pos_cm = pos*pix2cm
speed_cm = speed*pix2cm
# Align imaging and position data
pos_align, speed_align, PSAbool_align, time_interp = \
align_imaging_to_tracking(pos_cm, speed_cm, t_track, PSAbool, sr_image, align_from_end=align_from_end)
# Smooth speed data for legitimate thresholding, get limits of data
speed_sm = np.convolve(speed_align, np.ones(2*int(sr))/(2*sr), mode='same') # smooth speed
# Get data limits
if lims_method == 'auto': # automatic by default
lims = [np.min(pos_cm, axis=1), np.max(pos_cm, axis=1)]
elif lims_method == 'file': # file not yet enabled
print('lims_method=''file'' not enabled yet')
else: # grab array!
lims=lims_method
good = np.ones(len(speed_sm)) == 1
isrunning = good.copy()
isrunning[speed_sm < speed_thresh] = False
# Break up session into halves if necessary.
if half is not None:
# Identify # minutes and when half occurs
half_id = np.floor(len(isrunning) / 2).astype('int')
nminutes = np.ceil(len(isrunning) / sr_image / 60).astype(int)
# Now chop things up!
if half == 1:
isrunning[half_id:] = False
elif half == 2:
isrunning[:half_id] = False
elif half in ('odd', 'even'):
odd_even_bool = np.zeros_like(isrunning, dtype=bool)
start_minute = np.where([half == epoch for epoch in ['odd', 'even']])[0][0]
for a in range(start_minute, nminutes, 2):
odd_even_bool[a * 60 * sr_image:(a + 1) * 60 * sr_image] = 1
isrunning[~odd_even_bool] = False
# Use custom period for calculating placefields if specified
if isrunning_custom is not None:
isrunning = isrunning_custom
# Get the mouse's occupancy in each spatial bin
occmap, runoccmap, xEdges, yEdges, xBin, yBin = \
makeoccmap(pos_align, lims, good, isrunning, cmperbin)
# Get rid of non-running epochs
xrun = pos_align[0, isrunning]
yrun = pos_align[1, isrunning]
PSAboolrun = PSAbool_align[:, isrunning]
nGood = len(xrun)
# Construct place field and compute mutual information
neurons = list(range(0, nneurons))
tmap_us, tcounts, tmap_gauss = [], [], []
for idn, neuron in enumerate(neurons):
tmap_us_temp, tcounts_temp, tmap_gauss_temp = \
makeplacefield(PSAboolrun[neuron, :], xrun, yrun, xEdges, yEdges, runoccmap,
cmperbin=cmperbin)
tmap_us.append(tmap_us_temp)
tcounts.append(tcounts_temp)
tmap_gauss.append(tmap_gauss_temp)
# calculate mutual information
mi, _, _, _, _ = spatinfo(tmap_us, runoccmap, PSAboolrun)
# Shuffle to get p-value!
pval, tmap_sm_shuf = [], []
print('Shuffling to get placefield p-values')
for neuron in tqdm(np.arange(nneurons)):
rtmap, rtmap_sm = [], []
shifts = np.random.randint(0, nGood, nshuf)
for ns in np.arange(nshuf):
# circularly shift PSAbool to disassociate transients from mouse location
shuffled = np.roll(PSAboolrun[neuron, :], shifts[ns])
map_temp, _, sm_map_temp = makeplacefield(shuffled, xrun, yrun, xEdges, yEdges, runoccmap,
cmperbin=cmperbin)
rtmap.append(map_temp)
rtmap_sm.append(sm_map_temp)
# Calculate mutual information of randomized vectors
rmi, _, _, _, _ = spatinfo(rtmap, runoccmap, repmat(PSAboolrun[neuron, :], nshuf, 1))
# Calculate p-value
pval.append(1 - np.sum(mi[neuron] > rmi) / nshuf)
# Aggregate shuffled maps if specified (worried this might kill memory)
if keep_shuffled:
tmap_sm_shuf.append(rtmap_sm)
# save variables to working dirs as .pkl files in PFobject
PFobj = PlaceFieldObject(tmap_us, tmap_gauss, xrun, yrun, PSAboolrun, occmap, runoccmap,
xEdges, yEdges, xBin, yBin, tcounts, pval, mi, pos_align, PSAbool_align,
speed_sm, isrunning, cmperbin, speed_thresh, mouse, arena, day, list_dir,
nshuf, sr_image, tmap_sm_shuf)
if save_file is not None:
PFobj.save_data(filename=save_file)
return PFobj