def plot_perf_vs_contacts(session):
db = whiskvid.db.load_db()
# Load stuff
res = whiskvid.db.load_everything_from_session(session, db)
tac = res['tac']
trial_matrix = res['trial_matrix']
v2b_fit = res['v2b_fit']
# Get trial timings
trial_matrix['choice_time'] = MCwatch.behavior.misc.get_choice_times(
db.loc[session, 'bfile'])
# Add trials
tac = whiskvid.db.add_trials_to_tac(tac,
v2b_fit,
trial_matrix,
drop_late_contacts=True)
# Add # of contacts to trial_matrix
trial_matrix['n_contacts'] = tac.groupby('trial').apply(len)
trial_matrix.loc[trial_matrix['n_contacts'].isnull(), 'n_contacts'] = 0
# Plot histogram of contacts vs hit or error
f, ax = plt.subplots()
# Split on hits and errors and draw hist for each
tm_hit = my.pick_rows(trial_matrix, outcome='hit', isrnd=True)
tm_err = my.pick_rows(trial_matrix, outcome='error', isrnd=True)
ax.hist([
np.sqrt(tm_hit.n_contacts.values),
np.sqrt(tm_err.n_contacts.values),
])
ax.set_title(session)
# Plot perf vs some or none contacts
f, ax = plt.subplots()
# Split on whether contacts occurred
tm_n_contacts = trial_matrix[(trial_matrix.n_contacts == 0)
& trial_matrix.outcome.isin(['hit', 'error'])
& trial_matrix.isrnd]
tm_y_contacts = trial_matrix[(trial_matrix.n_contacts > 0)
& trial_matrix.outcome.isin(['hit', 'error'])
& trial_matrix.isrnd]
perf_n_contacts = tm_n_contacts.outcome == 'hit'
perf_y_contacts = tm_y_contacts.outcome == 'hit'
data = [perf_n_contacts, perf_y_contacts]
my.plot.vert_bar(
ax=ax,
bar_lengths=map(np.mean, data),
bar_errs=map(np.std, data),
bar_colors=('b', 'r'),
bar_labels=('none', 'some'),
tick_labels_rotation=0,
)
ax.set_ylim((0, 1))
ax.set_title(session)
def plot_perf_vs_contacts(session):
db = whiskvid.db.load_db()
# Load stuff
res = whiskvid.db.load_everything_from_session(session, db)
tac = res['tac']
trial_matrix = res['trial_matrix']
v2b_fit = res['v2b_fit']
# Get trial timings
trial_matrix['choice_time'] = MCwatch.behavior.misc.get_choice_times(
db.loc[session, 'bfile'])
# Add trials
tac = whiskvid.db.add_trials_to_tac(tac, v2b_fit, trial_matrix,
drop_late_contacts=True)
# Add # of contacts to trial_matrix
trial_matrix['n_contacts'] = tac.groupby('trial').apply(len)
trial_matrix.loc[trial_matrix['n_contacts'].isnull(), 'n_contacts'] = 0
# Plot histogram of contacts vs hit or error
f, ax = plt.subplots()
# Split on hits and errors and draw hist for each
tm_hit = my.pick_rows(trial_matrix, outcome='hit', isrnd=True)
tm_err = my.pick_rows(trial_matrix, outcome='error', isrnd=True)
ax.hist([
np.sqrt(tm_hit.n_contacts.values),
np.sqrt(tm_err.n_contacts.values),
])
ax.set_title(session)
# Plot perf vs some or none contacts
f, ax = plt.subplots()
# Split on whether contacts occurred
tm_n_contacts = trial_matrix[
(trial_matrix.n_contacts == 0) &
trial_matrix.outcome.isin(['hit', 'error']) &
trial_matrix.isrnd]
tm_y_contacts = trial_matrix[
(trial_matrix.n_contacts > 0) &
trial_matrix.outcome.isin(['hit', 'error']) &
trial_matrix.isrnd]
perf_n_contacts = tm_n_contacts.outcome == 'hit'
perf_y_contacts = tm_y_contacts.outcome == 'hit'
data = [perf_n_contacts, perf_y_contacts]
my.plot.vert_bar(ax=ax,
bar_lengths=map(np.mean, data),
bar_errs=map(np.std, data),
bar_colors=('b', 'r'),
bar_labels=('none', 'some'),
tick_labels_rotation=0,
)
ax.set_ylim((0, 1))
ax.set_title(session)
def compute_by_block(self):
"""By block"""
res = {}
for zt in self.zap_types:
for block in [1, 2, 3, 4]:
zdata = my.pick_rows(self.TI, zap=zt, block=block)['is_hit']
cdata = my.pick_rows(self.TI, zap=0, block=block)['is_hit']
# Skip if nothing
if len(zdata) < 5 or len(cdata) < 5:
continue
subres = compare(zdata, cdata)
res['block%d_zap%d' % (block, zt)] = subres
return res
def display_perf_by_servo(session=None, tm=None, ax=None, mean_meth='lr_pool'):
"""Plot perf by servo from single session into ax.
if session is not None, loads trial matrix from it.
if session is None, uses tm.
mean_meth: string
if 'lr_pool': sum together the trials on left and right before
averaging
if 'lr_mean': mean the performances on left and right
"""
# Get trial matrix
if session is not None:
tm = BeWatch.db.get_trial_matrix(session)
# Ax
if ax is None:
f, ax = plt.subplots()
# Pivot perf by servo pos and rewside
tm = my.pick_rows(tm, isrnd=True)
if len(tm) < 5:
return ax
# Group by side and servo and calculate perf
gobj = tm.groupby(['rewside', 'servo_pos'])
rec_l = []
for (rwsd, sp), subdf in gobj:
ntots = len(subdf)
nhits = np.sum(subdf.outcome == 'hit')
rec_l.append({'rewside': rwsd, 'servo_pos': sp,
'nhits': nhits, 'ntots': ntots})
resdf = pandas.DataFrame.from_records(rec_l)
resdf['perf'] = resdf['nhits'] / resdf['ntots']
if mean_meth == 'lr_average':
# mean left and right performances
meanperf = resdf.groupby('servo_pos')['perf'].mean()
elif mean_meth == 'lr_pool':
# combine L and R trials
lr_combo = resdf.groupby('servo_pos').sum()
meanperf = lr_combo['nhits'] / lr_combo['ntots']
else:
raise ValueError(str(mean_meth))
# Plot
colors = {'left': 'blue', 'right': 'red', 'mean': 'purple'}
xticks = resdf['servo_pos'].unique()
for rwsd, subperf in resdf.groupby('rewside'):
xax = subperf['servo_pos']
yax = subperf['perf']
ax.plot(xax, yax, color=colors[rwsd], marker='s', ls='-')
ax.plot(xax, meanperf.values, color=colors['mean'], marker='s', ls='-')
ax.set_xlim((resdf['servo_pos'].min() - 50, resdf['servo_pos'].max() + 50))
ax.set_xticks(xticks)
ax.plot(ax.get_xlim(), [.5, .5], 'k-')
ax.set_ylim((0, 1))
return ax
def numericate_trial_matrix(translated_trial_matrix):
"""Replaces strings with ints to allow anova
* Insert prevchoice column by shifting choice
* Dump trials unless choice is left or right, prevchoice is left or right,
and outcome is hit or error. This drops spoiled and current trials.
* Replace left with -1 and right with +1 in choice, prevchoice, and rewside
and intify those columns.
Return the result.
"""
# Copy and add a prevchoice
df = translated_trial_matrix.copy()
df['prevchoice'] = df['choice'].shift(1)
# Drop where choice, prevchoice are not equal to left or right
df = my.pick_rows(df,
choice=['left', 'right'], prevchoice=['left', 'right'],
rewside=['left', 'right'],
outcome=['hit', 'error'])
# Replace and intify
df['choice'] = df['choice'].replace(
{'left': -1, 'right': 1}).astype(np.int)
df['prevchoice'] = df['prevchoice'].replace(
{'left': -1, 'right': 1}).astype(np.int)
df['rewside'] = df['rewside'].replace(
{'left': -1, 'right': 1}).astype(np.int)
return df
def identify_state_change_times(behavior_filename=None, logfile_df=None,
state0=None, state1=None,
error_on_multiple_changes=False, warn_on_multiple_changes=True,
command='ST_CHG2'):
"""Return time that state changed from state0 to state1 on each trial
behavior_filename : name of logfile.
Only used if logfile_df is not None
logfile_df : result of read_logfile_into_df
state0 : state before change
If None, can be any
state1 : state after change
If None, can be any
Uses read_logfile_into_df to read the file
and get_commands_from_parsed_lines to parse the lines
and then parses the states in the resulting dataframe.
ST_CHG2 is used, because this is the time of the end of the last
call of the state before the change. ST_CHG gives you the time of the
start of that call.
If multiple times are found for a trial, only the first is returned.
If no times are found for a trial, there will be no entry for that trial
in the returned data.
Returns: pandas Series indexed by trial with the state change time
for each trial. The values will be a number of milliseconds
as an integer.
"""
# Get logfile_df
if logfile_df is None:
logfile_df = read_logfile_into_df(behavior_filename)
# Get the state change times
state_change_cmds = get_commands_from_parsed_lines(
logfile_df, command)
# Drop any from trial "-1"
state_change_cmds = state_change_cmds[state_change_cmds.trial != -1]
# Get the ones corresponding to servo retract
state_change_cmds = my.pick_rows(state_change_cmds,
arg0=state0, arg1=state1)
# Group by trial
gobj = state_change_cmds.groupby('trial')
# Error check
if (gobj.apply(len) != 1).any():
if error_on_multiple_changes:
raise ValueError("non-unique state change on some trials")
if warn_on_multiple_changes:
print "warning: non-unique state change on some trials"
# Take the first from each trial
time_by_trial = gobj.first()
return time_by_trial['time']
def identify_state_change_times(behavior_filename=None, logfile_df=None,
state0=None, state1=None,
error_on_multiple_changes=False, warn_on_multiple_changes=True,
command='ST_CHG2'):
"""Return time that state changed from state0 to state1 on each trial
behavior_filename : name of logfile.
Only used if logfile_df is not None
logfile_df : result of read_logfile_into_df
state0 : state before change
If None, can be any
state1 : state after change
If None, can be any
Uses read_logfile_into_df to read the file
and get_commands_from_parsed_lines to parse the lines
and then parses the states in the resulting dataframe.
ST_CHG2 is used, because this is the time of the end of the last
call of the state before the change. ST_CHG gives you the time of the
start of that call.
If multiple times are found for a trial, only the first is returned.
If no times are found for a trial, there will be no entry for that trial
in the returned data.
Returns: pandas Series indexed by trial with the state change time
for each trial. The values will be a number of milliseconds
as an integer.
"""
# Get logfile_df
if logfile_df is None:
logfile_df = read_logfile_into_df(behavior_filename)
# Get the state change times
state_change_cmds = get_commands_from_parsed_lines(
logfile_df, command)
# Drop any from trial "-1"
state_change_cmds = state_change_cmds[state_change_cmds.trial != -1]
# Get the ones corresponding to servo retract
state_change_cmds = my.pick_rows(state_change_cmds,
arg0=state0, arg1=state1)
# Group by trial
gobj = state_change_cmds.groupby('trial')
# Error check
if (gobj.apply(len) != 1).any():
if error_on_multiple_changes:
raise ValueError("non-unique state change on some trials")
if warn_on_multiple_changes:
print("warning: non-unique state change on some trials")
# Take the first from each trial
time_by_trial = gobj.first()
return time_by_trial['time']
def get_trial_parameters(parsed_lines, command_string=trial_param_token):
"""Returns the value of trial parameters"""
rec = {}
rows = my.pick_rows(parsed_lines, command=command_string)
for arg in rows['argument'].values:
name, value = arg.split()
rec[name.lower()] = int(value)
return rec
def get_trial_parameters(parsed_lines, command_string=trial_param_token):
"""Returns the value of trial parameters"""
rec = {}
rows = my.pick_rows(parsed_lines, command=command_string)
for arg in rows['argument'].values:
name, value = arg.split()
rec[name.lower()] = int(value)
return rec
def identify_state_change_times(parsed_df_by_trial, state0=None, state1=None,
show_warnings=True, error_on_multi=False, command='ST_CHG'):
"""Return time that state changed from state0 to state1
This should be replaced with identify_state_change_times_new, which
uses a combination of
ArduFSM.TrialSpeak.read_logfile_into_df
ArduFSM.TrialSpeak.get_commands_from_parsed_lines
parsed_df_by_trial : result of parse_lines_into_df_split_by_trial
(May be more efficient to rewrite this to operate on the whole thing?)
state0 and state1 : any argument that pick_rows can work on, so
13 works or [13, 14] works
command : the state change token
ST_CHG2 uses the time at the end, instead of beginning, of the loop
If multiple hits per trial found:
returns first one
If no hits per trial found:
return nan
"""
multi_warn_flag = False
res = []
# Iterate over trials
for df in parsed_df_by_trial:
# Get st_chg commands
st_chg_rows = my.pick_rows(df, command=command)
# Split the argument string and intify
if len(st_chg_rows) > 0:
split_arg = pandas.DataFrame(
st_chg_rows['argument'].str.split().tolist(),
dtype=np.int, columns=['state0', 'state1'],
index=st_chg_rows.index)
# Match to state0, state1
picked = my.pick(split_arg, state0=state0, state1=state1)
else:
picked = []
# Split on number of hits per trial
if len(picked) == 0:
res.append(np.nan)
elif len(picked) == 1:
res.append(df['time'][picked[0]])
else:
res.append(df['time'][picked[0]])
multi_warn_flag = True
if error_on_multi:
raise(ValueError("multiple events on this trial"))
if show_warnings and multi_warn_flag:
print "warning: multiple target state changes found on some trial"
return np.array(res, dtype=np.float) / 1000.0
def get_trial_release_time(parsed_lines):
"""Returns the time of trial release in seconds"""
rows = my.pick_rows(parsed_lines, command=trial_released_token)
if len(rows) > 1:
raise ValueError("too many trial relased lines")
elif len(rows) == 0:
return None
else:
return int(rows['time'].irow(0)) / 1000.
def get_trial_release_time(parsed_lines):
"""Returns the time of trial release in seconds"""
rows = my.pick_rows(parsed_lines, command=trial_released_token)
if len(rows) > 1:
raise ValueError("too many trial relased lines")
elif len(rows) == 0:
return None
else:
return int(rows['time'].iat[0]) / 1000.
def plot_tac(session):
db = whiskvid.db.load_db()
# Load stuff
res = whiskvid.db.load_everything_from_session(session, db)
tac = res['tac']
trial_matrix = res['trial_matrix']
v2b_fit = res['v2b_fit']
# Get trial timings
bfile = db.loc[session, 'bfile']
trial_matrix['choice_time'] = BeWatch.misc.get_choice_times(bfile)
# Add trials
tac = whiskvid.db.add_trials_to_tac(tac, v2b_fit, trial_matrix,
drop_late_contacts=True)
# Plot tac vs rewside
rewside2color = {'left': 'b', 'right': 'r'}
f, ax = plt.subplots()
gobj = my.pick_rows(tac,
choice=['left', 'right'], outcome='hit', isrnd=True).groupby('rewside')
for rewside, subtac in gobj:
ax.plot(subtac['tip_x'], subtac['tip_y'], 'o',
color=rewside2color[rewside], mec='none', alpha=1)
ax.set_xlim((0, db.loc[session, 'v_width']))
ax.set_ylim((db.loc[session, 'v_height'], 0))
ax.set_title(session)
my.plot.rescue_tick(f=f, x=4, y=4)
# Plot tac vs choice
choice2color = {'left': 'b', 'right': 'r'}
f, ax = plt.subplots()
gobj = my.pick_rows(tac,
choice=['left', 'right'], isrnd=True).groupby('choice')
for rewside, subtac in gobj:
ax.plot(subtac['tip_x'], subtac['tip_y'], 'o',
color=rewside2color[rewside], mec='none', alpha=1)
ax.set_xlim((0, db.loc[session, 'v_width']))
ax.set_ylim((db.loc[session, 'v_height'], 0))
ax.set_title(session)
my.plot.rescue_tick(f=f, x=4, y=4)
plt.show()
def get_trial_start_time(parsed_lines):
"""Returns the time of the start of the trial in seconds"""
rows = my.pick_rows(parsed_lines, command=start_trial_token)
if len(rows) > 1:
raise ValueError("too many trial start lines")
elif len(rows) == 0:
return None
else:
return int(rows['time'].irow(0)) / 1000.
def get_trial_start_time(parsed_lines):
"""Returns the time of the start of the trial in seconds"""
rows = my.pick_rows(parsed_lines, command=start_trial_token)
if len(rows) > 1:
raise ValueError("too many trial start lines")
elif len(rows) == 0:
return None
else:
return old_div(int(rows['time'].iat[0]), 1000.)
def get_commands_from_parsed_lines(parsed_lines, command,
arg2dtype=None):
"""Return only those lines that match "command" and set dtypes.
parsed_lines : result of read_logfile_into_df
command : 'ST_CHG', 'ST_CHG2', 'TCH', etc.
This is used to select rows from parsed_lines. For known arguments,
we can also use this to set arg2dtype.
arg2dtype : dict explaining which args to keep and what dtype to convert
e.g., {'arg0': np.int, 'arg1': np.float}
Returns:
DataFrame with one row for each matching command, and just the
requested columns. We always include 'time', 'command', and
'trial' if available
Can use something like this to group the result by trial and arg0:
tt2licks = lick_times.groupby(['trial', 'arg0']).groups
for (trial, lick_type) in tt2licks:
tt2licks[(trial, lick_type)] = \
ldf.loc[tt2licks[(trial, lick_type)], 'time'].values / 1000.
See BeWatch.misc for other examples of task-specific logic
"""
# Pick
res = my.pick_rows(parsed_lines, command=command)
# Decide which columns to keep and how to coerce
if command == 'ST_CHG2':
if arg2dtype is None:
arg2dtype = {'arg0': np.int, 'arg1': np.int}
elif command == 'ST_CHG':
if arg2dtype is None:
arg2dtype = {'arg0': np.int, 'arg1': np.int}
elif command == 'TCH':
arg2dtype = {'arg0': np.int}
if arg2dtype is None:
raise ValueError("must provide arg2dtype")
# Keep only the columns we want
keep_cols = ['time', 'command'] + sorted(arg2dtype.keys())
if 'trial' in res.columns:
keep_cols.append('trial')
res = res[keep_cols]
# Coerce dtypes
for argname, dtyp in arg2dtype.items():
try:
res[argname] = res[argname].astype(dtyp)
except ValueError:
print "warning: cannot coerce column %s to %r" % (argname, dtyp)
return res
def __init__(self, trial_types, **kwargs):
self.name = 'session starter'
self.params = {
'FD': 'X',
'RPB': 1,
}
self.trial_types = trial_types
# For simplicity, slice trial_types
# Later, might want to reimplement the choosing rule instead
lefts = my.pick_rows(self.trial_types, rewside='left')
closest_left = lefts.srvpos.idxmax()
rights = my.pick_rows(self.trial_types, rewside='right')
closest_right = rights.srvpos.idxmax()
# Because we maintain the indices, plotter will work correctly
# Not quite right, we don't currently use indices, but this is a TODO
self.picked_trial_types = self.trial_types.loc[
[closest_left, closest_right]].copy()
def compute_by_block_and_gonogo(self):
"""By block and go/nogo"""
res = {}
for zt in self.zap_types:
for block in [1, 2, 3, 4]:
for gng in ['go', 'nogo']:
subres = {}
zdata = my.pick_rows(self.TI, zap=zt, block=block,
go_or_nogo=gng)['is_hit']
cdata = my.pick_rows(self.TI, zap=0, block=block,
go_or_nogo=gng)['is_hit']
# Skip if nothing
if len(zdata) < 5 or len(cdata) < 5:
continue
subres = compare(zdata, cdata)
res['block%d_%s_zap%d' % (block, gng, zt)] = subres
return res
def get_commands_from_parsed_lines(parsed_lines, command, arg2dtype=None):
"""Return only those lines that match "command" and set dtypes.
parsed_lines : result of read_logfile_into_df
command : 'ST_CHG', 'ST_CHG2', 'TCH', etc.
This is used to select rows from parsed_lines. For known arguments,
we can also use this to set arg2dtype.
arg2dtype : dict explaining which args to keep and what dtype to convert
e.g., {'arg0': np.int, 'arg1': np.float}
Returns:
DataFrame with one row for each matching command, and just the
requested columns. We always include 'time', 'command', and
'trial' if available
Can use something like this to group the result by trial and arg0:
tt2licks = lick_times.groupby(['trial', 'arg0']).groups
for (trial, lick_type) in tt2licks:
tt2licks[(trial, lick_type)] = \
ldf.loc[tt2licks[(trial, lick_type)], 'time'].values / 1000.
See BeWatch.misc for other examples of task-specific logic
"""
# Pick
res = my.pick_rows(parsed_lines, command=command)
# Decide which columns to keep and how to coerce
if command == 'ST_CHG2':
if arg2dtype is None:
arg2dtype = {'arg0': np.int, 'arg1': np.int}
elif command == 'ST_CHG':
if arg2dtype is None:
arg2dtype = {'arg0': np.int, 'arg1': np.int}
elif command == 'TCH':
arg2dtype = {'arg0': np.int}
if arg2dtype is None:
raise ValueError("must provide arg2dtype")
# Keep only the columns we want
keep_cols = ['time', 'command'] + sorted(arg2dtype.keys())
if 'trial' in res.columns:
keep_cols.append('trial')
res = res[keep_cols]
# Coerce dtypes
for argname, dtyp in arg2dtype.items():
try:
res[argname] = res[argname].astype(dtyp)
except ValueError:
print "warning: cannot coerce column %s to %r" % (argname, dtyp)
return res
def __init__(self, trial_types, **kwargs):
self.name = 'session starter'
self.params = {
'FD': 'X',
'RPB': 1,
}
self.trial_types = trial_types
# For simplicity, slice trial_types
# Later, might want to reimplement the choosing rule instead
lefts = my.pick_rows(self.trial_types, rewside='left')
closest_left = lefts.srvpos.idxmax()
rights = my.pick_rows(self.trial_types, rewside='right')
closest_right = rights.srvpos.idxmax()
# Because we maintain the indices, plotter will work correctly
# Not quite right, we don't currently use indices, but this is a TODO
self.picked_trial_types = self.trial_types.loc[[
closest_left, closest_right
]].copy()
def display_overlays_by_rig_from_day(date=None, rigs=('B1', 'B2', 'B3', 'B4'),
overlay_meth='all'):
"""Plot all overlays from each rig to check positioning
The 'frames' dir needs to be filled out first. The easiest way to do
this is to run make_overlays_from_all_fits.
"""
# Load data
sbvdf = BeWatch.db.get_synced_behavior_and_video_df()
msdf = BeWatch.db.get_manual_sync_df()
sbvdf_dates = sbvdf['dt_end'].apply(lambda dt: dt.date())
# Set to most recent date in database if None
if date is None:
date = sbvdf_dates.max()
# Choose the ones to display
display_dates = sbvdf.ix[sbvdf_dates == date]
# Select by rigs and sort by rig and date
display_dates = my.pick_rows(display_dates, rig=rigs).sort(
['rig', 'dt_end'])
# Make a figure with rigs on columns
n_rows = display_dates.groupby('rig').apply(len).max()
n_cols = len(rigs)
f, axa = plt.subplots(n_rows, n_cols)
# Go through each entry and place into appropriate axis
rownum = np.zeros(n_cols)
for idx, sub_sbvdf_row in display_dates.iterrows():
# Figure out which row and column we're in
col = rigs.index(sub_sbvdf_row['rig'])
row = rownum[col]
rownum[col] = rownum[col] + 1
ax = axa[row, col]
# Title the subplot with the session
session = sub_sbvdf_row['session']
ax.set_title(session, size='small')
# Try to construct the meaned frames
if session in msdf.index:
# Syncing information available
BeWatch.overlays.make_overlays_from_fits(session,
verbose=True, ax=ax, ax_meth=overlay_meth)
else:
# No syncing information
ax.set_xticks([])
ax.set_yticks([])
return f
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next trial.
This object simply chooses randomly from all trial types, iid.
Returns in TrialSpeak. TODO: return straight from trial_types,
and let trial_setter handle the translation to TrialSpeak.
"""
res = {}
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=self.params['side'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(0, len(sub_trial_types))]
# Set the rest of the params
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
res['OPTO'] = NO
# if the last three trials were all forced this way, direct deliver
if len(trial_matrix) > n_dd_trials:
force_on_2afc = (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(trial_matrix['rewside'].values[-n_dd_trials:] ==
res['RWSD'])
and np.all(
trial_matrix['outcome'].values[-n_dd_trials:] == 'error'))
force_on_gng = (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(
trial_matrix['rewside'].values[-n_dd_trials:] == 'right')
and np.all(
trial_matrix['outcome'].values[-n_dd_trials:] == 'spoil'))
if force_on_2afc or force_on_gng:
res['DIRDEL'] = TrialSpeak.YES
# Opto on every Nth trial
if np.mod(len(trial_matrix), N_OPTO_TRIALS) == N_OPTO_TRIALS - 1:
res['OPTO'] = YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2, 'nogo': 3}[res['RWSD']]
return res
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
if len(trial_matrix) == 0:
idx = np.where(self.trial_types.rewside == 'right')[0][0]
#~ idx = self.trial_types.index[np.random.randint(0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'right'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['outcome'] == 'hit':
res['RWSD'] = {
'left': 'right',
'right': 'left'
}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(
0, len(sub_trial_types))]
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2}[res['RWSD']]
return res
def perfcount(trials_info, **kwargs):
"""Filter trials_info by kwargs and return performance"""
if 'nonrandom' not in kwargs:
kwargs = kwargs.copy()
kwargs['nonrandom'] = 0
data = my.pick_rows(trials_info, **kwargs)
nhits = np.sum(data.outcome == 'hit')
nerrs = np.sum(data.outcome == 'error')
nspos = np.sum(data.outcome == 'wrong_port')
nfut = np.sum(data.outcome == 'future_trial')
assert nhits + nerrs + nspos + nfut == len(data)
return nhits, nerrs, nspos
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next trial.
This object simply chooses randomly from all trial types, iid.
Returns in TrialSpeak. TODO: return straight from trial_types,
and let trial_setter handle the translation to TrialSpeak.
"""
res = {}
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=self.params['side'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(0, len(sub_trial_types))]
# Set the rest of the params
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
res['OPTO'] = NO
# if the last three trials were all forced this way, direct deliver
if len(trial_matrix) > n_dd_trials:
force_on_2afc = (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:]) and
np.all(trial_matrix['rewside'].values[-n_dd_trials:] == res['RWSD']) and
np.all(trial_matrix['outcome'].values[-n_dd_trials:] == 'error'))
force_on_gng = (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:]) and
np.all(trial_matrix['rewside'].values[-n_dd_trials:] == 'right') and
np.all(trial_matrix['outcome'].values[-n_dd_trials:] == 'spoil'))
if force_on_2afc or force_on_gng:
res['DIRDEL'] = TrialSpeak.YES
# Opto on every Nth trial
if np.mod(len(trial_matrix), N_OPTO_TRIALS) == N_OPTO_TRIALS - 1:
res['OPTO'] = YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2, 'nogo': 3}[res['RWSD']]
return res
def scatter_ess(
data,
axa=None,
metrics=('aov_essblock', 'aov_essangl', 'aov_esscross'),
):
metrics = list(metrics)
# Plot each region
for region in ['A1', 'PFC']:
# Choose axis
if axa is None:
f, ax = plt.subplots(figsize=(3, 3))
else:
ax = axa[region == 'PFC']
subdf = my.pick_rows(data, region=region)
# Plot ESS of block and angl
esss = subdf[metrics].values.T
# boxplot
box_shit = ax.boxplot(esss.T,
sym='',
positions=range(len(esss)),
widths=.5)
for key in ['caps', 'fliers', 'whiskers']:
for handle in box_shit[key]:
handle.set_visible(False)
handle.set_color('gray')
for line in box_shit['medians']:
line.set_lw(4)
for line in box_shit['boxes']:
line.set_color('gray')
# ESS
ax.plot(esss, 'x', ms=4, color='gray')
# Pretty
ax.set_xticks([0, 1, 2])
ax.set_xlim((-.5, 2.5))
ax.set_ylim((0, 1))
ax.set_yticks((0, .25, .5, .75, 1.0))
ax.set_ylabel('fraction of explainable variance')
#~ ax.set_title('region=' + region)
my.plot.despine(ax)
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
if len(trial_matrix) == 0:
idx = np.where(self.trial_types.rewside == 'right')[0][0]
#~ idx = self.trial_types.index[np.random.randint(0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'right'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['outcome'] == 'hit':
res['RWSD'] = {'left': 'right', 'right':'left'}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(0, len(sub_trial_types))]
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2}[res['RWSD']]
return res
def identify_state_change_times_new(behavior_filename,
state0=None,
state1=None,
error_on_multi=False,
command='ST_CHG'):
"""Return time that state changed from state0 to state1 on each trial
This is the most current way to do this.
Uses read_logfile_into_df to read the file
and get_commands_from_parsed_lines to parse the lines
and then parses the states in the resulting dataframe.
If multiple times are found for a trial, only the first is returned.
If no times are found for a trial, there will be no entry for that trial
in the returned data.
Returns: pandas Series indexed by trial with the state change time
for each trial. The values will be a number of milliseconds
as an integer.
"""
# Get the state change times
logfile_df = read_logfile_into_df(behavior_filename)
state_change_cmds = get_commands_from_parsed_lines(logfile_df, 'ST_CHG2')
# Drop any from trial "-1"
state_change_cmds = state_change_cmds[state_change_cmds.trial != -1]
# Get the ones corresponding to servo retract
state_change_cmds = my.pick_rows(state_change_cmds,
arg0=state0,
arg1=state1)
# Group by trial
gobj = state_change_cmds.groupby('trial')
# Error check
if error_on_multi:
if (gobj.apply(len) != 1).any():
raise ValueError("non-unique state change on some trials")
# Take the first from each trial
time_by_trial = gobj.first()
return time_by_trial['time']
def stacked_bar_ess(data,
axa=None,
metrics=('aov_essangl', 'aov_esscross', 'aov_essblock'),
col_list=('black', 'purple', 'orange')):
"""Stacked bar of ESS"""
# Which metrics and in which order (bottom to top)
metrics = list(metrics)
# Iterate over regions
for region in ['A1', 'PFC']:
if axa is None:
f, ax = plt.subplots(figsize=(3, 3))
else:
ax = axa[region == 'PFC']
# Break by prefblock
subdf = my.pick_rows(data, region=region)
# Plot ESS of block and angl
esss = subdf[metrics].values.T
# To make a stacked bar, do a cumsum
esss_cumsum = np.vstack(
[np.zeros_like(esss[0]),
np.cumsum(esss, axis=0)])
# And plot each on top of the previous
left = np.arange(len(subdf)) + 1
for nrow in range(esss.shape[0]):
ax.bar(left=left,
height=esss[nrow],
bottom=esss_cumsum[nrow],
color=col_list[nrow],
align='center')
# Pretty
ax.set_xlim((.5, left.max() + .5))
ax.set_ylim((0, 1))
ax.set_ylabel('fraction of explainable variance')
ax.set_xlabel('neuron')
# Line at 50%
ax.plot(ax.get_xlim(), [.5, .5], '-', lw=2, color='magenta')
def calculate_perf_metrics(trial_matrix, exclude_warmup=True):
if exclude_warmup:
trial_matrix = trial_matrix[~trial_matrix.warmup].copy()
if len(trial_matrix) == 0:
raise ValueError("no data in trial matrix")
# Perf metrics
rec_l = []
for opto in [False, True]:
for typ in ['go', 'nogo']:
subdf = my.pick_rows(trial_matrix, opto=opto, typ=typ)
n_hits = np.sum(subdf.correct)
n_tots = len(subdf)
rec_l.append({'opto': opto, 'typ': typ, 'n_hits': n_hits,
'n_tots': n_tots})
perfdf = pandas.DataFrame.from_records(rec_l)
perfdf['perf'] = perfdf['n_hits'] / perfdf['n_tots']
return perfdf
def identify_state_change_times_new(behavior_filename, state0=None, state1=None,
error_on_multi=False, command='ST_CHG'):
"""Return time that state changed from state0 to state1 on each trial
This is the most current way to do this.
Uses read_logfile_into_df to read the file
and get_commands_from_parsed_lines to parse the lines
and then parses the states in the resulting dataframe.
If multiple times are found for a trial, only the first is returned.
If no times are found for a trial, there will be no entry for that trial
in the returned data.
Returns: pandas Series indexed by trial with the state change time
for each trial. The values will be a number of milliseconds
as an integer.
"""
# Get the state change times
logfile_df = read_logfile_into_df(behavior_filename)
state_change_cmds = get_commands_from_parsed_lines(
logfile_df, 'ST_CHG2')
# Drop any from trial "-1"
state_change_cmds = state_change_cmds[state_change_cmds.trial != -1]
# Get the ones corresponding to servo retract
state_change_cmds = my.pick_rows(state_change_cmds,
arg0=state0, arg1=state1)
# Group by trial
gobj = state_change_cmds.groupby('trial')
# Error check
if error_on_multi:
if (gobj.apply(len) != 1).any():
raise ValueError("non-unique state change on some trials")
# Take the first from each trial
time_by_trial = gobj.first()
return time_by_trial['time']
def numericate_trial_matrix(translated_trial_matrix):
"""Replaces strings with ints to allow anova
* Insert prevchoice column by shifting choice
* Dump trials unless choice is left or right, prevchoice is left or right,
and outcome is hit or error. This drops spoiled and current trials.
* Replace left with -1 and right with +1 in choice, prevchoice, and rewside
and intify those columns.
Return the result.
"""
# Copy and add a prevchoice
df = translated_trial_matrix.copy()
df['prevchoice'] = df['choice'].shift(1)
# Drop where choice, prevchoice are not equal to left or right
df = my.pick_rows(df,
choice=['left', 'right'],
prevchoice=['left', 'right'],
rewside=['left', 'right'],
outcome=['hit', 'error'])
# Replace and intify
df['choice'] = df['choice'].replace({
'left': -1,
'right': 1
}).astype(np.int)
df['prevchoice'] = df['prevchoice'].replace({
'left': -1,
'right': 1
}).astype(np.int)
df['rewside'] = df['rewside'].replace({
'left': -1,
'right': 1
}).astype(np.int)
return df
def parse_lines_into_df_split_by_trial(lines, verbose=False):
"""Like parse_lines_into_df but split by trial
We drop everything before the first TRL_START token.
If there is no TRL_START token, return None.
This can be slow if each trial has to be processed separately.
Consider replacing this with read_logfile_into_df
"""
# Parse
df = parse_lines_into_df(lines)
# Debug
n_lost_lines = len(lines) - len(df)
if verbose and n_lost_lines != 0:
print("warning: lost %d lines" % n_lost_lines)
# Split by trial
trl_start_idxs = my.pick_rows(df, command=start_trial_token).index
# Return [empty df] if nothing
if len(trl_start_idxs) == 0:
return None
# Split df
# In each case, we include the first line but exclude the last
res = []
for nidx in range(len(trl_start_idxs) - 1):
# Slice out, including first but excluding last
slc = df.loc[trl_start_idxs[nidx]:trl_start_idxs[nidx + 1] - 1]
res.append(slc)
# Append anything left at the end (current trial, generally)
res.append(df.loc[trl_start_idxs[-1]:])
return res
def parse_lines_into_df_split_by_trial(lines, verbose=False):
"""Like parse_lines_into_df but split by trial
We drop everything before the first TRL_START token.
If there is no TRL_START token, return None.
This can be slow if each trial has to be processed separately.
Consider replacing this with read_logfile_into_df
"""
# Parse
df = parse_lines_into_df(lines)
# Debug
n_lost_lines = len(lines) - len(df)
if verbose and n_lost_lines != 0:
print "warning: lost %d lines" % n_lost_lines
# Split by trial
trl_start_idxs = my.pick_rows(df, command=start_trial_token).index
# Return [empty df] if nothing
if len(trl_start_idxs) == 0:
return None
# Split df
# In each case, we include the first line but exclude the last
res = []
for nidx in range(len(trl_start_idxs) - 1):
# Slice out, including first but excluding last
slc = df.ix[trl_start_idxs[nidx]:trl_start_idxs[nidx + 1] - 1]
res.append(slc)
# Append anything left at the end (current trial, generally)
res.append(df.ix[trl_start_idxs[-1]:])
return res
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
res['OPTO'] = NO
if len(trial_matrix) == 0:
# First trial, so pick at random from trial_types
if hasattr(self, 'picked_trial_types'):
idx = self.trial_types.index[np.random.randint(
0, len(self.picked_trial_types))]
else:
idx = self.trial_types.index[np.random.randint(
0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'left'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['choice'] == last_trial['rewside']:
res['RWSD'] = {
'left': 'right',
'right': 'left'
}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# ugly hack to get Session Starter working
if hasattr(self, 'picked_trial_types'):
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.picked_trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
else:
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(
0, len(sub_trial_types))]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
# if the last three trials were all forced this way, direct deliver
if len(trial_matrix) > n_dd_trials:
if (np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(trial_matrix['rewside'].
values[-n_dd_trials:] == res['RWSD'])
and np.all(trial_matrix['outcome'].
values[-n_dd_trials:] == 'error')):
res['DIRDEL'] = TrialSpeak.YES
# Only do opto on forced if requested
if OPTO_FORCED:
# Depends on how many targets
if N_OPTO_TARGETS == 2:
# Two-target version
# Randomly do nothing or target either location
rand_number = np.random.rand()
if rand_number < (1 / 3.):
res['OPTO'] = NO
elif rand_number < (2 / 3.):
res['OPTO'] = 4
else:
res['OPTO'] = 5
elif N_OPTO_TARGETS == 1:
# One-target version
if np.random.rand() < (1. / N_OPTO_TRIALS):
res['OPTO'] = YES
else:
raise ValueError(
"invalid N_OPTO_TARGETS: {}".format(N_OPTO_TARGETS))
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2}[res['RWSD']]
return res
def logreg_perf_vs_contacts(session):
trial_matrix = ArduFSM.TrialMatrix.make_trial_matrix_from_file(
db.loc[session, 'bfile'])
tac = whiskvid.db.Contacts.load(db.loc[session, 'tac'])
v2b_fit = db.loc[session, ['fit_v2b0', 'fit_v2b1']]
b2v_fit = db.loc[session, ['fit_b2v0', 'fit_b2v1']]
if np.any(pandas.isnull(v2b_fit.values)):
1/0
# Get trial timings
trial_matrix['choice_time'] = MCwatch.behavior.misc.get_choice_times(
db.loc[session, 'bfile'])
trial_matrix['vchoice_time'] = np.polyval(b2v_fit, trial_matrix['choice_time'])
# Add trials
tac = whiskvid.db.add_trials_to_tac(tac, v2b_fit, trial_matrix,
drop_late_contacts=True)
# Add # of contacts to trial_matrix
trial_matrix['n_contacts'] = tac.groupby('trial').apply(len)
trial_matrix.loc[trial_matrix['n_contacts'].isnull(), 'n_contacts'] = 0
# Drop the ones before video started
trial_matrix = trial_matrix[trial_matrix.vchoice_time > 0]
# Choose the random hits
lr_tm = my.pick_rows(trial_matrix, outcome=['hit', 'error'], isrnd=True)
# Choose the regularizations
C_l = [1, .1, .01]
# Setup input / output
input = lr_tm['n_contacts'].values[:, None]
output = (lr_tm['outcome'].values == 'hit').astype(np.int)
# Transform the input
input = np.sqrt(input)
# Values for plotting the decision function
plotl = np.linspace(0, input.max(), 100)
# Bins for actual data
bins = np.sqrt([0, 1, 4, 8, 16, 32, 64, 128])
#~ bins = np.linspace(0, input.max(), 4)
bin_centers = bins[:-1] + 0.5
# Extract perf of each bin of trials based on # of contacts
binned_input = np.searchsorted(bins, input.flatten())
bin_mean_l, bin_err_l = [], []
for nbin, bin in enumerate(bins):
mask = binned_input == nbin
if np.sum(mask) == 0:
bin_mean_l.append(np.nan)
bin_err_l.append(np.nan)
else:
hits = output[mask]
bin_mean_l.append(np.mean(hits))
bin_err_l.append(np.std(hits))
f, axa = plt.subplots(1, len(C_l), figsize=(12, 4))
for C, ax in zip(C_l, axa):
lr = scikits.learn.linear_model.LogisticRegression(C=C)
lr.fit(input, output)#, class_weight='auto')
ax.plot(plotl, lr.predict_proba(plotl[:, None])[:, 1])
ax.plot(plotl, np.ones_like(plotl) * 0.5)
ax.set_ylim((0, 1))
# plot data
ax.errorbar(x=bins, y=bin_mean_l, yerr=bin_err_l)
f.suptitle(session)
plt.show()
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next trial.
This object simply chooses randomly from all trial types, iid.
Returns in TrialSpeak. TODO: return straight from trial_types,
and let trial_setter handle the translation to TrialSpeak.
"""
res = {}
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=self.params['side'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(0, len(sub_trial_types))]
# Set the rest of the params
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
res['OPTO'] = NO
# Only do opto on forced if requested
if OPTO_FORCED:
# Depends on how many targets
if N_OPTO_TARGETS == 2:
# Two-target version
# Randomly do nothing or target either location
rand_number = np.random.rand()
if rand_number < (1 / 3.):
res['OPTO'] = NO
elif rand_number < (2 / 3.):
res['OPTO'] = 4
else:
res['OPTO'] = 5
elif N_OPTO_TARGETS == 1:
# One-target version
if np.random.rand() < (1. / N_OPTO_TRIALS):
res['OPTO'] = YES
else:
raise ValueError(
"invalid N_OPTO_TARGETS: {}".format(N_OPTO_TARGETS))
# if the last three trials were all forced this way, direct deliver
if len(trial_matrix) > n_dd_trials:
force_on_2afc = (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(trial_matrix['rewside'].values[-n_dd_trials:] ==
res['RWSD'])
and np.all(
trial_matrix['outcome'].values[-n_dd_trials:] == 'error'))
force_on_gng = (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(
trial_matrix['rewside'].values[-n_dd_trials:] == 'right')
and np.all(
trial_matrix['outcome'].values[-n_dd_trials:] == 'spoil'))
if force_on_2afc or force_on_gng:
res['DIRDEL'] = TrialSpeak.YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2, 'nogo': 3}[res['RWSD']]
return res
# angles and counts
full_data = pandas.load('full_data')
# Choose the sub-indexed trials for each session
session2subindex = {}
for ulabel in full_data.index.levels[0]:
# Skip if we already did this session
session = kkpandas.kkrs.ulabel2session_name(ulabel)
if session in session2subindex:
continue
# each block, sorted by angle
fdu = full_data.ix[ulabel]
fdu_LB = my.pick_rows(fdu, block='LB').sort('angl')
fdu_PB = my.pick_rows(fdu, block='PB').sort('angl')
# find how many we can include
# LB is always less than PB
for n_include in range(1, len(fdu)):
# Take the highest LB values
sub_LB = fdu_LB.angl.values[-n_include:]
# And the lowest PB values
sub_PB = fdu_PB.angl.values[:n_include]
# Break when the mean PB > mean LB
assert len(sub_LB) > 0
assert len(sub_PB) > 0
if sub_LB.mean() < sub_PB.mean():
from ns5_process import myutils, LBPB
import kkpandas
import pandas, os.path
import numpy as np
import my, my.dataload, scipy.stats, my.plot
import matplotlib.pyplot as plt
my.plot.publication_defaults()
my.plot.font_embed()
# Load data
unit_db = my.dataload.getstarted()['unit_db']
# Analyze only included units with audresps
units_to_analyze = my.pick_rows(unit_db, audresp=['good', 'weak', 'sustained'],
include=True)
# Load evoked responses from previous analysis
evoked_resps = pandas.load('evoked_resps')
# Load hold results
hold_results = pandas.load('../3-1-hold/hold_results')
# Mean the evoked responses
mer = evoked_resps[LBPB.mixed_stimnames].applymap(np.mean)
# Convert to hertz
mer = mer.divide(evoked_resps['evok_dt'], axis=0)
# Join the hold for subtraction
hr = hold_results[['mLB', 'mPB']].divide(hold_results['dt'], axis=0)
# This is only valid for indiv neurons
analysis = 'sua_cue'
# Previously there was a bug here where ulabel was set to index,
# but this is non-unique
res = pandas.load('decode_results_%s' % analysis)
# Add in information about hold period, region, and auditory-responsiveness
res = res.join(hold_results[['diffHz', 'p_adj']], on='ulabel')
res = res.join(unit_db[['region', 'audresp']], on='ulabel')
# Filter: keep only units for which there were 10 spikes total (across blocks)
keep_indexes = res[['block', 'n_spikes', 'ulabel']].pivot_table(
rows='ulabel', cols='block', values='n_spikes').sum(1) >= 10
keep_ulabels = keep_indexes.index[keep_indexes.values]
res = my.pick_rows(res, ulabel=keep_ulabels)
# Additional filtering: drop units with low trial count (in EITHER block)
# Sometimes erratic results on the decoding analysis from such units
MIN_TRIAL_COUNT = 25
keep_indexes = res.pivot_table(
rows='ulabel', cols='block', values='n_trials').min(1) >= MIN_TRIAL_COUNT
keep_ulabels = keep_indexes.index[keep_indexes.values]
res = my.pick_rows(res, ulabel=keep_ulabels)
# Assign prefblock
res['prefblock'] = 'ns'
res['prefblock'][res.p_adj.isnull()] = 'NA'
res['prefblock'][(res.p_adj < .05) & (res.diffHz > 0)] = 'PB'
res['prefblock'][(res.p_adj < .05) & (res.diffHz < 0)] = 'LB'
import matplotlib.pyplot as plt
import os.path
my.plot.font_embed()
my.plot.publication_defaults()
hold_results = pandas.load('hold_results')
# Rat names
gets = my.dataload.getstarted()
rec_l = []
for ratname, subdf1 in hold_results.groupby('ratname'):
for region in ['A1', 'PFC']:
subdf = my.pick_rows(subdf1, region=region)
if len(region) > 0:
nL = len(subdf[(subdf.p_adj < .05) & (subdf.mLB > subdf.mPB)])
nP = len(subdf[(subdf.p_adj < .05) & (subdf.mLB < subdf.mPB)])
nN = len(subdf[(subdf.p_adj >= .05)])
else:
nL, nP, nN = 0, 0, 0
rec_l.append((ratname, region, nL, nP, nN))
res = pandas.DataFrame.from_records(rec_l,
columns=('ratname', 'region', 'nL', 'nP', 'nN')).set_index(
['region', 'ratname'])
divby = res.sum(1).astype(np.float)
divby[divby < 8] = 0.
def logreg_perf_vs_contacts(session):
trial_matrix = ArduFSM.TrialMatrix.make_trial_matrix_from_file(
db.loc[session, 'bfile'])
tac = whiskvid.db.Contacts.load(db.loc[session, 'tac'])
v2b_fit = db.loc[session, ['fit_v2b0', 'fit_v2b1']]
b2v_fit = db.loc[session, ['fit_b2v0', 'fit_b2v1']]
if np.any(pandas.isnull(v2b_fit.values)):
1 / 0
# Get trial timings
trial_matrix['choice_time'] = MCwatch.behavior.misc.get_choice_times(
db.loc[session, 'bfile'])
trial_matrix['vchoice_time'] = np.polyval(b2v_fit,
trial_matrix['choice_time'])
# Add trials
tac = whiskvid.db.add_trials_to_tac(tac,
v2b_fit,
trial_matrix,
drop_late_contacts=True)
# Add # of contacts to trial_matrix
trial_matrix['n_contacts'] = tac.groupby('trial').apply(len)
trial_matrix.loc[trial_matrix['n_contacts'].isnull(), 'n_contacts'] = 0
# Drop the ones before video started
trial_matrix = trial_matrix[trial_matrix.vchoice_time > 0]
# Choose the random hits
lr_tm = my.pick_rows(trial_matrix, outcome=['hit', 'error'], isrnd=True)
# Choose the regularizations
C_l = [1, .1, .01]
# Setup input / output
input = lr_tm['n_contacts'].values[:, None]
output = (lr_tm['outcome'].values == 'hit').astype(np.int)
# Transform the input
input = np.sqrt(input)
# Values for plotting the decision function
plotl = np.linspace(0, input.max(), 100)
# Bins for actual data
bins = np.sqrt([0, 1, 4, 8, 16, 32, 64, 128])
#~ bins = np.linspace(0, input.max(), 4)
bin_centers = bins[:-1] + 0.5
# Extract perf of each bin of trials based on # of contacts
binned_input = np.searchsorted(bins, input.flatten())
bin_mean_l, bin_err_l = [], []
for nbin, bin in enumerate(bins):
mask = binned_input == nbin
if np.sum(mask) == 0:
bin_mean_l.append(np.nan)
bin_err_l.append(np.nan)
else:
hits = output[mask]
bin_mean_l.append(np.mean(hits))
bin_err_l.append(np.std(hits))
f, axa = plt.subplots(1, len(C_l), figsize=(12, 4))
for C, ax in zip(C_l, axa):
lr = scikits.learn.linear_model.LogisticRegression(C=C)
lr.fit(input, output) #, class_weight='auto')
ax.plot(plotl, lr.predict_proba(plotl[:, None])[:, 1])
ax.plot(plotl, np.ones_like(plotl) * 0.5)
ax.set_ylim((0, 1))
# plot data
ax.errorbar(x=bins, y=bin_mean_l, yerr=bin_err_l)
f.suptitle(session)
plt.show()
def get_trial_start_time(parsed_lines):
"""Returns the time of the start of the trial in seconds"""
rows = my.pick_rows(parsed_lines, command=start_trial_token)
if len(rows) != 1:
raise ValueError("trial start is not unique")
return int(rows['time'].irow(0)) / 1000.
trial_picker_kwargs=tpk,
folding_kwargs=folding_kwargs,
)
rec['dfolded'] = dfolded
# Store
rec_l.append(rec)
## Now latency distributions
# Iterate over stim groups and get latency distributions
for stim_group_name, grouped_snames in stim_groups.items():
# Pick the same trials that went into folded
distrs_tpk = tpk.copy()
distrs_tpk.pop('labels')
distrs_tpk.pop('label_kwargs')
subti = my.pick_rows(trials_info, stim_name=grouped_snames,
**distrs_tpk)
# Iterate over event and get latencies for each
for event_name in event_name_list:
# Store time deltas by trial for this event
distr = (subti[event_name] - subti[locking_event]).values
rec_l2.append({'ulabel': ulabel,
'event': event_name,
'group': stim_group_name,
'distr': distr})
# DataFrame both
resdf = pandas.DataFrame.from_records(rec_l).set_index('ulabel')
times_distr = pandas.DataFrame.from_records(rec_l2).set_index(
def make_trials_matrix_from_logfile_lines2(logfile_lines,
always_insert=('resp', 'outc')):
"""Parse out the parameters and outcomes from the lines in the logfile
This was written to be a more optimized version of
make_trials_matrix_from_logfile_lines and is used in trial_setter.
Should combine this with TrialMatrix.make_trial_matrix_from_logfile_lines
and just make one thing that does this function.
For each trial, the following parameters are extracted:
trial_start : time in seconds at which TRL_START was issued
trial_released: time in seconds at which trial was released
All parameters and results listed for each trial.
If the first trial has not started yet, an empty DataFrame is returned.
The columns in always_insert are always inserted, even if they
weren't present. They will be inserted with np.nan, so the dtype
should be numerical, not stringy. The main use-case is the response
columns which are missing during the first trial but which most
code assumes exists.
TODO: bug here where malformed lines cause pldf and lines not to match
up anymore.
"""
if len(logfile_lines) == 0:
return pandas.DataFrame(np.zeros((0, len(always_insert))),
columns=always_insert)
# Parse
pldf = parse_lines_into_df(logfile_lines)
if len(pldf) == 0:
return pandas.DataFrame(np.zeros((0, len(always_insert))),
columns=always_insert)
# Find the boundaries between trials in logfile_lines
trl_start_idxs = my.pick_rows(pldf,
command=start_trial_token).index
if len(trl_start_idxs) == 0:
return pandas.DataFrame(np.zeros((0, len(always_insert))),
columns=always_insert)
# Assign trial numbers. The first chunk of lines are pre-session setup,
# so subtract 1 to make that trial "-1".
# Use side = 'right' to place TRL_START itself correctly
pldf['trial'] = np.searchsorted(np.asarray(trl_start_idxs),
np.asarray(pldf.index), side='right') - 1
# Extract various things
trlps_by_trial = get_trial_parameters2(pldf, logfile_lines)
trlrs_by_trial = get_trial_results2(pldf, logfile_lines)
tt_by_trial = get_trial_timings(pldf, logfile_lines)
# Join
res = pandas.concat(
[trlps_by_trial, trlrs_by_trial, tt_by_trial], axis=1,
verify_integrity=True)
# Lower case the names
res.columns = [col.lower() for col in res.columns]
# rename timings names
res = res.rename(columns={
'trl_start': 'start_time',
'trl_released': 'release_time',
})
# Define duration
if 'release_time' in res.columns and 'start_time' in res.columns:
res['duration'] = res['release_time'] - res['start_time']
else:
res['release_time'] = pandas.Series([], dtype=np.float)
res['start_time'] = pandas.Series([], dtype=np.float)
res['duration'] = pandas.Series([], dtype=np.float)
# Reorder
ordered_cols = ['start_time', 'release_time', 'duration']
for col in sorted(res.columns):
if col not in ordered_cols:
ordered_cols.append(col)
res = res[ordered_cols]
# Avoid SettingWithCopyWarning below
res = res.copy()
# Insert always_insert
for col in always_insert:
if col not in res:
res[col] = np.nan
# Name index
res.index.name = 'trial'
return res
def make_plot(ax, ulabel):
"""Convenience function to scatter counts by angle with trend lines"""
# Hold period counts
counts_result = counts_results.ix[ulabel]
hold_result = hold_results.ix[ulabel]
# Test results
test_result = test_res.ix[ulabel]
# Parse into block and trial number and form spike_df
labels = np.concatenate([
counts_result['LB_trials'], counts_result['PB_trials']])
counts = np.concatenate([
counts_result['LB_counts'], counts_result['PB_counts']])
blocks = np.concatenate([
['LB'] * len(counts_result['LB_trials']),
['PB'] * len(counts_result['PB_trials'])])
spike_df = pandas.DataFrame({'btrial': labels, 'counts': counts,
'block': blocks}).set_index('btrial')
# Behavioral info on the session
session_name = unit_db['session_name'][ulabel]
# Vid tracking object
vts = vidtrack.Session(os.path.join(root_dir, session_name))
# Convert to dataframe of head position
location_df = vidtrack.vts_db2head_pos_df(vts, in_degrees=True)
# Join and dropna (errors, short holds, etc)
full_df = location_df.join(spike_df).dropna()
# Subtract off mean head angle
full_df['angl'] = full_df['angl'] - full_df['angl'].mean()
# Plot
block2color = {'LB': 'b', 'PB': 'r'}
# Fit to all data
m, b, r, p, se = scipy.stats.linregress(full_df['angl'],
np.sqrt(full_df['counts']))
sdump.append("all: %0.3f" % p)
xvals = mlab.prctile(full_df.angl, (5, 95))
ax.plot(xvals, np.polyval((m, b), xvals), '-', color='k', lw=3)
for block in ['LB', 'PB']:
subdf = my.pick_rows(full_df, block=block)
# Plot the individual points
ax.plot(subdf.angl, np.sqrt(subdf.counts), ',', mew=.5, mec=block2color[block])
ax.plot(subdf.angl, np.sqrt(subdf.counts), ls='none', marker='s',
ms=1, mew=.5, mec=block2color[block])
# Plot the linfits
m, b, r, p, se = scipy.stats.linregress(subdf['angl'],
np.sqrt(subdf['counts']))
sdump.append("block %s: %0.3f" % (block, p))
xvals = mlab.prctile(subdf.angl, (15, 85))
ax.plot(xvals, np.polyval((m, b), xvals), '-',
color=block2color[block], lw=3)
mmm = np.sqrt(full_df.counts.max())
ax.set_ylim((-.5, mmm + .5))
ax.set_yticks(list(range(int(mmm) + 1)))
ax.set_xticks((-45, -30, -15, 0, 15, 30, 45))
#~ ax.set_ylim((-.5, np.sqrt(full_df.counts.max()) + 0.5))
ax.set_xlabel('head angle (degrees)')
ax.set_ylabel('sqrt(trial spike count)')
my.plot.despine(ax)
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
res['OPTO'] = NO
if len(trial_matrix) == 0:
# First trial, so pick at random from trial_types
if hasattr(self, 'picked_trial_types'):
idx = self.trial_types.index[np.random.randint(0, len(self.picked_trial_types))]
else:
idx = self.trial_types.index[np.random.randint(0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'left'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['choice'] == last_trial['rewside']:
res['RWSD'] = {'left': 'right', 'right':'left'}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# ugly hack to get Session Starter working
if hasattr(self, 'picked_trial_types'):
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.picked_trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
else:
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(0, len(sub_trial_types))]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
# if the last three trials were all forced this way, direct deliver
if len(trial_matrix) > n_dd_trials:
if (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:]) and
np.all(trial_matrix['rewside'].values[-n_dd_trials:] == res['RWSD']) and
np.all(trial_matrix['outcome'].values[-n_dd_trials:] == 'error')):
res['DIRDEL'] = TrialSpeak.YES
# Only do opto on forced if requested
if OPTO_FORCED:
# Opto either periodic or random
if OPTO_PERIODIC:
# Every Nth trial exactly
if np.mod(len(trial_matrix), N_OPTO_TRIALS) == (
N_OPTO_TRIALS - 1):
res['OPTO'] = YES
else:
# With probability 1/N
if np.random.rand() < (1. / N_OPTO_TRIALS):
res['OPTO'] = YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2}[res['RWSD']]
return res
# Slice out the two gain ranges, using argsort
# These are the indexes into gains_l; also, can choose these values in n_gains_l
argsort_ggains = np.argsort(ggains)
undo_sort = np.argsort(argsort_ggains) # magic
range1 = undo_sort[:80] # check: gains_l[range1] is exponentially spaced
range2 = undo_sort[80:]
resdf['grange'] = 'narrow'
resdf['grange'][resdf['ngain'].isin(range1)] = 'broad'
assert resdf[resdf['grange'] == 'narrow']['ngain'].isin(range2).all()
## END OF BUG CHECKING
# We only want the BROAD and the N=320 runs
N = 320
grange = 'broad'
resdf = my.pick_rows(resdf, grange=grange, N=N)
# Pivot and prepare to average over nreps
pivdf2 = resdf.pivot_table(
rows=['n_noise_level', 'ngain'],
cols=['nrep'], values=['score1b', 'score2b']) / 4. / NT_TEST
# Median performance over nreps
medians = pivdf2.median(axis=1, level=0)
# To plot the data, we want noise level on the columns and gain on the rows
data = medians['score1b'].unstack('n_noise_level')
data2 = medians['score2b'].unstack('n_noise_level')
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
if len(trial_matrix) == 0:
# First trial, so pick at random from trial_types
if hasattr(self, 'picked_trial_types'):
idx = self.trial_types.index[np.random.randint(
0, len(self.picked_trial_types))]
else:
idx = self.trial_types.index[np.random.randint(
0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'right'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['choice'] == last_trial['rewside']:
res['RWSD'] = {
'nogo': 'right',
'right': 'nogo'
}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# ugly hack to get Session Starter working
if hasattr(self, 'picked_trial_types'):
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.picked_trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
else:
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(
0, len(sub_trial_types))]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
# if the last three trials were all NOGO-on-GO errors, direct deliver
if len(trial_matrix) > n_dd_trials:
if (np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(trial_matrix['rewside'].
values[-n_dd_trials:] == 'right')
and np.all(trial_matrix['outcome'].
values[-n_dd_trials:] == 'spoil')):
res['DIRDEL'] = TrialSpeak.YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2, 'nogo': 3}[res['RWSD']]
return res
def make_trials_matrix_from_logfile_lines2(logfile_lines,
always_insert=('resp', 'outc')):
"""Parse out the parameters and outcomes from the lines in the logfile
This was written to be a more optimized version of
make_trials_matrix_from_logfile_lines and is used in trial_setter.
Should combine this with TrialMatrix.make_trial_matrix_from_logfile_lines
and just make one thing that does this function.
For each trial, the following parameters are extracted:
trial_start : time in seconds at which TRL_START was issued
trial_released: time in seconds at which trial was released
All parameters and results listed for each trial.
If the first trial has not started yet, an empty DataFrame is returned.
The columns in always_insert are always inserted, even if they
weren't present. They will be inserted with np.nan, so the dtype
should be numerical, not stringy. The main use-case is the response
columns which are missing during the first trial but which most
code assumes exists.
TODO: bug here where malformed lines cause pldf and lines not to match
up anymore.
"""
if len(logfile_lines) == 0:
return pandas.DataFrame(np.zeros((0, len(always_insert))),
columns=always_insert)
# Parse
pldf = parse_lines_into_df(logfile_lines)
if len(pldf) == 0:
return pandas.DataFrame(np.zeros((0, len(always_insert))),
columns=always_insert)
# Find the boundaries between trials in logfile_lines
trl_start_idxs = my.pick_rows(pldf,
command=start_trial_token).index
if len(trl_start_idxs) == 0:
return pandas.DataFrame(np.zeros((0, len(always_insert))),
columns=always_insert)
# Assign trial numbers. The first chunk of lines are pre-session setup,
# so subtract 1 to make that trial "-1".
# Use side = 'right' to place TRL_START itself correctly
pldf['trial'] = np.searchsorted(np.asarray(trl_start_idxs),
np.asarray(pldf.index), side='right') - 1
# Extract various things
trlps_by_trial = get_trial_parameters2(pldf, logfile_lines)
trlrs_by_trial = get_trial_results2(pldf, logfile_lines)
tt_by_trial = get_trial_timings(pldf, logfile_lines)
# Join
res = pandas.concat(
[trlps_by_trial, trlrs_by_trial, tt_by_trial], axis=1,
verify_integrity=True)
# Lower case the names
res.columns = [col.lower() for col in res.columns]
# rename timings names
res = res.rename(columns={
'trl_start': 'start_time',
'trl_released': 'release_time',
})
# Define duration
if 'release_time' in res.columns and 'start_time' in res.columns:
res['duration'] = res['release_time'] - res['start_time']
else:
res['release_time'] = pandas.Series([], dtype=np.float)
res['start_time'] = pandas.Series([], dtype=np.float)
res['duration'] = pandas.Series([], dtype=np.float)
# Reorder
ordered_cols = ['start_time', 'release_time', 'duration']
for col in sorted(res.columns):
if col not in ordered_cols:
ordered_cols.append(col)
res = res[ordered_cols]
# Avoid SettingWithCopyWarning below
res = res.copy()
# Insert always_insert
for col in always_insert:
if col not in res:
res[col] = np.nan
# Name index
res.index.name = 'trial'
return res
def make_plot(ax, ulabel):
"""Convenience function to scatter counts by angle with trend lines"""
# Hold period counts
counts_result = counts_results.ix[ulabel]
hold_result = hold_results.ix[ulabel]
# Test results
test_result = test_res.ix[ulabel]
# Parse into block and trial number and form spike_df
labels = np.concatenate(
[counts_result['LB_trials'], counts_result['PB_trials']])
counts = np.concatenate(
[counts_result['LB_counts'], counts_result['PB_counts']])
blocks = np.concatenate([['LB'] * len(counts_result['LB_trials']),
['PB'] * len(counts_result['PB_trials'])])
spike_df = pandas.DataFrame({
'btrial': labels,
'counts': counts,
'block': blocks
}).set_index('btrial')
# Behavioral info on the session
session_name = unit_db['session_name'][ulabel]
# Vid tracking object
vts = vidtrack.Session(os.path.join(root_dir, session_name))
# Convert to dataframe of head position
location_df = vidtrack.vts_db2head_pos_df(vts, in_degrees=True)
# Join and dropna (errors, short holds, etc)
full_df = location_df.join(spike_df).dropna()
# Subtract off mean head angle
full_df['angl'] = full_df['angl'] - full_df['angl'].mean()
# Plot
block2color = {'LB': 'b', 'PB': 'r'}
# Fit to all data
m, b, r, p, se = scipy.stats.linregress(full_df['angl'],
np.sqrt(full_df['counts']))
sdump.append("all: %0.3f" % p)
xvals = mlab.prctile(full_df.angl, (5, 95))
ax.plot(xvals, np.polyval((m, b), xvals), '-', color='k', lw=3)
for block in ['LB', 'PB']:
subdf = my.pick_rows(full_df, block=block)
# Plot the individual points
ax.plot(subdf.angl,
np.sqrt(subdf.counts),
',',
mew=.5,
mec=block2color[block])
ax.plot(subdf.angl,
np.sqrt(subdf.counts),
ls='none',
marker='s',
ms=1,
mew=.5,
mec=block2color[block])
# Plot the linfits
m, b, r, p, se = scipy.stats.linregress(subdf['angl'],
np.sqrt(subdf['counts']))
sdump.append("block %s: %0.3f" % (block, p))
xvals = mlab.prctile(subdf.angl, (15, 85))
ax.plot(xvals,
np.polyval((m, b), xvals),
'-',
color=block2color[block],
lw=3)
mmm = np.sqrt(full_df.counts.max())
ax.set_ylim((-.5, mmm + .5))
ax.set_yticks(list(range(int(mmm) + 1)))
ax.set_xticks((-45, -30, -15, 0, 15, 30, 45))
#~ ax.set_ylim((-.5, np.sqrt(full_df.counts.max()) + 0.5))
ax.set_xlabel('head angle (degrees)')
ax.set_ylabel('sqrt(trial spike count)')
my.plot.despine(ax)
import my, my.dataload, my.plot, numpy as np
import matplotlib.pyplot as plt
import os.path
my.plot.font_embed()
my.plot.publication_defaults()
hold_results = pandas.load('hold_results')
# Rat names
gets = my.dataload.getstarted()
rec_l = []
for ratname, subdf1 in hold_results.groupby('ratname'):
for region in ['A1', 'PFC']:
subdf = my.pick_rows(subdf1, region=region)
if len(region) > 0:
nL = len(subdf[(subdf.p_adj < .05) & (subdf.mLB > subdf.mPB)])
nP = len(subdf[(subdf.p_adj < .05) & (subdf.mLB < subdf.mPB)])
nN = len(subdf[(subdf.p_adj >= .05)])
else:
nL, nP, nN = 0, 0, 0
rec_l.append((ratname, region, nL, nP, nN))
res = pandas.DataFrame.from_records(rec_l,
columns=('ratname', 'region', 'nL', 'nP',
'nN')).set_index(
['region', 'ratname'])
divby = res.sum(1).astype(np.float)
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
if len(trial_matrix) == 0:
# First trial, so pick at random from trial_types
if hasattr(self, 'picked_trial_types'):
idx = self.trial_types.index[np.random.randint(0, len(self.picked_trial_types))]
else:
idx = self.trial_types.index[np.random.randint(0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'right'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['choice'] == last_trial['rewside']:
res['RWSD'] = {'nogo': 'right', 'right':'nogo'}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# ugly hack to get Session Starter working
if hasattr(self, 'picked_trial_types'):
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.picked_trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
else:
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(0, len(sub_trial_types))]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
# if the last three trials were all NOGO-on-GO errors, direct deliver
if len(trial_matrix) > n_dd_trials:
if (
np.all(~trial_matrix['isrnd'].values[-n_dd_trials:]) and
np.all(trial_matrix['rewside'].values[-n_dd_trials:] == 'right') and
np.all(trial_matrix['outcome'].values[-n_dd_trials:] == 'spoil')):
res['DIRDEL'] = TrialSpeak.YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2, 'nogo': 3}[res['RWSD']]
return res
def read_logfile_into_df(logfile, nargs=4, add_trial_column=True):
"""Read logfile into a DataFrame
Something like this should probably be the preferred way to read the
lines into a structured data frame.
Use get_commands_from_parsed_lines to parse the arguments, eg,
converting to numbers.
Each line in the file will be a row in the data frame.
Each line is separated by whitespace into the different columns.
Thus, the first column will be "time", the second "command", and the
rest "arguments".
nargs : how many argument columns to add. Lines that contain more arguments
than this will be silently truncated! Lines with fewer will be padded
with None.
add_trial_column : optionally add a column for the trial number of
each line. Lines before the first trial begins have trial number -1.
The dtype will always be int for the time column and object (ie, string)
for every other column. This is to ensure consistency. You may want
to coerce certain columns into numeric dtypes.
"""
# Determine how many argument columns to use
arg_cols = ['arg%d' % n for n in range(nargs)]
all_cols = ['time', 'command'] + arg_cols
# Set dtypes
dtype_d = {'time': np.int, 'command': np.object}
for col in arg_cols:
dtype_d[col] = np.object
# Read. Important to avoid reading header of index or you can get
# weird errors here, like unnamed columns.
rdf = pandas.read_table(logfile, sep=' ', names=all_cols,
index_col=False, header=None)
if not np.all(rdf.columns == all_cols):
raise IOError("cannot read columns correctly from logfile")
# Convert dtypes. We have to do it here, because if done during reading
# it will crash on mal-formed dtypes. Could catch that error and then
# run this...
# Well this isn't that useful because it leaves dtypes messed up. Need
# to find and drop the problematic lines.
for col, dtyp in dtype_d.items():
try:
rdf[col] = rdf[col].astype(dtyp)
except ValueError:
print "warning: cannot coerce %s to %r" % (col, dtyp)
# Join on trial number
if add_trial_column:
# Find the boundaries between trials in logfile_lines
trl_start_idxs = my.pick_rows(rdf,
command=start_trial_token).index
if len(trl_start_idxs) > 0:
# Assign trial numbers. The first chunk of lines are
# pre-session setup, so subtract 1 to make that trial "-1".
# Use side = 'right' to place TRL_START itself correctly
rdf['trial'] = np.searchsorted(np.asarray(trl_start_idxs),
np.asarray(rdf.index), side='right') - 1
# Error check
# Very commonly the ACK TRL_RELEASED, SENH, AAR_L, and AAR_R commands
# are out of order. So ignore this for now.
# Somewhat commonly, there is a missing first digit of the time, for
# some reason.
rrdf = rdf[
~rdf.command.isin(['DBG', 'ACK', 'SENH']) &
~rdf.arg0.isin(['AAR_L', 'AAR_R'])
]
unsorted_times = rrdf['time'].values
bad_args = np.where(np.diff(unsorted_times) < 0)[0]
if len(bad_args) > 0:
first_bad_arg = bad_args[0]
print "bad args"
pre_bad_arg = np.max([first_bad_arg - 2, 0])
post_bad_arg = np.min([first_bad_arg + 2, len(rrdf)])
bad_rows = rrdf.ix[rrdf.index[pre_bad_arg]:rrdf.index[post_bad_arg]]
print bad_rows
raise ValueError("unsorted times in logfile, starting at line %d" %
bad_args[0])
return rdf
# This is only valid for indiv neurons
analysis = 'sua_cue'
# Previously there was a bug here where ulabel was set to index,
# but this is non-unique
res = pandas.load('decode_results_%s' % analysis)
# Add in information about hold period, region, and auditory-responsiveness
res = res.join(hold_results[['diffHz', 'p_adj']], on='ulabel')
res = res.join(unit_db[['region', 'audresp']], on='ulabel')
# Filter: keep only units for which there were 10 spikes total (across blocks)
keep_indexes = res[['block', 'n_spikes', 'ulabel']].pivot_table(
rows='ulabel', cols='block', values='n_spikes').sum(1) >= 10
keep_ulabels = keep_indexes.index[keep_indexes.values]
res = my.pick_rows(res, ulabel=keep_ulabels)
# Additional filtering: drop units with low trial count (in EITHER block)
# Sometimes erratic results on the decoding analysis from such units
MIN_TRIAL_COUNT = 25
keep_indexes = res.pivot_table(rows='ulabel', cols='block',
values='n_trials').min(1) >= MIN_TRIAL_COUNT
keep_ulabels = keep_indexes.index[keep_indexes.values]
res = my.pick_rows(res, ulabel=keep_ulabels)
# Assign prefblock
res['prefblock'] = 'ns'
res['prefblock'][res.p_adj.isnull()] = 'NA'
res['prefblock'][(res.p_adj < .05) & (res.diffHz > 0)] = 'PB'
res['prefblock'][(res.p_adj < .05) & (res.diffHz < 0)] = 'LB'
summary = counts[['dt']]
summary['mEv'] = counts['evok'].apply(np.mean)
summary['mSp'] = counts['pre'].apply(np.mean)
summary['mEvHz'] = summary['mEv'] / summary['dt']
summary['mSpHz'] = summary['mSp'] / .050
summary['diffHz'] = summary['mEvHz'] - summary['mSpHz']
summary['diffspks'] = summary['diffHz'] * summary['dt']
summary['ratio'] = summary['mEvHz'] / summary['mSpHz']
summary['region'] = unit_db['region'][summary.index]
summary['latency'] = 1000*unit_db[['audresp_t1', 'audresp_t2']].ix[
summary.index].mean(1)
assert unit_db['include'][summary.index].all()
# Comparison of prevalence of audresp cells across regions
sdump = ''
A1_cells = my.pick_rows(unit_db, region='A1', include=True)
PFC_cells = my.pick_rows(unit_db, region='PFC', include=True)
n_A1_cells, n_audresp_A1_cells = map(len,
[A1_cells, my.pick(A1_cells, audresp=['good', 'weak', 'sustained'])])
n_PFC_cells, n_audresp_PFC_cells = map(len,
[PFC_cells, my.pick(PFC_cells, audresp=['good', 'weak', 'sustained'])])
sdump += "* A1: %d/%d\n" % (n_audresp_A1_cells, n_A1_cells)
sdump += "* PFC: %d/%d\n" % (n_audresp_PFC_cells, n_PFC_cells)
sdump += "* p=%0.4f, Fisher's Exact Test\n" % scipy.stats.fisher_exact([
[n_audresp_A1_cells, n_A1_cells - n_audresp_A1_cells],
[n_audresp_PFC_cells, n_PFC_cells - n_audresp_PFC_cells],])[1]
with file('stat__comparison_of_prevalence_of_audresp_cells_across_regions', 'w') as fi:
fi.write(sdump)
print sdump
# Histogram of strengths
def generate_trial_params(self, trial_matrix):
"""Given trial matrix so far, generate params for next"""
res = {}
res['ISRND'] = NO
res['DIRDEL'] = TrialSpeak.NO
res['OPTO'] = NO
if len(trial_matrix) == 0:
# First trial, so pick at random from trial_types
if hasattr(self, 'picked_trial_types'):
idx = self.trial_types.index[np.random.randint(
0, len(self.picked_trial_types))]
else:
idx = self.trial_types.index[np.random.randint(
0, len(self.trial_types))]
res['RWSD'] = self.trial_types['rewside'][idx]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
else:
# Not the first trial
# First check that the last trial hasn't been released
assert trial_matrix['release_time'].isnull().iloc[-1]
# But that it has been responded
assert not trial_matrix['choice'].isnull().iloc[-1]
# Set side to left by default, and otherwise forced alt
if len(trial_matrix) < 2:
res['RWSD'] = 'left'
else:
# Get last trial
last_trial = trial_matrix.iloc[-1]
if last_trial['choice'] == last_trial['rewside']:
res['RWSD'] = {
'left': 'right',
'right': 'left'
}[last_trial['rewside']]
else:
res['RWSD'] = last_trial['rewside']
# Update the stored force dir
self.params['FD'] = res['RWSD']
# ugly hack to get Session Starter working
if hasattr(self, 'picked_trial_types'):
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.picked_trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
else:
# Choose from trials from the forced side
sub_trial_types = my.pick_rows(self.trial_types,
rewside=res['RWSD'])
assert len(sub_trial_types) > 0
idx = sub_trial_types.index[np.random.randint(
0, len(sub_trial_types))]
res['STPPOS'] = self.trial_types['stppos'][idx]
res['SRVPOS'] = self.trial_types['srvpos'][idx]
# if the last three trials were all forced this way, direct deliver
if len(trial_matrix) > n_dd_trials:
if (np.all(~trial_matrix['isrnd'].values[-n_dd_trials:])
and np.all(trial_matrix['rewside'].
values[-n_dd_trials:] == res['RWSD'])
and np.all(trial_matrix['outcome'].
values[-n_dd_trials:] == 'error')):
res['DIRDEL'] = TrialSpeak.YES
# Only do opto on forced if requested
if OPTO_FORCED:
# Opto either periodic or random
if OPTO_PERIODIC:
# Every Nth trial exactly
if np.mod(len(trial_matrix),
N_OPTO_TRIALS) == (N_OPTO_TRIALS - 1):
res['OPTO'] = YES
else:
# With probability 1/N
if np.random.rand() < (1. / N_OPTO_TRIALS):
res['OPTO'] = YES
# Untranslate the rewside
# This should be done more consistently, eg, use real phrases above here
# and only untranslate at this point.
res['RWSD'] = {'left': 1, 'right': 2}[res['RWSD']]
return res
summary = counts[['dt']]
summary['mEv'] = counts['evok'].apply(np.mean)
summary['mSp'] = counts['pre'].apply(np.mean)
summary['mEvHz'] = summary['mEv'] / summary['dt']
summary['mSpHz'] = summary['mSp'] / .050
summary['diffHz'] = summary['mEvHz'] - summary['mSpHz']
summary['diffspks'] = summary['diffHz'] * summary['dt']
summary['ratio'] = summary['mEvHz'] / summary['mSpHz']
summary['region'] = unit_db['region'][summary.index]
summary['latency'] = 1000 * unit_db[['audresp_t1', 'audresp_t2'
]].ix[summary.index].mean(1)
assert unit_db['include'][summary.index].all()
# Comparison of prevalence of audresp cells across regions
sdump = ''
A1_cells = my.pick_rows(unit_db, region='A1', include=True)
PFC_cells = my.pick_rows(unit_db, region='PFC', include=True)
n_A1_cells, n_audresp_A1_cells = map(
len, [A1_cells,
my.pick(A1_cells, audresp=['good', 'weak', 'sustained'])])
n_PFC_cells, n_audresp_PFC_cells = map(
len,
[PFC_cells,
my.pick(PFC_cells, audresp=['good', 'weak', 'sustained'])])
sdump += "* A1: %d/%d\n" % (n_audresp_A1_cells, n_A1_cells)
sdump += "* PFC: %d/%d\n" % (n_audresp_PFC_cells, n_PFC_cells)
sdump += "* p=%0.4f, Fisher's Exact Test\n" % scipy.stats.fisher_exact([
[n_audresp_A1_cells, n_A1_cells - n_audresp_A1_cells],
[n_audresp_PFC_cells, n_PFC_cells - n_audresp_PFC_cells],
])[1]
with file('stat__comparison_of_prevalence_of_audresp_cells_across_regions',
if 'angl by session' in plots_to_make:
#~ f, axa = plt.subplots(1, 3, figsize=(9.5, 3))
metric = 'angl'
sdump = ['head angle by block']
# One row per session
for idx in idxs:
f, ax = plt.subplots(1, 1, figsize=(3, 3))
# Get data for these session
df = full_data.ix[idx]
bins = np.linspace(df[metric].min(), df[metric].max(), 30)
# Slice out values for each block
to_hist = [
my.pick_rows(df, block=block)[metric].values
for block in ['LB', 'PB']]
# Subtract off session mean
sess_mean = np.mean(np.concatenate(to_hist))
to_hist = [a - sess_mean for a in to_hist]
sdump.append("Head angle diffs, %s" % idx)
sdump.append('means by block: ' + str(map(np.mean, to_hist)))
sdump.append('diff: ' + str(np.diff(map(np.mean, to_hist))))
heights, bbins, patches = ax.hist(
to_hist, color=['b', 'r'], histtype='step',
label=['LB', 'PB'])
ax.set_ylim((0, np.max(map(np.max, heights)) * 1.1))