def make(self, key):
water_res_num, sess_date = get_wr_sessdate(key)
sess_dir = store_stage / water_res_num / sess_date
sess_dir.mkdir(parents=True, exist_ok=True)
# -- Plotting --
fig = plt.figure(figsize=(20, 12))
fig.suptitle(f'{water_res_num}, session {key["session"]}')
gs = GridSpec(5,
1,
wspace=0.4,
hspace=0.4,
bottom=0.07,
top=0.95,
left=0.1,
right=0.9)
ax1 = fig.add_subplot(gs[0:3, :])
ax2 = fig.add_subplot(gs[3, :])
ax3 = fig.add_subplot(gs[4, :])
ax1.get_shared_x_axes().join(ax1, ax2, ax3)
# Plot settings
plot_setting = {'left lick': 'red', 'right lick': 'blue'}
# -- Get event times --
key_subject_id_session = (
experiment.Session() &
(lab.WaterRestriction()
& 'water_restriction_number="{}"'.format(water_res_num))
& 'session="{}"'.format(key['session'])).fetch1("KEY")
go_cue_times = (experiment.TrialEvent() & key_subject_id_session
& 'trial_event_type="go"').fetch(
'trial_event_time', order_by='trial').astype(float)
lick_times = pd.DataFrame(
(experiment.ActionEvent()
& key_subject_id_session).fetch(order_by='trial'))
trial_num = len(go_cue_times)
all_trial_num = np.arange(1, trial_num + 1).tolist()
all_trial_start = [[-x] for x in go_cue_times]
all_lick = dict()
for event_type in plot_setting:
all_lick[event_type] = []
for i, trial_start in enumerate(all_trial_start):
all_lick[event_type].append(
(lick_times[(lick_times['trial'] == i + 1) &
(lick_times['action_event_type'] == event_type
)]['action_event_time'].values.astype(float) +
trial_start).tolist())
# -- All licking events (Ordered by trials) --
ax1.plot([0, 0], [0, trial_num], 'k', lw=0.5) # Aligned by go cue
ax1.set(ylabel='Trial number', xlim=(-3, 3), xticks=[])
# Batch plotting to speed up
ax1.eventplot(lineoffsets=all_trial_num,
positions=all_trial_start,
color='k') # Aligned by go cue
for event_type in plot_setting:
ax1.eventplot(lineoffsets=all_trial_num,
positions=all_lick[event_type],
color=plot_setting[event_type],
linewidth=2) # Trial start
# -- Histogram of all licks --
for event_type in plot_setting:
sns.histplot(np.hstack(all_lick[event_type]),
binwidth=0.01,
alpha=0.5,
ax=ax2,
color=plot_setting[event_type],
label=event_type) # 10-ms window
ymax_tmp = max(ax2.get_ylim())
sns.histplot(-go_cue_times,
binwidth=0.01,
color='k',
ax=ax2,
label='trial start') # 10-ms window
ax2.axvline(x=0, color='k', lw=0.5)
ax2.set(ylim=(0, ymax_tmp), xticks=[],
title='All events') # Fix the ylim of left and right licks
ax2.legend()
# -- Histogram of reaction time (first lick after go cue) --
plot_setting = {'LEFT': 'red', 'RIGHT': 'blue'}
for water_port in plot_setting:
this_RT = (foraging_analysis.TrialStats() & key_subject_id_session
& (experiment.WaterPortChoice()
& 'water_port="{}"'.format(water_port))
).fetch('reaction_time').astype(float)
sns.histplot(this_RT,
binwidth=0.01,
alpha=0.5,
ax=ax3,
color=plot_setting[water_port],
label=water_port) # 10-ms window
ax3.axvline(x=0, color='k', lw=0.5)
ax3.set(xlabel='Time to Go Cue (s)',
title='First lick (reaction time)'
) # Fix the ylim of left and right licks
ax3.legend()
# ---- Save fig and insert ----
fn_prefix = f'{water_res_num}_{sess_date}_'
fig_dict = save_figs((fig, ), ('session_foraging_licking_psth', ),
sess_dir, fn_prefix)
plt.close('all')
self.insert1({**key, **fig_dict})
def make(self, key):
log.info('BehaviorIngest.make(): key: {key}'.format(key=key))
subject_id = key['subject_id']
h2o = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
ymd = key['session_date']
datestr = ymd.strftime('%Y%m%d')
log.info('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr))
# session record key
skey = {}
skey['subject_id'] = subject_id
skey['session_date'] = ymd
skey['username'] = self.get_session_user()
skey['rig'] = key['rig']
# File paths conform to the pattern:
# dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat
# which is, more generally:
# {h2o}/{training_protocol}/Session Data/{h2o}_{training protocol}_{YYYYMMDD}_{HHMMSS}.mat
path = pathlib.Path(key['rig_data_path'], key['subpath'])
if experiment.Session() & skey:
log.info("note: session exists for {h2o} on {d}".format(h2o=h2o,
d=ymd))
trial = namedtuple( # simple structure to track per-trial vars
'trial',
('ttype', 'stim', 'free', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times', 'trial_start'))
if os.stat(path).st_size / 1024 < 1000:
log.info('skipping file {} - too small'.format(path))
return
log.debug('loading file {}'.format(path))
mat = spio.loadmat(path, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
# parse session datetime
session_datetime_str = str('').join(
(str(SessionData['Info'][0]['SessionDate']), ' ',
str(SessionData['Info'][0]['SessionStartTime_UTC'])))
session_datetime = datetime.strptime(session_datetime_str,
'%d-%b-%Y %H:%M:%S')
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
AllTrialStarts = SessionData['TrialStartTimestamp'][0]
AllTrialStarts = AllTrialStarts - AllTrialStarts[0] # real 1st trial
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all((x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps,
AllTrialStarts, AllTrialStarts))))
# AllStimTrials optional special case
if 'StimTrials' in SessionData.dtype.fields:
log.debug('StimTrials detected in session - will include')
AllStimTrials = SessionData['StimTrials'][0]
assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('StimTrials not detected in session - will skip')
AllStimTrials = np.array(
[None for _ in enumerate(range(AllStateTimestamps.shape[0]))])
# AllFreeTrials optional special case
if 'FreeTrials' in SessionData.dtype.fields:
log.debug('FreeTrials detected in session - will include')
AllFreeTrials = SessionData['FreeTrials'][0]
assert (AllFreeTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('FreeTrials not detected in session - synthesizing')
AllFreeTrials = np.zeros(AllStateTimestamps.shape[0],
dtype=np.uint8)
trials = list(
zip(AllTrialTypes, AllStimTrials, AllFreeTrials, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData, AllEventData,
AllEventTimestamps, AllTrialStarts))
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
return
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
assert skey['session_date'] == session_datetime.date()
skey['session_date'] = session_datetime.date()
skey['session_time'] = session_datetime.time()
log.debug('synthesizing session ID')
session = (dj.U().aggr(experiment.Session()
& {
'subject_id': subject_id
},
n='max(session)').fetch1('n') or 0) + 1
log.info('generated session id: {session}'.format(session=session))
skey['session'] = session
key = dict(key, **skey)
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'corrected_trial_event', 'action_event', 'photostim',
'photostim_location', 'photostim_trial',
'photostim_trial_event')
}
i = 0 # trial numbering starts at 1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart, PreSamplePeriod, SamplePeriod, DelayPeriod
# EarlyLickDelay, EarlyLickSample, ResponseCue, GiveLeftDrop
# GiveRightDrop, GiveLeftDropShort, GiveRightDropShort
# AnswerPeriod, Reward, RewardConsumption, NoResponse
# TimeOut, StopLicking, StopLickingReturn, TrialEnd
#
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.warning('skipping trial {i}; missing {m}'.format(
i=i, m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.warning(
'skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.warning('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.warning('skipping {}: start/end mismatch: {}/{}'.format(
i, str(startindex), str(endindex)))
continue
try:
tkey['trial'] = i
tkey['trial_uid'] = i
tkey['start_time'] = t.trial_start
tkey['stop_time'] = t.trial_start + t.state_times[endindex][0]
except IndexError:
log.warning('skipping {}: IndexError: {}/{} -> {}'.format(
i, str(startindex), str(endindex), str(t.state_times)))
continue
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type >= 5 and
('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states
and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
# Determine free/autowater (Autowater 1 == enabled, 2 == disabled)
bkey['auto_water'] = True if gui['Autowater'][0] == 1 else False
bkey['free_water'] = t.free
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning(
'... trial {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning(
'... trial {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)
]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)
]
#
# Photostim Events
#
if t.stim:
log.debug('BehaviorIngest.make(): t.stim == {}'.format(t.stim))
rows['photostim_trial'].append(tkey)
delay_period_idx = np.where(
t.state_data == states['DelayPeriod'])[0][0]
rows['photostim_trial_event'].append(
dict(tkey,
photo_stim=t.stim,
photostim_event_id=len(rows['photostim_trial_event']),
photostim_event_time=t.state_times[delay_period_idx],
power=5.5))
# end of trial loop.
# Session Insertion
log.info('BehaviorIngest.make(): adding session record')
experiment.Session().insert1(skey)
# Behavior Insertion
log.info('BehaviorIngest.make(): bulk insert phase')
log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key))
self.insert1(key, ignore_extra_fields=True, allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(rows['trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(rows['behavior_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(rows['trial_note'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True,
skip_duplicates=True)
log.info('BehaviorIngest.make(): ... CorrectedTrialEvents')
BehaviorIngest().CorrectedTrialEvents().insert(
rows['corrected_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(rows['action_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
# Photostim Insertion
photostim_ids = np.unique(
[r['photo_stim'] for r in rows['photostim_trial_event']])
unknown_photostims = np.setdiff1d(photostim_ids,
list(photostims.keys()))
if unknown_photostims:
raise ValueError(
'Unknown photostim protocol: {}'.format(unknown_photostims))
if photostim_ids.size > 0:
log.info('BehaviorIngest.make(): ... experiment.Photostim')
for stim in photostim_ids:
experiment.Photostim.insert1(dict(skey, **photostims[stim]),
ignore_extra_fields=True)
experiment.Photostim.PhotostimLocation.insert(
(dict(
skey, **loc, photo_stim=photostims[stim]['photo_stim'])
for loc in photostims[stim]['locations']),
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrial')
experiment.PhotostimTrial.insert(rows['photostim_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrialEvent')
experiment.PhotostimEvent.insert(rows['photostim_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
# Behavior Ingest Insertion
log.info('BehaviorIngest.make(): ... BehaviorIngest.BehaviorFile')
BehaviorIngest.BehaviorFile().insert1(dict(
key, behavior_file=os.path.basename(key['subpath'])),
ignore_extra_fields=True,
allow_direct_insert=True)
def _export_recording(insert_key,
output_dir='./',
filename=None,
overwrite=False):
'''
Export a 'recording' (probe specific data + related events) to a file.
Parameters:
- insert_key: an ephys.ProbeInsertion.primary_key
currently: {'subject_id', 'session', 'insertion_number'})
- output_dir: directory to save the file at (default to be the current working directory)
- filename: an optional output file path string. If not provided,
filename will be autogenerated using the 'mkfilename'
function.
'''
if filename is None:
filename = mkfilename(insert_key)
filepath = pathlib.Path(output_dir) / filename
if filepath.exists() and not overwrite:
print('{} already exists, skipping...'.format(filepath))
return
print('exporting {} to {}'.format(insert_key, filepath))
print('fetching spike/behavior data')
insertion = (ephys.ProbeInsertion.InsertionLocation & insert_key).fetch1()
units = (ephys.Unit & insert_key).fetch()
trial_spikes = (ephys.Unit.TrialSpikes
& insert_key).fetch(order_by='trial asc')
behav = (experiment.BehaviorTrial & insert_key).fetch(order_by='trial asc')
trials = behav['trial']
exports = [
'neuron_single_units', 'neuron_unit_info', 'behavior_report',
'behavior_early_report', 'behavior_lick_times', 'task_trial_type',
'task_stimulation', 'task_cue_time'
]
edata = {k: None for k in exports}
print('reshaping/processing for export')
# neuron_single_units
# -------------------
# [[u0t0.spikes, ..., u0tN.spikes], ..., [uNt0.spikes, ..., uNtN.spikes]]
print('... neuron_single_units:', end='')
_su = defaultdict(list)
ts = trial_spikes[['unit', 'trial', 'spike_times']]
for u, t in ((u, t) for t in trials for u in units['unit']):
ud = ts[np.logical_and(ts['unit'] == u, ts['trial'] == t)]
if ud:
_su[u].append(ud['spike_times'][0])
else:
_su[u].append(np.array([]))
ndarray_object = np.empty((len(_su.keys()), 1), dtype=np.object)
for idx, i in enumerate(sorted(_su.keys())):
ndarray_object[idx, 0] = np.array(_su[i], ndmin=2).T
edata['neuron_single_units'] = ndarray_object
print('ok.')
# neuron_unit_info
# ----------------
#
# [[depth_in_um, cell_type, recording_location] ...]
print('... neuron_unit_info:', end='')
dv = float(insertion['depth']) if insertion['depth'] else np.nan
loc = (ephys.ProbeInsertion & insert_key).aggr(
ephys.ProbeInsertion.RecordableBrainRegion.proj(
brain_region='CONCAT(hemisphere, " ", brain_area)'),
brain_regions='GROUP_CONCAT(brain_region SEPARATOR ", ")').fetch1(
'brain_regions')
cell_types = {
u['unit']: u['cell_type']
for u in (ephys.UnitCellType & insert_key).fetch(as_dict=True)
}
_ui = []
for u in units:
typ = cell_types[u['unit']] if u['unit'] in cell_types else 'unknown'
_ui.append([u['unit_posy'] + dv, typ, loc])
edata['neuron_unit_info'] = np.array(_ui, dtype='O')
print('ok.')
# behavior_report
# ---------------
print('... behavior_report:', end='')
behavior_report_map = {'hit': 1, 'miss': 0, 'ignore': 0} # XXX: ignore ok?
edata['behavior_report'] = np.array(
[behavior_report_map[i] for i in behav['outcome']])
print('ok.')
# behavior_early_report
# ---------------------
print('... behavior_early_report:', end='')
early_report_map = {'early': 1, 'no early': 0}
edata['behavior_early_report'] = np.array(
[early_report_map[i] for i in behav['early_lick']])
print('ok.')
# behavior_touch_times
# --------------------
behavior_touch_times = None # NOQA no data (see ActionEventType())
# behavior_lick_times
# -------------------
print('... behavior_lick_times:', end='')
_lt = []
licks = (experiment.ActionEvent() & insert_key
& "action_event_type in ('left lick', 'right lick')").fetch()
for t in trials:
_lt.append([
float(i) for i in # decimal -> float
licks[licks['trial'] == t]['action_event_time']
] if t in licks['trial'] else [])
edata['behavior_lick_times'] = np.array(_lt)
behavior_whisker_angle = None # NOQA no data
behavior_whisker_dist2pol = None # NOQA no data
print('ok.')
# task_trial_type
# ---------------
print('... task_trial_type:', end='')
task_trial_type_map = {'left': 'l', 'right': 'r'}
edata['task_trial_type'] = np.array(
[task_trial_type_map[i] for i in behav['trial_instruction']],
dtype='O')
print('ok.')
# task_stimulation
# ----------------
print('... task_stimulation:', end='')
_ts = [] # [[power, type, on-time, off-time], ...]
photostim = (experiment.Photostim * experiment.PhotostimBrainRegion.proj(
stim_brain_region='CONCAT(stim_laterality, " ", stim_brain_area)')
& insert_key).fetch()
photostim_map = {}
photostim_dat = {}
photostim_keys = ['left ALM', 'right ALM', 'both ALM']
photostim_vals = [1, 2, 6]
# XXX: we don't detect duplicate presence of photostim_keys in data
for fk, rk in zip(photostim_keys, photostim_vals):
i = np.where(photostim['stim_brain_region'] == fk)[0][0]
j = photostim[i]['photo_stim']
photostim_map[j] = rk
photostim_dat[j] = photostim[i]
photostim_ev = (experiment.PhotostimEvent & insert_key).fetch()
for t in trials:
if t in photostim_ev['trial']:
ev = photostim_ev[np.where(photostim_ev['trial'] == t)]
ps = photostim_map[ev['photo_stim'][0]]
pd = photostim_dat[ev['photo_stim'][0]]
_ts.append([
float(ev['power']), ps,
float(ev['photostim_event_time']),
float(ev['photostim_event_time'] + pd['duration'])
])
else:
_ts.append([0, math.nan, math.nan, math.nan])
edata['task_stimulation'] = np.array(_ts)
print('ok.')
# task_pole_time
# --------------
task_pole_time = None # NOQA no data
# task_cue_time
# -------------
print('... task_cue_time:', end='')
_tct = (experiment.TrialEvent()
& {
**insert_key, 'trial_event_type': 'go'
}).fetch('trial_event_time')
edata['task_cue_time'] = np.array([float(i) for i in _tct])
print('ok.')
# savemat
# -------
print('... saving to {}:'.format(filepath), end='')
scio.savemat(filepath, edata)
print('ok.')
def make(self, key):
log.info('BehaviorIngest.make(): key: {key}'.format(key=key))
rigpaths = [
p for p in RigDataPath().fetch(order_by='rig_data_path')
if 'RRig' in p['rig']
] # change between TRig and RRig
subject_id = key['subject_id']
h2o = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
date = key['session_date']
datestr = date.strftime('%Y%m%d')
log.debug('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr))
# session record key
skey = {}
skey['subject_id'] = subject_id
skey['session_date'] = date
skey['username'] = '******' # username has to be changed
# e.g: dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat
# # p.split('/foo/bar')[1]
for rp in rigpaths:
root = rp['rig_data_path']
path = root
path = os.path.join(path, h2o)
# path = os.path.join(path, 'TW_autoTrain')
path = os.path.join(path, 'tw2')
path = os.path.join(path, 'Session Data')
path = os.path.join(
# path, '{h2o}_TW_autoTrain_{d}*.mat'.format(h2o=h2o, d=datestr)) # earlier program protocol
path,
'{h2o}_tw2_{d}*.mat'.format(
h2o=h2o, d=datestr)) # later program protocol
log.debug('rigpath {p}'.format(p=path))
matches = glob.glob(path)
if len(matches):
log.debug('found files, this is the rig')
skey['rig'] = rp['rig']
break
else:
log.info('no file matches found in {p}'.format(p=path))
if not len(matches):
log.warning('no file matches found for {h2o} / {d}'.format(
h2o=h2o, d=datestr))
return
#
# Find files & Check for split files
# XXX: not checking rig.. 2+ sessions on 2+ rigs possible for date?
#
if len(matches) > 1:
log.warning(
'split session case detected for {h2o} on {date}'.format(
h2o=h2o, date=date))
# session:date relationship is 1:1; skip if we have a session
if experiment.Session() & skey:
log.warning("Warning! session exists for {h2o} on {d}".format(
h2o=h2o, d=date))
return
#
# Extract trial data from file(s) & prepare trial loop
#
trials = zip()
trial = namedtuple( # simple structure to track per-trial vars
'trial', ('ttype', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times'))
for f in matches:
if os.stat(f).st_size / 1024 < 100:
log.info('skipping file {f} - too small'.format(f=f))
continue
mat = spio.loadmat(f, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all(
(x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps))))
z = zip(AllTrialTypes, AllTrialSettings, AllStateTimestamps,
AllStateNames, AllStateData, AllEventData,
AllEventTimestamps)
trials = chain(trials, z) # concatenate the files
trials = list(trials)
# all files were internally invalid or size < 100k
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
log.debug('synthesizing session ID')
session = (dj.U().aggr(experiment.Session() & {
'subject_id': subject_id
},
n='max(session)').fetch1('n') or 0) + 1
log.info('generated session id: {session}'.format(session=session))
skey['session'] = session
key = dict(key, **skey)
log.debug('BehaviorIngest.make(): adding session record')
experiment.Session().insert1(skey)
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'action_event')
}
i = -1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.info('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart
# PreSamplePeriod
# SamplePeriod
# DelayPeriod
# EarlyLickDelay
# EarlyLickSample
# ResponseCue
# GiveLeftDrop
# GiveRightDrop
# GiveLeftDropShort
# GiveRightDropShort
# AnswerPeriod
# Reward
# RewardConsumption
# NoResponse
# TimeOut
# StopLicking
# StopLickingReturn
# TrialEnd
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.info('skipping trial {i}; missing {m}'.format(i=i,
m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.info('skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.info('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endendex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.info('skipping trial {i}: start/end index error: {s}/{e}'.
format(i=i, s=str(startindex), e=str(endindex)))
continue
try:
tkey['trial'] = i
tkey['trial_uid'] = i
tkey['start_time'] = t.state_times[startindex][0]
except IndexError:
log.info('skipping trial {i}: error indexing {s}/{e} into {t}'.
format(i=i,
s=str(startindex),
e=str(endindex),
t=str(t.state_times)))
continue
log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay'
bkey['task_protocol'] = 1
# determine trial instruction
trial_instruction = 'left'
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type > 5 and
('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states
and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
# add behavior record
log.debug('BehaviorIngest.make(): BehaviorTrial()')
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
log.debug('BehaviorIngest.make(): TrialNote().insert1')
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#pdb.set_trace()
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
log.debug('BehaviorIngest.make(): presample')
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
log.debug('BehaviorIngest.make(): go')
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
log.debug('BehaviorIngest.make(): delay events')
for d in delayindex: # protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
if len(lickleft):
[
rows['action_event'].append(
dict(**tkey,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for l in lickleft
]
lickright = np.where(t.event_data == 70)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
if len(lickright):
[
rows['action_event'].append(
dict(**tkey,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for r in lickright
]
# end of trial loop.
log.info('BehaviorIngest.make(): bulk insert phase')
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(rows['trial'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(rows['behavior_trial'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(rows['trial_note'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(rows['action_event'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key))
self.insert1(key, ignore_extra_fields=True)
BehaviorIngest.BehaviorFile().insert(
(dict(key, behavior_file=f.split(root)[1]) for f in matches),
ignore_extra_fields=True)
def fix_session(session_key):
paths = behavior_ingest.RigDataPath.fetch(as_dict=True)
files = (behavior_ingest.BehaviorIngest *
behavior_ingest.BehaviorIngest.BehaviorFile
& session_key).fetch(as_dict=True, order_by='behavior_file asc')
filelist = []
for pf in [(p, f) for f in files for p in paths]:
p, f = pf
found = find_path(p['rig_data_path'], f['behavior_file'])
if found:
filelist.append(found)
if len(filelist) != len(files):
log.warning("behavior files missing in {} ({}/{}). skipping".format(
session_key, len(filelist), len(files)))
return False
log.info('filelist: {}'.format(filelist))
#
# Prepare PhotoStim
#
photosti_duration = 0.5 # (s) Hard-coded here
photostims = {
4: {
'photo_stim': 4,
'photostim_device': 'OBIS470',
'brain_location_name': 'left_alm',
'duration': photosti_duration
},
5: {
'photo_stim': 5,
'photostim_device': 'OBIS470',
'brain_location_name': 'right_alm',
'duration': photosti_duration
},
6: {
'photo_stim': 6,
'photostim_device': 'OBIS470',
'brain_location_name': 'both_alm',
'duration': photosti_duration
}
}
#
# Extract trial data from file(s) & prepare trial loop
#
trials = zip()
trial = namedtuple( # simple structure to track per-trial vars
'trial', ('ttype', 'stim', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times'))
for f in filelist:
if os.stat(f).st_size / 1024 < 1000:
log.info('skipping file {f} - too small'.format(f=f))
continue
log.debug('loading file {}'.format(f))
mat = spio.loadmat(f, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all(
(x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps))))
if 'StimTrials' in SessionData.dtype.fields:
log.debug('StimTrials detected in session - will include')
AllStimTrials = SessionData['StimTrials'][0]
assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('StimTrials not detected in session - will skip')
AllStimTrials = np.array(
[None for i in enumerate(range(AllStateTimestamps.shape[0]))])
z = zip(AllTrialTypes, AllStimTrials, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData, AllEventData,
AllEventTimestamps)
trials = chain(trials, z) # concatenate the files
trials = list(trials)
# all files were internally invalid or size < 100k
if not trials:
log.warning('skipping ., no valid files')
return False
key = session_key
skey = (experiment.Session & key).fetch1()
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'corrected_trial_event', 'action_event', 'photostim',
'photostim_location', 'photostim_trial',
'photostim_trial_event')
}
i = -1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart
# PreSamplePeriod
# SamplePeriod
# DelayPeriod
# EarlyLickDelay
# EarlyLickSample
# ResponseCue
# GiveLeftDrop
# GiveRightDrop
# GiveLeftDropShort
# GiveRightDropShort
# AnswerPeriod
# Reward
# RewardConsumption
# NoResponse
# TimeOut
# StopLicking
# StopLickingReturn
# TrialEnd
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod', 'DelayPeriod',
'ResponseCue', 'StopLicking', 'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.warning('skipping trial {i}; missing {m}'.format(i=i,
m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.warning('skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.warning('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.warning(
'skipping trial {i}: start/end index error: {s}/{e}'.format(
i=i, s=str(startindex), e=str(endindex)))
continue
try:
tkey['trial'] = i
tkey[
'trial_uid'] = i # Arseny has unique id to identify some trials
tkey['start_time'] = t.state_times[startindex][0]
tkey['stop_time'] = t.state_times[endindex][0]
except IndexError:
log.warning(
'skipping trial {i}: error indexing {s}/{e} into {t}'.format(
i=i,
s=str(startindex),
e=str(endindex),
t=str(t.state_times)))
continue
log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type > 5
and ('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... trial {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... trial {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)
]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)
]
# Photostim Events
#
# TODO:
#
# - base stimulation parameters:
#
# - should be loaded elsewhere - where
# - actual ccf locations - cannot be known apriori apparently?
# - Photostim.Profile: what is? fix/add
#
# - stim data
#
# - how retrieve power from file (didn't see) or should
# be statically coded here?
# - how encode stim type 6?
# - we have hemisphere as boolean or
# - but adding an event 4 and event 5 means querying
# is less straightforwrard (e.g. sessions with 5 & 6)
if t.stim:
log.debug('BehaviorIngest.make(): t.stim == {}'.format(t.stim))
rows['photostim_trial'].append(tkey)
delay_period_idx = np.where(
t.state_data == states['DelayPeriod'])[0][0]
rows['photostim_trial_event'].append(
dict(tkey,
**photostims[t.stim],
photostim_event_id=len(rows['photostim_trial_event']),
photostim_event_time=t.state_times[delay_period_idx],
power=5.5))
# end of trial loop.
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
fix_events = rows['trial_event']
ref_events = (experiment.TrialEvent() & skey).fetch(
order_by='trial, trial_event_id', as_dict=True)
if False:
for e in ref_events:
log.debug('ref_events: t: {}, e: {}, event_type: {}'.format(
e['trial'], e['trial_event_id'], e['trial_event_type']))
for e in fix_events:
log.debug('fix_events: t: {}, e: {}, type: {}'.format(
e['trial'], e['trial_event_id'], e['trial_event_type']))
log.info('deleting old events')
with dj.config(safemode=False):
log.info('... TrialEvent')
(experiment.TrialEvent() & session_key).delete()
log.info('... CorrectedTrialEvents')
(behavior_ingest.BehaviorIngest.CorrectedTrialEvents()
& session_key).delete_quick()
log.info('adding new records')
log.info('... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True,
skip_duplicates=True)
log.info('... CorrectedTrialEvents')
behavior_ingest.BehaviorIngest.CorrectedTrialEvents().insert(
rows['corrected_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
return True
def extract_trials(plottype='2lickport',
wr_name='FOR01',
sessions=(5, 11),
show_bias_check_trials=True,
kernel=np.ones(10) / 10,
filters=None,
local_matching={'calculate_local_matching': False}):
#%%
# =============================================================================
# plottype = '2lickport'
# wr_name = 'FOR11'
# sessions = (25,46)
# show_bias_check_trials = False
# kernel = np.ones(20)/20
# filters = {'ignore_rate_max':40}
# local_matching = {'calculate_local_matching': True,
# 'sliding_window':50,
# 'matching_window':500,
# 'matching_step':100}
# =============================================================================
movingwindow = local_matching['sliding_window']
fit_window = local_matching['matching_window']
fit_step = local_matching['matching_step']
subject_id = (lab.WaterRestriction()
& 'water_restriction_number = "{}"'.format(wr_name)
).fetch1('subject_id')
df_behaviortrial = pd.DataFrame(np.asarray(
(experiment.BehaviorTrial() * experiment.SessionTrial() *
experiment.TrialEvent() * experiment.SessionBlock() *
behavior_foraging.TrialReactionTime
& 'subject_id = {}'.format(subject_id)
& 'session >= {}'.format(sessions[0])
& 'session <= {}'.format(sessions[1])
& 'trial_event_type = "go"').fetch(
'session', 'trial', 'early_lick', 'trial_start_time',
'reaction_time', 'p_reward_left', 'p_reward_right',
'p_reward_middle', 'trial_event_time', 'trial_choice',
'outcome')).T,
columns=[
'session', 'trial', 'early_lick',
'trial_start_time', 'reaction_time',
'p_reward_left', 'p_reward_right',
'p_reward_middle', 'trial_event_time',
'trial_choice', 'outcome'
])
unique_sessions = df_behaviortrial['session'].unique()
df_behaviortrial['iti'] = np.nan
df_behaviortrial['delay'] = np.nan
df_behaviortrial['early_count'] = 0
df_behaviortrial.loc[df_behaviortrial['early_lick'] == 'early',
'early_count'] = 1
df_behaviortrial['ignore_rate'] = np.nan
df_behaviortrial['reaction_time_smoothed'] = np.nan
if type(filters) == dict:
df_behaviortrial['keep_trial'] = 1
for session in unique_sessions:
total_trials_so_far = (
behavior_foraging.SessionStats()
& 'subject_id = {}'.format(subject_id)
& 'session < {}'.format(session)).fetch('session_total_trial_num')
bias_check_trials_now = (behavior_foraging.SessionStats()
& 'subject_id = {}'.format(subject_id)
& 'session = {}'.format(session)
).fetch1('session_bias_check_trial_num')
total_trials_so_far = sum(total_trials_so_far)
gotime = df_behaviortrial.loc[df_behaviortrial['session'] == session,
'trial_event_time']
trialtime = df_behaviortrial.loc[df_behaviortrial['session'] ==
session, 'trial_start_time']
itis = np.concatenate([[np.nan],
np.diff(np.asarray(trialtime + gotime, float))])
df_behaviortrial.loc[df_behaviortrial['session'] == session,
'iti'] = itis
df_behaviortrial.loc[df_behaviortrial['session'] == session,
'delay'] = np.asarray(gotime, float)
df_behaviortrial.loc[df_behaviortrial['session'] == session,
'ignore_rate'] = np.convolve(
df_behaviortrial.loc[
df_behaviortrial['session'] == session,
'outcome'] == 'ignore', kernel, 'same')
reaction_time_interpolated = np.asarray(
pd.DataFrame(
np.asarray(
df_behaviortrial.loc[df_behaviortrial['session'] ==
session, 'reaction_time'].values,
float)).interpolate().values.ravel().tolist()) * 1000
df_behaviortrial.loc[df_behaviortrial['session'] == session,
'reaction_time_smoothed'] = np.convolve(
reaction_time_interpolated, kernel, 'same')
df_behaviortrial.loc[df_behaviortrial['session'] == session,
'trial'] += total_trials_so_far
if type(filters) == dict:
max_idx = (
df_behaviortrial.loc[df_behaviortrial['session'] == session,
'ignore_rate'] >
filters['ignore_rate_max'] / 100).idxmax()
session_first_trial_idx = (
df_behaviortrial['session'] == session).idxmax()
#print(max_idx)
if max_idx > session_first_trial_idx or df_behaviortrial[
'ignore_rate'][session_first_trial_idx] > filters[
'ignore_rate_max'] / 100:
df_behaviortrial.loc[df_behaviortrial.index.isin(
np.arange(max_idx, len(df_behaviortrial))) &
(df_behaviortrial['session'] == session),
'keep_trial'] = 0
#%
if type(filters) == dict:
trialstokeep = df_behaviortrial['keep_trial'] == 1
df_behaviortrial = df_behaviortrial[trialstokeep]
df_behaviortrial = df_behaviortrial.reset_index(drop=True)
if not show_bias_check_trials:
realtraining = (df_behaviortrial['p_reward_left'] <
1) & (df_behaviortrial['p_reward_right'] < 1) & (
(df_behaviortrial['p_reward_middle'] < 1)
| df_behaviortrial['p_reward_middle'].isnull())
df_behaviortrial = df_behaviortrial[realtraining]
df_behaviortrial = df_behaviortrial.reset_index(drop=True)
#% calculating local matching, bias, reward rate
kernel = np.ones(movingwindow) / movingwindow
p1 = np.asarray(
np.max([
df_behaviortrial['p_reward_right'],
df_behaviortrial['p_reward_left']
], 0), float)
p0 = np.asarray(
np.min([
df_behaviortrial['p_reward_right'],
df_behaviortrial['p_reward_left']
], 0), float)
m_star_greedy = np.floor(np.log(1 - p1) / np.log(1 - p0))
p_star_greedy = p1 + (1 - (1 - p0)**
(m_star_greedy + 1) - p1**2) / (m_star_greedy + 1)
local_reward_rate = np.convolve(df_behaviortrial['outcome'] == 'hit',
kernel, 'same')
max_reward_rate = np.convolve(p_star_greedy, kernel, 'same')
local_efficiency = local_reward_rate / max_reward_rate
choice_right = np.asarray(df_behaviortrial['trial_choice'] == 'right')
choice_left = np.asarray(df_behaviortrial['trial_choice'] == 'left')
choice_middle = np.asarray(df_behaviortrial['trial_choice'] == 'middle')
reward_rate_right = np.asarray(
(df_behaviortrial['trial_choice'] == 'right')
& (df_behaviortrial['outcome'] == 'hit'))
reward_rate_left = np.asarray((df_behaviortrial['trial_choice'] == 'left')
& (df_behaviortrial['outcome'] == 'hit'))
reward_rate_middle = np.asarray(
(df_behaviortrial['trial_choice'] == 'middle')
& (df_behaviortrial['outcome'] == 'hit'))
# =============================================================================
# choice_fraction_right = np.convolve(choice_right,kernel,'same')/np.convolve(choice_right+choice_left+choice_middle,kernel,'same')
# reward_fraction_right = np.convolve(reward_rate_right,kernel,'same')/local_reward_rate
# =============================================================================
choice_rate_right = np.convolve(
choice_right, kernel, 'same') / np.convolve(
choice_left + choice_middle, kernel, 'same')
reward_rate_right = np.convolve(
reward_rate_right, kernel, 'same') / np.convolve(
reward_rate_left + reward_rate_middle, kernel, 'same')
slopes = list()
intercepts = list()
trial_number = list()
for center_trial in np.arange(np.round(fit_window / 2),
len(df_behaviortrial), fit_step):
#%
reward_rates_now = reward_rate_right[
int(np.round(center_trial - fit_window /
2)):int(np.round(center_trial + fit_window / 2))]
choice_rates_now = choice_rate_right[
int(np.round(center_trial - fit_window /
2)):int(np.round(center_trial + fit_window / 2))]
todel = (reward_rates_now == 0) | (choice_rates_now == 0)
reward_rates_now = reward_rates_now[~todel]
choice_rates_now = choice_rates_now[~todel]
try:
slope_now, intercept_now = np.polyfit(np.log2(reward_rates_now),
np.log2(choice_rates_now), 1)
slopes.append(slope_now)
intercepts.append(intercept_now)
trial_number.append(center_trial)
except:
pass
df_behaviortrial['local_efficiency'] = local_efficiency
df_behaviortrial['local_matching_slope'] = np.nan
df_behaviortrial.loc[trial_number, 'local_matching_slope'] = slopes
df_behaviortrial['local_matching_bias'] = np.nan
df_behaviortrial.loc[trial_number, 'local_matching_bias'] = intercepts
#%%
return df_behaviortrial
def populatebehavior_core(IDs = None):
if IDs:
print('subject started:')
print(IDs.keys())
print(IDs.values())
rigpath_1 = 'E:/Projects/Ablation/datajoint/Behavior'
#df_surgery = pd.read_csv(dj.config['locations.metadata']+'Surgery.csv')
if IDs == None:
IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))}
for subject_now,subject_id_now in zip(IDs.keys(),IDs.values()): # iterating over subjects
print('subject: ',subject_now)
# =============================================================================
# if drop_last_session_for_mice_in_training:
# delete_last_session_before_upload = True
# else:
# delete_last_session_before_upload = False
# #df_wr = online_notebook.fetch_water_restriction_metadata(subject_now)
# =============================================================================
try:
df_wr = pd.read_csv(dj.config['locations.metadata_behavior']+subject_now+'.csv')
except:
print(subject_now + ' has no metadata available')
df_wr = pd.DataFrame()
for df_wr_row in df_wr.iterrows():
date_now = df_wr_row[1].Date.replace('-','')
print('subject: ',subject_now,' date: ',date_now)
session_date = datetime(int(date_now[0:4]),int(date_now[4:6]),int(date_now[6:8]))
if len(experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"' & 'session_date > "'+str(session_date)+'"') != 0: # if it is not the last
print('session already imported, skipping: ' + str(session_date))
dotheupload = False
elif len(experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"' & 'session_date = "'+str(session_date)+'"') != 0: # if it is the last
dotheupload = False
else: # reuploading new session that is not present on the server
dotheupload = True
# if dotheupload is True, meaning that there are new mat file hasn't been uploaded
# => needs to find which mat file hasn't been uploaded
if dotheupload:
found = set()
rigpath_2 = subject_now
rigpath_3 = rigpath_1 + '/' + rigpath_2
rigpath = pathlib.Path(rigpath_3)
def buildrec(rigpath, root, f):
try:
fullpath = pathlib.Path(root, f)
subpath = fullpath.relative_to(rigpath)
fsplit = subpath.stem.split('_')
h2o = fsplit[0]
ymd = fsplit[-2:-1][0]
animal = IDs[h2o]
if ymd == date_now:
return {
'subject_id': animal,
'session_date': date(int(ymd[0:4]), int(ymd[4:6]), int(ymd[6:8])),
'rig_data_path': rigpath.as_posix(),
'subpath': subpath.as_posix(),
}
except:
pass
for root, dirs, files in os.walk(rigpath):
for f in files:
r = buildrec(rigpath, root, f)
if r:
found.add(r['subpath'])
file = r
# now start insert data
path = pathlib.Path(file['rig_data_path'], file['subpath'])
mat = spio.loadmat(path, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
# session record key
skey = {}
skey['subject_id'] = file['subject_id']
skey['session_date'] = file['session_date']
skey['username'] = '******'
#skey['rig'] = key['rig']
trial = namedtuple( # simple structure to track per-trial vars
'trial', ('ttype', 'settings', 'state_times',
'state_names', 'state_data', 'event_data',
'event_times', 'trial_start'))
# parse session datetime
session_datetime_str = str('').join((str(SessionData['Info'][0]['SessionDate']),' ', str(SessionData['Info'][0]['SessionStartTime_UTC'])))
session_datetime = datetime.strptime(session_datetime_str, '%d-%b-%Y %H:%M:%S')
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
AllTrialStarts = SessionData['TrialStartTimestamp'][0]
AllTrialStarts = AllTrialStarts - AllTrialStarts[0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
trials = list(zip(AllTrialTypes, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData,
AllEventData, AllEventTimestamps, AllTrialStarts))
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
return
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
assert skey['session_date'] == session_datetime.date()
skey['session_date'] = session_datetime.date()
#skey['session_time'] = session_datetime.time()
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"' & 'session_date = "'+str(file['session_date'])+'"') == 0:
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"') == 0:
skey['session'] = 1
else:
skey['session'] = len((experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"').fetch()['session']) + 1
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {k: list() for k in ('trial', 'behavior_trial', 'trial_note',
'trial_event', 'corrected_trial_event',
'action_event')} #, 'photostim',
#'photostim_location', 'photostim_trial',
#'photostim_trial_event')}
i = 0 # trial numbering starts at 1
for t in trials:
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
states = {k: (v+1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing) and missing =='PreSamplePeriod':
log.warning('skipping trial {i}; missing {m}'.format(i=i, m=missing))
continue
gui = t.settings['GUI'].flatten()
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"' & 'session_date = "'+str(file['session_date'])+'"') == 0:
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"') == 0:
skey['session'] = 1
else:
skey['session'] = len((experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"').fetch()['session']) + 1
#
# Top-level 'Trial' record
#
protocol_type = gui['ProtocolType'][0]
tkey = dict(skey)
has_presample = 1
try:
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
has_presample = 1
except:
startindex = np.where(t.state_data == states['SamplePeriod'])[0]
has_presample = 0
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not(len(startindex) and len(endindex)):
log.warning('skipping {}: start/end mismatch: {}/{}'.format(i, str(startindex), str(endindex)))
continue
try:
tkey['trial'] = i
tkey['trial_uid'] = i
tkey['trial_start_time'] = t.trial_start
tkey['trial_stop_time'] = t.trial_start + t.state_times[endindex][0]
except IndexError:
log.warning('skipping {}: IndexError: {}/{} -> {}'.format(i, str(startindex), str(endindex), str(t.state_times)))
continue
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif t.ttype == 2:
trial_instruction = 'catch_right_autowater'
elif t.ttype == 3:
trial_instruction = 'catch_left_autowater'
elif t.ttype == 4:
trial_instruction = 'catch_right_noDelay'
elif t.ttype == 5:
trial_instruction = 'catch_left_noDelay'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
elif t.ttype == 2:
trial_instruction = 'catch_left_autowater'
elif t.ttype == 3:
trial_instruction = 'catch_right_autowater'
elif t.ttype == 4:
trial_instruction = 'catch_left_noDelay'
elif t.ttype == 5:
trial_instruction = 'catch_right_noDelay'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type >= 5 and ('EarlyLickSample' in states and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
if has_presample == 1:
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]]- t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... trial {} bad duration, no last_edur'.format(i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... trial {} duration using last_edur {}'.format(i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[rows['action_event'].append(
dict(tkey, action_event_id=action_event_count+idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[rows['action_event'].append(
dict(tkey, action_event_id=action_event_count+idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)]
# end of trial loop..
# Session Insertion
log.info('BehaviorIngest.make(): adding session record')
skey['session_date'] = df_wr_row[1].Date
skey['rig'] = 'Old Recording rig'
skey['username'] = '******'
experiment.Session().insert1(skey,skip_duplicates=True)
# Behavior Insertion
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(
rows['trial'], ignore_extra_fields=True, allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(
rows['behavior_trial'], ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(
rows['trial_note'], ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(
rows['trial_event'], ignore_extra_fields=True,
allow_direct_insert=True, skip_duplicates=True)
# log.info('BehaviorIngest.make(): ... CorrectedTrialEvents')
# BehaviorIngest().CorrectedTrialEvents().insert(
# rows['corrected_trial_event'], ignore_extra_fields=True,
# allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(
rows['action_event'], ignore_extra_fields=True,
allow_direct_insert=True)
#%% for ingest tracking
if IDs:
print('subject started:')
print(IDs.keys())
print(IDs.values())
rigpath_tracking_1 = 'E:/Projects/Ablation/datajoint/video/'
rigpath_tracking_2 = subject_now
VideoDate1 = str(df_wr_row[1].VideoDate)
if len(VideoDate1)==5:
VideoDate = '0'+ VideoDate1
elif len(VideoDate1)==7:
VideoDate = '0'+ VideoDate1
rigpath_tracking_3 = rigpath_tracking_1 + rigpath_tracking_2 + '/' + rigpath_tracking_2 + '_'+ VideoDate + '_front'
rigpath_tracking = pathlib.Path(rigpath_tracking_3)
#df_surgery = pd.read_csv(dj.config['locations.metadata']+'Surgery.csv')
if IDs == None:
IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))}
h2o = subject_now
session = df_wr_row[1].Date
trials = (experiment.SessionTrial() & session).fetch('trial')
log.info('got session: {} ({} trials)'.format(session, len(trials)))
#sdate = session['session_date']
#sdate_sml = date_now #"{}{:02d}{:02d}".format(sdate.year, sdate.month, sdate.day)
paths = rigpath_tracking
devices = tracking.TrackingDevice().fetch(as_dict=True)
# paths like: <root>/<h2o>/YYYY-MM-DD/tracking
tracking_files = []
for d in (d for d in devices):
tdev = d['tracking_device']
tpos = d['tracking_position']
tdat = paths
log.info('checking {} for tracking data'.format(tdat))
# if not tpath.exists():
# log.warning('tracking path {} n/a - skipping'.format(tpath))
# continue
#
# camtrial = '{}_{}_{}.txt'.format(h2o, sdate_sml, tpos)
# campath = tpath / camtrial
#
# log.info('trying camera position trial map: {}'.format(campath))
#
# if not campath.exists():
# log.info('skipping {} - does not exist'.format(campath))
# continue
#
# tmap = load_campath(campath) # file:trial
# n_tmap = len(tmap)
# log.info('loading tracking data for {} trials'.format(n_tmap))
i = 0
VideoTrialNum = df_wr_row[1].VideoTrialNum
#tpath = pathlib.Path(tdat, h2o, VideoDate, 'tracking')
ppp = list(range(0,VideoTrialNum))
for tt in reversed(range(VideoTrialNum)): # load tracking for trial
i += 1
# if i % 50 == 0:
# log.info('item {}/{}, trial #{} ({:.2f}%)'
# .format(i, n_tmap, t, (i/n_tmap)*100))
# else:
# log.debug('item {}/{}, trial #{} ({:.2f}%)'
# .format(i, n_tmap, t, (i/n_tmap)*100))
# ex: dl59_side_1-0000.csv / h2o_position_tn-0000.csv
tfile = '{}_{}_{}_{}-*.csv'.format(h2o, VideoDate ,tpos, tt)
tfull = list(tdat.glob(tfile))
if not tfull or len(tfull) > 1:
log.info('file mismatch: file: {} trial: ({})'.format(
tt, tfull))
continue
tfull = tfull[-1]
trk = load_tracking(tfull)
recs = {}
#key_source = experiment.Session - tracking.Tracking
rec_base = dict(trial=ppp[tt], tracking_device=tdev)
#print(rec_base)
for k in trk:
if k == 'samples':
recs['tracking'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**rec_base,
'tracking_samples': len(trk['samples']['ts']),
}
else:
rec = dict(rec_base)
for attr in trk[k]:
rec_key = '{}_{}'.format(k, attr)
rec[rec_key] = np.array(trk[k][attr])
recs[k] = rec
tracking.Tracking.insert1(
recs['tracking'], allow_direct_insert=True)
#if len(recs['nose']) > 3000:
#continue
recs['nose'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['nose'],
}
#print(recs['nose']['nose_x'])
if 'nose' in recs:
tracking.Tracking.NoseTracking.insert1(
recs['nose'], allow_direct_insert=True)
recs['tongue_mid'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['tongue_mid'],
}
if 'tongue_mid' in recs:
tracking.Tracking.TongueTracking.insert1(
recs['tongue_mid'], allow_direct_insert=True)
recs['jaw'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['jaw'],
}
if 'jaw' in recs:
tracking.Tracking.JawTracking.insert1(
recs['jaw'], allow_direct_insert=True)
recs['tongue_left'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['tongue_left'],
}
if 'tongue_left' in recs:
tracking.Tracking.LeftTongueTracking.insert1(
recs['tongue_left'], allow_direct_insert=True)
recs['tongue_right'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['tongue_right'],
}
if 'tongue_right' in recs:
tracking.Tracking.RightTongueTracking.insert1(
recs['tongue_right'], allow_direct_insert=True)
# fmap = {'paw_left_x': 'left_paw_x', # remap field names
# 'paw_left_y': 'left_paw_y',
# 'paw_left_likelihood': 'left_paw_likelihood'}
# tracking.Tracking.LeftPawTracking.insert1({
# **{k: v for k, v in recs['paw_left'].items()
# if k not in fmap},
# **{fmap[k]: v for k, v in recs['paw_left'].items()
# if k in fmap}}, allow_direct_insert=True)
recs['right_lickport'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['right_lickport'],
}
if 'right_lickport' in recs:
tracking.Tracking.RightLickPortTracking.insert1(
recs['right_lickport'], allow_direct_insert=True)
# fmap = {'paw_right_x': 'right_paw_x', # remap field names
# 'paw_right_y': 'right_paw_y',
# 'paw_right_likelihood': 'right_paw_likelihood'}
#
# tracking.Tracking.RightPawTracking.insert1({
# **{k: v for k, v in recs['paw_right'].items()
# if k not in fmap},
# **{fmap[k]: v for k, v in recs['paw_right'].items()
# if k in fmap}}, allow_direct_insert=True)
recs['left_lickport'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['left_lickport'],
}
if 'left_lickport' in recs:
tracking.Tracking.LeftLickPortTracking.insert1(
recs['left_lickport'], allow_direct_insert=True)
# tracking_files.append({**key, 'trial': tmap[t], 'tracking_device': tdev,
# 'tracking_file': str(tfull.relative_to(tdat))})
#
# log.info('... completed {}/{} items.'.format(i, n_tmap))
#
# self.insert1(key)
# self.TrackingFile.insert(tracking_files)
#
tracking.VideoFiducialsTrial.populate()
bottom_tongue.Camera_pixels.populate()
print('start!')
bottom_tongue.VideoTongueTrial.populate()
sessiontrialdata={ 'subject_id':skey['subject_id'],
'session':skey['session'],
'trial': tt
}
if len(bottom_tongue.VideoTongueTrial* experiment.Session & experiment.BehaviorTrial & 'session_date = "'+str(file['session_date'])+'"' &{'trial':tt})==0:
print('trial couldn''t be exported, deleting trial')
print(tt)
dj.config['safemode'] = False
(experiment.SessionTrial()&sessiontrialdata).delete()
dj.config['safemode'] = True
log.info('... done.')
def make(self, key):
log.info('BehaviorIngest.make(): key: {key}'.format(key=key))
subject_id = key['subject_id']
h2o = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
date = key['session_date']
datestr = date.strftime('%Y%m%d')
log.info('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr))
# session record key
skey = {}
skey['subject_id'] = subject_id
skey['session_date'] = date
skey['username'] = self.get_session_user()
# File paths conform to the pattern:
# dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat
# which is, more generally:
# {h2o}/{training_protocol}/Session Data/{h2o}_{training protocol}_{YYYYMMDD}_{HHMMSS}.mat
root = pathlib.Path(key['rig_data_path'],
os.path.dirname(key['subpath']))
path = root / '{h2o}_*_{d}*.mat'.format(h2o=h2o, d=datestr)
log.info('rigpath {p}'.format(p=path))
matches = sorted(
root.glob('{h2o}_*_{d}*.mat'.format(h2o=h2o, d=datestr)))
if matches:
log.info('found files: {}, this is the rig'.format(matches))
skey['rig'] = key['rig']
else:
log.info('no file matches found in {p}'.format(p=path))
if not len(matches):
log.warning('no file matches found for {h2o} / {d}'.format(
h2o=h2o, d=datestr))
return
#
# Find files & Check for split files
# XXX: not checking rig.. 2+ sessions on 2+ rigs possible for date?
#
if len(matches) > 1:
log.warning(
'split session case detected for {h2o} on {date}'.format(
h2o=h2o, date=date))
# session:date relationship is 1:1; skip if we have a session
if experiment.Session() & skey:
log.warning("Warning! session exists for {h2o} on {d}".format(
h2o=h2o, d=date))
return
#
# Prepare PhotoStim
#
photosti_duration = 0.5 # (s) Hard-coded here
photostims = {
4: {
'photo_stim': 4,
'photostim_device': 'OBIS470',
'brain_location_name': 'left_alm',
'duration': photosti_duration
},
5: {
'photo_stim': 5,
'photostim_device': 'OBIS470',
'brain_location_name': 'right_alm',
'duration': photosti_duration
},
6: {
'photo_stim': 6,
'photostim_device': 'OBIS470',
'brain_location_name': 'both_alm',
'duration': photosti_duration
}
}
#
# Extract trial data from file(s) & prepare trial loop
#
trials = zip()
trial = namedtuple( # simple structure to track per-trial vars
'trial',
('ttype', 'stim', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times'))
for f in matches:
if os.stat(f).st_size / 1024 < 1000:
log.info('skipping file {f} - too small'.format(f=f))
continue
log.debug('loading file {}'.format(f))
mat = spio.loadmat(f, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all(
(x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps))))
if 'StimTrials' in SessionData.dtype.fields:
log.debug('StimTrials detected in session - will include')
AllStimTrials = SessionData['StimTrials'][0]
assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('StimTrials not detected in session - will skip')
AllStimTrials = np.array([
None for i in enumerate(range(AllStateTimestamps.shape[0]))
])
z = zip(AllTrialTypes, AllStimTrials, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData,
AllEventData, AllEventTimestamps)
trials = chain(trials, z) # concatenate the files
trials = list(trials)
# all files were internally invalid or size < 100k
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
return
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
log.debug('synthesizing session ID')
session = (dj.U().aggr(experiment.Session() & {
'subject_id': subject_id
},
n='max(session)').fetch1('n') or 0) + 1
log.info('generated session id: {session}'.format(session=session))
skey['session'] = session
key = dict(key, **skey)
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'corrected_trial_event', 'action_event', 'photostim',
'photostim_location', 'photostim_trial',
'photostim_trial_event')
}
i = -1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart
# PreSamplePeriod
# SamplePeriod
# DelayPeriod
# EarlyLickDelay
# EarlyLickSample
# ResponseCue
# GiveLeftDrop
# GiveRightDrop
# GiveLeftDropShort
# GiveRightDropShort
# AnswerPeriod
# Reward
# RewardConsumption
# NoResponse
# TimeOut
# StopLicking
# StopLickingReturn
# TrialEnd
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.warning('skipping trial {i}; missing {m}'.format(
i=i, m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.warning(
'skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.warning('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.warning(
'skipping trial {i}: start/end index error: {s}/{e}'.
format(i=i, s=str(startindex), e=str(endindex)))
continue
try:
tkey['trial'] = i
tkey[
'trial_uid'] = i # Arseny has unique id to identify some trials
tkey['start_time'] = t.state_times[startindex][0]
tkey['stop_time'] = t.state_times[endindex][0]
except IndexError:
log.warning(
'skipping trial {i}: error indexing {s}/{e} into {t}'.
format(i=i,
s=str(startindex),
e=str(endindex),
t=str(t.state_times)))
continue
log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type > 5 and
('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states
and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning(
'... trial {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning(
'... trial {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)
]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)
]
# Photostim Events
#
# TODO:
#
# - base stimulation parameters:
#
# - should be loaded elsewhere - where
# - actual ccf locations - cannot be known apriori apparently?
# - Photostim.Profile: what is? fix/add
#
# - stim data
#
# - how retrieve power from file (didn't see) or should
# be statically coded here?
# - how encode stim type 6?
# - we have hemisphere as boolean or
# - but adding an event 4 and event 5 means querying
# is less straightforwrard (e.g. sessions with 5 & 6)
if t.stim:
log.info('BehaviorIngest.make(): t.stim == {}'.format(t.stim))
rows['photostim_trial'].append(tkey)
delay_period_idx = np.where(
t.state_data == states['DelayPeriod'])[0][0]
rows['photostim_trial_event'].append(
dict(tkey,
**photostims[t.stim],
photostim_event_id=len(rows['photostim_trial_event']),
photostim_event_time=t.state_times[delay_period_idx],
power=5.5))
# end of trial loop.
# Session Insertion
log.info('BehaviorIngest.make(): adding session record')
experiment.Session().insert1(skey)
# Behavior Insertion
log.info('BehaviorIngest.make(): bulk insert phase')
log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key))
self.insert1(key, ignore_extra_fields=True, allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(rows['trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(rows['behavior_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(rows['trial_note'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True,
skip_duplicates=True)
log.info('BehaviorIngest.make(): ... CorrectedTrialEvents')
BehaviorIngest().CorrectedTrialEvents().insert(
rows['corrected_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(rows['action_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
BehaviorIngest.BehaviorFile().insert(
(dict(key, behavior_file=f.name) for f in matches),
ignore_extra_fields=True,
allow_direct_insert=True)
# Photostim Insertion
photostim_ids = set(
[r['photo_stim'] for r in rows['photostim_trial_event']])
if photostim_ids:
log.info('BehaviorIngest.make(): ... experiment.Photostim')
experiment.Photostim.insert(
(dict(skey, **photostims[stim]) for stim in photostim_ids),
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrial')
experiment.PhotostimTrial.insert(rows['photostim_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrialEvent')
experiment.PhotostimEvent.insert(rows['photostim_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)