def read_bitcode(bitcode_dir, h2o, skey):
"""
Load bitcode file from specified dir - example bitcode format: e.g. 'SC022_030319_Imec3_bitcode.mat'
:return: sync_behav, sync_ephys, trial_fix, trial_go, trial_start
"""
bitcode_dir = pathlib.Path(bitcode_dir)
try:
bf_path = next(bitcode_dir.glob('{}_*bitcode.mat'.format(h2o)))
except StopIteration:
raise FileNotFoundError('No bitcode for {} found in {}'.format(
h2o, bitcode_dir))
log.info('.... loading bitcode file: {}'.format(str(bf_path)))
bf = spio.loadmat(str(bf_path))
trial_start = bf['sTrig'].flatten() # trial start
trial_go = bf['goCue'].flatten() # trial go cues
# check if there are `FreeWater` trials (i.e. no trial_go), if so, set those with trial_go value of NaN
if len(trial_go) < len(trial_start):
if len(experiment.BehaviorTrial & skey) != len(trial_start):
raise BitCodeError(
'Mismatch sTrig ({} elements) and total behavior trials ({} trials)'
.format(len(trial_start),
len(experiment.BehaviorTrial & skey)))
if len(experiment.BehaviorTrial & skey
& 'free_water = 0') != len(trial_go):
raise BitCodeError(
'Mismatch goCue ({} elements) and non-FreeWater trials ({} trials)'
.format(
len(trial_go),
len(experiment.BehaviorTrial & skey & 'free_water = 0')))
all_tr = (experiment.BehaviorTrial & skey).fetch('trial',
order_by='trial')
no_free_water_tr = (experiment.BehaviorTrial & skey
& 'free_water = 0').fetch('trial',
order_by='trial')
is_go_trial = np.in1d(all_tr, no_free_water_tr)
trial_go_full = np.full_like(trial_start, np.nan)
trial_go_full[is_go_trial] = trial_go
trial_go = trial_go_full
sync_ephys = bf['bitCodeS'] # ephys sync codes
sync_behav = (
experiment.TrialNote() # behavior sync codes
& {
**skey, 'trial_note_type': 'bitcode'
}).fetch('trial_note', order_by='trial')
trial_fix = bf['trialNum'].flatten() if 'trialNum' in bf else None
return sync_behav, sync_ephys, trial_fix, trial_go, trial_start
def _load_v4(self, sinfo, rigpath, dpath, fpath):
'''
Ephys data loader for JRClust v4 files.
Arguments:
- sinfo: lab.WaterRestriction * lab.Subject * experiment.Session
- rigpath: rig path root
- dpath: expanded rig data path (rigpath/h2o/YYYY-MM-DD)
- fpath: file path under dpath
'''
h2o = sinfo['water_restriction_number']
skey = {
k: v
for k, v in sinfo.items() if k in experiment.Session.primary_key
}
probe = fpath.parts[0]
imec = 'Imec{}'.format(int(probe) - 1) # probe key substring
ef_path = pathlib.Path(dpath, fpath)
log.info('.. jrclust v4 data load:')
log.info('.... sinfo: {}'.format(sinfo))
log.info('.... probe: {}'.format(probe))
log.info('.... loading ef_path: {}'.format(str(ef_path)))
ef = h5py.File(str(pathlib.Path(dpath, fpath))) # ephys file
# bitcode path (ex: 'SC022_030319_Imec3_bitcode.mat')
bf_path = list(
pathlib.Path(dpath, probe).glob('{}_*bitcode.mat'.format(h2o)))[0]
log.info('.... loading bf_path: {}'.format(str(bf_path)))
bf = spio.loadmat(str(bf_path))
# extract unit data
hz = bf['{}_SR'.format(imec)][0][0] # sampling rate
spikes = ef['spikeTimes'][0] # spikes times
spike_sites = ef['spikeSites'][0] # spike electrode
units = ef['spikeClusters'][0] # spike:unit id
unit_wav = ef['meanWfLocalRaw'] # waveform
unit_notes = ef['clusterNotes'] # curation notes
unit_notes = self._decode_notes(ef, unit_notes[:].flatten())
unit_xpos = ef['clusterCentroids'][0] # x position
unit_ypos = ef['clusterCentroids'][1] # y position
unit_amp = ef['unitVppRaw'][0] # amplitude
unit_snr = ef['unitSNR'][0] # signal to noise
vmax_unit_site = ef['clusterSites'] # max amplitude site
vmax_unit_site = np.array(vmax_unit_site[:].flatten(), dtype=np.int64)
start_idx, go_idx = (s.format(imec) for s in ('sTrig{}', 'goCue{}'))
trial_start = bf[start_idx].flatten() - 7500 # trial start
trial_go = bf[go_idx].flatten() # trial go cues
sync_ephys = bf['bitCodeS'] # ephys sync codes
sync_behav = (
experiment.TrialNote() # behavior sync codes
& {
**skey, 'trial_note_type': 'bitcode'
}).fetch('trial_note', order_by='trial')
trial_fix = bf['trialNum'] if 'trialNum' in bf else None
data = {
'sinfo': sinfo,
'rigpath': rigpath,
'ef_path': ef_path,
'probe': probe,
'skey': skey,
'method': 'jrclust_v4',
'hz': hz,
'spikes': spikes,
'spike_sites': spike_sites,
'units': units,
'unit_wav': unit_wav,
'unit_notes': unit_notes,
'unit_xpos': unit_xpos,
'unit_ypos': unit_ypos,
'unit_amp': unit_amp,
'unit_snr': unit_snr,
'vmax_unit_site': vmax_unit_site,
'trial_start': trial_start,
'trial_go': trial_go,
'sync_ephys': sync_ephys,
'sync_behav': sync_behav,
'trial_fix': trial_fix,
}
return data
def _load_v3(self, sinfo, rigpath, dpath, fpath):
'''
Ephys data loader for JRClust v4 files.
Arguments:
- sinfo: lab.WaterRestriction * lab.Subject * experiment.Session
- rigpath: rig path root
- dpath: expanded rig data path (rigpath/h2o/YYYY-MM-DD)
- fpath: file path under dpath
Returns:
- tbd
'''
h2o = sinfo['water_restriction_number']
skey = {
k: v
for k, v in sinfo.items() if k in experiment.Session.primary_key
}
probe = fpath.parts[0]
ef_path = pathlib.Path(dpath, fpath)
bf_path = pathlib.Path(dpath, probe, '{}_bitcode.mat'.format(h2o))
log.info('.. jrclust v3 data load:')
log.info('.... sinfo: {}'.format(sinfo))
log.info('.... probe: {}'.format(probe))
log.info('.... loading ef_path: {}'.format(str(ef_path)))
ef = h5py.File(str(pathlib.Path(dpath, fpath))) # ephys file
log.info('.... loading bf_path: {}'.format(str(bf_path)))
bf = spio.loadmat(
pathlib.Path(dpath, probe,
'{}_bitcode.mat'.format(h2o))) # bitcode file
# extract unit data
hz = ef['P']['sRateHz'][0][0] # sampling rate
spikes = ef['viTime_spk'][0] # spike times
spike_sites = ef['viSite_spk'][0] # spike electrode
units = ef['S_clu']['viClu'][0] # spike:unit id
unit_wav = ef['S_clu']['trWav_raw_clu'] # waveform
unit_notes = ef['S_clu']['csNote_clu'][0] # curation notes
unit_notes = self._decode_notes(ef, unit_notes)
unit_xpos = ef['S_clu']['vrPosX_clu'][0] # x position
unit_ypos = ef['S_clu']['vrPosY_clu'][0] # y position
unit_amp = ef['S_clu']['vrVpp_uv_clu'][0] # amplitude
unit_snr = ef['S_clu']['vrSnr_clu'][0] # signal to noise
vmax_unit_site = ef['S_clu']['viSite_clu'] # max amplitude site
vmax_unit_site = np.array(vmax_unit_site[:].flatten(), dtype=np.int64)
trial_start = bf['sTrig'].flatten() - 7500 # start of trials
trial_go = bf['goCue'].flatten() # go cues
sync_ephys = bf['bitCodeS'].flatten() # ephys sync codes
sync_behav = (
experiment.TrialNote() # behavior sync codes
& {
**skey, 'trial_note_type': 'bitcode'
}).fetch('trial_note', order_by='trial')
trial_fix = bf['trialNum'] if 'trialNum' in bf else None
data = {
'sinfo': sinfo,
'rigpath': rigpath,
'ef_path': ef_path,
'probe': probe,
'skey': skey,
'method': 'jrclust',
'hz': hz,
'spikes': spikes,
'spike_sites': spike_sites,
'units': units,
'unit_wav': unit_wav,
'unit_notes': unit_notes,
'unit_xpos': unit_xpos,
'unit_ypos': unit_ypos,
'unit_amp': unit_amp,
'unit_snr': unit_snr,
'vmax_unit_site': vmax_unit_site,
'trial_start': trial_start,
'trial_go': trial_go,
'sync_ephys': sync_ephys,
'sync_behav': sync_behav,
'trial_fix': trial_fix,
}
return data
def make(self, key):
log.info('EphysIngest().make(): key: {k}'.format(k=key))
#
# Find Ephys Recording
#
key = (experiment.Session & key).fetch1()
rigpath = EphysDataPath().fetch1('data_path')
date = key['session_date'].strftime('%Y-%m-%d')
subject_id = key['subject_id']
water = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
file = '{h2o}ap_imec3_opt3_jrc.mat'.format(
h2o=water) # current file naming format
# file = '{h2o}_g0_t0.imec.ap_imec3_opt3_jrc.mat'.format(h2o=water) # some older files
# subpath = os.path.join('Spike', date, file)
fullpath = os.path.join(rigpath, date, file)
if not os.path.exists(fullpath):
log.info('EphysIngest().make(): skipping - no file in %s' %
fullpath)
return
log.info('EphysIngest().make(): found ephys recording in %s' %
fullpath)
#
# Find corresponding BehaviorIngest
#
# ... we are keying times, sessions, etc from behavior ingest;
# so lookup behavior ingest for session id, quit with warning otherwise
try:
behavior = (ingestBehavior.BehaviorIngest() & key).fetch1()
except dj.DataJointError:
log.warning('EphysIngest().make(): skip - behavior ingest error')
return
log.info('behavior for ephys: {b}'.format(b=behavior))
#
# Prepare ElectrodeGroup configuration
#
# HACK / TODO: assuming single specific ElectrodeGroup for all tests;
# better would be to have this encoded in filename or similar.
ekey = {
'subject_id': behavior['subject_id'],
'session': behavior['session'],
'electrode_group': 1,
}
log.debug('inserting electrode group')
ephys.ElectrodeGroup().insert1(dict(ekey, probe_part_no=15131808323))
ephys.ElectrodeGroup().make(ekey) # note: no locks; is dj.Manual
log.debug('extracting spike data')
f = h5py.File(fullpath, 'r')
ind = np.argsort(f['S_clu']['viClu'][0]) # index sorted by cluster
cluster_ids = f['S_clu']['viClu'][0][ind] # cluster (unit) number
ind = ind[np.where(
cluster_ids > 0)[0]] # get rid of the -ve noise clusters
cluster_ids = cluster_ids[np.where(
cluster_ids > 0)[0]] # get rid of the -ve noise clusters
trWav_raw_clu = f['S_clu']['trWav_raw_clu'] # spike waveform
# trWav_raw_clu1 = np.concatenate((trWav_raw_clu[0:1][:][:],trWav_raw_clu),axis=0) # add a spike waveform to cluster 0, not necessary anymore after the previous step
csNote_clu = f['S_clu']['csNote_clu'][0] # manual sorting note
strs = ["all" for x in range(len(csNote_clu))
] # all units are "all" by definition
for iU in range(
0, len(csNote_clu)): # read the manual curation of each unit
log.debug('extracting spike indicators {s}:{u}'.format(
s=behavior['session'], u=iU))
unitQ = f[csNote_clu[iU]]
str1 = ''.join(chr(i) for i in unitQ[:])
if str1 == 'single': # definitions in unit quality
strs[iU] = 'good'
elif str1 == 'multi':
strs[iU] = 'multi'
spike_times = f['viTime_spk'][0][ind] # spike times
viSite_spk = f['viSite_spk'][0][ind] # electrode site for the spike
viT_offset_file = f[
'viT_offset_file'][:] # start of each trial, subtract this number for each trial
sRateHz = f['P']['sRateHz'][0] # sampling rate
spike_trials = np.ones(len(spike_times)) * (
len(viT_offset_file) - 1) # every spike is in the last trial
spike_times2 = np.copy(spike_times)
for i in range(len(viT_offset_file) - 1, 0,
-1): #find the trials each unit has a spike in
log.debug('locating trials with spikes {s}:{t}'.format(
s=behavior['session'], t=i))
spike_trials[spike_times < viT_offset_file[
i]] = i - 1 # Get the trial number of each spike
spike_times2[(spike_times >= viT_offset_file[i - 1]) & (
spike_times < viT_offset_file[i])] = spike_times[
(spike_times >= viT_offset_file[i - 1])
& (spike_times < viT_offset_file[i])] - viT_offset_file[
i - 1] # subtract the viT_offset_file from each trial
spike_times2[np.where(
spike_times2 >= viT_offset_file[-1])] = spike_times[np.where(
spike_times2 >= viT_offset_file[-1])] - viT_offset_file[
-1] # subtract the viT_offset_file from each trial
spike_times2 = spike_times2 / sRateHz # divide the sampling rate, sRateHz
clu_ids_diff = np.diff(cluster_ids) # where the units seperate
clu_ids_diff = np.where(
clu_ids_diff != 0)[0] + 1 # separate the spike_times
units = np.split(
spike_times, clu_ids_diff
) / sRateHz # sub arrays of spike_times for each unit (for ephys.Unit())
trialunits = np.split(
spike_trials,
clu_ids_diff) # sub arrays of spike_trials for each unit
unit_ids = np.arange(len(clu_ids_diff) + 1) # unit number
trialunits1 = [] # array of unit number (for ephys.Unit.UnitTrial())
trialunits2 = [] # array of trial number
for i in range(0, len(trialunits)): # loop through each unit
log.debug('aggregating trials with units {s}:{t}'.format(
s=behavior['session'], t=i))
trialunits2 = np.append(trialunits2, np.unique(
trialunits[i])) # add the trials that a unit is in
trialunits1 = np.append(trialunits1,
np.zeros(len(np.unique(trialunits[i]))) +
i) # add the unit numbers
log.debug(
'inserting units for session {s}'.format(s=behavior['session']))
ephys.Unit().insert(
list(
dict(ekey,
unit=x,
unit_uid=x,
unit_quality=strs[x],
spike_times=units[x],
waveform=trWav_raw_clu[x][0])
for x in unit_ids)) # batch insert the units
file = '{h2o}_bitcode.mat'.format(
h2o=water) # fetch the bitcode and realign
# subpath = os.path.join('Spike', date, file)
fullpath = os.path.join(rigpath, date, file)
log.debug('opening bitcode for {s} ({f})'.format(s=behavior['session'],
f=fullpath))
#pdb.set_trace()
mat = spio.loadmat(fullpath, squeeze_me=True) # load the bitcode file
bitCodeE = mat['bitCodeS'].flatten() # bitCodeS is the char variable
trialNote = experiment.TrialNote()
bitCodeB = (trialNote & {
'subject_id': ekey['subject_id']
} & {
'session': ekey['session']
} & {
'trial_note_type': 'bitcode'
}).fetch(
'trial_note',
order_by='trial') # fetch the bitcode from the behavior trialNote
if len(bitCodeB) < len(
bitCodeE
): # behavior file is shorter; e.g. seperate protocols were used; Bpod trials missing due to crash; session restarted
startB = np.where(bitCodeE == bitCodeB[0])[0]
elif len(bitCodeB) > len(
bitCodeE
): # behavior file is longer; e.g. only some trials are sorted, the bitcode.mat should reflect this; Sometimes SpikeGLX can skip a trial, I need to check the last trial
startE = np.where(bitCodeB == bitCodeE[0])[0]
startB = -startE
else:
startB = 0
startE = 0
log.debug('extracting trial unit information {s} ({f})'.format(
s=behavior['session'], f=fullpath))
trialunits2 = trialunits2 - startB # behavior has less trials if startB is +ve, behavior has more trials if startB is -ve
indT = np.where(trialunits2 > -1)[0] # get rid of the -ve trials
trialunits1 = trialunits1[indT]
trialunits2 = trialunits2[indT]
spike_trials = spike_trials - startB # behavior has less trials if startB is +ve, behavior has more trials if startB is -ve
indT = np.where(spike_trials > -1)[0] # get rid of the -ve trials
cluster_ids = cluster_ids[indT]
spike_times2 = spike_times2[indT]
viSite_spk = viSite_spk[indT]
spike_trials = spike_trials[indT]
trialunits = np.asarray(trialunits) # convert the list to an array
trialunits = trialunits - startB
# split units based on which trial they are in (for ephys.TrialSpikes())
trialPerUnit = np.copy(units) # list of trial index for each unit
for i in unit_ids: # loop through each unit, maybe this can be avoid?
log.debug('.. unit information {u}'.format(u=i))
indT = np.where(trialunits[i] > -1)[0] # get rid of the -ve trials
trialunits[i] = trialunits[i][indT]
units[i] = units[i][indT]
trialidx = np.argsort(trialunits[i]) # index of the sorted trials
trialunits[i] = np.sort(
trialunits[i]) # sort the trials for a given unit
trial_ids_diff = np.diff(
trialunits[i]) # where the trial index seperate
trial_ids_diff = np.where(trial_ids_diff != 0)[0] + 1
units[i] = units[i][
trialidx] # sort the spike times based on the trial mapping
units[i] = np.split(
units[i],
trial_ids_diff) # separate the spike_times based on trials
trialPerUnit[i] = np.arange(0, len(trial_ids_diff) + 1,
dtype=int) # list of trial index
log.debug('inserting UnitTrial information')
ephys.Unit.UnitTrial().insert(
list(
dict(ekey, unit=trialunits1[x], trial=trialunits2[x])
for x in range(0, len(trialunits2)))
) # batch insert the TrialUnit (key, unit, trial)
log.debug('inserting UnitSpike information')
ephys.Unit.UnitSpike().insert(
list(
dict(ekey,
unit=cluster_ids[x] - 1,
spike_time=spike_times2[x],
electrode=viSite_spk[x],
trial=spike_trials[x])
for x in range(0, len(spike_times2))),
skip_duplicates=True
) # batch insert the Spikes (key, unit, spike_time, electrode, trial)
# TODO: 2D batch insert
# pdb.set_trace()
l = [] # list of trialSpikes to be inserted
for x in zip(unit_ids, trialPerUnit): # loop through the units
for i in x[1]: # loop through the trials for each unit
l.append(
dict(ekey,
unit=x[0],
trial=int(trialunits2[x[1]][i]),
spike_times=units[x[0]][x[1][i]])) # create the list
ephys.TrialSpikes().insert(
l, skip_duplicates=True) # batch insert TrialSpikes
log.debug('inserting file load information')
self.insert1(key, ignore_extra_fields=True)
EphysIngest.EphysFile().insert1(dict(key, ephys_file=fullpath),
ignore_extra_fields=True)
def make(self, key):
log.info('BehaviorIngest.make(): key: {key}'.format(key=key))
subject_id = key['subject_id']
h2o = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
ymd = key['session_date']
datestr = ymd.strftime('%Y%m%d')
log.info('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr))
# session record key
skey = {}
skey['subject_id'] = subject_id
skey['session_date'] = ymd
skey['username'] = self.get_session_user()
skey['rig'] = key['rig']
# File paths conform to the pattern:
# dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat
# which is, more generally:
# {h2o}/{training_protocol}/Session Data/{h2o}_{training protocol}_{YYYYMMDD}_{HHMMSS}.mat
path = pathlib.Path(key['rig_data_path'], key['subpath'])
if experiment.Session() & skey:
log.info("note: session exists for {h2o} on {d}".format(h2o=h2o,
d=ymd))
trial = namedtuple( # simple structure to track per-trial vars
'trial',
('ttype', 'stim', 'free', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times', 'trial_start'))
if os.stat(path).st_size / 1024 < 1000:
log.info('skipping file {} - too small'.format(path))
return
log.debug('loading file {}'.format(path))
mat = spio.loadmat(path, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
# parse session datetime
session_datetime_str = str('').join(
(str(SessionData['Info'][0]['SessionDate']), ' ',
str(SessionData['Info'][0]['SessionStartTime_UTC'])))
session_datetime = datetime.strptime(session_datetime_str,
'%d-%b-%Y %H:%M:%S')
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
AllTrialStarts = SessionData['TrialStartTimestamp'][0]
AllTrialStarts = AllTrialStarts - AllTrialStarts[0] # real 1st trial
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all((x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps,
AllTrialStarts, AllTrialStarts))))
# AllStimTrials optional special case
if 'StimTrials' in SessionData.dtype.fields:
log.debug('StimTrials detected in session - will include')
AllStimTrials = SessionData['StimTrials'][0]
assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('StimTrials not detected in session - will skip')
AllStimTrials = np.array(
[None for _ in enumerate(range(AllStateTimestamps.shape[0]))])
# AllFreeTrials optional special case
if 'FreeTrials' in SessionData.dtype.fields:
log.debug('FreeTrials detected in session - will include')
AllFreeTrials = SessionData['FreeTrials'][0]
assert (AllFreeTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('FreeTrials not detected in session - synthesizing')
AllFreeTrials = np.zeros(AllStateTimestamps.shape[0],
dtype=np.uint8)
trials = list(
zip(AllTrialTypes, AllStimTrials, AllFreeTrials, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData, AllEventData,
AllEventTimestamps, AllTrialStarts))
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
return
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
assert skey['session_date'] == session_datetime.date()
skey['session_date'] = session_datetime.date()
skey['session_time'] = session_datetime.time()
log.debug('synthesizing session ID')
session = (dj.U().aggr(experiment.Session()
& {
'subject_id': subject_id
},
n='max(session)').fetch1('n') or 0) + 1
log.info('generated session id: {session}'.format(session=session))
skey['session'] = session
key = dict(key, **skey)
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'corrected_trial_event', 'action_event', 'photostim',
'photostim_location', 'photostim_trial',
'photostim_trial_event')
}
i = 0 # trial numbering starts at 1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart, PreSamplePeriod, SamplePeriod, DelayPeriod
# EarlyLickDelay, EarlyLickSample, ResponseCue, GiveLeftDrop
# GiveRightDrop, GiveLeftDropShort, GiveRightDropShort
# AnswerPeriod, Reward, RewardConsumption, NoResponse
# TimeOut, StopLicking, StopLickingReturn, TrialEnd
#
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.warning('skipping trial {i}; missing {m}'.format(
i=i, m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.warning(
'skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.warning('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.warning('skipping {}: start/end mismatch: {}/{}'.format(
i, str(startindex), str(endindex)))
continue
try:
tkey['trial'] = i
tkey['trial_uid'] = i
tkey['start_time'] = t.trial_start
tkey['stop_time'] = t.trial_start + t.state_times[endindex][0]
except IndexError:
log.warning('skipping {}: IndexError: {}/{} -> {}'.format(
i, str(startindex), str(endindex), str(t.state_times)))
continue
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type >= 5 and
('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states
and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
# Determine free/autowater (Autowater 1 == enabled, 2 == disabled)
bkey['auto_water'] = True if gui['Autowater'][0] == 1 else False
bkey['free_water'] = t.free
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning(
'... trial {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning(
'... trial {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)
]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)
]
#
# Photostim Events
#
if t.stim:
log.debug('BehaviorIngest.make(): t.stim == {}'.format(t.stim))
rows['photostim_trial'].append(tkey)
delay_period_idx = np.where(
t.state_data == states['DelayPeriod'])[0][0]
rows['photostim_trial_event'].append(
dict(tkey,
photo_stim=t.stim,
photostim_event_id=len(rows['photostim_trial_event']),
photostim_event_time=t.state_times[delay_period_idx],
power=5.5))
# end of trial loop.
# Session Insertion
log.info('BehaviorIngest.make(): adding session record')
experiment.Session().insert1(skey)
# Behavior Insertion
log.info('BehaviorIngest.make(): bulk insert phase')
log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key))
self.insert1(key, ignore_extra_fields=True, allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(rows['trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(rows['behavior_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(rows['trial_note'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True,
skip_duplicates=True)
log.info('BehaviorIngest.make(): ... CorrectedTrialEvents')
BehaviorIngest().CorrectedTrialEvents().insert(
rows['corrected_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(rows['action_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
# Photostim Insertion
photostim_ids = np.unique(
[r['photo_stim'] for r in rows['photostim_trial_event']])
unknown_photostims = np.setdiff1d(photostim_ids,
list(photostims.keys()))
if unknown_photostims:
raise ValueError(
'Unknown photostim protocol: {}'.format(unknown_photostims))
if photostim_ids.size > 0:
log.info('BehaviorIngest.make(): ... experiment.Photostim')
for stim in photostim_ids:
experiment.Photostim.insert1(dict(skey, **photostims[stim]),
ignore_extra_fields=True)
experiment.Photostim.PhotostimLocation.insert(
(dict(
skey, **loc, photo_stim=photostims[stim]['photo_stim'])
for loc in photostims[stim]['locations']),
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrial')
experiment.PhotostimTrial.insert(rows['photostim_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrialEvent')
experiment.PhotostimEvent.insert(rows['photostim_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
# Behavior Ingest Insertion
log.info('BehaviorIngest.make(): ... BehaviorIngest.BehaviorFile')
BehaviorIngest.BehaviorFile().insert1(dict(
key, behavior_file=os.path.basename(key['subpath'])),
ignore_extra_fields=True,
allow_direct_insert=True)
def make(self, key):
log.info('BehaviorIngest.make(): key: {key}'.format(key=key))
rigpaths = [
p for p in RigDataPath().fetch(order_by='rig_data_path')
if 'RRig' in p['rig']
] # change between TRig and RRig
subject_id = key['subject_id']
h2o = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
date = key['session_date']
datestr = date.strftime('%Y%m%d')
log.debug('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr))
# session record key
skey = {}
skey['subject_id'] = subject_id
skey['session_date'] = date
skey['username'] = '******' # username has to be changed
# e.g: dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat
# # p.split('/foo/bar')[1]
for rp in rigpaths:
root = rp['rig_data_path']
path = root
path = os.path.join(path, h2o)
# path = os.path.join(path, 'TW_autoTrain')
path = os.path.join(path, 'tw2')
path = os.path.join(path, 'Session Data')
path = os.path.join(
# path, '{h2o}_TW_autoTrain_{d}*.mat'.format(h2o=h2o, d=datestr)) # earlier program protocol
path,
'{h2o}_tw2_{d}*.mat'.format(
h2o=h2o, d=datestr)) # later program protocol
log.debug('rigpath {p}'.format(p=path))
matches = glob.glob(path)
if len(matches):
log.debug('found files, this is the rig')
skey['rig'] = rp['rig']
break
else:
log.info('no file matches found in {p}'.format(p=path))
if not len(matches):
log.warning('no file matches found for {h2o} / {d}'.format(
h2o=h2o, d=datestr))
return
#
# Find files & Check for split files
# XXX: not checking rig.. 2+ sessions on 2+ rigs possible for date?
#
if len(matches) > 1:
log.warning(
'split session case detected for {h2o} on {date}'.format(
h2o=h2o, date=date))
# session:date relationship is 1:1; skip if we have a session
if experiment.Session() & skey:
log.warning("Warning! session exists for {h2o} on {d}".format(
h2o=h2o, d=date))
return
#
# Extract trial data from file(s) & prepare trial loop
#
trials = zip()
trial = namedtuple( # simple structure to track per-trial vars
'trial', ('ttype', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times'))
for f in matches:
if os.stat(f).st_size / 1024 < 100:
log.info('skipping file {f} - too small'.format(f=f))
continue
mat = spio.loadmat(f, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all(
(x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps))))
z = zip(AllTrialTypes, AllTrialSettings, AllStateTimestamps,
AllStateNames, AllStateData, AllEventData,
AllEventTimestamps)
trials = chain(trials, z) # concatenate the files
trials = list(trials)
# all files were internally invalid or size < 100k
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
log.debug('synthesizing session ID')
session = (dj.U().aggr(experiment.Session() & {
'subject_id': subject_id
},
n='max(session)').fetch1('n') or 0) + 1
log.info('generated session id: {session}'.format(session=session))
skey['session'] = session
key = dict(key, **skey)
log.debug('BehaviorIngest.make(): adding session record')
experiment.Session().insert1(skey)
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'action_event')
}
i = -1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.info('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart
# PreSamplePeriod
# SamplePeriod
# DelayPeriod
# EarlyLickDelay
# EarlyLickSample
# ResponseCue
# GiveLeftDrop
# GiveRightDrop
# GiveLeftDropShort
# GiveRightDropShort
# AnswerPeriod
# Reward
# RewardConsumption
# NoResponse
# TimeOut
# StopLicking
# StopLickingReturn
# TrialEnd
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.info('skipping trial {i}; missing {m}'.format(i=i,
m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.info('skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.info('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endendex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.info('skipping trial {i}: start/end index error: {s}/{e}'.
format(i=i, s=str(startindex), e=str(endindex)))
continue
try:
tkey['trial'] = i
tkey['trial_uid'] = i
tkey['start_time'] = t.state_times[startindex][0]
except IndexError:
log.info('skipping trial {i}: error indexing {s}/{e} into {t}'.
format(i=i,
s=str(startindex),
e=str(endindex),
t=str(t.state_times)))
continue
log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay'
bkey['task_protocol'] = 1
# determine trial instruction
trial_instruction = 'left'
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type > 5 and
('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states
and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
# add behavior record
log.debug('BehaviorIngest.make(): BehaviorTrial()')
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
log.debug('BehaviorIngest.make(): TrialNote().insert1')
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#pdb.set_trace()
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
log.debug('BehaviorIngest.make(): presample')
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
log.debug('BehaviorIngest.make(): go')
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
log.debug('BehaviorIngest.make(): delay events')
for d in delayindex: # protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
if len(lickleft):
[
rows['action_event'].append(
dict(**tkey,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for l in lickleft
]
lickright = np.where(t.event_data == 70)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
if len(lickright):
[
rows['action_event'].append(
dict(**tkey,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for r in lickright
]
# end of trial loop.
log.info('BehaviorIngest.make(): bulk insert phase')
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(rows['trial'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(rows['behavior_trial'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(rows['trial_note'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(rows['action_event'],
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key))
self.insert1(key, ignore_extra_fields=True)
BehaviorIngest.BehaviorFile().insert(
(dict(key, behavior_file=f.split(root)[1]) for f in matches),
ignore_extra_fields=True)
def populatebehavior_core(IDs = None):
if IDs:
print('subject started:')
print(IDs.keys())
print(IDs.values())
rigpath_1 = 'E:/Projects/Ablation/datajoint/Behavior'
#df_surgery = pd.read_csv(dj.config['locations.metadata']+'Surgery.csv')
if IDs == None:
IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))}
for subject_now,subject_id_now in zip(IDs.keys(),IDs.values()): # iterating over subjects
print('subject: ',subject_now)
# =============================================================================
# if drop_last_session_for_mice_in_training:
# delete_last_session_before_upload = True
# else:
# delete_last_session_before_upload = False
# #df_wr = online_notebook.fetch_water_restriction_metadata(subject_now)
# =============================================================================
try:
df_wr = pd.read_csv(dj.config['locations.metadata_behavior']+subject_now+'.csv')
except:
print(subject_now + ' has no metadata available')
df_wr = pd.DataFrame()
for df_wr_row in df_wr.iterrows():
date_now = df_wr_row[1].Date.replace('-','')
print('subject: ',subject_now,' date: ',date_now)
session_date = datetime(int(date_now[0:4]),int(date_now[4:6]),int(date_now[6:8]))
if len(experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"' & 'session_date > "'+str(session_date)+'"') != 0: # if it is not the last
print('session already imported, skipping: ' + str(session_date))
dotheupload = False
elif len(experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"' & 'session_date = "'+str(session_date)+'"') != 0: # if it is the last
dotheupload = False
else: # reuploading new session that is not present on the server
dotheupload = True
# if dotheupload is True, meaning that there are new mat file hasn't been uploaded
# => needs to find which mat file hasn't been uploaded
if dotheupload:
found = set()
rigpath_2 = subject_now
rigpath_3 = rigpath_1 + '/' + rigpath_2
rigpath = pathlib.Path(rigpath_3)
def buildrec(rigpath, root, f):
try:
fullpath = pathlib.Path(root, f)
subpath = fullpath.relative_to(rigpath)
fsplit = subpath.stem.split('_')
h2o = fsplit[0]
ymd = fsplit[-2:-1][0]
animal = IDs[h2o]
if ymd == date_now:
return {
'subject_id': animal,
'session_date': date(int(ymd[0:4]), int(ymd[4:6]), int(ymd[6:8])),
'rig_data_path': rigpath.as_posix(),
'subpath': subpath.as_posix(),
}
except:
pass
for root, dirs, files in os.walk(rigpath):
for f in files:
r = buildrec(rigpath, root, f)
if r:
found.add(r['subpath'])
file = r
# now start insert data
path = pathlib.Path(file['rig_data_path'], file['subpath'])
mat = spio.loadmat(path, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
# session record key
skey = {}
skey['subject_id'] = file['subject_id']
skey['session_date'] = file['session_date']
skey['username'] = '******'
#skey['rig'] = key['rig']
trial = namedtuple( # simple structure to track per-trial vars
'trial', ('ttype', 'settings', 'state_times',
'state_names', 'state_data', 'event_data',
'event_times', 'trial_start'))
# parse session datetime
session_datetime_str = str('').join((str(SessionData['Info'][0]['SessionDate']),' ', str(SessionData['Info'][0]['SessionStartTime_UTC'])))
session_datetime = datetime.strptime(session_datetime_str, '%d-%b-%Y %H:%M:%S')
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
AllTrialStarts = SessionData['TrialStartTimestamp'][0]
AllTrialStarts = AllTrialStarts - AllTrialStarts[0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
trials = list(zip(AllTrialTypes, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData,
AllEventData, AllEventTimestamps, AllTrialStarts))
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
return
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
assert skey['session_date'] == session_datetime.date()
skey['session_date'] = session_datetime.date()
#skey['session_time'] = session_datetime.time()
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"' & 'session_date = "'+str(file['session_date'])+'"') == 0:
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"') == 0:
skey['session'] = 1
else:
skey['session'] = len((experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"').fetch()['session']) + 1
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {k: list() for k in ('trial', 'behavior_trial', 'trial_note',
'trial_event', 'corrected_trial_event',
'action_event')} #, 'photostim',
#'photostim_location', 'photostim_trial',
#'photostim_trial_event')}
i = 0 # trial numbering starts at 1
for t in trials:
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
states = {k: (v+1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing) and missing =='PreSamplePeriod':
log.warning('skipping trial {i}; missing {m}'.format(i=i, m=missing))
continue
gui = t.settings['GUI'].flatten()
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"' & 'session_date = "'+str(file['session_date'])+'"') == 0:
if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"') == 0:
skey['session'] = 1
else:
skey['session'] = len((experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"').fetch()['session']) + 1
#
# Top-level 'Trial' record
#
protocol_type = gui['ProtocolType'][0]
tkey = dict(skey)
has_presample = 1
try:
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
has_presample = 1
except:
startindex = np.where(t.state_data == states['SamplePeriod'])[0]
has_presample = 0
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not(len(startindex) and len(endindex)):
log.warning('skipping {}: start/end mismatch: {}/{}'.format(i, str(startindex), str(endindex)))
continue
try:
tkey['trial'] = i
tkey['trial_uid'] = i
tkey['trial_start_time'] = t.trial_start
tkey['trial_stop_time'] = t.trial_start + t.state_times[endindex][0]
except IndexError:
log.warning('skipping {}: IndexError: {}/{} -> {}'.format(i, str(startindex), str(endindex), str(t.state_times)))
continue
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif t.ttype == 2:
trial_instruction = 'catch_right_autowater'
elif t.ttype == 3:
trial_instruction = 'catch_left_autowater'
elif t.ttype == 4:
trial_instruction = 'catch_right_noDelay'
elif t.ttype == 5:
trial_instruction = 'catch_left_noDelay'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
elif t.ttype == 2:
trial_instruction = 'catch_left_autowater'
elif t.ttype == 3:
trial_instruction = 'catch_right_autowater'
elif t.ttype == 4:
trial_instruction = 'catch_left_noDelay'
elif t.ttype == 5:
trial_instruction = 'catch_right_noDelay'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type >= 5 and ('EarlyLickSample' in states and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
if has_presample == 1:
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]]- t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... trial {} bad duration, no last_edur'.format(i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... trial {} duration using last_edur {}'.format(i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[rows['action_event'].append(
dict(tkey, action_event_id=action_event_count+idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[rows['action_event'].append(
dict(tkey, action_event_id=action_event_count+idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)]
# end of trial loop..
# Session Insertion
log.info('BehaviorIngest.make(): adding session record')
skey['session_date'] = df_wr_row[1].Date
skey['rig'] = 'Old Recording rig'
skey['username'] = '******'
experiment.Session().insert1(skey,skip_duplicates=True)
# Behavior Insertion
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(
rows['trial'], ignore_extra_fields=True, allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(
rows['behavior_trial'], ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(
rows['trial_note'], ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(
rows['trial_event'], ignore_extra_fields=True,
allow_direct_insert=True, skip_duplicates=True)
# log.info('BehaviorIngest.make(): ... CorrectedTrialEvents')
# BehaviorIngest().CorrectedTrialEvents().insert(
# rows['corrected_trial_event'], ignore_extra_fields=True,
# allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(
rows['action_event'], ignore_extra_fields=True,
allow_direct_insert=True)
#%% for ingest tracking
if IDs:
print('subject started:')
print(IDs.keys())
print(IDs.values())
rigpath_tracking_1 = 'E:/Projects/Ablation/datajoint/video/'
rigpath_tracking_2 = subject_now
VideoDate1 = str(df_wr_row[1].VideoDate)
if len(VideoDate1)==5:
VideoDate = '0'+ VideoDate1
elif len(VideoDate1)==7:
VideoDate = '0'+ VideoDate1
rigpath_tracking_3 = rigpath_tracking_1 + rigpath_tracking_2 + '/' + rigpath_tracking_2 + '_'+ VideoDate + '_front'
rigpath_tracking = pathlib.Path(rigpath_tracking_3)
#df_surgery = pd.read_csv(dj.config['locations.metadata']+'Surgery.csv')
if IDs == None:
IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))}
h2o = subject_now
session = df_wr_row[1].Date
trials = (experiment.SessionTrial() & session).fetch('trial')
log.info('got session: {} ({} trials)'.format(session, len(trials)))
#sdate = session['session_date']
#sdate_sml = date_now #"{}{:02d}{:02d}".format(sdate.year, sdate.month, sdate.day)
paths = rigpath_tracking
devices = tracking.TrackingDevice().fetch(as_dict=True)
# paths like: <root>/<h2o>/YYYY-MM-DD/tracking
tracking_files = []
for d in (d for d in devices):
tdev = d['tracking_device']
tpos = d['tracking_position']
tdat = paths
log.info('checking {} for tracking data'.format(tdat))
# if not tpath.exists():
# log.warning('tracking path {} n/a - skipping'.format(tpath))
# continue
#
# camtrial = '{}_{}_{}.txt'.format(h2o, sdate_sml, tpos)
# campath = tpath / camtrial
#
# log.info('trying camera position trial map: {}'.format(campath))
#
# if not campath.exists():
# log.info('skipping {} - does not exist'.format(campath))
# continue
#
# tmap = load_campath(campath) # file:trial
# n_tmap = len(tmap)
# log.info('loading tracking data for {} trials'.format(n_tmap))
i = 0
VideoTrialNum = df_wr_row[1].VideoTrialNum
#tpath = pathlib.Path(tdat, h2o, VideoDate, 'tracking')
ppp = list(range(0,VideoTrialNum))
for tt in reversed(range(VideoTrialNum)): # load tracking for trial
i += 1
# if i % 50 == 0:
# log.info('item {}/{}, trial #{} ({:.2f}%)'
# .format(i, n_tmap, t, (i/n_tmap)*100))
# else:
# log.debug('item {}/{}, trial #{} ({:.2f}%)'
# .format(i, n_tmap, t, (i/n_tmap)*100))
# ex: dl59_side_1-0000.csv / h2o_position_tn-0000.csv
tfile = '{}_{}_{}_{}-*.csv'.format(h2o, VideoDate ,tpos, tt)
tfull = list(tdat.glob(tfile))
if not tfull or len(tfull) > 1:
log.info('file mismatch: file: {} trial: ({})'.format(
tt, tfull))
continue
tfull = tfull[-1]
trk = load_tracking(tfull)
recs = {}
#key_source = experiment.Session - tracking.Tracking
rec_base = dict(trial=ppp[tt], tracking_device=tdev)
#print(rec_base)
for k in trk:
if k == 'samples':
recs['tracking'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**rec_base,
'tracking_samples': len(trk['samples']['ts']),
}
else:
rec = dict(rec_base)
for attr in trk[k]:
rec_key = '{}_{}'.format(k, attr)
rec[rec_key] = np.array(trk[k][attr])
recs[k] = rec
tracking.Tracking.insert1(
recs['tracking'], allow_direct_insert=True)
#if len(recs['nose']) > 3000:
#continue
recs['nose'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['nose'],
}
#print(recs['nose']['nose_x'])
if 'nose' in recs:
tracking.Tracking.NoseTracking.insert1(
recs['nose'], allow_direct_insert=True)
recs['tongue_mid'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['tongue_mid'],
}
if 'tongue_mid' in recs:
tracking.Tracking.TongueTracking.insert1(
recs['tongue_mid'], allow_direct_insert=True)
recs['jaw'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['jaw'],
}
if 'jaw' in recs:
tracking.Tracking.JawTracking.insert1(
recs['jaw'], allow_direct_insert=True)
recs['tongue_left'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['tongue_left'],
}
if 'tongue_left' in recs:
tracking.Tracking.LeftTongueTracking.insert1(
recs['tongue_left'], allow_direct_insert=True)
recs['tongue_right'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['tongue_right'],
}
if 'tongue_right' in recs:
tracking.Tracking.RightTongueTracking.insert1(
recs['tongue_right'], allow_direct_insert=True)
# fmap = {'paw_left_x': 'left_paw_x', # remap field names
# 'paw_left_y': 'left_paw_y',
# 'paw_left_likelihood': 'left_paw_likelihood'}
# tracking.Tracking.LeftPawTracking.insert1({
# **{k: v for k, v in recs['paw_left'].items()
# if k not in fmap},
# **{fmap[k]: v for k, v in recs['paw_left'].items()
# if k in fmap}}, allow_direct_insert=True)
recs['right_lickport'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['right_lickport'],
}
if 'right_lickport' in recs:
tracking.Tracking.RightLickPortTracking.insert1(
recs['right_lickport'], allow_direct_insert=True)
# fmap = {'paw_right_x': 'right_paw_x', # remap field names
# 'paw_right_y': 'right_paw_y',
# 'paw_right_likelihood': 'right_paw_likelihood'}
#
# tracking.Tracking.RightPawTracking.insert1({
# **{k: v for k, v in recs['paw_right'].items()
# if k not in fmap},
# **{fmap[k]: v for k, v in recs['paw_right'].items()
# if k in fmap}}, allow_direct_insert=True)
recs['left_lickport'] = {
'subject_id' : skey['subject_id'],
'session' : skey['session'],
**recs['left_lickport'],
}
if 'left_lickport' in recs:
tracking.Tracking.LeftLickPortTracking.insert1(
recs['left_lickport'], allow_direct_insert=True)
# tracking_files.append({**key, 'trial': tmap[t], 'tracking_device': tdev,
# 'tracking_file': str(tfull.relative_to(tdat))})
#
# log.info('... completed {}/{} items.'.format(i, n_tmap))
#
# self.insert1(key)
# self.TrackingFile.insert(tracking_files)
#
tracking.VideoFiducialsTrial.populate()
bottom_tongue.Camera_pixels.populate()
print('start!')
bottom_tongue.VideoTongueTrial.populate()
sessiontrialdata={ 'subject_id':skey['subject_id'],
'session':skey['session'],
'trial': tt
}
if len(bottom_tongue.VideoTongueTrial* experiment.Session & experiment.BehaviorTrial & 'session_date = "'+str(file['session_date'])+'"' &{'trial':tt})==0:
print('trial couldn''t be exported, deleting trial')
print(tt)
dj.config['safemode'] = False
(experiment.SessionTrial()&sessiontrialdata).delete()
dj.config['safemode'] = True
log.info('... done.')
def make(self, key):
# import pdb; pdb.set_trace()
q_all_trial = experiment.SessionTrial & key
q_block = experiment.SessionBlock & key
q_hit = experiment.BehaviorTrial & key & 'outcome = "hit"'
q_miss = experiment.BehaviorTrial & key & 'outcome = "miss"'
q_auto_water = experiment.TrialNote & key & 'trial_note_type = "autowater"'
q_actual_finished = q_hit.proj() + q_miss.proj() - q_auto_water.proj(
) # Real finished trial = 'hit' or 'miss' but not 'autowater'
session_stats = {
'session_total_trial_num':
len(q_all_trial),
'session_block_num':
len(q_block),
'session_hit_num':
len(q_hit),
'session_miss_num':
len(q_miss),
'session_ignore_num':
len(experiment.BehaviorTrial & key & 'outcome = "ignore"'),
'session_early_lick_ratio':
len(experiment.BehaviorTrial & key & 'early_lick="early"') /
(len(q_hit) + len(q_miss)),
'session_autowater_num':
len(q_auto_water),
'session_pure_choices_num':
len(q_actual_finished)
}
if session_stats['session_total_trial_num'] > 0:
session_stats['session_length'] = float(
((experiment.SessionTrial() & key).fetch('stop_time')).max())
else:
session_stats['session_length'] = 0
# -- Double dipping ratio --
q_double_dipping = TrialStats & key & 'double_dipping = 1'
session_stats.update(
session_double_dipping_ratio_hit=len(q_double_dipping & q_hit) /
len(q_hit))
# Double dipping in missed trial is detected only for sessions later than the first day of using new lickport retraction logic
if (experiment.Session & key & 'session_date > "2020-08-11"'):
session_stats.update(
session_double_dipping_ratio_miss=len(q_double_dipping
& q_miss) / len(q_miss),
session_double_dipping_ratio=len(q_double_dipping
& q_actual_finished) /
len(q_actual_finished))
# -- Session-wise foraging efficiency and schedule stats (2lp only) --
if len(experiment.BehaviorTrial & key & 'task="foraging"'):
# Get reward rate (hit but not autowater) / (hit but not autowater + miss but not autowater)
q_pure_hit_num = q_hit.proj() - q_auto_water.proj()
reward_rate = len(q_pure_hit_num) / len(q_actual_finished)
q_actual_finished_reward_prob = (
experiment.SessionTrial *
experiment.SessionBlock.BlockTrial # Session-block-trial
* experiment.SessionBlock.
WaterPortRewardProbability # Block-trial-p_reward
& q_actual_finished) # Select finished trials
# Get reward probability (only pure finished trials)
p_Ls = (q_actual_finished_reward_prob & 'water_port="left"').fetch(
'reward_probability',
order_by='trial').astype(float) # Note 'order_by'!!!
p_Rs = (q_actual_finished_reward_prob
& 'water_port="right"').fetch(
'reward_probability', order_by='trial').astype(float)
# Recover actual random numbers
random_number_Ls = np.empty(len(q_all_trial))
random_number_Ls[:] = np.nan
random_number_Rs = random_number_Ls.copy()
rand_seed_starts = (experiment.TrialNote() & key
& 'trial_note_type="random_seed_start"').fetch(
'trial', 'trial_note', order_by='trial')
if len(rand_seed_starts[0]): # Random seed exists
for start_idx, start_seed in zip(
rand_seed_starts[0],
rand_seed_starts[1]): # For each pybpod session
# Must be exactly the same as the pybpod protocol
# https://github.com/hanhou/Foraging-Pybpod/blob/5e19e1d227657ed19e27c6e1221495e9f180c323/pybpod_protocols/Foraging_baptize_by_fire_new_lickport_retraction.py#L478
np.random.seed(int(start_seed))
random_number_L_this = np.random.uniform(0., 1.,
2000).tolist()
random_number_R_this = np.random.uniform(0., 1.,
2000).tolist()
# Fill in random numbers
random_number_Ls[
start_idx -
1:] = random_number_L_this[:len(random_number_Ls) -
start_idx + 1]
random_number_Rs[
start_idx -
1:] = random_number_R_this[:len(random_number_Rs) -
start_idx + 1]
# Select finished trials
actual_finished_idx = q_actual_finished.fetch(
'trial', order_by='trial') - 1
random_number_Ls = random_number_Ls[actual_finished_idx]
random_number_Rs = random_number_Rs[actual_finished_idx]
else: # No random seed (backward compatibility)
print(f'No random seeds for {key}')
random_number_Ls = None
random_number_Rs = None
# Compute foraging efficiency
for_eff_optimal, for_eff_optimal_random_seed = foraging_eff(
reward_rate, p_Ls, p_Rs, random_number_Ls, random_number_Rs)
# Reward schedule stats
if (SessionTaskProtocol
& key).fetch1('session_real_foraging'): # Real foraging
p_contrast = np.max([p_Ls, p_Rs], axis=0) / np.min(
[p_Ls, p_Rs], axis=0)
p_contrast[np.isinf(
p_contrast)] = np.nan # A arbitrary huge number
p_contrast_mean = np.nanmean(p_contrast)
else:
p_contrast_mean = 100
session_stats.update(session_foraging_eff_optimal=for_eff_optimal,
session_foraging_eff_optimal_random_seed=
for_eff_optimal_random_seed,
session_mean_reward_sum=np.nanmean(p_Ls +
p_Rs),
session_mean_reward_contrast=p_contrast_mean)
self.insert1({**key, **session_stats})
def make(self, key):
log.info('BehaviorIngest.make(): key: {key}'.format(key=key))
subject_id = key['subject_id']
h2o = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
date = key['session_date']
datestr = date.strftime('%Y%m%d')
log.info('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr))
# session record key
skey = {}
skey['subject_id'] = subject_id
skey['session_date'] = date
skey['username'] = self.get_session_user()
# File paths conform to the pattern:
# dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat
# which is, more generally:
# {h2o}/{training_protocol}/Session Data/{h2o}_{training protocol}_{YYYYMMDD}_{HHMMSS}.mat
root = pathlib.Path(key['rig_data_path'],
os.path.dirname(key['subpath']))
path = root / '{h2o}_*_{d}*.mat'.format(h2o=h2o, d=datestr)
log.info('rigpath {p}'.format(p=path))
matches = sorted(
root.glob('{h2o}_*_{d}*.mat'.format(h2o=h2o, d=datestr)))
if matches:
log.info('found files: {}, this is the rig'.format(matches))
skey['rig'] = key['rig']
else:
log.info('no file matches found in {p}'.format(p=path))
if not len(matches):
log.warning('no file matches found for {h2o} / {d}'.format(
h2o=h2o, d=datestr))
return
#
# Find files & Check for split files
# XXX: not checking rig.. 2+ sessions on 2+ rigs possible for date?
#
if len(matches) > 1:
log.warning(
'split session case detected for {h2o} on {date}'.format(
h2o=h2o, date=date))
# session:date relationship is 1:1; skip if we have a session
if experiment.Session() & skey:
log.warning("Warning! session exists for {h2o} on {d}".format(
h2o=h2o, d=date))
return
#
# Prepare PhotoStim
#
photosti_duration = 0.5 # (s) Hard-coded here
photostims = {
4: {
'photo_stim': 4,
'photostim_device': 'OBIS470',
'brain_location_name': 'left_alm',
'duration': photosti_duration
},
5: {
'photo_stim': 5,
'photostim_device': 'OBIS470',
'brain_location_name': 'right_alm',
'duration': photosti_duration
},
6: {
'photo_stim': 6,
'photostim_device': 'OBIS470',
'brain_location_name': 'both_alm',
'duration': photosti_duration
}
}
#
# Extract trial data from file(s) & prepare trial loop
#
trials = zip()
trial = namedtuple( # simple structure to track per-trial vars
'trial',
('ttype', 'stim', 'settings', 'state_times', 'state_names',
'state_data', 'event_data', 'event_times'))
for f in matches:
if os.stat(f).st_size / 1024 < 1000:
log.info('skipping file {f} - too small'.format(f=f))
continue
log.debug('loading file {}'.format(f))
mat = spio.loadmat(f, squeeze_me=True)
SessionData = mat['SessionData'].flatten()
AllTrialTypes = SessionData['TrialTypes'][0]
AllTrialSettings = SessionData['TrialSettings'][0]
RawData = SessionData['RawData'][0].flatten()
AllStateNames = RawData['OriginalStateNamesByNumber'][0]
AllStateData = RawData['OriginalStateData'][0]
AllEventData = RawData['OriginalEventData'][0]
AllStateTimestamps = RawData['OriginalStateTimestamps'][0]
AllEventTimestamps = RawData['OriginalEventTimestamps'][0]
# verify trial-related data arrays are all same length
assert (all(
(x.shape[0] == AllStateTimestamps.shape[0]
for x in (AllTrialTypes, AllTrialSettings, AllStateNames,
AllStateData, AllEventData, AllEventTimestamps))))
if 'StimTrials' in SessionData.dtype.fields:
log.debug('StimTrials detected in session - will include')
AllStimTrials = SessionData['StimTrials'][0]
assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0])
else:
log.debug('StimTrials not detected in session - will skip')
AllStimTrials = np.array([
None for i in enumerate(range(AllStateTimestamps.shape[0]))
])
z = zip(AllTrialTypes, AllStimTrials, AllTrialSettings,
AllStateTimestamps, AllStateNames, AllStateData,
AllEventData, AllEventTimestamps)
trials = chain(trials, z) # concatenate the files
trials = list(trials)
# all files were internally invalid or size < 100k
if not trials:
log.warning('skipping date {d}, no valid files'.format(d=date))
return
#
# Trial data seems valid; synthesize session id & add session record
# XXX: note - later breaks can result in Sessions without valid trials
#
log.debug('synthesizing session ID')
session = (dj.U().aggr(experiment.Session() & {
'subject_id': subject_id
},
n='max(session)').fetch1('n') or 0) + 1
log.info('generated session id: {session}'.format(session=session))
skey['session'] = session
key = dict(key, **skey)
#
# Actually load the per-trial data
#
log.info('BehaviorIngest.make(): trial parsing phase')
# lists of various records for batch-insert
rows = {
k: list()
for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event',
'corrected_trial_event', 'action_event', 'photostim',
'photostim_location', 'photostim_trial',
'photostim_trial_event')
}
i = -1
for t in trials:
#
# Misc
#
t = trial(*t) # convert list of items to a 'trial' structure
i += 1 # increment trial counter
log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i))
# covert state data names into a lookup dictionary
#
# names (seem to be? are?):
#
# Trigtrialstart
# PreSamplePeriod
# SamplePeriod
# DelayPeriod
# EarlyLickDelay
# EarlyLickSample
# ResponseCue
# GiveLeftDrop
# GiveRightDrop
# GiveLeftDropShort
# GiveRightDropShort
# AnswerPeriod
# Reward
# RewardConsumption
# NoResponse
# TimeOut
# StopLicking
# StopLickingReturn
# TrialEnd
states = {k: (v + 1) for v, k in enumerate(t.state_names)}
required_states = ('PreSamplePeriod', 'SamplePeriod',
'DelayPeriod', 'ResponseCue', 'StopLicking',
'TrialEnd')
missing = list(k for k in required_states if k not in states)
if len(missing):
log.warning('skipping trial {i}; missing {m}'.format(
i=i, m=missing))
continue
gui = t.settings['GUI'].flatten()
# ProtocolType - only ingest protocol >= 3
#
# 1 Water-Valve-Calibration 2 Licking 3 Autoassist
# 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed
#
if 'ProtocolType' not in gui.dtype.names:
log.warning(
'skipping trial {i}; protocol undefined'.format(i=i))
continue
protocol_type = gui['ProtocolType'][0]
if gui['ProtocolType'][0] < 3:
log.warning('skipping trial {i}; protocol {n} < 3'.format(
i=i, n=gui['ProtocolType'][0]))
continue
#
# Top-level 'Trial' record
#
tkey = dict(skey)
startindex = np.where(t.state_data == states['PreSamplePeriod'])[0]
# should be only end of 1st StopLicking;
# rest of data is irrelevant w/r/t separately ingested ephys
endindex = np.where(t.state_data == states['StopLicking'])[0]
log.debug('states\n' + str(states))
log.debug('state_data\n' + str(t.state_data))
log.debug('startindex\n' + str(startindex))
log.debug('endindex\n' + str(endindex))
if not (len(startindex) and len(endindex)):
log.warning(
'skipping trial {i}: start/end index error: {s}/{e}'.
format(i=i, s=str(startindex), e=str(endindex)))
continue
try:
tkey['trial'] = i
tkey[
'trial_uid'] = i # Arseny has unique id to identify some trials
tkey['start_time'] = t.state_times[startindex][0]
tkey['stop_time'] = t.state_times[endindex][0]
except IndexError:
log.warning(
'skipping trial {i}: error indexing {s}/{e} into {t}'.
format(i=i,
s=str(startindex),
e=str(endindex),
t=str(t.state_times)))
continue
log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg
log.debug('tkey' + str(tkey))
rows['trial'].append(tkey)
#
# Specific BehaviorTrial information for this trial
#
bkey = dict(tkey)
bkey['task'] = 'audio delay' # hard-coded here
bkey['task_protocol'] = 1 # hard-coded here
# determine trial instruction
trial_instruction = 'left' # hard-coded here
if gui['Reversal'][0] == 1:
if t.ttype == 1:
trial_instruction = 'left'
elif t.ttype == 0:
trial_instruction = 'right'
elif gui['Reversal'][0] == 2:
if t.ttype == 1:
trial_instruction = 'right'
elif t.ttype == 0:
trial_instruction = 'left'
bkey['trial_instruction'] = trial_instruction
# determine early lick
early_lick = 'no early'
if (protocol_type >= 5 and 'EarlyLickDelay' in states
and np.any(t.state_data == states['EarlyLickDelay'])):
early_lick = 'early'
if (protocol_type > 5 and
('EarlyLickSample' in states
and np.any(t.state_data == states['EarlyLickSample']))):
early_lick = 'early'
bkey['early_lick'] = early_lick
# determine outcome
outcome = 'ignore'
if ('Reward' in states
and np.any(t.state_data == states['Reward'])):
outcome = 'hit'
elif ('TimeOut' in states
and np.any(t.state_data == states['TimeOut'])):
outcome = 'miss'
elif ('NoResponse' in states
and np.any(t.state_data == states['NoResponse'])):
outcome = 'ignore'
bkey['outcome'] = outcome
rows['behavior_trial'].append(bkey)
#
# Add 'protocol' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'protocol #'
nkey['trial_note'] = str(protocol_type)
rows['trial_note'].append(nkey)
#
# Add 'autolearn' note
#
nkey = dict(tkey)
nkey['trial_note_type'] = 'autolearn'
nkey['trial_note'] = str(gui['Autolearn'][0])
rows['trial_note'].append(nkey)
#
# Add 'bitcode' note
#
if 'randomID' in gui.dtype.names:
nkey = dict(tkey)
nkey['trial_note_type'] = 'bitcode'
nkey['trial_note'] = str(gui['randomID'][0])
rows['trial_note'].append(nkey)
#
# Add presample event
#
log.debug('BehaviorIngest.make(): presample')
ekey = dict(tkey)
sampleindex = np.where(t.state_data == states['SamplePeriod'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'presample'
ekey['trial_event_time'] = t.state_times[startindex][0]
ekey['duration'] = (t.state_times[sampleindex[0]] -
t.state_times[startindex])[0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing presample duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial
rows['trial_event'].append(ekey)
#
# Add other 'sample' events
#
log.debug('BehaviorIngest.make(): sample events')
last_dur = None
for s in sampleindex: # in protocol > 6 ~-> n>1
# todo: batch events
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'sample'
ekey['trial_event_time'] = t.state_times[s]
ekey['duration'] = gui['SamplePeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning(
'... trial {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning(
'... trial {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
rows['trial_event'].append(ekey)
#
# Add 'delay' events
#
log.debug('BehaviorIngest.make(): delay events')
last_dur = None
delayindex = np.where(t.state_data == states['DelayPeriod'])[0]
for d in delayindex: # protocol > 6 ~-> n>1
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'delay'
ekey['trial_event_time'] = t.state_times[d]
ekey['duration'] = gui['DelayPeriod'][0]
if math.isnan(ekey['duration']) and last_dur is None:
log.warning('... {} bad duration, no last_edur'.format(
i, last_dur))
ekey['duration'] = 0.0 # FIXDUR: cross-trial check
rows['corrected_trial_event'].append(ekey)
elif math.isnan(ekey['duration']) and last_dur is not None:
log.warning('... {} duration using last_edur {}'.format(
i, last_dur))
ekey['duration'] = last_dur
rows['corrected_trial_event'].append(ekey)
else:
last_dur = ekey['duration'] # only track 'good' values.
log.debug('delay event duration: {}'.format(ekey['duration']))
rows['trial_event'].append(ekey)
#
# Add 'go' event
#
log.debug('BehaviorIngest.make(): go')
ekey = dict(tkey)
responseindex = np.where(t.state_data == states['ResponseCue'])[0]
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'go'
ekey['trial_event_time'] = t.state_times[responseindex][0]
ekey['duration'] = gui['AnswerPeriod'][0]
if math.isnan(ekey['duration']):
log.debug('BehaviorIngest.make(): fixing go duration')
ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials
rows['corrected_trial_event'].append(ekey)
rows['trial_event'].append(ekey)
#
# Add 'trialEnd' events
#
log.debug('BehaviorIngest.make(): trialend events')
last_dur = None
trialendindex = np.where(t.state_data == states['TrialEnd'])[0]
ekey = dict(tkey)
ekey['trial_event_id'] = len(rows['trial_event'])
ekey['trial_event_type'] = 'trialend'
ekey['trial_event_time'] = t.state_times[trialendindex][0]
ekey['duration'] = 0.0
rows['trial_event'].append(ekey)
#
# Add lick events
#
lickleft = np.where(t.event_data == 69)[0]
log.debug('... lickleft: {r}'.format(r=str(lickleft)))
action_event_count = len(rows['action_event'])
if len(lickleft):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='left lick',
action_event_time=t.event_times[l]))
for idx, l in enumerate(lickleft)
]
lickright = np.where(t.event_data == 71)[0]
log.debug('... lickright: {r}'.format(r=str(lickright)))
action_event_count = len(rows['action_event'])
if len(lickright):
[
rows['action_event'].append(
dict(tkey,
action_event_id=action_event_count + idx,
action_event_type='right lick',
action_event_time=t.event_times[r]))
for idx, r in enumerate(lickright)
]
# Photostim Events
#
# TODO:
#
# - base stimulation parameters:
#
# - should be loaded elsewhere - where
# - actual ccf locations - cannot be known apriori apparently?
# - Photostim.Profile: what is? fix/add
#
# - stim data
#
# - how retrieve power from file (didn't see) or should
# be statically coded here?
# - how encode stim type 6?
# - we have hemisphere as boolean or
# - but adding an event 4 and event 5 means querying
# is less straightforwrard (e.g. sessions with 5 & 6)
if t.stim:
log.info('BehaviorIngest.make(): t.stim == {}'.format(t.stim))
rows['photostim_trial'].append(tkey)
delay_period_idx = np.where(
t.state_data == states['DelayPeriod'])[0][0]
rows['photostim_trial_event'].append(
dict(tkey,
**photostims[t.stim],
photostim_event_id=len(rows['photostim_trial_event']),
photostim_event_time=t.state_times[delay_period_idx],
power=5.5))
# end of trial loop.
# Session Insertion
log.info('BehaviorIngest.make(): adding session record')
experiment.Session().insert1(skey)
# Behavior Insertion
log.info('BehaviorIngest.make(): bulk insert phase')
log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key))
self.insert1(key, ignore_extra_fields=True, allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.Session.Trial')
experiment.SessionTrial().insert(rows['trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial')
experiment.BehaviorTrial().insert(rows['behavior_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialNote')
experiment.TrialNote().insert(rows['trial_note'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.TrialEvent')
experiment.TrialEvent().insert(rows['trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True,
skip_duplicates=True)
log.info('BehaviorIngest.make(): ... CorrectedTrialEvents')
BehaviorIngest().CorrectedTrialEvents().insert(
rows['corrected_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.ActionEvent')
experiment.ActionEvent().insert(rows['action_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
BehaviorIngest.BehaviorFile().insert(
(dict(key, behavior_file=f.name) for f in matches),
ignore_extra_fields=True,
allow_direct_insert=True)
# Photostim Insertion
photostim_ids = set(
[r['photo_stim'] for r in rows['photostim_trial_event']])
if photostim_ids:
log.info('BehaviorIngest.make(): ... experiment.Photostim')
experiment.Photostim.insert(
(dict(skey, **photostims[stim]) for stim in photostim_ids),
ignore_extra_fields=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrial')
experiment.PhotostimTrial.insert(rows['photostim_trial'],
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('BehaviorIngest.make(): ... experiment.PhotostimTrialEvent')
experiment.PhotostimEvent.insert(rows['photostim_trial_event'],
ignore_extra_fields=True,
allow_direct_insert=True)
def make(self, key):
'''
Ephys .make() function
'''
log.info('EphysIngest().make(): key: {k}'.format(k=key))
#
# Find corresponding BehaviorIngest
#
# ... we are keying times, sessions, etc from behavior ingest;
# so lookup behavior ingest for session id, quit with warning otherwise
try:
behavior = (behavior_ingest.BehaviorIngest() & key).fetch1()
except dj.DataJointError:
log.warning('EphysIngest().make(): skip - behavior ingest error')
return
log.info('behavior for ephys: {b}'.format(b=behavior))
#
# Find Ephys Recording
#
key = (experiment.Session & key).fetch1()
rigpath = EphysDataPath().fetch1('data_path')
date = key['session_date'].strftime('%Y-%m-%d')
subject_id = key['subject_id']
water = (lab.WaterRestriction() & {
'subject_id': subject_id
}).fetch1('water_restriction_number')
for probe in range(1, 3):
# TODO: should code include actual logic to pick these up still?
# file = '{h2o}_g0_t0.imec.ap_imec3_opt3_jrc.mat'.format(h2o=water) # some older files
# subpath = os.path.join('{}-{}'.format(date, probe), file)
# file = '{h2o}ap_imec3_opt3_jrc.mat'.format(h2o=water) # current file naming format
epfile = '{h2o}_g0_*.imec.ap_imec3_opt3_jrc.mat'.format(
h2o=water) # current file naming format
epfullpath = pathlib.Path(rigpath, water, date, str(probe))
ephys_files = list(epfullpath.glob(epfile))
if len(ephys_files) != 1:
log.info(
'EphysIngest().make(): skipping probe {} - incorrect files found: {}/{}'
.format(probe, epfullpath, ephys_files))
continue
epfullpath = ephys_files[0]
epsubpath = epfullpath.relative_to(rigpath)
log.info(
'EphysIngest().make(): found probe {} ephys recording in {}'.
format(probe, epfullpath))
#
# Prepare ProbeInsertion configuration
#
# HACK / TODO: assuming single specific ProbeInsertion for all tests;
# better would be to have this encoded in filename or similar.
probe_part_no = '15131808323' # hard-coded here
ekey = {
'subject_id': behavior['subject_id'],
'session': behavior['session'],
'insertion_number': probe
}
# ElectrodeConfig - add electrode group and group member (hard-coded to be the first 384 electrode)
electrode_group = {'probe': probe_part_no, 'electrode_group': 0}
electrode_group_member = [{
**electrode_group, 'electrode': chn
} for chn in range(1, 385)]
electrode_config_name = 'npx_first384' # user-friendly name - npx probe config with the first 384 channels
electrode_config_hash = dict_to_hash({
**electrode_group,
**{
str(idx): k
for idx, k in enumerate(electrode_group_member)
}
})
# extract ElectrodeConfig, check DB to reference if exists, else create
if ({
'probe': probe_part_no,
'electrode_config_name': electrode_config_name
} not in lab.ElectrodeConfig()):
log.info(
'create Neuropixels electrode configuration (lab.ElectrodeConfig)'
)
lab.ElectrodeConfig.insert1({
'probe':
probe_part_no,
'electrode_config_hash':
electrode_config_hash,
'electrode_config_name':
electrode_config_name
})
lab.ElectrodeConfig.ElectrodeGroup.insert1({
'electrode_config_name':
electrode_config_name,
**electrode_group
})
lab.ElectrodeConfig.Electrode.insert({
'electrode_config_name': electrode_config_name,
**member
} for member in electrode_group_member)
log.info('inserting probe insertion')
ephys.ProbeInsertion.insert1(
dict(ekey,
probe=probe_part_no,
electrode_config_name=electrode_config_name))
#
# Extract spike data
#
log.info('extracting spike data')
f = h5py.File(epfullpath, 'r')
cluster_ids = f['S_clu']['viClu'][0] # cluster (unit) number
trWav_raw_clu = f['S_clu']['trWav_raw_clu'] # spike waveform
# trWav_raw_clu1 = np.concatenate((trWav_raw_clu[0:1][:][:],trWav_raw_clu),axis=0) # add a spike waveform to cluster 0, not necessary anymore after the previous step
csNote_clu = f['S_clu']['csNote_clu'][0] # manual sorting note
viSite_clu = f['S_clu'][
'viSite_clu'][:] # site of the unit with the largest amplitude
vrPosX_clu = f['S_clu']['vrPosX_clu'][0] # x position of the unit
vrPosY_clu = f['S_clu']['vrPosY_clu'][0] # y position of the unit
vrVpp_uv_clu = f['S_clu']['vrVpp_uv_clu'][
0] # amplitude of the unit
vrSnr_clu = f['S_clu']['vrSnr_clu'][0] # snr of the unit
strs = ["all" for x in range(len(csNote_clu))
] # all units are "all" by definition
for iU in range(
0,
len(csNote_clu)): # read the manual curation of each unit
log.debug('extracting spike indicators {s}:{u}'.format(
s=behavior['session'], u=iU))
unitQ = f[csNote_clu[iU]]
str1 = ''.join(chr(i) for i in unitQ[:])
if str1 == 'single': # definitions in unit quality
strs[iU] = 'good'
elif str1 == 'ok':
strs[iU] = 'ok'
elif str1 == 'multi':
strs[iU] = 'multi'
spike_times = f['viTime_spk'][0] # spike times
viSite_spk = f['viSite_spk'][0] # electrode site for the spike
sRateHz = f['P']['sRateHz'][0] # sampling rate
# get rid of the -ve noise clusters
non_neg_cluster_idx = cluster_ids > 0
cluster_ids = cluster_ids[non_neg_cluster_idx]
spike_times = spike_times[non_neg_cluster_idx]
viSite_spk = viSite_spk[non_neg_cluster_idx]
file = '{h2o}_bitcode.mat'.format(
h2o=water) # fetch the bitcode and realign
# subpath = os.path.join('{}-{}'.format(date, probe), file)
bcsubpath = pathlib.Path(water, date, str(probe), file)
bcfullpath = rigpath / bcsubpath
log.info('opening bitcode for session {s} probe {p} ({f})'.format(
s=behavior['session'], p=probe, f=bcfullpath))
mat = spio.loadmat(bcfullpath,
squeeze_me=True) # load the bitcode file
log.info('extracting spike information {s} probe {p} ({f})'.format(
s=behavior['session'], p=probe, f=bcfullpath))
bitCodeE = mat['bitCodeS'].flatten(
) # bitCodeS is the char variable
goCue = mat['goCue'].flatten() # bitCodeS is the char variable
viT_offset_file = mat['sTrig'].flatten(
) - 7500 # start of each trial, subtract this number for each trial
trialNote = experiment.TrialNote()
bitCodeB = (trialNote & {
'subject_id': ekey['subject_id']
} & {
'session': ekey['session']
} & {
'trial_note_type': 'bitcode'
}).fetch('trial_note', order_by='trial'
) # fetch the bitcode from the behavior trialNote
# check ephys/bitcode match to determine trial numbering method
bitCodeB_0 = np.where(bitCodeB == bitCodeE[0])[0][0]
bitCodeB_ext = bitCodeB[bitCodeB_0:][:len(bitCodeE)]
spike_trials_fix = None
if not np.all(np.equal(bitCodeE, bitCodeB_ext)):
log.info('ephys/bitcode trial mismatch - attempting fix')
if 'trialNum' in mat:
spike_trials_fix = mat['trialNum']
else:
raise Exception('Bitcode Mismatch')
spike_trials = np.full_like(
spike_times,
(len(viT_offset_file) - 1)) # every spike is in the last trial
spike_times2 = np.copy(spike_times)
for i in range(len(viT_offset_file) - 1, 0,
-1): #find the trials each unit has a spike in
log.debug('locating trials with spikes {s}:{t}'.format(
s=behavior['session'], t=i))
spike_trials[(spike_times >= viT_offset_file[i - 1]) &
(spike_times < viT_offset_file[i]
)] = i - 1 # Get the trial number of each spike
spike_times2[(spike_times >= viT_offset_file[i - 1]) & (
spike_times < viT_offset_file[i])] = spike_times[
(spike_times >= viT_offset_file[i - 1])
& (spike_times < viT_offset_file[i])] - goCue[
i - 1] # subtract the goCue from each trial
spike_trials[np.where(spike_times2 >= viT_offset_file[-1]
)] = len(viT_offset_file) - 1
spike_times2[np.where(
spike_times2 >= viT_offset_file[-1])] = spike_times[np.where(
spike_times2 >= viT_offset_file[-1])] - goCue[
-1] # subtract the goCue from the last trial
spike_times2 = spike_times2 / sRateHz # divide the sampling rate, sRateHz
# at this point, spike-times are aligned to go-cue for that respective trial
unit_trial_spks = {
u: (spike_trials[cluster_ids == u],
spike_times2[cluster_ids == u])
for u in set(cluster_ids)
}
trial_start_time = viT_offset_file / sRateHz
log.info('inserting units for session {s}'.format(
s=behavior['session']))
#pdb.set_trace()
# Unit - with JRclust clustering method
ekey['clustering_method'] = 'jrclust'
def build_unit_insert():
for u_id, (u, (u_spk_trials, u_spk_times)) in enumerate(
unit_trial_spks.items()):
# unit spike times - realign back to trial-start, relative to 1st trial
spk_times = sorted(u_spk_times +
(goCue / sRateHz)[u_spk_trials] +
trial_start_time[u_spk_trials])
yield (dict(ekey,
unit=u,
unit_uid=u,
unit_quality=strs[u_id],
electrode_config_name=electrode_config_name,
probe=probe_part_no,
electrode_group=0,
electrode=int(viSite_clu[u_id]),
unit_posx=vrPosX_clu[u_id],
unit_posy=vrPosY_clu[u_id],
unit_amp=vrVpp_uv_clu[u_id],
unit_snr=vrSnr_clu[u_id],
spike_times=spk_times,
waveform=trWav_raw_clu[u_id][0]))
ephys.Unit.insert(build_unit_insert(), allow_direct_insert=True)
# UnitTrial
log.info('inserting UnitTrial information')
if spike_trials_fix is None:
if len(bitCodeB) < len(
bitCodeE
): # behavior file is shorter; e.g. seperate protocols were used; Bpod trials missing due to crash; session restarted
startB = np.where(bitCodeE == bitCodeB[0])[0].squeeze()
elif len(bitCodeB) > len(
bitCodeE
): # behavior file is longer; e.g. only some trials are sorted, the bitcode.mat should reflect this; Sometimes SpikeGLX can skip a trial, I need to check the last trial
startE = np.where(bitCodeB == bitCodeE[0])[0].squeeze()
startB = -startE
else:
startB = 0
startE = 0
spike_trials_fix = np.arange(spike_trials.max() + 1)
else: # XXX: under test
startB = 0
startE = 0
spike_trials_fix -= 1
with InsertBuffer(ephys.Unit.UnitTrial,
10000,
skip_duplicates=True,
allow_direct_insert=True) as ib:
for x, (u_spk_trials, u_spk_times) in unit_trial_spks.items():
ib.insert(
dict(ekey, unit=x, trial=spike_trials_fix[tr] - startB)
for tr in set(spike_trials))
if ib.flush():
log.debug('... UnitTrial spike')
# TrialSpike
with InsertBuffer(ephys.TrialSpikes,
10000,
skip_duplicates=True,
allow_direct_insert=True) as ib:
for x, (u_spk_trials, u_spk_times) in unit_trial_spks.items():
ib.insert(
dict(ekey,
unit=x,
spike_times=u_spk_times[u_spk_trials == tr],
trial=spike_trials_fix[tr] - startB)
for tr in set(spike_trials))
if ib.flush():
log.debug('... TrialSpike spike')
log.info('inserting file load information')
self.insert1(key,
ignore_extra_fields=True,
skip_duplicates=True,
allow_direct_insert=True)
EphysIngest.EphysFile().insert1(dict(
key,
probe_insertion_number=probe,
ephys_file=epsubpath.as_posix()),
ignore_extra_fields=True,
allow_direct_insert=True)
log.info('ephys ingest for {} complete'.format(key))