def main(filename):
'''
Add the missing timeseries data:
all_poke_epoch
poke_epoch
response_ts
signal_epoch - not implemented
trial_epoch
Also creates some epoch data never seen before:
all_spout_epoch
'''
with tables.openFile(filename, 'a') as fh:
data = h5.p_get_node(fh.root, '*/data')
TTL_fs = data.contact.poke_TTL._v_attrs['fs']
trial_start = (data.trial_log.cols.start * TTL_fs).astype('i')
trials = len(trial_start)
if 'all_poke_epoch' not in data.contact:
all_poke_epoch = epochs(data.contact.poke_TTL[:])
node = fh.createArray(data.contact, 'all_poke_epoch',
all_poke_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created all_poke_epoch'
else:
print 'all_poke_epoch already exists'
if 'poke_epoch' not in data.contact:
# If we are inside this statement, it is highly probable that we
# already had to compute all_poke_epoch above. Howver, just in case
# we didn't, let's load it back in from the file.
all_poke_epoch = data.contact.all_poke_epoch[:]
# Use broadcasting to do away with a for loop and find out which
# poke epochs bracket the start of a trial.
trial_start = trial_start[np.newaxis].T
mask = (all_poke_epoch[:,0] <= trial_start) & \
(all_poke_epoch[:,1] > trial_start)
# Check to see if we have any continuous nose-pokes that triggered
# more than one trial (these are not actually continuous nose-poke,
# the sampling rate was so low that we simply did not detect the
# discontinuity). If so, we need to break down this nose-poke into
# two nose-pokes. We know that the subject broke the nose-poke
# when the response window went high, so we'll simply insert a zero
# into the correct place in the nose-poke TTL signal and rerun the
# script.
double_mask = mask.sum(0) > 1
if double_mask.any():
for lb, ub in all_poke_epoch[double_mask]:
i = ts(edge_rising(data.contact.response_TTL[lb:ub]))
data.contact.poke_TTL[lb+i] = False
data.contact.all_poke_epoch._f_remove()
print 'Updated poke_TTL. Please rerun script.'
return
mask = mask.any(0)
poke_epoch = all_poke_epoch[mask]
if len(poke_epoch) != trials:
raise ValueError, "Unable to winnow down poke epoch list"
node = fh.createArray(data.contact, 'poke_epoch', poke_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created poke_epoch'
else:
print 'poke_epoch already exists'
if 'response_ts' not in data.contact:
response_ts = epochs(data.contact.response_TTL[:])[:,1]
if len(response_ts) != trials:
ts_end = data.trial_log.cols.ts_end[:]
incomplete_response_ts = response_ts
response_ts = np.empty(len(ts_end))
# Find out which response_ts values are missing. If no
# response_ts is within 500 msec of the ts_end value, we know
# that the response_ts is missing.
delta = np.abs(incomplete_response_ts-ts_end[np.newaxis].T)
# Boolean mask indicating which trials we have a valid
# response_ts for.
valid = np.any(delta < (0.5 * TTL_fs), 1)
response_ts[valid] = incomplete_response_ts
# The trials for which we don't have a valid response_ts based
# on the response_TTL should be discarded due to a faulty spout
# sensor. So, let's just use the ts_end timestamp instead. The
# ts_end timestamp is sampled at the same fs as the
# response_TTL, so no conversion is needed.
response_ts[~valid] = ts_end[~valid]
node = fh.createArray(data.contact, 'response_ts', response_ts)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created response_ts'
else:
print 'response_ts already exists'
# The logic for this is slightly more complicated because early versions
# of the appetitive dt paradigm set the signal duration to 0 for the
# nogo. This effectively means that there's no signal_TTL during these
# nogos, so what is the "epoch" in this case? I may work this out
# later. I don't really use signal_epoch (we already know when the
# signal is presented based on the trial start timestamp).
#
#if 'signal_epoch' not in data.contact:
# signal_epoch = get_epochs(data.contact.signal_TTL[:])
# print len(signal_epoch), trials
# if len(signal_epoch) != trials:
# raise ValueError, 'Unable to compute signal epoch'
# node = fh.createArray(data.contact, 'signal_epoch', signal_epoch)
# node._v_attrs['fs'] = TTL_fs
# node._v_attrs['t0'] = 0
if 'all_spout_epoch' not in data.contact:
all_spout_epoch = epochs(data.contact.spout_TTL[:])
node = fh.createArray(data.contact, 'all_spout_epoch',
all_spout_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created all_spout_epoch'
else:
print 'all_spout_epoch already exists'
if 'trial_epoch' not in data.contact:
trial_epoch = zip(data.trial_log.cols.ts_start,
data.trial_log.cols.ts_end)
if len(trial_epoch) != trials:
raise ValueError, 'Unable to compute trial epoch'
node = fh.createArray(data.contact, 'trial_epoch', trial_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created trial_epoch'
else:
print 'trial_epoch already exists'
# Return from this function if there is no physiology data to work on
if 'physiology' not in data:
return
if 'epoch' not in data.physiology:
epoch = epochs(data.physiology.sweep[:])
node = fh.createArray(data.physiology, 'epoch', epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created physiology epoch'
else:
print 'physiology epoch already exists'
# If the ts data is present, it is more likely to be of a higher
# resolution than the epoch data.
if 'ts' not in data.physiology:
timestamps = data.physiology.epoch[:,0]
node = fh.createArray(data.physiology, 'ts', timestamps)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created physiology ts'
else:
print 'physiology ts already exists'
# Make sure the node flavor is set to Numpy
for series in ('contact/all_poke_epoch',
'contact/poke_epoch',
'contact/trial_epoch',
'contact/all_spout_epoch',
'contact/signal_epoch'
'contact/response_ts',
'physiology/epoch',
'physiology/ts'):
try:
node = h5.p_get_node(data, series)
if node.flavor != 'numpy':
node.flavor = 'numpy'
print 'Updated node flavor for ', series
except tables.NoSuchNodeError:
pass
def censor_spikes(ext_filename, rms_pad=0.5, block_pad=5, artifact_pad=0.5,
artifact_mode='single', zero_pad=1.25, force_overwrite=False):
'''
Censor extracted spikes based on three criteria::
noise floor
artifacts
block
Once the spikes have been censored, you can use the physiology review GUI
(using the overla extracted spikes feature) to inspect the censored result.
'''
with tables.openFile(ext_filename, 'a') as fh:
enode = fh.root.event_data
if 'censored' in fh.root.event_data:
if not force_overwrite:
raise IOError, 'Censored data already exists'
else:
fh.root.event_data.censored._f_remove()
fh.root.censor._f_remove(recursive=True)
# Create some UUID information that we can reference from other files that
# are derived from this one. If I re-censor spikes, the UUID will
# change. This means we can check to see whether sorted spike data
# (obtained from the extracted spiketimes file) is from the current
# version of the extracted times file.
fh.setNodeAttr(fh.root, 'censor_uuid', str(uuid.uuid1()))
fh.setNodeAttr(fh.root, 'last_censored', time.time())
cnode = fh.createGroup('/', 'censor')
censored = fh.createCArray(enode, 'censored', tables.BoolAtom(),
enode.timestamps.shape)
channel_indices = enode.channel_indices[:]
lb = fh.root.block_data.trial_log.cols.start[0]-block_pad
ub = fh.root.block_data.trial_log.cols.end[-1]+block_pad
if artifact_mode == 'all':
# Censor based on transients occuring on *any* channel
artifact_mask = np.any(enode.artifacts[:], axis=-1)
artifact_ts = enode.timestamps[artifact_mask]
artifact_epochs = np.c_[artifact_ts-artifact_pad,
artifact_ts+artifact_pad]
artifact_epochs = smooth_epochs(artifact_epochs)
for i, ch in enumerate(enode._v_attrs.extracted_channels-1):
mask = i == channel_indices
timestamps = enode.timestamps[mask]
if len(timestamps) == 0:
# This usually means the channel was used solely for artifact
# reject (or the user set the spike threshold wrong)
continue
if np.diff(timestamps).min() < 0:
raise ValueError, 'Timestamps are not ordered!'
# Censor based on RMS artifacts
rms = fh.root.rms.rms[ch]
rms_th = np.median(rms)/0.6745
rms_epochs = epochs(rms >= rms_th)
rms_epochs = rms_epochs / fh.root.rms.rms._v_attrs.fs
rms_epochs += [[-rms_pad, rms_pad]]
rms_epochs = smooth_epochs(rms_epochs)
# Censor based on block start/end
#block_censor = (timestamps < lb) | (timestamps > ub)
zero_epochs = epochs(rms == 0)
zero_epochs = zero_epochs / fh.root.rms.rms._v_attrs.fs
zero_epochs += [[-zero_pad, zero_pad]]
zero_epochs = smooth_epochs(zero_epochs)
if len(zero_epochs) > 0:
print zero_epochs
if artifact_mode == 'single':
# Censor based on transients in data. Need to recompute this
# for each channel.
artifact = enode.artifacts[:,i][mask].astype('bool')
artifact_ts = timestamps[artifact]
artifact_epochs = np.c_[artifact_ts-artifact_pad,
artifact_ts+artifact_pad]
artifact_epochs = smooth_epochs(artifact_epochs)
#j = np.searchsorted(artifact_epochs[:,0], timestamps)
#k = np.searchsorted(artifact_epochs[:,1], timestamps)
#artifact_censor = ~(j == k)
cenode = fh.createGroup(cnode, 'extracted_{}'.format(i))
cenode._v_attrs['channel'] = ch+1 # 1-based channel number
# Save the rms censor data
array = fh.createEArray(cenode, 'rms_epochs', shape=(0, 2),
atom=tables.Float32Atom(),
title='Censored RMS epochs')
array.append(rms_epochs)
array._v_attrs['rms_pad'] = rms_pad
array._v_attrs['rms_threshold'] = rms_th
# Save the zero censor data
array = fh.createEArray(cenode, 'zero_epochs', shape=(0, 2),
atom=tables.Float32Atom(),
title='Censored RMS epochs')
array.append(zero_epochs)
array._v_attrs['zero_pad'] = zero_pad
# Save the artifact reject data
array = fh.createEArray(cenode, 'artifact_epochs', shape=(0, 2),
atom=tables.Float32Atom(),
title='Censored artifact epochs')
array.append(artifact_epochs)
array._v_attrs['artifact_pad'] = artifact_pad
array._v_attrs['artifact_mode'] = artifact_mode
# Save the block reject data
cenode._v_attrs['block_start'] = lb
cenode._v_attrs['block_end'] = ub
all_epochs = np.r_[[(0, lb)],
rms_epochs,
artifact_epochs,
zero_epochs,
[(ub, np.inf)]]
all_epochs.sort()
all_epochs = smooth_epochs(all_epochs)
fh.createArray(cenode, 'censored_epochs', all_epochs,
title='All censored epochs')
# Update the censor mask
censored[mask] = epochs_contain(all_epochs, timestamps)
# Save some metadata regarding the censoring process (this is redundant
# with the data stored in the subnodes, but provided here as well)
cnode._v_attrs['rms_pad'] = rms_pad
cnode._v_attrs['block_pad'] = block_pad
cnode._v_attrs['artifact_pad'] = artifact_pad
cnode._v_attrs['artifact_mode'] = artifact_mode
