def main(filename, trials):
with tables.openFile(filename, 'a') as fh:
lengths = [len(h5.p_get_node(fh.root, p)) for p in paths]
#print lengths
# Make sure that each node has the same number of trials saved. If not,
# then we need to look at the file by hand to make sure that we are
# removing the correct data.
if len(set(lengths)) != 1:
raise ValueError, 'unequal elements in each array'
for path in paths:
node = h5.p_get_node(fh.root, path)
node.truncate(trials)
def main(filename):
with tables.openFile(filename, 'a') as fh:
data = h5.p_get_node(fh.root, '*/data')
#trial_start = data.trial_log.cols.start[:]
poke_node = data.contact.poke_epoch
poke_epoch = poke_node[:]/poke_node._v_attrs['fs']
bad_epoch = np.flatnonzero((poke_epoch[:,1] - poke_epoch[:,0]) < 0)[0]
## First check
#extra_trial = None
#for i, (start, poke) in enumerate(zip(trial_start, poke_epoch[:,0])):
# if start-poke > 1.25:
# extra_trial = i
# break
#extra_trial -= 1
# Check for negative signals. This seems to be an indicator the epoch
# data is incorrect.
#signal_node = data.contact.signal_epoch
#signal_epoch = signal_node[:]/signal_node._v_attrs['fs']
#print signal_epoch[:,1]-signal_epoch[:,0]
##print signal_epoch[383:387,1] - signal_epoch[383:387,0]
#bad_epoch = np.flatnonzero((signal_epoch[:,1]-signal_epoch[:,0]) < 0)[0]
print bad_epoch
print divmod(poke_epoch[bad_epoch][1], 60.0)
for node_name in node_names:
node = data.contact._f_getChild(node_name)
node._f_rename(node_name + '_old')
values = np.delete(node[:], bad_epoch, axis=0)
new_node = fh.createArray(data.contact, node_name, values)
for attr in node._v_attrs._v_attrnamesuser:
new_node._v_attrs[attr] = node._v_attrs[attr]
def compute_rms(ext_filename, force_overwrite=False):
'''
Add running measurement of RMS noise floor to the extracted spiketimes file.
This metric is required for many of the spike processing routines; however,
this is such a slow (possibly inefficient) algorithm that it was broken out
into a separate function.
'''
processing = {}
with tables.openFile(ext_filename, 'a') as fh:
raw_filename = ext_filename.replace('extracted', 'raw')
if 'rms' in fh.root:
if not force_overwrite:
raise IOError, 'Already contains RMS data'
else:
fh.root.rms._f_remove(recursive=True)
processing['filter_freq_lp'] = fh.root.filter._v_attrs.fc_lowpass
processing['filter_freq_hp'] = fh.root.filter._v_attrs.fc_highpass
processing['filter_order'] = fh.root.filter._v_attrs.filter_order
processing['filter_btype'] = fh.root.filter._v_attrs.filter_btype
processing['bad_channels'] = fh.root.filter.bad_channels[:]-1
processing['diff_mode'] = fh.root.filter._v_attrs.diff_mode
#channels = fh.root.event_data._v_attrs.extracted_channels[:]-1
with tables.openFile(raw_filename, 'r') as fh_raw:
input_node = h5.p_get_node(fh_raw.root, '*')
output_node = fh.createGroup('/', 'rms')
running_rms(input_node, output_node, 1, 0.25, processing=processing,
algorithm='median', progress_callback=update_progress)
def main(filename):
'''
Removes artifacts typically found at the beginning and end of an experiment
(e.g. when putting the animal in the cage or when the headstage falls off at
the end of the experiment).
'''
with tables.openFile(filename, 'a') as fh:
exp_node = h5.p_get_node(fh, '*')
start = exp_node.data.trial_log.cols.start[0] - 5
end = exp_node.data.trial_log.cols.end[-1] + 5
if start > 0:
analysis.zero_waveform(exp_node, start)
analysis.truncate_waveform(exp_node, end)
def main(filename):
"""
Validate timeseries data
poke_epoch
response_ts
signal_epoch - not implemented
trial_epoch
"""
with tables.openFile(filename, "r") as fh:
data = h5.p_get_node(fh.root, "*/data")
trial_start = data.trial_log.cols.start[:]
trial_end = data.trial_log.cols.end[:]
poke_node = data.contact.poke_epoch
poke_epoch = poke_node[:] / poke_node._v_attrs["fs"]
# Make sure that all poke starts are before the start of the trial
try:
if not np.all(poke_epoch[:, 0] < trial_start):
raise ValueError, "poke epoch"
# Make sure that all pokes occured within 1 second of the trial start
if not np.all((poke_epoch[:, 0] - trial_start) < 1.25):
raise ValueError, "poke epoch"
# Make sure that the end of the poke did not occur before the start of
# the next trial
if not np.all(poke_epoch[:-1, 1] < trial_start[1:]):
print poke_epoch[:-1, 1] < trial_start[1:]
raise ValueError, "poke epoch"
except ValueError:
trials = len(trial_start)
dt = poke_epoch[:trials, 0] - trial_start
print np.flatnonzero(dt < -2)[0]
raise
response = data.contact.response_ts
response_ts = response[:] / response._v_attrs["fs"]
fs_error = data.contact.response_TTL._v_attrs["fs"] ** -1
if not np.all(response_ts > trial_start):
print np.flatnonzero(~(response_ts > trial_start))[0]
raise ValueError, "response ts"
if not np.all(response_ts <= (trial_end + 2 * fs_error)):
print fs_error
print response_ts - (trial_end + fs_error)
raise ValueError, "response ts"
def extract_spikes(raw_filename, template=None, force_overwrite=False):
'''
Extract spikes from raw data based on information stored in the channel
metadata table. Use the review physiology GUI to configure and save the
settings for spike extraction.
'''
ext_filename = raw_filename.replace('raw', 'extracted')
if template is None:
template = raw_filename
kwargs = io.create_extract_arguments(template)
if path.exists(ext_filename):
if not force_overwrite:
raise IOError, 'Extracted file already exists'
else:
ext_kwargs = io.create_extract_arguments(ext_filename)
# Remove the extracted data information in preparation for the
# reprocessing of the file.
fh_out = tables.openFile(ext_filename, 'a')
fh_out.root.event_data._f_remove(recursive=True)
fh_out.root.filter._f_remove(recursive=True)
fh_out.root.block_data._f_remove(recursive=True)
fh_out.root.censor._f_remove(recursive=True)
# Check to see if the filtering or referencing data has changed. If
# not, we can keep the RMS data stored in the extracted file
# otherwise we need to discard that data and start fresh.
if ext_kwargs['processing'] != kwargs['processing']:
fh_out.root.rms._f_remove(recursive=True)
else:
print 'Discarding RMS data'
fh_out = tables.openFile(ext_filename, 'w')
fh_in = tables.openFile(raw_filename, 'r')
kwargs['input_node'] = h5.p_get_node(fh_in, '*')
kwargs['output_node'] = fh_out.root
kwargs['progress_callback'] = io.update_progress
analysis.extract_spikes(**kwargs)
fh_in.close()
fh_out.close()
return ext_filename
def main(filename):
'''
Add the missing timeseries data:
all_poke_epoch
poke_epoch
response_ts
signal_epoch - not implemented
trial_epoch
Also creates some epoch data never seen before:
all_spout_epoch
'''
with tables.openFile(filename, 'a') as fh:
data = h5.p_get_node(fh.root, '*/data')
TTL_fs = data.contact.poke_TTL._v_attrs['fs']
trial_start = (data.trial_log.cols.start * TTL_fs).astype('i')
trials = len(trial_start)
if 'all_poke_epoch' not in data.contact:
all_poke_epoch = epochs(data.contact.poke_TTL[:])
node = fh.createArray(data.contact, 'all_poke_epoch',
all_poke_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created all_poke_epoch'
else:
print 'all_poke_epoch already exists'
if 'poke_epoch' not in data.contact:
# If we are inside this statement, it is highly probable that we
# already had to compute all_poke_epoch above. Howver, just in case
# we didn't, let's load it back in from the file.
all_poke_epoch = data.contact.all_poke_epoch[:]
# Use broadcasting to do away with a for loop and find out which
# poke epochs bracket the start of a trial.
trial_start = trial_start[np.newaxis].T
mask = (all_poke_epoch[:,0] <= trial_start) & \
(all_poke_epoch[:,1] > trial_start)
# Check to see if we have any continuous nose-pokes that triggered
# more than one trial (these are not actually continuous nose-poke,
# the sampling rate was so low that we simply did not detect the
# discontinuity). If so, we need to break down this nose-poke into
# two nose-pokes. We know that the subject broke the nose-poke
# when the response window went high, so we'll simply insert a zero
# into the correct place in the nose-poke TTL signal and rerun the
# script.
double_mask = mask.sum(0) > 1
if double_mask.any():
for lb, ub in all_poke_epoch[double_mask]:
i = ts(edge_rising(data.contact.response_TTL[lb:ub]))
data.contact.poke_TTL[lb+i] = False
data.contact.all_poke_epoch._f_remove()
print 'Updated poke_TTL. Please rerun script.'
return
mask = mask.any(0)
poke_epoch = all_poke_epoch[mask]
if len(poke_epoch) != trials:
raise ValueError, "Unable to winnow down poke epoch list"
node = fh.createArray(data.contact, 'poke_epoch', poke_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created poke_epoch'
else:
print 'poke_epoch already exists'
if 'response_ts' not in data.contact:
response_ts = epochs(data.contact.response_TTL[:])[:,1]
if len(response_ts) != trials:
ts_end = data.trial_log.cols.ts_end[:]
incomplete_response_ts = response_ts
response_ts = np.empty(len(ts_end))
# Find out which response_ts values are missing. If no
# response_ts is within 500 msec of the ts_end value, we know
# that the response_ts is missing.
delta = np.abs(incomplete_response_ts-ts_end[np.newaxis].T)
# Boolean mask indicating which trials we have a valid
# response_ts for.
valid = np.any(delta < (0.5 * TTL_fs), 1)
response_ts[valid] = incomplete_response_ts
# The trials for which we don't have a valid response_ts based
# on the response_TTL should be discarded due to a faulty spout
# sensor. So, let's just use the ts_end timestamp instead. The
# ts_end timestamp is sampled at the same fs as the
# response_TTL, so no conversion is needed.
response_ts[~valid] = ts_end[~valid]
node = fh.createArray(data.contact, 'response_ts', response_ts)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created response_ts'
else:
print 'response_ts already exists'
# The logic for this is slightly more complicated because early versions
# of the appetitive dt paradigm set the signal duration to 0 for the
# nogo. This effectively means that there's no signal_TTL during these
# nogos, so what is the "epoch" in this case? I may work this out
# later. I don't really use signal_epoch (we already know when the
# signal is presented based on the trial start timestamp).
#
#if 'signal_epoch' not in data.contact:
# signal_epoch = get_epochs(data.contact.signal_TTL[:])
# print len(signal_epoch), trials
# if len(signal_epoch) != trials:
# raise ValueError, 'Unable to compute signal epoch'
# node = fh.createArray(data.contact, 'signal_epoch', signal_epoch)
# node._v_attrs['fs'] = TTL_fs
# node._v_attrs['t0'] = 0
if 'all_spout_epoch' not in data.contact:
all_spout_epoch = epochs(data.contact.spout_TTL[:])
node = fh.createArray(data.contact, 'all_spout_epoch',
all_spout_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created all_spout_epoch'
else:
print 'all_spout_epoch already exists'
if 'trial_epoch' not in data.contact:
trial_epoch = zip(data.trial_log.cols.ts_start,
data.trial_log.cols.ts_end)
if len(trial_epoch) != trials:
raise ValueError, 'Unable to compute trial epoch'
node = fh.createArray(data.contact, 'trial_epoch', trial_epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created trial_epoch'
else:
print 'trial_epoch already exists'
# Return from this function if there is no physiology data to work on
if 'physiology' not in data:
return
if 'epoch' not in data.physiology:
epoch = epochs(data.physiology.sweep[:])
node = fh.createArray(data.physiology, 'epoch', epoch)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created physiology epoch'
else:
print 'physiology epoch already exists'
# If the ts data is present, it is more likely to be of a higher
# resolution than the epoch data.
if 'ts' not in data.physiology:
timestamps = data.physiology.epoch[:,0]
node = fh.createArray(data.physiology, 'ts', timestamps)
node._v_attrs['fs'] = TTL_fs
node._v_attrs['t0'] = 0
print 'created physiology ts'
else:
print 'physiology ts already exists'
# Make sure the node flavor is set to Numpy
for series in ('contact/all_poke_epoch',
'contact/poke_epoch',
'contact/trial_epoch',
'contact/all_spout_epoch',
'contact/signal_epoch'
'contact/response_ts',
'physiology/epoch',
'physiology/ts'):
try:
node = h5.p_get_node(data, series)
if node.flavor != 'numpy':
node.flavor = 'numpy'
print 'Updated node flavor for ', series
except tables.NoSuchNodeError:
pass
