def main(filename, trials): with tables.openFile(filename, 'a') as fh: lengths = [len(h5.p_get_node(fh.root, p)) for p in paths] #print lengths # Make sure that each node has the same number of trials saved. If not, # then we need to look at the file by hand to make sure that we are # removing the correct data. if len(set(lengths)) != 1: raise ValueError, 'unequal elements in each array' for path in paths: node = h5.p_get_node(fh.root, path) node.truncate(trials)
def main(filename): with tables.openFile(filename, 'a') as fh: data = h5.p_get_node(fh.root, '*/data') #trial_start = data.trial_log.cols.start[:] poke_node = data.contact.poke_epoch poke_epoch = poke_node[:]/poke_node._v_attrs['fs'] bad_epoch = np.flatnonzero((poke_epoch[:,1] - poke_epoch[:,0]) < 0)[0] ## First check #extra_trial = None #for i, (start, poke) in enumerate(zip(trial_start, poke_epoch[:,0])): # if start-poke > 1.25: # extra_trial = i # break #extra_trial -= 1 # Check for negative signals. This seems to be an indicator the epoch # data is incorrect. #signal_node = data.contact.signal_epoch #signal_epoch = signal_node[:]/signal_node._v_attrs['fs'] #print signal_epoch[:,1]-signal_epoch[:,0] ##print signal_epoch[383:387,1] - signal_epoch[383:387,0] #bad_epoch = np.flatnonzero((signal_epoch[:,1]-signal_epoch[:,0]) < 0)[0] print bad_epoch print divmod(poke_epoch[bad_epoch][1], 60.0) for node_name in node_names: node = data.contact._f_getChild(node_name) node._f_rename(node_name + '_old') values = np.delete(node[:], bad_epoch, axis=0) new_node = fh.createArray(data.contact, node_name, values) for attr in node._v_attrs._v_attrnamesuser: new_node._v_attrs[attr] = node._v_attrs[attr]
def compute_rms(ext_filename, force_overwrite=False): ''' Add running measurement of RMS noise floor to the extracted spiketimes file. This metric is required for many of the spike processing routines; however, this is such a slow (possibly inefficient) algorithm that it was broken out into a separate function. ''' processing = {} with tables.openFile(ext_filename, 'a') as fh: raw_filename = ext_filename.replace('extracted', 'raw') if 'rms' in fh.root: if not force_overwrite: raise IOError, 'Already contains RMS data' else: fh.root.rms._f_remove(recursive=True) processing['filter_freq_lp'] = fh.root.filter._v_attrs.fc_lowpass processing['filter_freq_hp'] = fh.root.filter._v_attrs.fc_highpass processing['filter_order'] = fh.root.filter._v_attrs.filter_order processing['filter_btype'] = fh.root.filter._v_attrs.filter_btype processing['bad_channels'] = fh.root.filter.bad_channels[:]-1 processing['diff_mode'] = fh.root.filter._v_attrs.diff_mode #channels = fh.root.event_data._v_attrs.extracted_channels[:]-1 with tables.openFile(raw_filename, 'r') as fh_raw: input_node = h5.p_get_node(fh_raw.root, '*') output_node = fh.createGroup('/', 'rms') running_rms(input_node, output_node, 1, 0.25, processing=processing, algorithm='median', progress_callback=update_progress)
def main(filename): ''' Removes artifacts typically found at the beginning and end of an experiment (e.g. when putting the animal in the cage or when the headstage falls off at the end of the experiment). ''' with tables.openFile(filename, 'a') as fh: exp_node = h5.p_get_node(fh, '*') start = exp_node.data.trial_log.cols.start[0] - 5 end = exp_node.data.trial_log.cols.end[-1] + 5 if start > 0: analysis.zero_waveform(exp_node, start) analysis.truncate_waveform(exp_node, end)
def main(filename): """ Validate timeseries data poke_epoch response_ts signal_epoch - not implemented trial_epoch """ with tables.openFile(filename, "r") as fh: data = h5.p_get_node(fh.root, "*/data") trial_start = data.trial_log.cols.start[:] trial_end = data.trial_log.cols.end[:] poke_node = data.contact.poke_epoch poke_epoch = poke_node[:] / poke_node._v_attrs["fs"] # Make sure that all poke starts are before the start of the trial try: if not np.all(poke_epoch[:, 0] < trial_start): raise ValueError, "poke epoch" # Make sure that all pokes occured within 1 second of the trial start if not np.all((poke_epoch[:, 0] - trial_start) < 1.25): raise ValueError, "poke epoch" # Make sure that the end of the poke did not occur before the start of # the next trial if not np.all(poke_epoch[:-1, 1] < trial_start[1:]): print poke_epoch[:-1, 1] < trial_start[1:] raise ValueError, "poke epoch" except ValueError: trials = len(trial_start) dt = poke_epoch[:trials, 0] - trial_start print np.flatnonzero(dt < -2)[0] raise response = data.contact.response_ts response_ts = response[:] / response._v_attrs["fs"] fs_error = data.contact.response_TTL._v_attrs["fs"] ** -1 if not np.all(response_ts > trial_start): print np.flatnonzero(~(response_ts > trial_start))[0] raise ValueError, "response ts" if not np.all(response_ts <= (trial_end + 2 * fs_error)): print fs_error print response_ts - (trial_end + fs_error) raise ValueError, "response ts"
def extract_spikes(raw_filename, template=None, force_overwrite=False): ''' Extract spikes from raw data based on information stored in the channel metadata table. Use the review physiology GUI to configure and save the settings for spike extraction. ''' ext_filename = raw_filename.replace('raw', 'extracted') if template is None: template = raw_filename kwargs = io.create_extract_arguments(template) if path.exists(ext_filename): if not force_overwrite: raise IOError, 'Extracted file already exists' else: ext_kwargs = io.create_extract_arguments(ext_filename) # Remove the extracted data information in preparation for the # reprocessing of the file. fh_out = tables.openFile(ext_filename, 'a') fh_out.root.event_data._f_remove(recursive=True) fh_out.root.filter._f_remove(recursive=True) fh_out.root.block_data._f_remove(recursive=True) fh_out.root.censor._f_remove(recursive=True) # Check to see if the filtering or referencing data has changed. If # not, we can keep the RMS data stored in the extracted file # otherwise we need to discard that data and start fresh. if ext_kwargs['processing'] != kwargs['processing']: fh_out.root.rms._f_remove(recursive=True) else: print 'Discarding RMS data' fh_out = tables.openFile(ext_filename, 'w') fh_in = tables.openFile(raw_filename, 'r') kwargs['input_node'] = h5.p_get_node(fh_in, '*') kwargs['output_node'] = fh_out.root kwargs['progress_callback'] = io.update_progress analysis.extract_spikes(**kwargs) fh_in.close() fh_out.close() return ext_filename
def main(filename): ''' Add the missing timeseries data: all_poke_epoch poke_epoch response_ts signal_epoch - not implemented trial_epoch Also creates some epoch data never seen before: all_spout_epoch ''' with tables.openFile(filename, 'a') as fh: data = h5.p_get_node(fh.root, '*/data') TTL_fs = data.contact.poke_TTL._v_attrs['fs'] trial_start = (data.trial_log.cols.start * TTL_fs).astype('i') trials = len(trial_start) if 'all_poke_epoch' not in data.contact: all_poke_epoch = epochs(data.contact.poke_TTL[:]) node = fh.createArray(data.contact, 'all_poke_epoch', all_poke_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created all_poke_epoch' else: print 'all_poke_epoch already exists' if 'poke_epoch' not in data.contact: # If we are inside this statement, it is highly probable that we # already had to compute all_poke_epoch above. Howver, just in case # we didn't, let's load it back in from the file. all_poke_epoch = data.contact.all_poke_epoch[:] # Use broadcasting to do away with a for loop and find out which # poke epochs bracket the start of a trial. trial_start = trial_start[np.newaxis].T mask = (all_poke_epoch[:,0] <= trial_start) & \ (all_poke_epoch[:,1] > trial_start) # Check to see if we have any continuous nose-pokes that triggered # more than one trial (these are not actually continuous nose-poke, # the sampling rate was so low that we simply did not detect the # discontinuity). If so, we need to break down this nose-poke into # two nose-pokes. We know that the subject broke the nose-poke # when the response window went high, so we'll simply insert a zero # into the correct place in the nose-poke TTL signal and rerun the # script. double_mask = mask.sum(0) > 1 if double_mask.any(): for lb, ub in all_poke_epoch[double_mask]: i = ts(edge_rising(data.contact.response_TTL[lb:ub])) data.contact.poke_TTL[lb+i] = False data.contact.all_poke_epoch._f_remove() print 'Updated poke_TTL. Please rerun script.' return mask = mask.any(0) poke_epoch = all_poke_epoch[mask] if len(poke_epoch) != trials: raise ValueError, "Unable to winnow down poke epoch list" node = fh.createArray(data.contact, 'poke_epoch', poke_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created poke_epoch' else: print 'poke_epoch already exists' if 'response_ts' not in data.contact: response_ts = epochs(data.contact.response_TTL[:])[:,1] if len(response_ts) != trials: ts_end = data.trial_log.cols.ts_end[:] incomplete_response_ts = response_ts response_ts = np.empty(len(ts_end)) # Find out which response_ts values are missing. If no # response_ts is within 500 msec of the ts_end value, we know # that the response_ts is missing. delta = np.abs(incomplete_response_ts-ts_end[np.newaxis].T) # Boolean mask indicating which trials we have a valid # response_ts for. valid = np.any(delta < (0.5 * TTL_fs), 1) response_ts[valid] = incomplete_response_ts # The trials for which we don't have a valid response_ts based # on the response_TTL should be discarded due to a faulty spout # sensor. So, let's just use the ts_end timestamp instead. The # ts_end timestamp is sampled at the same fs as the # response_TTL, so no conversion is needed. response_ts[~valid] = ts_end[~valid] node = fh.createArray(data.contact, 'response_ts', response_ts) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created response_ts' else: print 'response_ts already exists' # The logic for this is slightly more complicated because early versions # of the appetitive dt paradigm set the signal duration to 0 for the # nogo. This effectively means that there's no signal_TTL during these # nogos, so what is the "epoch" in this case? I may work this out # later. I don't really use signal_epoch (we already know when the # signal is presented based on the trial start timestamp). # #if 'signal_epoch' not in data.contact: # signal_epoch = get_epochs(data.contact.signal_TTL[:]) # print len(signal_epoch), trials # if len(signal_epoch) != trials: # raise ValueError, 'Unable to compute signal epoch' # node = fh.createArray(data.contact, 'signal_epoch', signal_epoch) # node._v_attrs['fs'] = TTL_fs # node._v_attrs['t0'] = 0 if 'all_spout_epoch' not in data.contact: all_spout_epoch = epochs(data.contact.spout_TTL[:]) node = fh.createArray(data.contact, 'all_spout_epoch', all_spout_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created all_spout_epoch' else: print 'all_spout_epoch already exists' if 'trial_epoch' not in data.contact: trial_epoch = zip(data.trial_log.cols.ts_start, data.trial_log.cols.ts_end) if len(trial_epoch) != trials: raise ValueError, 'Unable to compute trial epoch' node = fh.createArray(data.contact, 'trial_epoch', trial_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created trial_epoch' else: print 'trial_epoch already exists' # Return from this function if there is no physiology data to work on if 'physiology' not in data: return if 'epoch' not in data.physiology: epoch = epochs(data.physiology.sweep[:]) node = fh.createArray(data.physiology, 'epoch', epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created physiology epoch' else: print 'physiology epoch already exists' # If the ts data is present, it is more likely to be of a higher # resolution than the epoch data. if 'ts' not in data.physiology: timestamps = data.physiology.epoch[:,0] node = fh.createArray(data.physiology, 'ts', timestamps) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created physiology ts' else: print 'physiology ts already exists' # Make sure the node flavor is set to Numpy for series in ('contact/all_poke_epoch', 'contact/poke_epoch', 'contact/trial_epoch', 'contact/all_spout_epoch', 'contact/signal_epoch' 'contact/response_ts', 'physiology/epoch', 'physiology/ts'): try: node = h5.p_get_node(data, series) if node.flavor != 'numpy': node.flavor = 'numpy' print 'Updated node flavor for ', series except tables.NoSuchNodeError: pass