def main(filename): ''' Add the missing timeseries data: all_poke_epoch poke_epoch response_ts signal_epoch - not implemented trial_epoch Also creates some epoch data never seen before: all_spout_epoch ''' with tables.openFile(filename, 'a') as fh: data = h5.p_get_node(fh.root, '*/data') TTL_fs = data.contact.poke_TTL._v_attrs['fs'] trial_start = (data.trial_log.cols.start * TTL_fs).astype('i') trials = len(trial_start) if 'all_poke_epoch' not in data.contact: all_poke_epoch = epochs(data.contact.poke_TTL[:]) node = fh.createArray(data.contact, 'all_poke_epoch', all_poke_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created all_poke_epoch' else: print 'all_poke_epoch already exists' if 'poke_epoch' not in data.contact: # If we are inside this statement, it is highly probable that we # already had to compute all_poke_epoch above. Howver, just in case # we didn't, let's load it back in from the file. all_poke_epoch = data.contact.all_poke_epoch[:] # Use broadcasting to do away with a for loop and find out which # poke epochs bracket the start of a trial. trial_start = trial_start[np.newaxis].T mask = (all_poke_epoch[:,0] <= trial_start) & \ (all_poke_epoch[:,1] > trial_start) # Check to see if we have any continuous nose-pokes that triggered # more than one trial (these are not actually continuous nose-poke, # the sampling rate was so low that we simply did not detect the # discontinuity). If so, we need to break down this nose-poke into # two nose-pokes. We know that the subject broke the nose-poke # when the response window went high, so we'll simply insert a zero # into the correct place in the nose-poke TTL signal and rerun the # script. double_mask = mask.sum(0) > 1 if double_mask.any(): for lb, ub in all_poke_epoch[double_mask]: i = ts(edge_rising(data.contact.response_TTL[lb:ub])) data.contact.poke_TTL[lb+i] = False data.contact.all_poke_epoch._f_remove() print 'Updated poke_TTL. Please rerun script.' return mask = mask.any(0) poke_epoch = all_poke_epoch[mask] if len(poke_epoch) != trials: raise ValueError, "Unable to winnow down poke epoch list" node = fh.createArray(data.contact, 'poke_epoch', poke_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created poke_epoch' else: print 'poke_epoch already exists' if 'response_ts' not in data.contact: response_ts = epochs(data.contact.response_TTL[:])[:,1] if len(response_ts) != trials: ts_end = data.trial_log.cols.ts_end[:] incomplete_response_ts = response_ts response_ts = np.empty(len(ts_end)) # Find out which response_ts values are missing. If no # response_ts is within 500 msec of the ts_end value, we know # that the response_ts is missing. delta = np.abs(incomplete_response_ts-ts_end[np.newaxis].T) # Boolean mask indicating which trials we have a valid # response_ts for. valid = np.any(delta < (0.5 * TTL_fs), 1) response_ts[valid] = incomplete_response_ts # The trials for which we don't have a valid response_ts based # on the response_TTL should be discarded due to a faulty spout # sensor. So, let's just use the ts_end timestamp instead. The # ts_end timestamp is sampled at the same fs as the # response_TTL, so no conversion is needed. response_ts[~valid] = ts_end[~valid] node = fh.createArray(data.contact, 'response_ts', response_ts) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created response_ts' else: print 'response_ts already exists' # The logic for this is slightly more complicated because early versions # of the appetitive dt paradigm set the signal duration to 0 for the # nogo. This effectively means that there's no signal_TTL during these # nogos, so what is the "epoch" in this case? I may work this out # later. I don't really use signal_epoch (we already know when the # signal is presented based on the trial start timestamp). # #if 'signal_epoch' not in data.contact: # signal_epoch = get_epochs(data.contact.signal_TTL[:]) # print len(signal_epoch), trials # if len(signal_epoch) != trials: # raise ValueError, 'Unable to compute signal epoch' # node = fh.createArray(data.contact, 'signal_epoch', signal_epoch) # node._v_attrs['fs'] = TTL_fs # node._v_attrs['t0'] = 0 if 'all_spout_epoch' not in data.contact: all_spout_epoch = epochs(data.contact.spout_TTL[:]) node = fh.createArray(data.contact, 'all_spout_epoch', all_spout_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created all_spout_epoch' else: print 'all_spout_epoch already exists' if 'trial_epoch' not in data.contact: trial_epoch = zip(data.trial_log.cols.ts_start, data.trial_log.cols.ts_end) if len(trial_epoch) != trials: raise ValueError, 'Unable to compute trial epoch' node = fh.createArray(data.contact, 'trial_epoch', trial_epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created trial_epoch' else: print 'trial_epoch already exists' # Return from this function if there is no physiology data to work on if 'physiology' not in data: return if 'epoch' not in data.physiology: epoch = epochs(data.physiology.sweep[:]) node = fh.createArray(data.physiology, 'epoch', epoch) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created physiology epoch' else: print 'physiology epoch already exists' # If the ts data is present, it is more likely to be of a higher # resolution than the epoch data. if 'ts' not in data.physiology: timestamps = data.physiology.epoch[:,0] node = fh.createArray(data.physiology, 'ts', timestamps) node._v_attrs['fs'] = TTL_fs node._v_attrs['t0'] = 0 print 'created physiology ts' else: print 'physiology ts already exists' # Make sure the node flavor is set to Numpy for series in ('contact/all_poke_epoch', 'contact/poke_epoch', 'contact/trial_epoch', 'contact/all_spout_epoch', 'contact/signal_epoch' 'contact/response_ts', 'physiology/epoch', 'physiology/ts'): try: node = h5.p_get_node(data, series) if node.flavor != 'numpy': node.flavor = 'numpy' print 'Updated node flavor for ', series except tables.NoSuchNodeError: pass
def censor_spikes(ext_filename, rms_pad=0.5, block_pad=5, artifact_pad=0.5, artifact_mode='single', zero_pad=1.25, force_overwrite=False): ''' Censor extracted spikes based on three criteria:: noise floor artifacts block Once the spikes have been censored, you can use the physiology review GUI (using the overla extracted spikes feature) to inspect the censored result. ''' with tables.openFile(ext_filename, 'a') as fh: enode = fh.root.event_data if 'censored' in fh.root.event_data: if not force_overwrite: raise IOError, 'Censored data already exists' else: fh.root.event_data.censored._f_remove() fh.root.censor._f_remove(recursive=True) # Create some UUID information that we can reference from other files that # are derived from this one. If I re-censor spikes, the UUID will # change. This means we can check to see whether sorted spike data # (obtained from the extracted spiketimes file) is from the current # version of the extracted times file. fh.setNodeAttr(fh.root, 'censor_uuid', str(uuid.uuid1())) fh.setNodeAttr(fh.root, 'last_censored', time.time()) cnode = fh.createGroup('/', 'censor') censored = fh.createCArray(enode, 'censored', tables.BoolAtom(), enode.timestamps.shape) channel_indices = enode.channel_indices[:] lb = fh.root.block_data.trial_log.cols.start[0]-block_pad ub = fh.root.block_data.trial_log.cols.end[-1]+block_pad if artifact_mode == 'all': # Censor based on transients occuring on *any* channel artifact_mask = np.any(enode.artifacts[:], axis=-1) artifact_ts = enode.timestamps[artifact_mask] artifact_epochs = np.c_[artifact_ts-artifact_pad, artifact_ts+artifact_pad] artifact_epochs = smooth_epochs(artifact_epochs) for i, ch in enumerate(enode._v_attrs.extracted_channels-1): mask = i == channel_indices timestamps = enode.timestamps[mask] if len(timestamps) == 0: # This usually means the channel was used solely for artifact # reject (or the user set the spike threshold wrong) continue if np.diff(timestamps).min() < 0: raise ValueError, 'Timestamps are not ordered!' # Censor based on RMS artifacts rms = fh.root.rms.rms[ch] rms_th = np.median(rms)/0.6745 rms_epochs = epochs(rms >= rms_th) rms_epochs = rms_epochs / fh.root.rms.rms._v_attrs.fs rms_epochs += [[-rms_pad, rms_pad]] rms_epochs = smooth_epochs(rms_epochs) # Censor based on block start/end #block_censor = (timestamps < lb) | (timestamps > ub) zero_epochs = epochs(rms == 0) zero_epochs = zero_epochs / fh.root.rms.rms._v_attrs.fs zero_epochs += [[-zero_pad, zero_pad]] zero_epochs = smooth_epochs(zero_epochs) if len(zero_epochs) > 0: print zero_epochs if artifact_mode == 'single': # Censor based on transients in data. Need to recompute this # for each channel. artifact = enode.artifacts[:,i][mask].astype('bool') artifact_ts = timestamps[artifact] artifact_epochs = np.c_[artifact_ts-artifact_pad, artifact_ts+artifact_pad] artifact_epochs = smooth_epochs(artifact_epochs) #j = np.searchsorted(artifact_epochs[:,0], timestamps) #k = np.searchsorted(artifact_epochs[:,1], timestamps) #artifact_censor = ~(j == k) cenode = fh.createGroup(cnode, 'extracted_{}'.format(i)) cenode._v_attrs['channel'] = ch+1 # 1-based channel number # Save the rms censor data array = fh.createEArray(cenode, 'rms_epochs', shape=(0, 2), atom=tables.Float32Atom(), title='Censored RMS epochs') array.append(rms_epochs) array._v_attrs['rms_pad'] = rms_pad array._v_attrs['rms_threshold'] = rms_th # Save the zero censor data array = fh.createEArray(cenode, 'zero_epochs', shape=(0, 2), atom=tables.Float32Atom(), title='Censored RMS epochs') array.append(zero_epochs) array._v_attrs['zero_pad'] = zero_pad # Save the artifact reject data array = fh.createEArray(cenode, 'artifact_epochs', shape=(0, 2), atom=tables.Float32Atom(), title='Censored artifact epochs') array.append(artifact_epochs) array._v_attrs['artifact_pad'] = artifact_pad array._v_attrs['artifact_mode'] = artifact_mode # Save the block reject data cenode._v_attrs['block_start'] = lb cenode._v_attrs['block_end'] = ub all_epochs = np.r_[[(0, lb)], rms_epochs, artifact_epochs, zero_epochs, [(ub, np.inf)]] all_epochs.sort() all_epochs = smooth_epochs(all_epochs) fh.createArray(cenode, 'censored_epochs', all_epochs, title='All censored epochs') # Update the censor mask censored[mask] = epochs_contain(all_epochs, timestamps) # Save some metadata regarding the censoring process (this is redundant # with the data stored in the subnodes, but provided here as well) cnode._v_attrs['rms_pad'] = rms_pad cnode._v_attrs['block_pad'] = block_pad cnode._v_attrs['artifact_pad'] = artifact_pad cnode._v_attrs['artifact_mode'] = artifact_mode