def read_protocol(self): """ Read the protocol waveform of the file, if present; function works with ABF2 only. Protocols can be reconstructed from the ABF1 header. Returns: list of segments (one for every episode) with list of analog signls (one for every DAC). """ header = self.read_header() if header['fFileVersionNumber'] < 2.: raise IOError("Protocol section is only present in ABF2 files.") nADC = header['sections']['ADCSection']['llNumEntries'] # n ADC chans nDAC = header['sections']['DACSection']['llNumEntries'] # n DAC chans nSam = header['protocol']['lNumSamplesPerEpisode']/nADC # samples/ep nEpi = header['lActualEpisodes'] sampling_rate = 1.e6/header['protocol']['fADCSequenceInterval'] * pq.Hz # Make a list of segments with analog signals with just holding levels # List of segments relates to number of episodes, as for recorded data segments = [] for epiNum in range(nEpi): seg = Segment(index=epiNum) # One analog signal for each DAC in segment (episode) for DACNum in range(nDAC): t_start = 0 * pq.s # TODO: Possibly check with episode array name = header['listDACInfo'][DACNum]['DACChNames'] unit = header['listDACInfo'][DACNum]['DACChUnits'].\ replace(b'\xb5', b'u') # \xb5 is µ signal = np.ones(nSam) *\ header['listDACInfo'][DACNum]['fDACHoldingLevel'] *\ pq.Quantity(1, unit) anaSig = AnalogSignal(signal, sampling_rate=sampling_rate, t_start=t_start, name=str(name), channel_index=DACNum) # If there are epoch infos for this DAC if DACNum in header['dictEpochInfoPerDAC']: # Save last sample index i_last = int(nSam * 15625 / 10**6) # TODO guess for first holding # Go over EpochInfoPerDAC and change the analog signal # according to the epochs epochInfo = header['dictEpochInfoPerDAC'][DACNum] for epochNum, epoch in iteritems(epochInfo): i_begin = i_last i_end = i_last + epoch['lEpochInitDuration'] +\ epoch['lEpochDurationInc'] * epiNum dif = i_end-i_begin anaSig[i_begin:i_end] = np.ones(len(range(dif))) *\ pq.Quantity(1, unit) * (epoch['fEpochInitLevel'] + epoch['fEpochLevelInc'] * epiNum) i_last += epoch['lEpochInitDuration'] seg.analogsignals.append(anaSig) segments.append(seg) return segments
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True): """ Read in a segment. Arguments: load_spike_waveform : load or not waveform of spikes (default True) """ fid = open(self.filename, 'rb') globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) # metadatas seg = Segment() seg.rec_datetime = datetime.datetime( globalHeader.pop('Year'), globalHeader.pop('Month'), globalHeader.pop('Day'), globalHeader.pop('Hour'), globalHeader.pop('Minute'), globalHeader.pop('Second') ) seg.file_origin = os.path.basename(self.filename) for key, val in globalHeader.iteritems(): seg.annotate(**{key: val}) if not cascade: return seg ## Step 1 : read headers # dsp channels header = spikes and waveforms dspChannelHeaders = {} maxunit = 0 maxchan = 0 for _ in range(globalHeader['NumDSPChannels']): # channel is 1 based channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None) channelHeader['Template'] = np.array(channelHeader['Template']).reshape((5,64)) channelHeader['Boxes'] = np.array(channelHeader['Boxes']).reshape((5,2,4)) dspChannelHeaders[channelHeader['Channel']] = channelHeader maxunit = max(channelHeader['NUnits'], maxunit) maxchan = max(channelHeader['Channel'], maxchan) # event channel header eventHeaders = { } for _ in range(globalHeader['NumEventChannels']): eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None) eventHeaders[eventHeader['Channel']] = eventHeader # slow channel header = signal slowChannelHeaders = {} for _ in range(globalHeader['NumSlowChannels']): slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None) slowChannelHeaders[slowChannelHeader['Channel']] = slowChannelHeader ## Step 2 : a first loop for counting size # signal nb_samples = np.zeros(len(slowChannelHeaders)) sample_positions = np.zeros(len(slowChannelHeaders)) t_starts = np.zeros(len(slowChannelHeaders), dtype='f') #spiketimes and waveform nb_spikes = np.zeros((maxchan+1, maxunit+1) ,dtype='i') wf_sizes = np.zeros((maxchan+1, maxunit+1, 2) ,dtype='i') # eventarrays nb_events = { } #maxstrsizeperchannel = { } for chan, h in iteritems(eventHeaders): nb_events[chan] = 0 #maxstrsizeperchannel[chan] = 0 start = fid.tell() while fid.tell() !=-1 : # read block header dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None) if dataBlockHeader is None : break chan = dataBlockHeader['Channel'] unit = dataBlockHeader['Unit'] n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform'] time = (dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp']) if dataBlockHeader['Type'] == 1: nb_spikes[chan,unit] +=1 wf_sizes[chan,unit,:] = [n1,n2] fid.seek(n1*n2*2,1) elif dataBlockHeader['Type'] ==4: #event nb_events[chan] += 1 elif dataBlockHeader['Type'] == 5: #continuous signal fid.seek(n2*2, 1) if n2> 0: nb_samples[chan] += n2 if nb_samples[chan] ==0: t_starts[chan] = time ## Step 3: allocating memory and 2 loop for reading if not lazy if not lazy: # allocating mem for signal sigarrays = { } for chan, h in iteritems(slowChannelHeaders): sigarrays[chan] = np.zeros(nb_samples[chan]) # allocating mem for SpikeTrain stimearrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object) swfarrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object) for (chan, unit), _ in np.ndenumerate(nb_spikes): stimearrays[chan,unit] = np.zeros(nb_spikes[chan,unit], dtype = 'f') if load_spike_waveform: n1,n2 = wf_sizes[chan, unit,:] swfarrays[chan, unit] = np.zeros( (nb_spikes[chan, unit], n1, n2 ) , dtype = 'f4' ) pos_spikes = np.zeros(nb_spikes.shape, dtype = 'i') # allocating mem for event eventpositions = { } evarrays = { } for chan, nb in iteritems(nb_events): evarrays[chan] = { 'times': np.zeros(nb, dtype='f'), 'labels': np.zeros(nb, dtype='S4') } eventpositions[chan]=0 fid.seek(start) while fid.tell() !=-1 : dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None) if dataBlockHeader is None : break chan = dataBlockHeader['Channel'] n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform'] time = dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp'] time/= globalHeader['ADFrequency'] if n2 <0: break if dataBlockHeader['Type'] == 1: #spike unit = dataBlockHeader['Unit'] pos = pos_spikes[chan,unit] stimearrays[chan, unit][pos] = time if load_spike_waveform and n1*n2 != 0 : swfarrays[chan,unit][pos,:,:] = np.fromstring( fid.read(n1*n2*2) , dtype = 'i2').reshape(n1,n2).astype('f4') else: fid.seek(n1*n2*2,1) pos_spikes[chan,unit] +=1 elif dataBlockHeader['Type'] == 4: # event pos = eventpositions[chan] evarrays[chan]['times'][pos] = time evarrays[chan]['labels'][pos] = dataBlockHeader['Unit'] eventpositions[chan]+= 1 elif dataBlockHeader['Type'] == 5: #signal data = np.fromstring( fid.read(n2*2) , dtype = 'i2').astype('f4') sigarrays[chan][sample_positions[chan] : sample_positions[chan]+data.size] = data sample_positions[chan] += data.size ## Step 4: create neo object for chan, h in iteritems(eventHeaders): if lazy: times = [] labels = None else: times = evarrays[chan]['times'] labels = evarrays[chan]['labels'] ea = EventArray( times*pq.s, labels=labels, channel_name=eventHeaders[chan]['Name'], channel_index=chan ) if lazy: ea.lazy_shape = nb_events[chan] seg.eventarrays.append(ea) for chan, h in iteritems(slowChannelHeaders): if lazy: signal = [ ] else: if globalHeader['Version'] ==100 or globalHeader['Version'] ==101 : gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*1000.) elif globalHeader['Version'] ==102 : gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain']) elif globalHeader['Version'] >= 103: gain = globalHeader['SlowMaxMagnitudeMV']/(.5*(2**globalHeader['BitsPerSpikeSample'])*\ slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain']) signal = sigarrays[chan]*gain anasig = AnalogSignal(signal*pq.V, sampling_rate = float(slowChannelHeaders[chan]['ADFreq'])*pq.Hz, t_start = t_starts[chan]*pq.s, channel_index = slowChannelHeaders[chan]['Channel'], channel_name = slowChannelHeaders[chan]['Name'], ) if lazy: anasig.lazy_shape = nb_samples[chan] seg.analogsignals.append(anasig) for (chan, unit), value in np.ndenumerate(nb_spikes): if nb_spikes[chan, unit] == 0: continue if lazy: times = [ ] waveforms = None t_stop = 0 else: times = stimearrays[chan,unit] t_stop = times.max() if load_spike_waveform: if globalHeader['Version'] <103: gain = 3000./(2048*dspChannelHeaders[chan]['Gain']*1000.) elif globalHeader['Version'] >=103 and globalHeader['Version'] <105: gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*1000.) elif globalHeader['Version'] >105: gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*globalHeader['SpikePreAmpGain']) waveforms = swfarrays[chan, unit] * gain * pq.V else: waveforms = None sptr = SpikeTrain( times, units='s', t_stop=t_stop*pq.s, waveforms=waveforms ) sptr.annotate(unit_name = dspChannelHeaders[chan]['Name']) sptr.annotate(channel_index = chan) for key, val in dspChannelHeaders[chan].iteritems(): sptr.annotate(**{key: val}) if lazy: sptr.lazy_shape = nb_spikes[chan,unit] seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg
def generate_one_simple_segment( seg_name = 'segment 0', supported_objects = [ ], nb_analogsignal = 4, t_start = 0.*s, sampling_rate = 10*kHz, duration = 6.*s, nb_spiketrain = 6, spikerate_range = [.5*Hz, 12*Hz], event_array_types = { 'stim' : ['a', 'b', 'c' , 'd'], 'enter_zone' : [ 'one', 'two'], 'color' : ['black', 'yellow', 'green' ], }, event_array_size_range = [5, 20], epoch_array_types = { 'animal state' : ['Sleep', 'Freeze', 'Escape', ], 'light' : ['dark', 'lighted', ], }, epoch_array_duration_range = [.5, 3., ], ): seg = Segment(name = seg_name) if AnalogSignal in supported_objects: for a in range(nb_analogsignal): anasig = AnalogSignal(rand(int(sampling_rate * duration)), sampling_rate=sampling_rate, t_start=t_start, units=mV, channel_index=a, name='sig %d for segment %s' % (a, seg.name)) seg.analogsignals.append(anasig) if SpikeTrain in supported_objects: for s in range(nb_spiketrain): spikerate = rand()*np.diff(spikerate_range)+spikerate_range[0].magnitude #sptr = SpikeTrain( rand(int((spikerate*duration).simplified))*duration , t_start = t_start, t_stop = t_start+duration) #, name = 'spiketrain %d'%s) spikes = rand(int((spikerate*duration).simplified)) spikes.sort() # spikes are supposed to be an ascending sequence sptr = SpikeTrain( spikes*duration, t_start = t_start, t_stop = t_start+duration) sptr.annotations['channel_index'] = s seg.spiketrains.append(sptr) if EventArray in supported_objects: for name, labels in iteritems(event_array_types): ea_size = rand()*np.diff(event_array_size_range)+event_array_size_range[0] ea = EventArray( #name = name, times = rand(ea_size)*duration, labels = np.array( labels, dtype='S')[(rand(ea_size)*len(labels)).astype('i')], ) seg.eventarrays.append(ea) if EpochArray in supported_objects: for name, labels in iteritems(epoch_array_types): t = 0 times, durations = [ ], [ ] while t<duration: times.append(t) dur = (rand()*np.diff(epoch_array_duration_range)+epoch_array_duration_range[0]) durations.append(dur) t = t+dur epa = EpochArray( #name = name, times = pq.Quantity(times, units = pq.s), durations = pq.Quantity([x[0] for x in durations], units = pq.s), labels = np.array( labels, dtype='S')[(rand(len(times))*len(labels)).astype('i')], ) seg.epocharrays.append(epa) # TODO : Spike, Event, Epoch return seg
def generate_one_simple_segment( seg_name='segment 0', supported_objects=[], nb_analogsignal=4, t_start=0. * pq.s, sampling_rate=10 * pq.kHz, duration=6. * pq.s, nb_spiketrain=6, spikerate_range=[.5 * pq.Hz, 12 * pq.Hz], event_types={ 'stim': ['a', 'b', 'c', 'd'], 'enter_zone': ['one', 'two'], 'color': ['black', 'yellow', 'green'], }, event_size_range=[5, 20], epoch_types={ 'animal state': ['Sleep', 'Freeze', 'Escape'], 'light': ['dark', 'lighted'] }, epoch_duration_range=[.5, 3.], ): if supported_objects and Segment not in supported_objects: raise ValueError('Segment must be in supported_objects') seg = Segment(name=seg_name) if AnalogSignal in supported_objects: for a in range(nb_analogsignal): anasig = AnalogSignal(rand(int(sampling_rate * duration)), sampling_rate=sampling_rate, t_start=t_start, units=pq.mV, channel_index=a, name='sig %d for segment %s' % (a, seg.name)) seg.analogsignals.append(anasig) if SpikeTrain in supported_objects: for s in range(nb_spiketrain): spikerate = rand() * np.diff(spikerate_range) spikerate += spikerate_range[0].magnitude #spikedata = rand(int((spikerate*duration).simplified))*duration #sptr = SpikeTrain(spikedata, # t_start=t_start, t_stop=t_start+duration) # #, name = 'spiketrain %d'%s) spikes = rand(int((spikerate * duration).simplified)) spikes.sort() # spikes are supposed to be an ascending sequence sptr = SpikeTrain(spikes * duration, t_start=t_start, t_stop=t_start + duration) sptr.annotations['channel_index'] = s seg.spiketrains.append(sptr) if Event in supported_objects: for name, labels in iteritems(event_types): evt_size = rand() * np.diff(event_size_range) evt_size += event_size_range[0] evt_size = int(evt_size) labels = np.array(labels, dtype='S') labels = labels[(rand(evt_size) * len(labels)).astype('i')] evt = Event(times=rand(evt_size) * duration, labels=labels) seg.events.append(evt) if Epoch in supported_objects: for name, labels in iteritems(epoch_types): t = 0 times = [] durations = [] while t < duration: times.append(t) dur = rand() * np.diff(epoch_duration_range) dur += epoch_duration_range[0] durations.append(dur) t = t + dur labels = np.array(labels, dtype='S') labels = labels[(rand(len(times)) * len(labels)).astype('i')] epc = Epoch( times=pq.Quantity(times, units=pq.s), durations=pq.Quantity([x[0] for x in durations], units=pq.s), labels=labels, ) seg.epochs.append(epc) # TODO : Spike, Event seg.create_many_to_one_relationship() return seg
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) create_many_to_one_relationship(bl) return bl
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) bl.create_many_to_one_relationship() return bl
def generate_one_simple_segment(seg_name='segment 0', supported_objects=[], nb_analogsignal=4, t_start=0.*pq.s, sampling_rate=10*pq.kHz, duration=6.*pq.s, nb_spiketrain=6, spikerate_range=[.5*pq.Hz, 12*pq.Hz], event_types={'stim': ['a', 'b', 'c', 'd'], 'enter_zone': ['one', 'two'], 'color': ['black', 'yellow', 'green'], }, event_size_range=[5, 20], epoch_types={'animal state': ['Sleep', 'Freeze', 'Escape'], 'light': ['dark', 'lighted'] }, epoch_duration_range=[.5, 3.], ): if supported_objects and Segment not in supported_objects: raise ValueError('Segment must be in supported_objects') seg = Segment(name=seg_name) if AnalogSignal in supported_objects: for a in range(nb_analogsignal): anasig = AnalogSignal(rand(int(sampling_rate * duration)), sampling_rate=sampling_rate, t_start=t_start, units=pq.mV, channel_index=a, name='sig %d for segment %s' % (a, seg.name)) seg.analogsignals.append(anasig) if SpikeTrain in supported_objects: for s in range(nb_spiketrain): spikerate = rand()*np.diff(spikerate_range) spikerate += spikerate_range[0].magnitude #spikedata = rand(int((spikerate*duration).simplified))*duration #sptr = SpikeTrain(spikedata, # t_start=t_start, t_stop=t_start+duration) # #, name = 'spiketrain %d'%s) spikes = rand(int((spikerate*duration).simplified)) spikes.sort() # spikes are supposed to be an ascending sequence sptr = SpikeTrain(spikes*duration, t_start=t_start, t_stop=t_start+duration) sptr.annotations['channel_index'] = s seg.spiketrains.append(sptr) if Event in supported_objects: for name, labels in iteritems(event_types): evt_size = rand()*np.diff(event_size_range) evt_size += event_size_range[0] labels = np.array(labels, dtype='S') labels = labels[(rand(evt_size)*len(labels)).astype('i')] evt = Event(times=rand(evt_size)*duration, labels=labels) seg.events.append(evt) if Epoch in supported_objects: for name, labels in iteritems(epoch_types): t = 0 times = [] durations = [] while t < duration: times.append(t) dur = rand()*np.diff(epoch_duration_range) dur += epoch_duration_range[0] durations.append(dur) t = t+dur labels = np.array(labels, dtype='S') labels = labels[(rand(len(times))*len(labels)).astype('i')] epc = Epoch(times=pq.Quantity(times, units=pq.s), durations=pq.Quantity([x[0] for x in durations], units=pq.s), labels=labels, ) seg.epochs.append(epc) # TODO : Spike, Event seg.create_many_to_one_relationship() return seg
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True): """ """ fid = open(self.filename, "rb") globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) # metadatas seg = Segment() seg.rec_datetime = datetime.datetime( globalHeader["Year"], globalHeader["Month"], globalHeader["Day"], globalHeader["Hour"], globalHeader["Minute"], globalHeader["Second"], ) seg.file_origin = os.path.basename(self.filename) seg.annotate(plexon_version=globalHeader["Version"]) if not cascade: return seg ## Step 1 : read headers # dsp channels header = sipkes and waveforms dspChannelHeaders = {} maxunit = 0 maxchan = 0 for _ in range(globalHeader["NumDSPChannels"]): # channel is 1 based channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None) channelHeader["Template"] = np.array(channelHeader["Template"]).reshape((5, 64)) channelHeader["Boxes"] = np.array(channelHeader["Boxes"]).reshape((5, 2, 4)) dspChannelHeaders[channelHeader["Channel"]] = channelHeader maxunit = max(channelHeader["NUnits"], maxunit) maxchan = max(channelHeader["Channel"], maxchan) # event channel header eventHeaders = {} for _ in range(globalHeader["NumEventChannels"]): eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None) eventHeaders[eventHeader["Channel"]] = eventHeader # slow channel header = signal slowChannelHeaders = {} for _ in range(globalHeader["NumSlowChannels"]): slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None) slowChannelHeaders[slowChannelHeader["Channel"]] = slowChannelHeader ## Step 2 : a first loop for counting size # signal nb_samples = np.zeros(len(slowChannelHeaders)) sample_positions = np.zeros(len(slowChannelHeaders)) t_starts = np.zeros(len(slowChannelHeaders), dtype="f") # spiketimes and waveform nb_spikes = np.zeros((maxchan + 1, maxunit + 1), dtype="i") wf_sizes = np.zeros((maxchan + 1, maxunit + 1, 2), dtype="i") # eventarrays nb_events = {} # maxstrsizeperchannel = { } for chan, h in iteritems(eventHeaders): nb_events[chan] = 0 # maxstrsizeperchannel[chan] = 0 start = fid.tell() while fid.tell() != -1: # read block header dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None) if dataBlockHeader is None: break chan = dataBlockHeader["Channel"] unit = dataBlockHeader["Unit"] n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"] time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"] if dataBlockHeader["Type"] == 1: nb_spikes[chan, unit] += 1 wf_sizes[chan, unit, :] = [n1, n2] fid.seek(n1 * n2 * 2, 1) elif dataBlockHeader["Type"] == 4: # event nb_events[chan] += 1 elif dataBlockHeader["Type"] == 5: # continuous signal fid.seek(n2 * 2, 1) if n2 > 0: nb_samples[chan] += n2 if nb_samples[chan] == 0: t_starts[chan] = time ## Step 3: allocating memory and 2 loop for reading if not lazy if not lazy: # allocating mem for signal sigarrays = {} for chan, h in iteritems(slowChannelHeaders): sigarrays[chan] = np.zeros(nb_samples[chan]) # allocating mem for SpikeTrain stimearrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object) swfarrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object) for (chan, unit), _ in np.ndenumerate(nb_spikes): stimearrays[chan, unit] = np.zeros(nb_spikes[chan, unit], dtype="f") if load_spike_waveform: n1, n2 = wf_sizes[chan, unit, :] swfarrays[chan, unit] = np.zeros((nb_spikes[chan, unit], n1, n2), dtype="f4") pos_spikes = np.zeros(nb_spikes.shape, dtype="i") # allocating mem for event eventpositions = {} evarrays = {} for chan, nb in iteritems(nb_events): evarrays[chan] = np.zeros(nb, dtype="f") eventpositions[chan] = 0 fid.seek(start) while fid.tell() != -1: dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None) if dataBlockHeader is None: break chan = dataBlockHeader["Channel"] n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"] time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"] time /= globalHeader["ADFrequency"] if n2 < 0: break if dataBlockHeader["Type"] == 1: # spike unit = dataBlockHeader["Unit"] pos = pos_spikes[chan, unit] stimearrays[chan, unit][pos] = time if load_spike_waveform and n1 * n2 != 0: swfarrays[chan, unit][pos, :, :] = ( np.fromstring(fid.read(n1 * n2 * 2), dtype="i2").reshape(n1, n2).astype("f4") ) else: fid.seek(n1 * n2 * 2, 1) pos_spikes[chan, unit] += 1 elif dataBlockHeader["Type"] == 4: # event pos = eventpositions[chan] evarrays[chan][pos] = time eventpositions[chan] += 1 elif dataBlockHeader["Type"] == 5: # signal data = np.fromstring(fid.read(n2 * 2), dtype="i2").astype("f4") sigarrays[chan][sample_positions[chan] : sample_positions[chan] + data.size] = data sample_positions[chan] += data.size ## Step 3: create neo object for chan, h in iteritems(eventHeaders): if lazy: times = [] else: times = evarrays[chan] ea = EventArray(times * pq.s, channel_name=eventHeaders[chan]["Name"], channel_index=chan) if lazy: ea.lazy_shape = nb_events[chan] seg.eventarrays.append(ea) for chan, h in iteritems(slowChannelHeaders): if lazy: signal = [] else: if globalHeader["Version"] == 100 or globalHeader["Version"] == 101: gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * 1000.0) elif globalHeader["Version"] == 102: gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"]) elif globalHeader["Version"] >= 103: gain = globalHeader["SlowMaxMagnitudeMV"] / ( 0.5 * (2 ** globalHeader["BitsPerSpikeSample"]) * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"] ) signal = sigarrays[chan] * gain anasig = AnalogSignal( signal * pq.V, sampling_rate=float(slowChannelHeaders[chan]["ADFreq"]) * pq.Hz, t_start=t_starts[chan] * pq.s, channel_index=slowChannelHeaders[chan]["Channel"], channel_name=slowChannelHeaders[chan]["Name"], ) if lazy: anasig.lazy_shape = nb_samples[chan] seg.analogsignals.append(anasig) for (chan, unit), value in np.ndenumerate(nb_spikes): if nb_spikes[chan, unit] == 0: continue if lazy: times = [] waveforms = None t_stop = 0 else: times = stimearrays[chan, unit] t_stop = times.max() if load_spike_waveform: if globalHeader["Version"] < 103: gain = 3000.0 / (2048 * dspChannelHeaders[chan]["Gain"] * 1000.0) elif globalHeader["Version"] >= 103 and globalHeader["Version"] < 105: gain = globalHeader["SpikeMaxMagnitudeMV"] / ( 0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * 1000.0 ) elif globalHeader["Version"] > 105: gain = globalHeader["SpikeMaxMagnitudeMV"] / ( 0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * globalHeader["SpikePreAmpGain"] ) waveforms = swfarrays[chan, unit] * gain * pq.V else: waveforms = None sptr = SpikeTrain(times, units="s", t_stop=t_stop * pq.s, waveforms=waveforms) sptr.annotate(unit_name=dspChannelHeaders[chan]["Name"]) sptr.annotate(channel_index=chan) if lazy: sptr.lazy_shape = nb_spikes[chan, unit] seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg