def read_segment( self, lazy=False, delimiter="\t", t_start=0. * pq.s, unit=pq.s, ): """ delimiter is the columns delimiter in file "\t" or one space or two space or "," or ";" t_start is the time start of all spiketrain 0 by default. unit is the unit of spike times, can be a str or directly a quantity. """ assert not lazy, "Do not support lazy" unit = pq.Quantity(1, unit) seg = Segment(file_origin=os.path.basename(self.filename)) f = open(self.filename, "Ur") for i, line in enumerate(f): alldata = line[:-1].split(delimiter) if alldata[-1] == "": alldata = alldata[:-1] if alldata[0] == "": alldata = alldata[1:] spike_times = np.array(alldata).astype("f") t_stop = spike_times.max() * unit sptr = SpikeTrain(spike_times * unit, t_start=t_start, t_stop=t_stop) sptr.annotate(channel_index=i) seg.spiketrains.append(sptr) f.close() seg.create_many_to_one_relationship() return seg
def read_spiketrain(self , # the 2 first key arguments are imposed by neo.io API lazy = False, cascade = True, segment_duration = 15., t_start = -1, channel_index = 0, ): """ With this IO SpikeTrain can e acces directly with its channel number """ # There are 2 possibles behaviour for a SpikeTrain # holding many Spike instance or directly holding spike times # we choose here the first : if not HAVE_SCIPY: raise SCIPY_ERR num_spike_by_spiketrain = 40 sr = 10000. if lazy: times = [ ] else: times = (np.random.rand(num_spike_by_spiketrain)*segment_duration + t_start) # create a spiketrain spiketr = SpikeTrain(times, t_start = t_start*pq.s, t_stop = (t_start+segment_duration)*pq.s , units = pq.s, name = 'it is a spiketrain from exampleio', ) if lazy: # we add the attribute lazy_shape with the size if loaded spiketr.lazy_shape = (num_spike_by_spiketrain,) # ours spiketrains also hold the waveforms: # 1 generate a fake spike shape (2d array if trodness >1) w1 = -stats.nct.pdf(np.arange(11,60,4), 5,20)[::-1]/3. w2 = stats.nct.pdf(np.arange(11,60,2), 5,20) w = np.r_[ w1 , w2 ] w = -w/max(w) if not lazy: # in the neo API the waveforms attr is 3 D in case tetrode # in our case it is mono electrode so dim 1 is size 1 waveforms = np.tile( w[np.newaxis,np.newaxis,:], ( num_spike_by_spiketrain ,1, 1) ) waveforms *= np.random.randn(*waveforms.shape)/6+1 spiketr.waveforms = waveforms*pq.mV spiketr.sampling_rate = sr * pq.Hz spiketr.left_sweep = 1.5* pq.s # for attributes out of neo you can annotate spiketr.annotate(channel_index = channel_index) return spiketr
def _extract_spikes(self, data, metadata, channel_index): spiketrain = None spike_times = self._extract_array(data, channel_index) if len(spike_times) > 0: spiketrain = SpikeTrain(spike_times, units=pq.ms, t_stop=spike_times.max()) spiketrain.annotate(label=metadata["label"], channel_index=channel_index, dt=metadata["dt"]) return spiketrain
def spiketrain_from_raw(channel: AnalogSignal, threshold: Quantity) -> SpikeTrain: assert channel.signal.shape[1] == 1 spikes = find_spikes(channel, threshold) signal = channel.signal.squeeze() times = np.array([channel_index_to_time(channel, start) for start, _ in spikes]) * channel.sampling_period.units waveforms = np.array([[signal[start:stop]] for start, stop in spikes]) * channel.units name = f"{channel.name}#{threshold}" result = SpikeTrain(name=name, times=times, units=channel.units, t_start=channel.t_start, t_stop=channel.t_stop, waveforms=waveforms, sampling_rate=channel.sampling_rate, sort=True) result.annotate(from_channel=channel.name, threshold=threshold) return result
def _extract_spikes(self, data, metadata, channel_index, lazy): spiketrain = None if lazy: if channel_index in data[:, 1]: spiketrain = SpikeTrain([], units=pq.ms, t_stop=0.0) spiketrain.lazy_shape = None else: spike_times = self._extract_array(data, channel_index) if len(spike_times) > 0: spiketrain = SpikeTrain(spike_times, units=pq.ms, t_stop=spike_times.max()) if spiketrain is not None: spiketrain.annotate(label=metadata["label"], channel_index=channel_index, dt=metadata["dt"]) return spiketrain
def create_all_annotated(cls): times = cls.rquant(1, pq.s) signal = cls.rquant(1, pq.V) blk = Block() blk.annotate(**cls.rdict(3)) cls.populate_dates(blk) seg = Segment() seg.annotate(**cls.rdict(4)) cls.populate_dates(seg) blk.segments.append(seg) asig = AnalogSignal(signal=signal, sampling_rate=pq.Hz) asig.annotate(**cls.rdict(2)) seg.analogsignals.append(asig) isig = IrregularlySampledSignal(times=times, signal=signal, time_units=pq.s) isig.annotate(**cls.rdict(2)) seg.irregularlysampledsignals.append(isig) epoch = Epoch(times=times, durations=times) epoch.annotate(**cls.rdict(4)) seg.epochs.append(epoch) event = Event(times=times) event.annotate(**cls.rdict(4)) seg.events.append(event) spiketrain = SpikeTrain(times=times, t_stop=pq.s, units=pq.s) d = cls.rdict(6) d["quantity"] = pq.Quantity(10, "mV") d["qarray"] = pq.Quantity(range(10), "mA") spiketrain.annotate(**d) seg.spiketrains.append(spiketrain) chx = ChannelIndex(name="achx", index=[1, 2], channel_ids=[0, 10]) chx.annotate(**cls.rdict(5)) blk.channel_indexes.append(chx) unit = Unit() unit.annotate(**cls.rdict(2)) chx.units.append(unit) return blk
def read_segment( self, lazy=False, cascade=True, delimiter='\t', t_start=0. * pq.s, unit=pq.s, ): """ Arguments: delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';' t_start : time start of all spiketrain 0 by default unit : unit of spike times, can be a str or directly a Quantities """ unit = pq.Quantity(1, unit) seg = Segment(file_origin=os.path.basename(self.filename)) if not cascade: return seg f = open(self.filename, 'Ur') for i, line in enumerate(f): alldata = line[:-1].split(delimiter) if alldata[-1] == '': alldata = alldata[:-1] if alldata[0] == '': alldata = alldata[1:] if lazy: spike_times = [] t_stop = t_start else: spike_times = np.array(alldata).astype('f') t_stop = spike_times.max() * unit sptr = SpikeTrain(spike_times * unit, t_start=t_start, t_stop=t_stop) if lazy: sptr.lazy_shape = len(alldata) sptr.annotate(channel_index=i) seg.spiketrains.append(sptr) f.close() seg.create_many_to_one_relationship() return seg
def create_all_annotated(cls): times = cls.rquant(1, pq.s) signal = cls.rquant(1, pq.V) blk = Block() blk.annotate(**cls.rdict(3)) seg = Segment() seg.annotate(**cls.rdict(4)) blk.segments.append(seg) asig = AnalogSignal(signal=signal, sampling_rate=pq.Hz) asig.annotate(**cls.rdict(2)) seg.analogsignals.append(asig) isig = IrregularlySampledSignal(times=times, signal=signal, time_units=pq.s) isig.annotate(**cls.rdict(2)) seg.irregularlysampledsignals.append(isig) epoch = Epoch(times=times, durations=times) epoch.annotate(**cls.rdict(4)) seg.epochs.append(epoch) event = Event(times=times) event.annotate(**cls.rdict(4)) seg.events.append(event) spiketrain = SpikeTrain(times=times, t_stop=pq.s, units=pq.s) d = cls.rdict(6) d["quantity"] = pq.Quantity(10, "mV") d["qarray"] = pq.Quantity(range(10), "mA") spiketrain.annotate(**d) seg.spiketrains.append(spiketrain) chx = ChannelIndex(name="achx", index=[1, 2], channel_ids=[0, 10]) chx.annotate(**cls.rdict(5)) blk.channel_indexes.append(chx) unit = Unit() unit.annotate(**cls.rdict(2)) chx.units.append(unit) return blk
def read_segment(self, lazy = False, cascade = True, delimiter = '\t', t_start = 0.*pq.s, unit = pq.s, ): """ Arguments: delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';' t_start : time start of all spiketrain 0 by default unit : unit of spike times, can be a str or directly a Quantities """ unit = pq.Quantity(1, unit) seg = Segment(file_origin = os.path.basename(self.filename)) if not cascade: return seg f = open(self.filename, 'Ur') for i,line in enumerate(f) : alldata = line[:-1].split(delimiter) if alldata[-1] == '': alldata = alldata[:-1] if alldata[0] == '': alldata = alldata[1:] if lazy: spike_times = [ ] t_stop = t_start else: spike_times = np.array(alldata).astype('f') t_stop = spike_times.max()*unit sptr = SpikeTrain(spike_times*unit, t_start=t_start, t_stop=t_stop) if lazy: sptr.lazy_shape = len(alldata) sptr.annotate(channel_index = i) seg.spiketrains.append(sptr) f.close() seg.create_many_to_one_relationship() return seg
def read_segment( self, lazy=False, delimiter='\t', t_start=0. * pq.s, unit=pq.s, ): """ Arguments: delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';' t_start : time start of all spiketrain 0 by default unit : unit of spike times, can be a str or directly a Quantities """ assert not lazy, 'Do not support lazy' unit = pq.Quantity(1, unit) seg = Segment(file_origin=os.path.basename(self.filename)) with open(self.filename, 'r', newline=None) as f: for i, line in enumerate(f): alldata = line[:-1].split(delimiter) if alldata[-1] == '': alldata = alldata[:-1] if alldata[0] == '': alldata = alldata[1:] spike_times = np.array(alldata).astype('f') t_stop = spike_times.max() * unit sptr = SpikeTrain(spike_times * unit, t_start=t_start, t_stop=t_stop) sptr.annotate(channel_index=i) seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg
def read_segment(self, blockname=None, lazy=False, cascade=True, sortname=''): """ Read a single segment from the tank. Note that TDT blocks are Neo segments, and TDT tanks are Neo blocks, so here the 'blockname' argument refers to the TDT block's name, which will be the Neo segment name. 'sortname' is used to specify the external sortcode generated by offline spike sorting. if sortname=='PLX', there should be a ./sort/PLX/*.SortResult file in the tdt block, which stores the sortcode for every spike; defaults to '', which uses the original online sort """ if not blockname: blockname = os.listdir(self.dirname)[0] if blockname == 'TempBlk': return None if not self.is_tdtblock(blockname): return None # if not a tdt block subdir = os.path.join(self.dirname, blockname) if not os.path.isdir(subdir): return None seg = Segment(name=blockname) tankname = os.path.basename(self.dirname) #TSQ is the global index tsq_filename = os.path.join(subdir, tankname + '_' + blockname + '.tsq') dt = [ ('size', 'int32'), ('evtype', 'int32'), ('code', 'S4'), ('channel', 'uint16'), ('sortcode', 'uint16'), ('timestamp', 'float64'), ('eventoffset', 'int64'), ('dataformat', 'int32'), ('frequency', 'float32'), ] tsq = np.fromfile(tsq_filename, dtype=dt) #0x8801: 'EVTYPE_MARK' give the global_start global_t_start = tsq[tsq['evtype'] == 0x8801]['timestamp'][0] #TEV is the old data file try: tev_filename = os.path.join(subdir, tankname + '_' + blockname + '.tev') #tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead tev_array = np.fromfile(tev_filename, dtype='uint8') except IOError: tev_filename = None #if there exists an external sortcode in ./sort/[sortname]/*.SortResult (generated after offline sortting) sortresult_filename = None if sortname is not '': try: for file in os.listdir(os.path.join(subdir, 'sort', sortname)): if file.endswith(".SortResult"): sortresult_filename = os.path.join( subdir, 'sort', sortname, file) # get new sortcode newsorcode = np.fromfile(sortresult_filename, 'int8')[ 1024:] # the first 1024 byte is file header # update the sort code with the info from this file tsq['sortcode'][1:-1] = newsorcode # print('sortcode updated') break except OSError: sortresult_filename = None except IOError: sortresult_filename = None for type_code, type_label in tdt_event_type: mask1 = tsq['evtype'] == type_code codes = np.unique(tsq[mask1]['code']) for code in codes: mask2 = mask1 & (tsq['code'] == code) channels = np.unique(tsq[mask2]['channel']) for channel in channels: mask3 = mask2 & (tsq['channel'] == channel) if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']: if lazy: times = [] * pq.s labels = np.array([], dtype=str) else: times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s labels = tsq[mask3]['eventoffset'].view( 'float64').astype('S') ea = Event(times=times, name=code, channel_index=int(channel), labels=labels) if lazy: ea.lazy_shape = np.sum(mask3) seg.events.append(ea) elif type_label == 'EVTYPE_SNIP': sortcodes = np.unique(tsq[mask3]['sortcode']) for sortcode in sortcodes: mask4 = mask3 & (tsq['sortcode'] == sortcode) nb_spike = np.sum(mask4) sr = tsq[mask4]['frequency'][0] waveformsize = tsq[mask4]['size'][0] - 10 if lazy: times = [] * pq.s waveforms = None else: times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s dt = np.dtype( data_formats[tsq[mask3]['dataformat'][0]]) waveforms = get_chunks( tsq[mask4]['size'], tsq[mask4]['eventoffset'], tev_array).view(dt) waveforms = waveforms.reshape( nb_spike, -1, waveformsize) waveforms = waveforms * pq.mV if nb_spike > 0: # t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not t_start = 0 * pq.s t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s else: t_start = 0 * pq.s t_stop = 0 * pq.s st = SpikeTrain( times=times, name='Chan{0} Code{1}'.format( channel, sortcode), t_start=t_start, t_stop=t_stop, waveforms=waveforms, left_sweep=waveformsize / 2. / sr * pq.s, sampling_rate=sr * pq.Hz, ) st.annotate(channel_index=channel) if lazy: st.lazy_shape = nb_spike seg.spiketrains.append(st) elif type_label == 'EVTYPE_STREAM': dt = np.dtype( data_formats[tsq[mask3]['dataformat'][0]]) shape = np.sum(tsq[mask3]['size'] - 10) sr = tsq[mask3]['frequency'][0] if lazy: signal = [] else: if PY3K: signame = code.decode('ascii') else: signame = code sev_filename = os.path.join( subdir, tankname + '_' + blockname + '_' + signame + '_ch' + str(channel) + '.sev') try: #sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead sig_array = np.fromfile(sev_filename, dtype='uint8') except IOError: sig_array = tev_array signal = get_chunks(tsq[mask3]['size'], tsq[mask3]['eventoffset'], sig_array).view(dt) anasig = AnalogSignal( signal=signal * pq.V, name='{0} {1}'.format(code, channel), sampling_rate=sr * pq.Hz, t_start=(tsq[mask3]['timestamp'][0] - global_t_start) * pq.s, channel_index=int(channel)) if lazy: anasig.lazy_shape = shape seg.analogsignals.append(anasig) return seg
def read_block( self, lazy=False, cascade=True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade: return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname, blockname) if not os.path.isdir(subdir): continue seg = Segment(name=blockname) bl.segments.append(seg) #TSQ is the global index tsq_filename = os.path.join(subdir, tankname + '_' + blockname + '.tsq') dt = [ ('size', 'int32'), ('evtype', 'int32'), ('code', 'S4'), ('channel', 'uint16'), ('sortcode', 'uint16'), ('timestamp', 'float64'), ('eventoffset', 'int64'), ('dataformat', 'int32'), ('frequency', 'float32'), ] tsq = np.fromfile(tsq_filename, dtype=dt) #0x8801: 'EVTYPE_MARK' give the global_start global_t_start = tsq[tsq['evtype'] == 0x8801]['timestamp'][0] #TEV is the old data file if os.path.exists( os.path.join(subdir, tankname + '_' + blockname + '.tev')): tev_filename = os.path.join( subdir, tankname + '_' + blockname + '.tev') #tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead tev_array = np.fromfile(tev_filename, dtype='uint8') else: tev_filename = None for type_code, type_label in tdt_event_type: mask1 = tsq['evtype'] == type_code codes = np.unique(tsq[mask1]['code']) for code in codes: mask2 = mask1 & (tsq['code'] == code) channels = np.unique(tsq[mask2]['channel']) for channel in channels: mask3 = mask2 & (tsq['channel'] == channel) if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']: if lazy: times = [] * pq.s labels = np.array([], dtype=str) else: times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s labels = tsq[mask3]['eventoffset'].view( 'float64').astype('S') ea = EventArray(times=times, name=code, channel_index=int(channel), labels=labels) if lazy: ea.lazy_shape = np.sum(mask3) seg.eventarrays.append(ea) elif type_label == 'EVTYPE_SNIP': sortcodes = np.unique(tsq[mask3]['sortcode']) for sortcode in sortcodes: mask4 = mask3 & (tsq['sortcode'] == sortcode) nb_spike = np.sum(mask4) sr = tsq[mask4]['frequency'][0] waveformsize = tsq[mask4]['size'][0] - 10 if lazy: times = [] * pq.s waveforms = None else: times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) waveforms = get_chunks( tsq[mask4]['size'], tsq[mask4]['eventoffset'], tev_array).view(dt) waveforms = waveforms.reshape( nb_spike, -1, waveformsize) waveforms = waveforms * pq.mV if nb_spike > 0: # t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not t_start = 0 * pq.s t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s else: t_start = 0 * pq.s t_stop = 0 * pq.s st = SpikeTrain( times=times, name='Chan{} Code{}'.format( channel, sortcode), t_start=t_start, t_stop=t_stop, waveforms=waveforms, left_sweep=waveformsize / 2. / sr * pq.s, sampling_rate=sr * pq.Hz, ) st.annotate(channel_index=channel) if lazy: st.lazy_shape = nb_spike seg.spiketrains.append(st) elif type_label == 'EVTYPE_STREAM': dt = np.dtype( data_formats[tsq[mask3]['dataformat'][0]]) shape = np.sum(tsq[mask3]['size'] - 10) sr = tsq[mask3]['frequency'][0] if lazy: signal = [] else: if PY3K: signame = code.decode('ascii') else: signame = code sev_filename = os.path.join( subdir, tankname + '_' + blockname + '_' + signame + '_ch' + str(channel) + '.sev') if os.path.exists(sev_filename): #sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead sig_array = np.fromfile(sev_filename, dtype='uint8') else: sig_array = tev_array signal = get_chunks(tsq[mask3]['size'], tsq[mask3]['eventoffset'], sig_array).view(dt) anasig = AnalogSignal( signal=signal * pq.V, name='{} {}'.format(code, channel), sampling_rate=sr * pq.Hz, t_start=(tsq[mask3]['timestamp'][0] - global_t_start) * pq.s, channel_index=int(channel)) if lazy: anasig.lazy_shape = shape seg.analogsignals.append(anasig) bl.create_many_to_one_relationship() return bl
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True): """ Read in a segment. Arguments: load_spike_waveform : load or not waveform of spikes (default True) """ fid = open(self.filename, 'rb') globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) # metadatas seg = Segment() seg.rec_datetime = datetime.datetime( globalHeader.pop('Year'), globalHeader.pop('Month'), globalHeader.pop('Day'), globalHeader.pop('Hour'), globalHeader.pop('Minute'), globalHeader.pop('Second') ) seg.file_origin = os.path.basename(self.filename) for key, val in globalHeader.iteritems(): seg.annotate(**{key: val}) if not cascade: return seg ## Step 1 : read headers # dsp channels header = spikes and waveforms dspChannelHeaders = {} maxunit = 0 maxchan = 0 for _ in range(globalHeader['NumDSPChannels']): # channel is 1 based channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None) channelHeader['Template'] = np.array(channelHeader['Template']).reshape((5,64)) channelHeader['Boxes'] = np.array(channelHeader['Boxes']).reshape((5,2,4)) dspChannelHeaders[channelHeader['Channel']] = channelHeader maxunit = max(channelHeader['NUnits'], maxunit) maxchan = max(channelHeader['Channel'], maxchan) # event channel header eventHeaders = { } for _ in range(globalHeader['NumEventChannels']): eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None) eventHeaders[eventHeader['Channel']] = eventHeader # slow channel header = signal slowChannelHeaders = {} for _ in range(globalHeader['NumSlowChannels']): slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None) slowChannelHeaders[slowChannelHeader['Channel']] = slowChannelHeader ## Step 2 : a first loop for counting size # signal nb_samples = np.zeros(len(slowChannelHeaders)) sample_positions = np.zeros(len(slowChannelHeaders)) t_starts = np.zeros(len(slowChannelHeaders), dtype='f') #spiketimes and waveform nb_spikes = np.zeros((maxchan+1, maxunit+1) ,dtype='i') wf_sizes = np.zeros((maxchan+1, maxunit+1, 2) ,dtype='i') # eventarrays nb_events = { } #maxstrsizeperchannel = { } for chan, h in iteritems(eventHeaders): nb_events[chan] = 0 #maxstrsizeperchannel[chan] = 0 start = fid.tell() while fid.tell() !=-1 : # read block header dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None) if dataBlockHeader is None : break chan = dataBlockHeader['Channel'] unit = dataBlockHeader['Unit'] n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform'] time = (dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp']) if dataBlockHeader['Type'] == 1: nb_spikes[chan,unit] +=1 wf_sizes[chan,unit,:] = [n1,n2] fid.seek(n1*n2*2,1) elif dataBlockHeader['Type'] ==4: #event nb_events[chan] += 1 elif dataBlockHeader['Type'] == 5: #continuous signal fid.seek(n2*2, 1) if n2> 0: nb_samples[chan] += n2 if nb_samples[chan] ==0: t_starts[chan] = time ## Step 3: allocating memory and 2 loop for reading if not lazy if not lazy: # allocating mem for signal sigarrays = { } for chan, h in iteritems(slowChannelHeaders): sigarrays[chan] = np.zeros(nb_samples[chan]) # allocating mem for SpikeTrain stimearrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object) swfarrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object) for (chan, unit), _ in np.ndenumerate(nb_spikes): stimearrays[chan,unit] = np.zeros(nb_spikes[chan,unit], dtype = 'f') if load_spike_waveform: n1,n2 = wf_sizes[chan, unit,:] swfarrays[chan, unit] = np.zeros( (nb_spikes[chan, unit], n1, n2 ) , dtype = 'f4' ) pos_spikes = np.zeros(nb_spikes.shape, dtype = 'i') # allocating mem for event eventpositions = { } evarrays = { } for chan, nb in iteritems(nb_events): evarrays[chan] = { 'times': np.zeros(nb, dtype='f'), 'labels': np.zeros(nb, dtype='S4') } eventpositions[chan]=0 fid.seek(start) while fid.tell() !=-1 : dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None) if dataBlockHeader is None : break chan = dataBlockHeader['Channel'] n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform'] time = dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp'] time/= globalHeader['ADFrequency'] if n2 <0: break if dataBlockHeader['Type'] == 1: #spike unit = dataBlockHeader['Unit'] pos = pos_spikes[chan,unit] stimearrays[chan, unit][pos] = time if load_spike_waveform and n1*n2 != 0 : swfarrays[chan,unit][pos,:,:] = np.fromstring( fid.read(n1*n2*2) , dtype = 'i2').reshape(n1,n2).astype('f4') else: fid.seek(n1*n2*2,1) pos_spikes[chan,unit] +=1 elif dataBlockHeader['Type'] == 4: # event pos = eventpositions[chan] evarrays[chan]['times'][pos] = time evarrays[chan]['labels'][pos] = dataBlockHeader['Unit'] eventpositions[chan]+= 1 elif dataBlockHeader['Type'] == 5: #signal data = np.fromstring( fid.read(n2*2) , dtype = 'i2').astype('f4') sigarrays[chan][sample_positions[chan] : sample_positions[chan]+data.size] = data sample_positions[chan] += data.size ## Step 4: create neo object for chan, h in iteritems(eventHeaders): if lazy: times = [] labels = None else: times = evarrays[chan]['times'] labels = evarrays[chan]['labels'] ea = EventArray( times*pq.s, labels=labels, channel_name=eventHeaders[chan]['Name'], channel_index=chan ) if lazy: ea.lazy_shape = nb_events[chan] seg.eventarrays.append(ea) for chan, h in iteritems(slowChannelHeaders): if lazy: signal = [ ] else: if globalHeader['Version'] ==100 or globalHeader['Version'] ==101 : gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*1000.) elif globalHeader['Version'] ==102 : gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain']) elif globalHeader['Version'] >= 103: gain = globalHeader['SlowMaxMagnitudeMV']/(.5*(2**globalHeader['BitsPerSpikeSample'])*\ slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain']) signal = sigarrays[chan]*gain anasig = AnalogSignal(signal*pq.V, sampling_rate = float(slowChannelHeaders[chan]['ADFreq'])*pq.Hz, t_start = t_starts[chan]*pq.s, channel_index = slowChannelHeaders[chan]['Channel'], channel_name = slowChannelHeaders[chan]['Name'], ) if lazy: anasig.lazy_shape = nb_samples[chan] seg.analogsignals.append(anasig) for (chan, unit), value in np.ndenumerate(nb_spikes): if nb_spikes[chan, unit] == 0: continue if lazy: times = [ ] waveforms = None t_stop = 0 else: times = stimearrays[chan,unit] t_stop = times.max() if load_spike_waveform: if globalHeader['Version'] <103: gain = 3000./(2048*dspChannelHeaders[chan]['Gain']*1000.) elif globalHeader['Version'] >=103 and globalHeader['Version'] <105: gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*1000.) elif globalHeader['Version'] >105: gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*globalHeader['SpikePreAmpGain']) waveforms = swfarrays[chan, unit] * gain * pq.V else: waveforms = None sptr = SpikeTrain( times, units='s', t_stop=t_stop*pq.s, waveforms=waveforms ) sptr.annotate(unit_name = dspChannelHeaders[chan]['Name']) sptr.annotate(channel_index = chan) for key, val in dspChannelHeaders[chan].iteritems(): sptr.annotate(**{key: val}) if lazy: sptr.lazy_shape = nb_spikes[chan,unit] seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg
def read_one_channel_event_or_spike(self, fid, channel_num, header, lazy=True): # return SPikeTrain or Event channelHeader = header.channelHeaders[channel_num] if channelHeader.firstblock < 0: return if channelHeader.kind not in [2, 3, 4, 5, 6, 7, 8]: return # # Step 1 : type of blocks if channelHeader.kind in [2, 3, 4]: # Event data fmt = [('tick', 'i4')] elif channelHeader.kind in [5]: # Marker data fmt = [('tick', 'i4'), ('marker', 'i4')] elif channelHeader.kind in [6]: # AdcMark data fmt = [('tick', 'i4'), ('marker', 'i4'), ('adc', 'S%d' % channelHeader.n_extra)] elif channelHeader.kind in [7]: # RealMark data fmt = [('tick', 'i4'), ('marker', 'i4'), ('real', 'S%d' % channelHeader.n_extra)] elif channelHeader.kind in [8]: # TextMark data fmt = [('tick', 'i4'), ('marker', 'i4'), ('label', 'S%d' % channelHeader.n_extra)] dt = np.dtype(fmt) ## Step 2 : first read for allocating mem fid.seek(channelHeader.firstblock) totalitems = 0 for _ in range(channelHeader.blocks): blockHeader = HeaderReader(fid, np.dtype(blockHeaderDesciption)) totalitems += blockHeader.items if blockHeader.succ_block > 0: fid.seek(blockHeader.succ_block) #~ print 'totalitems' , totalitems if lazy: if channelHeader.kind in [2, 3, 4, 5, 8]: ea = Event() ea.annotate(channel_index=channel_num) ea.lazy_shape = totalitems return ea elif channelHeader.kind in [6, 7]: # correct value for t_stop to be put in later sptr = SpikeTrain([] * pq.s, t_stop=1e99) sptr.annotate(channel_index=channel_num, ced_unit = 0) sptr.lazy_shape = totalitems return sptr else: alltrigs = np.zeros(totalitems, dtype=dt) ## Step 3 : read fid.seek(channelHeader.firstblock) pos = 0 for _ in range(channelHeader.blocks): blockHeader = HeaderReader( fid, np.dtype(blockHeaderDesciption)) # read all events in block trigs = np.fromstring( fid.read(blockHeader.items * dt.itemsize), dtype=dt) alltrigs[pos:pos + trigs.size] = trigs pos += trigs.size if blockHeader.succ_block > 0: fid.seek(blockHeader.succ_block) ## Step 3 convert in neo standard class: eventarrays or spiketrains alltimes = alltrigs['tick'].astype( 'f') * header.us_per_time * header.dtime_base * pq.s if channelHeader.kind in [2, 3, 4, 5, 8]: #events ea = Event(alltimes) ea.annotate(channel_index=channel_num) if channelHeader.kind >= 5: # Spike2 marker is closer to label sens of neo ea.labels = alltrigs['marker'].astype('S32') if channelHeader.kind == 8: ea.annotate(extra_labels=alltrigs['label']) return ea elif channelHeader.kind in [6, 7]: # spiketrains # waveforms if channelHeader.kind == 6: waveforms = np.fromstring(alltrigs['adc'].tostring(), dtype='i2') waveforms = waveforms.astype( 'f4') * channelHeader.scale / 6553.6 + \ channelHeader.offset elif channelHeader.kind == 7: waveforms = np.fromstring(alltrigs['real'].tostring(), dtype='f4') if header.system_id >= 6 and channelHeader.interleave > 1: waveforms = waveforms.reshape( (alltimes.size, -1, channelHeader.interleave)) waveforms = waveforms.swapaxes(1, 2) else: waveforms = waveforms.reshape((alltimes.size, 1, -1)) if header.system_id in [1, 2, 3, 4, 5]: sample_interval = (channelHeader.divide * header.us_per_time * header.time_per_adc) * 1e-6 else: sample_interval = (channelHeader.l_chan_dvd * header.us_per_time * header.dtime_base) if channelHeader.unit in unit_convert: unit = pq.Quantity(1, unit_convert[channelHeader.unit]) else: #print channelHeader.unit try: unit = pq.Quantity(1, channelHeader.unit) except: unit = pq.Quantity(1, '') if len(alltimes) > 0: # can get better value from associated AnalogSignal(s) ? t_stop = alltimes.max() else: t_stop = 0.0 if not self.ced_units: sptr = SpikeTrain(alltimes, waveforms = waveforms*unit, sampling_rate = (1./sample_interval)*pq.Hz, t_stop = t_stop ) sptr.annotate(channel_index = channel_num, ced_unit = 0) return [sptr] sptrs = [] for i in set(alltrigs['marker'] & 255): sptr = SpikeTrain(alltimes[alltrigs['marker'] == i], waveforms = waveforms[alltrigs['marker'] == i]*unit, sampling_rate = (1./sample_interval)*pq.Hz, t_stop = t_stop ) sptr.annotate(channel_index = channel_num, ced_unit = i) sptrs.append(sptr) return sptrs
def read_nev(self, filename_nev, seg, lazy, cascade, load_waveforms = False): # basic header dt = [('header_id','S8'), ('ver_major','uint8'), ('ver_minor','uint8'), ('additionnal_flag', 'uint16'), # Read flags, currently basically unused ('header_size', 'uint32'), #i.e. index of first data ('packet_size', 'uint32'),# Read number of packet bytes, i.e. byte per sample ('sampling_rate', 'uint32'),# Read time resolution in Hz of time stamps, i.e. data packets ('waveform_sampling_rate', 'uint32'),# Read sampling frequency of waveforms in Hz ('window_datetime', 'S16'), ('application', 'S32'), # ('comments', 'S256'), # comments ('num_ext_header', 'uint32') #Read number of extended headers ] nev_header = h = np.fromfile(filename_nev, count = 1, dtype = dt)[0] version = '{0}.{1}'.format(h['ver_major'], h['ver_minor']) assert h['header_id'].decode('ascii') == 'NEURALEV' or version == '2.1', 'Unsupported version {0}'.format(version) version = '{0}.{1}'.format(h['ver_major'], h['ver_minor']) seg.annotate(blackrock_version = version) seg.rec_datetime = get_window_datetime(nev_header['window_datetime']) sr = float(h['sampling_rate']) wsr = float(h['waveform_sampling_rate']) if not cascade: return # extented header # this consist in N block with code 8bytes + 24 data bytes # the data bytes depend on the code and need to be converted cafilename_nsx, segse by case raw_ext_header = np.memmap(filename_nev, offset = np.dtype(dt).itemsize, dtype = [('code', 'S8'), ('data', 'S24')], shape = h['num_ext_header']) # this is for debuging ext_header = { } for code, dt_ext in ext_nev_header_codes.items(): sel = raw_ext_header['code']==code ext_header[code] = raw_ext_header[sel].view(dt_ext) # channel label neuelbl_header = ext_header['NEUEVLBL'] # Sometimes when making the channel labels we have only one channel and so must address it differently. try: channel_labels = dict(zip(neuelbl_header['channel_id'], neuelbl_header['channel_label'])) except TypeError: channel_labels = dict([(neuelbl_header['channel_id'], neuelbl_header['channel_label'])]) # TODO ext_header['DIGLABEL'] is there only one label ???? because no id in that case # TODO ECOMMENT + CCOMMENT for annotations # TODO NEUEVFLT for annotations # read data packet and markers dt0 = [('samplepos', 'uint32'), ('id', 'uint16'), ('value', 'S{0}'.format(h['packet_size']-6)), ] data = np.memmap( filename_nev, offset = h['header_size'], dtype = dt0) all_ids = np.unique(data['id']) t_start = 0*pq.s t_stop = data['samplepos'][-1]/sr*pq.s # read event (digital 9+ analog+comment) def create_event_array_trig_or_analog(selection, name, labelmode = None): if lazy: times = [ ] labels = np.array([ ], dtype = 'S') else: times = data_trigger['samplepos'][selection].astype(float)/sr if labelmode == 'digital_port': labels = data_trigger['digital_port'][selection].astype('S2') elif labelmode is None: label = None ev = EventArray(times= times*pq.s, labels= labels, name=name) if lazy: ev.lazy_shape = np.sum(is_digital) seg.eventarrays.append(ev) mask = (data['id']==0) dt_trig = [('samplepos', 'uint32'), ('id', 'uint16'), ('reason', 'uint8'), ('reserved0', 'uint8'), ('digital_port', 'uint16'), ('reserved1', 'S{0}'.format(h['packet_size']-10)), ] data_trigger = data.view(dt_trig)[mask] # Digital Triggers (PaquetID 0) is_digital = (data_trigger ['reason']&1)>0 create_event_array_trig_or_analog(is_digital, 'Digital trigger', labelmode = 'digital_port' ) # Analog Triggers (PaquetID 0) if version in ['2.1', '2.2' ]: for i in range(5): is_analog = (data_trigger ['reason']&(2**(i+1)))>0 create_event_array_trig_or_analog(is_analog, 'Analog trigger {0}'.format(i), labelmode = None) # Comments mask = (data['id']==0xFFF) dt_comments = [('samplepos', 'uint32'), ('id', 'uint16'), ('charset', 'uint8'), ('reserved0', 'uint8'), ('color', 'uint32'), ('comment', 'S{0}'.format(h['packet_size']-12)), ] data_comments = data.view(dt_comments)[mask] if data_comments.size>0: if lazy: times = [ ] labels = [ ] else: times = data_comments['samplepos'].astype(float)/sr labels = data_comments['comment'].astype('S') ev = EventArray(times= times*pq.s, labels= labels, name='Comments') if lazy: ev.lazy_shape = np.sum(is_digital) seg.eventarrays.append(ev) # READ Spike channel channel_ids = all_ids[(all_ids>0) & (all_ids<=2048)] # get the dtype of waveform (this is stupidly complicated) if nev_header['additionnal_flag']&0x1: #dtype_waveforms = { k:'int16' for k in channel_ids } dtype_waveforms = dict( (k,'int16') for k in channel_ids) else: # there is a code electrodes by electrodes given the approiate dtype neuewav_header = ext_header['NEUEVWAV'] dtype_waveform = dict(zip(neuewav_header['channel_id'], neuewav_header['num_bytes_per_waveform'])) dtypes_conv = { 0: 'int8', 1 : 'int8', 2: 'int16', 4 : 'int32' } #dtype_waveforms = { k:dtypes_conv[v] for k,v in dtype_waveform.items() } dtype_waveforms = dict( (k,dtypes_conv[v]) for k,v in dtype_waveform.items() ) dt2 = [('samplepos', 'uint32'), ('id', 'uint16'), ('cluster', 'uint8'), ('reserved0', 'uint8'), ('waveform','uint8',(h['packet_size']-8, )), ] data_spike = data.view(dt2) for channel_id in channel_ids: data_spike_chan = data_spike[data['id']==channel_id] cluster_ids = np.unique(data_spike_chan['cluster']) for cluster_id in cluster_ids: if cluster_id==0: name = 'unclassified' elif cluster_id==255: name = 'noise' else: name = 'Cluster {0}'.format(cluster_id) name = 'Channel {0} '.format(channel_id)+name data_spike_chan_clus = data_spike_chan[data_spike_chan['cluster']==cluster_id] n_spike = data_spike_chan_clus.size waveforms, w_sampling_rate, left_sweep = None, None, None if lazy: times = [ ] else: times = data_spike_chan_clus['samplepos'].astype(float)/sr if load_waveforms: dtype_waveform = dtype_waveforms[channel_id] waveform_size = (h['packet_size']-8)/np.dtype(dtype_waveform).itemsize waveforms = data_spike_chan_clus['waveform'].flatten().view(dtype_waveform) waveforms = waveforms.reshape(n_spike,1, waveform_size) waveforms =waveforms*pq.uV w_sampling_rate = wsr*pq.Hz left_sweep = waveform_size//2/sr*pq.s st = SpikeTrain(times = times*pq.s, name = name, t_start = t_start, t_stop =t_stop, waveforms = waveforms, sampling_rate = w_sampling_rate, left_sweep = left_sweep) st.annotate(channel_index = int(channel_id)) if lazy: st.lazy_shape = n_spike seg.spiketrains.append(st)
def _find_action_potentials_on_tracks(ap_tracks: Iterable[APTrack], el_stimuli: Event, signal: Union[IrregularlySampledSignal, AnalogSignal], window_size: Quantity = Quantity( 0.003, "s"), sampling_rate=None) -> SpikeTrain: # TODO implement this function for analog signals and make it reusable if isinstance(signal, IrregularlySampledSignal) and sampling_rate is None: raise ValueError( "If an irregularly sampled signal is passed, you need to set the sampling rate!" ) elif isinstance(signal, AnalogSignal): sampling_rate = signal.sampling_rate # initialize our list of action_potentials ap_times = [] ap_waveforms = [] # iterate over all the tracks for track_idx, ap_track in enumerate(ap_tracks): # first, get the template of our current AP track if ap_track.ap_template == None: warnings.warn( f"""No AP template for AP track no. {track_idx}! Cannot extract APs for this track.""" ) continue else: ap_template = ap_track.ap_template try: # then, slide the template over the window, calculating cross correlation for all the datapoints for sweep_idx, ap_latency in tqdm(zip(ap_track.sweep_idcs, ap_track.latencies), total = len(ap_track), \ desc = f"""Processing AP Track {track_idx} with {len(ap_track)} latencies."""): # we need the time of the main pulse and add the latency to define the point around which we want to search window_center_time = el_stimuli.times[sweep_idx] + ap_latency # now, we define the indices of the first and last data points that we consider for our windowing if isinstance(signal, IrregularlySampledSignal): signal: IrregularlySampledSignal # find first signal index first_signal_idx = bisect.bisect_left( signal.times, (window_center_time - window_size - ap_template.duration / 2)) # find last signal index last_signal_idx = bisect.bisect_left( signal.times, (window_center_time + window_size + ap_template.duration / 2)) elif isinstance(signal, AnalogSignal): # TODO Check why this function is much slower for analog signals first_signal_idx = floor( (window_center_time - window_size - (ap_template.duration / 2.0)) * signal.sampling_rate) last_signal_idx = floor( (window_center_time + window_size + (ap_template.duration / 2.0)) * signal.sampling_rate) # slide the template over the window correlations = sliding_window_normalized_cross_correlation( signal[first_signal_idx:last_signal_idx], ap_template.signal_template) # then, retrieve the index for which we had the maximum correlation max_correlation_idx = np.argmax( correlations) + first_signal_idx # finally, append the starting time and ap_times.append(signal.times[max_correlation_idx]) ap_waveforms.append( signal[max_correlation_idx:max_correlation_idx + len(ap_template)]) except Exception as ex: traceback.print_exc() # sort the APs and return spiketrain object ap_times = sorted(ap_times) # build the waveforms array num_aps = len(ap_waveforms) max_len = max([len(ap) for ap in ap_waveforms]) waveforms = np.zeros(shape=(num_aps, 1, max_len), dtype=np.float64) * signal.units for ap_idx, ap_waveform in enumerate(ap_waveforms): waveforms[ap_idx, 0, 0:len(ap_waveform)] = ap_waveform.ravel() result = SpikeTrain(times=Quantity(ap_times, "s"), t_start=signal.t_start, t_stop=signal.t_stop, name="APs from tracks", waveforms=waveforms, sampling_rate=sampling_rate) result.annotate(id=f"{TypeID.ACTION_POTENTIAL.value}.0", type_id=TypeID.ACTION_POTENTIAL.value) return result
def read_segment(self, lazy=False, cascade=True): fid = open(self.filename, 'rb') global_header = HeaderReader(fid, GlobalHeader).read_f(offset=0) # ~ print globalHeader #~ print 'version' , globalHeader['version'] seg = Segment() seg.file_origin = os.path.basename(self.filename) seg.annotate(neuroexplorer_version=global_header['version']) seg.annotate(comment=global_header['comment']) if not cascade: return seg offset = 544 for i in range(global_header['nvar']): entity_header = HeaderReader(fid, EntityHeader).read_f( offset=offset + i * 208) entity_header['name'] = entity_header['name'].replace('\x00', '') #print 'i',i, entityHeader['type'] if entity_header['type'] == 0: # neuron if lazy: spike_times = [] * pq.s else: spike_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) spike_times = spike_times.astype('f8') / global_header[ 'freq'] * pq.s sptr = SpikeTrain( times=spike_times, t_start=global_header['tbeg'] / global_header['freq'] * pq.s, t_stop=global_header['tend'] / global_header['freq'] * pq.s, name=entity_header['name']) if lazy: sptr.lazy_shape = entity_header['n'] sptr.annotate(channel_index=entity_header['WireNumber']) seg.spiketrains.append(sptr) if entity_header['type'] == 1: # event if lazy: event_times = [] * pq.s else: event_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) event_times = event_times.astype('f8') / global_header[ 'freq'] * pq.s labels = np.array([''] * event_times.size, dtype='S') evar = Event(times=event_times, labels=labels, channel_name=entity_header['name']) if lazy: evar.lazy_shape = entity_header['n'] seg.events.append(evar) if entity_header['type'] == 2: # interval if lazy: start_times = [] * pq.s stop_times = [] * pq.s else: start_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) start_times = start_times.astype('f8') / global_header[ 'freq'] * pq.s stop_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset'] + entity_header['n'] * 4) stop_times = stop_times.astype('f') / global_header[ 'freq'] * pq.s epar = Epoch(times=start_times, durations=stop_times - start_times, labels=np.array([''] * start_times.size, dtype='S'), channel_name=entity_header['name']) if lazy: epar.lazy_shape = entity_header['n'] seg.epochs.append(epar) if entity_header['type'] == 3: # spiketrain and wavefoms if lazy: spike_times = [] * pq.s waveforms = None else: spike_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) spike_times = spike_times.astype('f8') / global_header[ 'freq'] * pq.s waveforms = np.memmap(self.filename, np.dtype('i2'), 'r', shape=(entity_header['n'], 1, entity_header['NPointsWave']), offset=entity_header['offset'] + entity_header['n'] * 4) waveforms = (waveforms.astype('f') * entity_header['ADtoMV'] + entity_header['MVOffset']) * pq.mV t_stop = global_header['tend'] / global_header['freq'] * pq.s if spike_times.size > 0: t_stop = max(t_stop, max(spike_times)) sptr = SpikeTrain( times=spike_times, t_start=global_header['tbeg'] / global_header['freq'] * pq.s, #~ t_stop = max(globalHeader['tend']/ #~ globalHeader['freq']*pq.s,max(spike_times)), t_stop=t_stop, name=entity_header['name'], waveforms=waveforms, sampling_rate=entity_header['WFrequency'] * pq.Hz, left_sweep=0 * pq.ms) if lazy: sptr.lazy_shape = entity_header['n'] sptr.annotate(channel_index=entity_header['WireNumber']) seg.spiketrains.append(sptr) if entity_header['type'] == 4: # popvectors pass if entity_header['type'] == 5: # analog timestamps = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) timestamps = timestamps.astype('f8') / global_header['freq'] fragment_starts = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) fragment_starts = fragment_starts.astype('f8') / global_header[ 'freq'] t_start = timestamps[0] - fragment_starts[0] / float( entity_header['WFrequency']) del timestamps, fragment_starts if lazy: signal = [] * pq.mV else: signal = np.memmap(self.filename, np.dtype('i2'), 'r', shape=(entity_header['NPointsWave']), offset=entity_header['offset']) signal = signal.astype('f') signal *= entity_header['ADtoMV'] signal += entity_header['MVOffset'] signal = signal * pq.mV ana_sig = AnalogSignal( signal=signal, t_start=t_start * pq.s, sampling_rate=entity_header['WFrequency'] * pq.Hz, name=entity_header['name'], channel_index=entity_header['WireNumber']) if lazy: ana_sig.lazy_shape = entity_header['NPointsWave'] seg.analogsignals.append(ana_sig) if entity_header['type'] == 6: # markers : TO TEST if lazy: times = [] * pq.s labels = np.array([], dtype='S') markertype = None else: times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) times = times.astype('f8') / global_header['freq'] * pq.s fid.seek(entity_header['offset'] + entity_header['n'] * 4) markertype = fid.read(64).replace('\x00', '') labels = np.memmap( self.filename, np.dtype( 'S' + str(entity_header['MarkerLength'])), 'r', shape=(entity_header['n']), offset=entity_header['offset'] + entity_header['n'] * 4 + 64) ea = Event(times=times, labels=labels.view(np.ndarray), name=entity_header['name'], channel_index=entity_header['WireNumber'], marker_type=markertype) if lazy: ea.lazy_shape = entity_header['n'] seg.events.append(ea) seg.create_many_to_one_relationship() return seg
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True): """ """ fid = open(self.filename, "rb") globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) # metadatas seg = Segment() seg.rec_datetime = datetime.datetime( globalHeader["Year"], globalHeader["Month"], globalHeader["Day"], globalHeader["Hour"], globalHeader["Minute"], globalHeader["Second"], ) seg.file_origin = os.path.basename(self.filename) seg.annotate(plexon_version=globalHeader["Version"]) if not cascade: return seg ## Step 1 : read headers # dsp channels header = sipkes and waveforms dspChannelHeaders = {} maxunit = 0 maxchan = 0 for _ in range(globalHeader["NumDSPChannels"]): # channel is 1 based channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None) channelHeader["Template"] = np.array(channelHeader["Template"]).reshape((5, 64)) channelHeader["Boxes"] = np.array(channelHeader["Boxes"]).reshape((5, 2, 4)) dspChannelHeaders[channelHeader["Channel"]] = channelHeader maxunit = max(channelHeader["NUnits"], maxunit) maxchan = max(channelHeader["Channel"], maxchan) # event channel header eventHeaders = {} for _ in range(globalHeader["NumEventChannels"]): eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None) eventHeaders[eventHeader["Channel"]] = eventHeader # slow channel header = signal slowChannelHeaders = {} for _ in range(globalHeader["NumSlowChannels"]): slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None) slowChannelHeaders[slowChannelHeader["Channel"]] = slowChannelHeader ## Step 2 : a first loop for counting size # signal nb_samples = np.zeros(len(slowChannelHeaders)) sample_positions = np.zeros(len(slowChannelHeaders)) t_starts = np.zeros(len(slowChannelHeaders), dtype="f") # spiketimes and waveform nb_spikes = np.zeros((maxchan + 1, maxunit + 1), dtype="i") wf_sizes = np.zeros((maxchan + 1, maxunit + 1, 2), dtype="i") # eventarrays nb_events = {} # maxstrsizeperchannel = { } for chan, h in iteritems(eventHeaders): nb_events[chan] = 0 # maxstrsizeperchannel[chan] = 0 start = fid.tell() while fid.tell() != -1: # read block header dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None) if dataBlockHeader is None: break chan = dataBlockHeader["Channel"] unit = dataBlockHeader["Unit"] n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"] time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"] if dataBlockHeader["Type"] == 1: nb_spikes[chan, unit] += 1 wf_sizes[chan, unit, :] = [n1, n2] fid.seek(n1 * n2 * 2, 1) elif dataBlockHeader["Type"] == 4: # event nb_events[chan] += 1 elif dataBlockHeader["Type"] == 5: # continuous signal fid.seek(n2 * 2, 1) if n2 > 0: nb_samples[chan] += n2 if nb_samples[chan] == 0: t_starts[chan] = time ## Step 3: allocating memory and 2 loop for reading if not lazy if not lazy: # allocating mem for signal sigarrays = {} for chan, h in iteritems(slowChannelHeaders): sigarrays[chan] = np.zeros(nb_samples[chan]) # allocating mem for SpikeTrain stimearrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object) swfarrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object) for (chan, unit), _ in np.ndenumerate(nb_spikes): stimearrays[chan, unit] = np.zeros(nb_spikes[chan, unit], dtype="f") if load_spike_waveform: n1, n2 = wf_sizes[chan, unit, :] swfarrays[chan, unit] = np.zeros((nb_spikes[chan, unit], n1, n2), dtype="f4") pos_spikes = np.zeros(nb_spikes.shape, dtype="i") # allocating mem for event eventpositions = {} evarrays = {} for chan, nb in iteritems(nb_events): evarrays[chan] = np.zeros(nb, dtype="f") eventpositions[chan] = 0 fid.seek(start) while fid.tell() != -1: dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None) if dataBlockHeader is None: break chan = dataBlockHeader["Channel"] n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"] time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"] time /= globalHeader["ADFrequency"] if n2 < 0: break if dataBlockHeader["Type"] == 1: # spike unit = dataBlockHeader["Unit"] pos = pos_spikes[chan, unit] stimearrays[chan, unit][pos] = time if load_spike_waveform and n1 * n2 != 0: swfarrays[chan, unit][pos, :, :] = ( np.fromstring(fid.read(n1 * n2 * 2), dtype="i2").reshape(n1, n2).astype("f4") ) else: fid.seek(n1 * n2 * 2, 1) pos_spikes[chan, unit] += 1 elif dataBlockHeader["Type"] == 4: # event pos = eventpositions[chan] evarrays[chan][pos] = time eventpositions[chan] += 1 elif dataBlockHeader["Type"] == 5: # signal data = np.fromstring(fid.read(n2 * 2), dtype="i2").astype("f4") sigarrays[chan][sample_positions[chan] : sample_positions[chan] + data.size] = data sample_positions[chan] += data.size ## Step 3: create neo object for chan, h in iteritems(eventHeaders): if lazy: times = [] else: times = evarrays[chan] ea = EventArray(times * pq.s, channel_name=eventHeaders[chan]["Name"], channel_index=chan) if lazy: ea.lazy_shape = nb_events[chan] seg.eventarrays.append(ea) for chan, h in iteritems(slowChannelHeaders): if lazy: signal = [] else: if globalHeader["Version"] == 100 or globalHeader["Version"] == 101: gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * 1000.0) elif globalHeader["Version"] == 102: gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"]) elif globalHeader["Version"] >= 103: gain = globalHeader["SlowMaxMagnitudeMV"] / ( 0.5 * (2 ** globalHeader["BitsPerSpikeSample"]) * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"] ) signal = sigarrays[chan] * gain anasig = AnalogSignal( signal * pq.V, sampling_rate=float(slowChannelHeaders[chan]["ADFreq"]) * pq.Hz, t_start=t_starts[chan] * pq.s, channel_index=slowChannelHeaders[chan]["Channel"], channel_name=slowChannelHeaders[chan]["Name"], ) if lazy: anasig.lazy_shape = nb_samples[chan] seg.analogsignals.append(anasig) for (chan, unit), value in np.ndenumerate(nb_spikes): if nb_spikes[chan, unit] == 0: continue if lazy: times = [] waveforms = None t_stop = 0 else: times = stimearrays[chan, unit] t_stop = times.max() if load_spike_waveform: if globalHeader["Version"] < 103: gain = 3000.0 / (2048 * dspChannelHeaders[chan]["Gain"] * 1000.0) elif globalHeader["Version"] >= 103 and globalHeader["Version"] < 105: gain = globalHeader["SpikeMaxMagnitudeMV"] / ( 0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * 1000.0 ) elif globalHeader["Version"] > 105: gain = globalHeader["SpikeMaxMagnitudeMV"] / ( 0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * globalHeader["SpikePreAmpGain"] ) waveforms = swfarrays[chan, unit] * gain * pq.V else: waveforms = None sptr = SpikeTrain(times, units="s", t_stop=t_stop * pq.s, waveforms=waveforms) sptr.annotate(unit_name=dspChannelHeaders[chan]["Name"]) sptr.annotate(channel_index=chan) if lazy: sptr.lazy_shape = nb_spikes[chan, unit] seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) bl.create_many_to_one_relationship() return bl
def read_segment(self, blockname=None, lazy=False, cascade=True, sortname=''): """ Read a single segment from the tank. Note that TDT blocks are Neo segments, and TDT tanks are Neo blocks, so here the 'blockname' argument refers to the TDT block's name, which will be the Neo segment name. sortname is used to specify the external sortcode generated by offline spike sorting, if sortname=='PLX', there should be a ./sort/PLX/*.SortResult file in the tdt block, which stores the sortcode for every spike, default to '', which uses the original online sort """ if not blockname: blockname = os.listdir(self.dirname)[0] if blockname == 'TempBlk': return None if not self.is_tdtblock(blockname): return None # if not a tdt block subdir = os.path.join(self.dirname, blockname) if not os.path.isdir(subdir): return None seg = Segment(name=blockname) tankname = os.path.basename(self.dirname) #TSQ is the global index tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq') dt = [('size','int32'), ('evtype','int32'), ('code','S4'), ('channel','uint16'), ('sortcode','uint16'), ('timestamp','float64'), ('eventoffset','int64'), ('dataformat','int32'), ('frequency','float32'), ] tsq = np.fromfile(tsq_filename, dtype=dt) #0x8801: 'EVTYPE_MARK' give the global_start global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0] #TEV is the old data file try: tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev') #tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead tev_array = np.fromfile(tev_filename, dtype='uint8') except IOError: tev_filename = None #if exists an external sortcode in ./sort/[sortname]/*.SortResult (generated after offline sortting) sortresult_filename = None if sortname is not '': try: for file in os.listdir(os.path.join(subdir, 'sort', sortname)): if file.endswith(".SortResult"): sortresult_filename = os.path.join(subdir, 'sort', sortname, file) # get new sortcode newsorcode = np.fromfile(sortresult_filename,'int8')[1024:] # the first 1024 byte is file header # update the sort code with the info from this file tsq['sortcode'][1:-1]=newsorcode # print('sortcode updated') break except OSError: sortresult_filename = None except IOError: sortresult_filename = None for type_code, type_label in tdt_event_type: mask1 = tsq['evtype']==type_code codes = np.unique(tsq[mask1]['code']) for code in codes: mask2 = mask1 & (tsq['code']==code) channels = np.unique(tsq[mask2]['channel']) for channel in channels: mask3 = mask2 & (tsq['channel']==channel) if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']: if lazy: times = [ ]*pq.s labels = np.array([ ], dtype=str) else: times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s labels = tsq[mask3]['eventoffset'].view('float64').astype('S') ea = Event(times=times, name=code, channel_index=int(channel), labels=labels) if lazy: ea.lazy_shape = np.sum(mask3) seg.events.append(ea) elif type_label == 'EVTYPE_SNIP': sortcodes = np.unique(tsq[mask3]['sortcode']) for sortcode in sortcodes: mask4 = mask3 & (tsq['sortcode']==sortcode) nb_spike = np.sum(mask4) sr = tsq[mask4]['frequency'][0] waveformsize = tsq[mask4]['size'][0]-10 if lazy: times = [ ]*pq.s waveforms = None else: times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt) waveforms = waveforms.reshape(nb_spike, -1, waveformsize) waveforms = waveforms * pq.mV if nb_spike > 0: # t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not t_start = 0 *pq.s t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s else: t_start = 0 *pq.s t_stop = 0 *pq.s st = SpikeTrain(times = times, name = 'Chan{0} Code{1}'.format(channel,sortcode), t_start = t_start, t_stop = t_stop, waveforms = waveforms, left_sweep = waveformsize/2./sr * pq.s, sampling_rate = sr * pq.Hz, ) st.annotate(channel_index=channel) if lazy: st.lazy_shape = nb_spike seg.spiketrains.append(st) elif type_label == 'EVTYPE_STREAM': dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) shape = np.sum(tsq[mask3]['size']-10) sr = tsq[mask3]['frequency'][0] if lazy: signal = [ ] else: if PY3K: signame = code.decode('ascii') else: signame = code sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev') try: #sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead sig_array = np.fromfile(sev_filename, dtype='uint8') except IOError: sig_array = tev_array signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt) anasig = AnalogSignal(signal = signal* pq.V, name = '{0} {1}'.format(code, channel), sampling_rate = sr * pq.Hz, t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s, channel_index = int(channel) ) if lazy: anasig.lazy_shape = shape seg.analogsignals.append(anasig) return seg
def read_nev(self, filename_nev, seg, lazy, cascade, load_waveforms=False): # basic hedaer dt = [ ('header_id', 'S8'), ('ver_major', 'uint8'), ('ver_minor', 'uint8'), ('additionnal_flag', 'uint16'), # Read flags, currently basically unused ('header_size', 'uint32'), #i.e. index of first data ('packet_size', 'uint32'), # Read number of packet bytes, i.e. byte per sample ('sampling_rate', 'uint32' ), # Read time resolution in Hz of time stamps, i.e. data packets ('waveform_sampling_rate', 'uint32'), # Read sampling frequency of waveforms in Hz ('window_datetime', 'S16'), ('application', 'S32'), # ('comments', 'S256'), # comments ('num_ext_header', 'uint32') #Read number of extended headers ] nev_header = h = np.fromfile(filename_nev, count=1, dtype=dt)[0] version = '{}.{}'.format(h['ver_major'], h['ver_minor']) assert h['header_id'].decode( 'ascii' ) == 'NEURALEV' or version == '2.1', 'Unsupported version {}'.format( version) version = '{}.{}'.format(h['ver_major'], h['ver_minor']) seg.annotate(blackrock_version=version) seg.rec_datetime = get_window_datetime(nev_header['window_datetime']) sr = float(h['sampling_rate']) wsr = float(h['waveform_sampling_rate']) if not cascade: return # extented header # this consist in N block with code 8bytes + 24 data bytes # the data bytes depend on the code and need to be converted cafilename_nsx, segse by case raw_ext_header = np.memmap(filename_nev, offset=np.dtype(dt).itemsize, dtype=[('code', 'S8'), ('data', 'S24')], shape=h['num_ext_header']) # this is for debuging ext_header = {} for code, dt_ext in ext_nev_header_codes.items(): sel = raw_ext_header['code'] == code ext_header[code] = raw_ext_header[sel].view(dt_ext) # channel label neuelbl_header = ext_header['NEUEVLBL'] channel_labels = dict( zip(neuelbl_header['channel_id'], neuelbl_header['channel_label'])) # TODO ext_header['DIGLABEL'] is there only one label ???? because no id in that case # TODO ECOMMENT + CCOMMENT for annotations # TODO NEUEVFLT for annotations # read data packet and markers dt0 = [ ('samplepos', 'uint32'), ('id', 'uint16'), ('value', 'S{}'.format(h['packet_size'] - 6)), ] data = np.memmap(filename_nev, offset=h['header_size'], dtype=dt0) all_ids = np.unique(data['id']) t_start = 0 * pq.s t_stop = data['samplepos'][-1] / sr * pq.s # read event (digital 9+ analog+comment) def create_event_array_trig_or_analog(selection, name, labelmode=None): if lazy: times = [] labels = np.array([], dtype='S') else: times = data_trigger['samplepos'][selection].astype(float) / sr if labelmode == 'digital_port': labels = data_trigger['digital_port'][selection].astype( 'S2') elif labelmode is None: label = None ev = EventArray(times=times * pq.s, labels=labels, name=name) if lazy: ev.lazy_shape = np.sum(is_digital) seg.eventarrays.append(ev) mask = (data['id'] == 0) dt_trig = [ ('samplepos', 'uint32'), ('id', 'uint16'), ('reason', 'uint8'), ('reserved0', 'uint8'), ('digital_port', 'uint16'), ('reserved1', 'S{}'.format(h['packet_size'] - 10)), ] data_trigger = data.view(dt_trig)[mask] # Digital Triggers (PaquetID 0) is_digital = (data_trigger['reason'] & 1) > 0 create_event_array_trig_or_analog(is_digital, 'Digital trigger', labelmode='digital_port') # Analog Triggers (PaquetID 0) if version in ['2.1', '2.2']: for i in range(5): is_analog = (data_trigger['reason'] & (2**(i + 1))) > 0 create_event_array_trig_or_analog( is_analog, 'Analog trigger {}'.format(i), labelmode=None) # Comments mask = (data['id'] == 0xFFF) dt_comments = [ ('samplepos', 'uint32'), ('id', 'uint16'), ('charset', 'uint8'), ('reserved0', 'uint8'), ('color', 'uint32'), ('comment', 'S{}'.format(h['packet_size'] - 12)), ] data_comments = data.view(dt_comments)[mask] if data_comments.size > 0: if lazy: times = [] labels = [] else: times = data_comments['samplepos'].astype(float) / sr labels = data_comments['comment'].astype('S') ev = EventArray(times=times * pq.s, labels=labels, name='Comments') if lazy: ev.lazy_shape = np.sum(is_digital) seg.eventarrays.append(ev) # READ Spike channel channel_ids = all_ids[(all_ids > 0) & (all_ids <= 2048)] # get the dtype of waveform (this is stupidly complicated) if nev_header['additionnal_flag'] & 0x1: #dtype_waveforms = { k:'int16' for k in channel_ids } dtype_waveforms = dict((k, 'int16') for k in channel_ids) else: # there is a code electrodes by electrodes given the approiate dtype neuewav_header = ext_header['NEUEVWAV'] dtype_waveform = dict( zip(neuewav_header['channel_id'], neuewav_header['num_bytes_per_waveform'])) dtypes_conv = {0: 'int8', 1: 'int8', 2: 'int16', 4: 'int32'} #dtype_waveforms = { k:dtypes_conv[v] for k,v in dtype_waveform.items() } dtype_waveforms = dict( (k, dtypes_conv[v]) for k, v in dtype_waveform.items()) dt2 = [ ('samplepos', 'uint32'), ('id', 'uint16'), ('cluster', 'uint8'), ('reserved0', 'uint8'), ('waveform', 'uint8', (h['packet_size'] - 8, )), ] data_spike = data.view(dt2) for channel_id in channel_ids: data_spike_chan = data_spike[data['id'] == channel_id] cluster_ids = np.unique(data_spike_chan['cluster']) for cluster_id in cluster_ids: if cluster_id == 0: name = 'unclassified' elif cluster_id == 255: name = 'noise' else: name = 'Cluster {}'.format(cluster_id) name = 'Channel {} '.format(channel_id) + name data_spike_chan_clus = data_spike_chan[ data_spike_chan['cluster'] == cluster_id] n_spike = data_spike_chan_clus.size waveforms, w_sampling_rate, left_sweep = None, None, None if lazy: times = [] else: times = data_spike_chan_clus['samplepos'].astype( float) / sr if load_waveforms: dtype_waveform = dtype_waveforms[channel_id] waveform_size = (h['packet_size'] - 8) / np.dtype(dtype_waveform).itemsize waveforms = data_spike_chan_clus['waveform'].flatten( ).view(dtype_waveform) waveforms = waveforms.reshape(n_spike, 1, waveform_size) waveforms = waveforms * pq.uV w_sampling_rate = wsr * pq.Hz left_sweep = waveform_size // 2 / sr * pq.s st = SpikeTrain(times=times * pq.s, name=name, t_start=t_start, t_stop=t_stop, waveforms=waveforms, sampling_rate=w_sampling_rate, left_sweep=left_sweep) st.annotate(channel_index=int(channel_id)) if lazy: st.lazy_shape = n_spike seg.spiketrains.append(st)
def readOneChannelEventOrSpike(self, fid, channel_num, header, lazy=True): # return SPikeTrain or EventArray channelHeader = header.channelHeaders[channel_num] if channelHeader.firstblock < 0: return if channelHeader.kind not in [2, 3, 4, 5, 6, 7, 8]: return ## Step 1 : type of blocks if channelHeader.kind in [2, 3, 4]: # Event data fmt = [('tick', 'i4')] elif channelHeader.kind in [5]: # Marker data fmt = [('tick', 'i4'), ('marker', 'i4')] elif channelHeader.kind in [6]: # AdcMark data fmt = [('tick', 'i4'), ('marker', 'i4'), ('adc', 'S%d' % channelHeader.n_extra)] elif channelHeader.kind in [7]: # RealMark data fmt = [('tick', 'i4'), ('marker', 'i4'), ('real', 'S%d' % channelHeader.n_extra)] elif channelHeader.kind in [8]: # TextMark data fmt = [('tick', 'i4'), ('marker', 'i4'), ('label', 'S%d' % channelHeader.n_extra)] dt = np.dtype(fmt) ## Step 2 : first read for allocating mem fid.seek(channelHeader.firstblock) totalitems = 0 for _ in range(channelHeader.blocks): blockHeader = HeaderReader(fid, np.dtype(blockHeaderDesciption)) totalitems += blockHeader.items if blockHeader.succ_block > 0: fid.seek(blockHeader.succ_block) #~ print 'totalitems' , totalitems if lazy: if channelHeader.kind in [2, 3, 4, 5, 8]: ea = EventArray() ea.annotate(channel_index=channel_num) ea.lazy_shape = totalitems return [ea] elif channelHeader.kind in [6, 7]: sptr = SpikeTrain( [] * pq.s, t_stop=1e99) # correct value for t_stop to be put in later sptr.annotate(channel_index=channel_num, ced_unit=0) sptr.lazy_shape = totalitems return [sptr] else: alltrigs = np.zeros(totalitems, dtype=dt) ## Step 3 : read fid.seek(channelHeader.firstblock) pos = 0 for _ in range(channelHeader.blocks): blockHeader = HeaderReader(fid, np.dtype(blockHeaderDesciption)) # read all events in block trigs = np.fromstring(fid.read(blockHeader.items * dt.itemsize), dtype=dt) alltrigs[pos:pos + trigs.size] = trigs pos += trigs.size if blockHeader.succ_block > 0: fid.seek(blockHeader.succ_block) ## Step 3 convert in neo standard class : eventarrays or spiketrains alltimes = alltrigs['tick'].astype( 'f') * header.us_per_time * header.dtime_base * pq.s if channelHeader.kind in [2, 3, 4, 5, 8]: #events ea = EventArray() ea.annotate(channel_index=channel_num) ea.times = alltimes if channelHeader.kind >= 5: # Spike2 marker is closer to label sens of neo ea.labels = alltrigs['marker'].astype('S32') if channelHeader.kind == 8: ea.annotate(extra_labels=alltrigs['label']) return [ea] elif channelHeader.kind in [6, 7]: # spiketrains # waveforms if channelHeader.kind == 6: waveforms = np.fromstring(alltrigs['adc'].tostring(), dtype='i2') waveforms = waveforms.astype( 'f4' ) * channelHeader.scale / 6553.6 + channelHeader.offset elif channelHeader.kind == 7: waveforms = np.fromstring(alltrigs['real'].tostring(), dtype='f4') if header.system_id >= 6 and channelHeader.interleave > 1: waveforms = waveforms.reshape( (alltimes.size, -1, channelHeader.interleave)) waveforms = waveforms.swapaxes(1, 2) else: waveforms = waveforms.reshape((alltimes.size, 1, -1)) if header.system_id in [1, 2, 3, 4, 5]: sample_interval = (channelHeader.divide * header.us_per_time * header.time_per_adc) * 1e-6 else: sample_interval = (channelHeader.l_chan_dvd * header.us_per_time * header.dtime_base) if channelHeader.unit in unit_convert: unit = pq.Quantity(1, unit_convert[channelHeader.unit]) else: #print channelHeader.unit try: unit = pq.Quantity(1, channelHeader.unit) except: unit = pq.Quantity(1, '') if len(alltimes) > 0: t_stop = alltimes.max( ) # can get better value from associated AnalogSignal(s) ? else: t_stop = 0.0 if not self.ced_units: sptr = SpikeTrain(alltimes, waveforms=waveforms * unit, sampling_rate=(1. / sample_interval) * pq.Hz, t_stop=t_stop) sptr.annotate(channel_index=channel_num, ced_unit=0) return [sptr] sptrs = [] for i in set(alltrigs['marker'] & 255): sptr = SpikeTrain( alltimes[alltrigs['marker'] == i], waveforms=waveforms[alltrigs['marker'] == i] * unit, sampling_rate=(1. / sample_interval) * pq.Hz, t_stop=t_stop) sptr.annotate(channel_index=channel_num, ced_unit=i) sptrs.append(sptr) return sptrs
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) create_many_to_one_relationship(bl) return bl
def read_segment( self, lazy=False, cascade=True, ): fid = open(self.filename, 'rb') globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) #~ print globalHeader #~ print 'version' , globalHeader['version'] seg = Segment() seg.file_origin = os.path.basename(self.filename) seg.annotate(neuroexplorer_version=globalHeader['version']) seg.annotate(comment=globalHeader['comment']) if not cascade: return seg offset = 544 for i in range(globalHeader['nvar']): entityHeader = HeaderReader( fid, EntityHeader).read_f(offset=offset + i * 208) entityHeader['name'] = entityHeader['name'].replace('\x00', '') #print 'i',i, entityHeader['type'] if entityHeader['type'] == 0: # neuron if lazy: spike_times = [] * pq.s else: spike_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) spike_times = spike_times.astype( 'f8') / globalHeader['freq'] * pq.s sptr = SpikeTrain( times=spike_times, t_start=globalHeader['tbeg'] / globalHeader['freq'] * pq.s, t_stop=globalHeader['tend'] / globalHeader['freq'] * pq.s, name=entityHeader['name'], ) if lazy: sptr.lazy_shape = entityHeader['n'] sptr.annotate(channel_index=entityHeader['WireNumber']) seg.spiketrains.append(sptr) if entityHeader['type'] == 1: # event if lazy: event_times = [] * pq.s else: event_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) event_times = event_times.astype( 'f8') / globalHeader['freq'] * pq.s labels = np.array([''] * event_times.size, dtype='S') evar = EventArray(times=event_times, labels=labels, channel_name=entityHeader['name']) if lazy: evar.lazy_shape = entityHeader['n'] seg.eventarrays.append(evar) if entityHeader['type'] == 2: # interval if lazy: start_times = [] * pq.s stop_times = [] * pq.s else: start_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) start_times = start_times.astype( 'f8') / globalHeader['freq'] * pq.s stop_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'] + entityHeader['n'] * 4, ) stop_times = stop_times.astype( 'f') / globalHeader['freq'] * pq.s epar = EpochArray(times=start_times, durations=stop_times - start_times, labels=np.array([''] * start_times.size, dtype='S'), channel_name=entityHeader['name']) if lazy: epar.lazy_shape = entityHeader['n'] seg.epocharrays.append(epar) if entityHeader['type'] == 3: # spiketrain and wavefoms if lazy: spike_times = [] * pq.s waveforms = None else: spike_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) spike_times = spike_times.astype( 'f8') / globalHeader['freq'] * pq.s waveforms = np.memmap( self.filename, np.dtype('i2'), 'r', shape=(entityHeader['n'], 1, entityHeader['NPointsWave']), offset=entityHeader['offset'] + entityHeader['n'] * 4, ) waveforms = (waveforms.astype('f') * entityHeader['ADtoMV'] + entityHeader['MVOffset']) * pq.mV t_stop = globalHeader['tend'] / globalHeader['freq'] * pq.s if spike_times.size > 0: t_stop = max(t_stop, max(spike_times)) sptr = SpikeTrain( times=spike_times, t_start=globalHeader['tbeg'] / globalHeader['freq'] * pq.s, #~ t_stop = max(globalHeader['tend']/globalHeader['freq']*pq.s,max(spike_times)), t_stop=t_stop, name=entityHeader['name'], waveforms=waveforms, sampling_rate=entityHeader['WFrequency'] * pq.Hz, left_sweep=0 * pq.ms, ) if lazy: sptr.lazy_shape = entityHeader['n'] sptr.annotate(channel_index=entityHeader['WireNumber']) seg.spiketrains.append(sptr) if entityHeader['type'] == 4: # popvectors pass if entityHeader['type'] == 5: # analog timestamps = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) timestamps = timestamps.astype('f8') / globalHeader['freq'] fragmentStarts = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) fragmentStarts = fragmentStarts.astype( 'f8') / globalHeader['freq'] t_start = timestamps[0] - fragmentStarts[0] / float( entityHeader['WFrequency']) del timestamps, fragmentStarts if lazy: signal = [] * pq.mV else: signal = np.memmap( self.filename, np.dtype('i2'), 'r', shape=(entityHeader['NPointsWave']), offset=entityHeader['offset'], ) signal = signal.astype('f') signal *= entityHeader['ADtoMV'] signal += entityHeader['MVOffset'] signal = signal * pq.mV anaSig = AnalogSignal( signal=signal, t_start=t_start * pq.s, sampling_rate=entityHeader['WFrequency'] * pq.Hz, name=entityHeader['name'], channel_index=entityHeader['WireNumber']) if lazy: anaSig.lazy_shape = entityHeader['NPointsWave'] seg.analogsignals.append(anaSig) if entityHeader['type'] == 6: # markers : TO TEST if lazy: times = [] * pq.s labels = np.array([], dtype='S') markertype = None else: times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) times = times.astype('f8') / globalHeader['freq'] * pq.s fid.seek(entityHeader['offset'] + entityHeader['n'] * 4) markertype = fid.read(64).replace('\x00', '') labels = np.memmap( self.filename, np.dtype('S' + str(entityHeader['MarkerLength'])), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'] + entityHeader['n'] * 4 + 64) ea = EventArray(times=times, labels=labels.view(np.ndarray), name=entityHeader['name'], channel_index=entityHeader['WireNumber'], marker_type=markertype) if lazy: ea.lazy_shape = entityHeader['n'] seg.eventarrays.append(ea) create_many_to_one_relationship(seg) return seg
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) #TSQ is the global index tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq') dt = [('size','int32'), ('evtype','int32'), ('code','S4'), ('channel','uint16'), ('sortcode','uint16'), ('timestamp','float64'), ('eventoffset','int64'), ('dataformat','int32'), ('frequency','float32'), ] tsq = np.fromfile(tsq_filename, dtype = dt) #0x8801: 'EVTYPE_MARK' give the global_start global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0] #TEV is the old data file if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev') #tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead tev_array = np.fromfile(tev_filename, dtype = 'uint8') else: tev_filename = None for type_code, type_label in tdt_event_type: mask1 = tsq['evtype']==type_code codes = np.unique(tsq[mask1]['code']) for code in codes: mask2 = mask1 & (tsq['code']==code) channels = np.unique(tsq[mask2]['channel']) for channel in channels: mask3 = mask2 & (tsq['channel']==channel) if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']: if lazy: times = [ ]*pq.s labels = np.array([ ], dtype = str) else: times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s labels = tsq[mask3]['eventoffset'].view('float64').astype('S') ea = EventArray(times = times, name = code , channel_index = int(channel), labels = labels) if lazy: ea.lazy_shape = np.sum(mask3) seg.eventarrays.append(ea) elif type_label == 'EVTYPE_SNIP': sortcodes = np.unique(tsq[mask3]['sortcode']) for sortcode in sortcodes: mask4 = mask3 & (tsq['sortcode']==sortcode) nb_spike = np.sum(mask4) sr = tsq[mask4]['frequency'][0] waveformsize = tsq[mask4]['size'][0]-10 if lazy: times = [ ]*pq.s waveforms = None else: times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt) waveforms = waveforms.reshape(nb_spike, -1, waveformsize) waveforms = waveforms * pq.mV if nb_spike>0: # t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not t_start = 0 *pq.s t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s else: t_start = 0 *pq.s t_stop = 0 *pq.s st = SpikeTrain(times = times, name = 'Chan{} Code{}'.format(channel,sortcode), t_start = t_start, t_stop = t_stop, waveforms = waveforms, left_sweep = waveformsize/2./sr * pq.s, sampling_rate = sr * pq.Hz, ) st.annotate(channel_index = channel) if lazy: st.lazy_shape = nb_spike seg.spiketrains.append(st) elif type_label == 'EVTYPE_STREAM': dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) shape = np.sum(tsq[mask3]['size']-10) sr = tsq[mask3]['frequency'][0] if lazy: signal = [ ] else: if PY3K: signame = code.decode('ascii') else: signame = code sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev') if os.path.exists(sev_filename): #sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead sig_array = np.fromfile(sev_filename, dtype = 'uint8') else: sig_array = tev_array signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt) anasig = AnalogSignal(signal = signal* pq.V, name = '{} {}'.format(code, channel), sampling_rate= sr * pq.Hz, t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s, channel_index = int(channel)) if lazy: anasig.lazy_shape = shape seg.analogsignals.append(anasig) bl.create_many_to_one_relationship() return bl