def read_segment( self, lazy=False, cascade=True, ): fid = open(self.filename, 'rb') globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) #~ print globalHeader #~ print 'version' , globalHeader['version'] seg = Segment() seg.file_origin = os.path.basename(self.filename) seg.annotate(neuroexplorer_version=globalHeader['version']) seg.annotate(comment=globalHeader['comment']) if not cascade: return seg offset = 544 for i in range(globalHeader['nvar']): entityHeader = HeaderReader( fid, EntityHeader).read_f(offset=offset + i * 208) entityHeader['name'] = entityHeader['name'].replace('\x00', '') #print 'i',i, entityHeader['type'] if entityHeader['type'] == 0: # neuron if lazy: spike_times = [] * pq.s else: spike_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) spike_times = spike_times.astype( 'f8') / globalHeader['freq'] * pq.s sptr = SpikeTrain( times=spike_times, t_start=globalHeader['tbeg'] / globalHeader['freq'] * pq.s, t_stop=globalHeader['tend'] / globalHeader['freq'] * pq.s, name=entityHeader['name'], ) if lazy: sptr.lazy_shape = entityHeader['n'] sptr.annotate(channel_index=entityHeader['WireNumber']) seg.spiketrains.append(sptr) if entityHeader['type'] == 1: # event if lazy: event_times = [] * pq.s else: event_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) event_times = event_times.astype( 'f8') / globalHeader['freq'] * pq.s labels = np.array([''] * event_times.size, dtype='S') evar = EventArray(times=event_times, labels=labels, channel_name=entityHeader['name']) if lazy: evar.lazy_shape = entityHeader['n'] seg.eventarrays.append(evar) if entityHeader['type'] == 2: # interval if lazy: start_times = [] * pq.s stop_times = [] * pq.s else: start_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) start_times = start_times.astype( 'f8') / globalHeader['freq'] * pq.s stop_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'] + entityHeader['n'] * 4, ) stop_times = stop_times.astype( 'f') / globalHeader['freq'] * pq.s epar = EpochArray(times=start_times, durations=stop_times - start_times, labels=np.array([''] * start_times.size, dtype='S'), channel_name=entityHeader['name']) if lazy: epar.lazy_shape = entityHeader['n'] seg.epocharrays.append(epar) if entityHeader['type'] == 3: # spiketrain and wavefoms if lazy: spike_times = [] * pq.s waveforms = None else: spike_times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) spike_times = spike_times.astype( 'f8') / globalHeader['freq'] * pq.s waveforms = np.memmap( self.filename, np.dtype('i2'), 'r', shape=(entityHeader['n'], 1, entityHeader['NPointsWave']), offset=entityHeader['offset'] + entityHeader['n'] * 4, ) waveforms = (waveforms.astype('f') * entityHeader['ADtoMV'] + entityHeader['MVOffset']) * pq.mV t_stop = globalHeader['tend'] / globalHeader['freq'] * pq.s if spike_times.size > 0: t_stop = max(t_stop, max(spike_times)) sptr = SpikeTrain( times=spike_times, t_start=globalHeader['tbeg'] / globalHeader['freq'] * pq.s, #~ t_stop = max(globalHeader['tend']/globalHeader['freq']*pq.s,max(spike_times)), t_stop=t_stop, name=entityHeader['name'], waveforms=waveforms, sampling_rate=entityHeader['WFrequency'] * pq.Hz, left_sweep=0 * pq.ms, ) if lazy: sptr.lazy_shape = entityHeader['n'] sptr.annotate(channel_index=entityHeader['WireNumber']) seg.spiketrains.append(sptr) if entityHeader['type'] == 4: # popvectors pass if entityHeader['type'] == 5: # analog timestamps = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) timestamps = timestamps.astype('f8') / globalHeader['freq'] fragmentStarts = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) fragmentStarts = fragmentStarts.astype( 'f8') / globalHeader['freq'] t_start = timestamps[0] - fragmentStarts[0] / float( entityHeader['WFrequency']) del timestamps, fragmentStarts if lazy: signal = [] * pq.mV else: signal = np.memmap( self.filename, np.dtype('i2'), 'r', shape=(entityHeader['NPointsWave']), offset=entityHeader['offset'], ) signal = signal.astype('f') signal *= entityHeader['ADtoMV'] signal += entityHeader['MVOffset'] signal = signal * pq.mV anaSig = AnalogSignal( signal=signal, t_start=t_start * pq.s, sampling_rate=entityHeader['WFrequency'] * pq.Hz, name=entityHeader['name'], channel_index=entityHeader['WireNumber']) if lazy: anaSig.lazy_shape = entityHeader['NPointsWave'] seg.analogsignals.append(anaSig) if entityHeader['type'] == 6: # markers : TO TEST if lazy: times = [] * pq.s labels = np.array([], dtype='S') markertype = None else: times = np.memmap( self.filename, np.dtype('i4'), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'], ) times = times.astype('f8') / globalHeader['freq'] * pq.s fid.seek(entityHeader['offset'] + entityHeader['n'] * 4) markertype = fid.read(64).replace('\x00', '') labels = np.memmap( self.filename, np.dtype('S' + str(entityHeader['MarkerLength'])), 'r', shape=(entityHeader['n']), offset=entityHeader['offset'] + entityHeader['n'] * 4 + 64) ea = EventArray(times=times, labels=labels.view(np.ndarray), name=entityHeader['name'], channel_index=entityHeader['WireNumber'], marker_type=markertype) if lazy: ea.lazy_shape = entityHeader['n'] seg.eventarrays.append(ea) create_many_to_one_relationship(seg) return seg
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True): """ Read in a segment. Arguments: load_spike_waveform : load or not waveform of spikes (default True) """ fid = open(self.filename, 'rb') globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) # metadatas seg = Segment() seg.rec_datetime = datetime.datetime( globalHeader.pop('Year'), globalHeader.pop('Month'), globalHeader.pop('Day'), globalHeader.pop('Hour'), globalHeader.pop('Minute'), globalHeader.pop('Second') ) seg.file_origin = os.path.basename(self.filename) for key, val in globalHeader.iteritems(): seg.annotate(**{key: val}) if not cascade: return seg ## Step 1 : read headers # dsp channels header = spikes and waveforms dspChannelHeaders = {} maxunit = 0 maxchan = 0 for _ in range(globalHeader['NumDSPChannels']): # channel is 1 based channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None) channelHeader['Template'] = np.array(channelHeader['Template']).reshape((5,64)) channelHeader['Boxes'] = np.array(channelHeader['Boxes']).reshape((5,2,4)) dspChannelHeaders[channelHeader['Channel']] = channelHeader maxunit = max(channelHeader['NUnits'], maxunit) maxchan = max(channelHeader['Channel'], maxchan) # event channel header eventHeaders = { } for _ in range(globalHeader['NumEventChannels']): eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None) eventHeaders[eventHeader['Channel']] = eventHeader # slow channel header = signal slowChannelHeaders = {} for _ in range(globalHeader['NumSlowChannels']): slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None) slowChannelHeaders[slowChannelHeader['Channel']] = slowChannelHeader ## Step 2 : a first loop for counting size # signal nb_samples = np.zeros(len(slowChannelHeaders)) sample_positions = np.zeros(len(slowChannelHeaders)) t_starts = np.zeros(len(slowChannelHeaders), dtype='f') #spiketimes and waveform nb_spikes = np.zeros((maxchan+1, maxunit+1) ,dtype='i') wf_sizes = np.zeros((maxchan+1, maxunit+1, 2) ,dtype='i') # eventarrays nb_events = { } #maxstrsizeperchannel = { } for chan, h in iteritems(eventHeaders): nb_events[chan] = 0 #maxstrsizeperchannel[chan] = 0 start = fid.tell() while fid.tell() !=-1 : # read block header dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None) if dataBlockHeader is None : break chan = dataBlockHeader['Channel'] unit = dataBlockHeader['Unit'] n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform'] time = (dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp']) if dataBlockHeader['Type'] == 1: nb_spikes[chan,unit] +=1 wf_sizes[chan,unit,:] = [n1,n2] fid.seek(n1*n2*2,1) elif dataBlockHeader['Type'] ==4: #event nb_events[chan] += 1 elif dataBlockHeader['Type'] == 5: #continuous signal fid.seek(n2*2, 1) if n2> 0: nb_samples[chan] += n2 if nb_samples[chan] ==0: t_starts[chan] = time ## Step 3: allocating memory and 2 loop for reading if not lazy if not lazy: # allocating mem for signal sigarrays = { } for chan, h in iteritems(slowChannelHeaders): sigarrays[chan] = np.zeros(nb_samples[chan]) # allocating mem for SpikeTrain stimearrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object) swfarrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object) for (chan, unit), _ in np.ndenumerate(nb_spikes): stimearrays[chan,unit] = np.zeros(nb_spikes[chan,unit], dtype = 'f') if load_spike_waveform: n1,n2 = wf_sizes[chan, unit,:] swfarrays[chan, unit] = np.zeros( (nb_spikes[chan, unit], n1, n2 ) , dtype = 'f4' ) pos_spikes = np.zeros(nb_spikes.shape, dtype = 'i') # allocating mem for event eventpositions = { } evarrays = { } for chan, nb in iteritems(nb_events): evarrays[chan] = { 'times': np.zeros(nb, dtype='f'), 'labels': np.zeros(nb, dtype='S4') } eventpositions[chan]=0 fid.seek(start) while fid.tell() !=-1 : dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None) if dataBlockHeader is None : break chan = dataBlockHeader['Channel'] n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform'] time = dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp'] time/= globalHeader['ADFrequency'] if n2 <0: break if dataBlockHeader['Type'] == 1: #spike unit = dataBlockHeader['Unit'] pos = pos_spikes[chan,unit] stimearrays[chan, unit][pos] = time if load_spike_waveform and n1*n2 != 0 : swfarrays[chan,unit][pos,:,:] = np.fromstring( fid.read(n1*n2*2) , dtype = 'i2').reshape(n1,n2).astype('f4') else: fid.seek(n1*n2*2,1) pos_spikes[chan,unit] +=1 elif dataBlockHeader['Type'] == 4: # event pos = eventpositions[chan] evarrays[chan]['times'][pos] = time evarrays[chan]['labels'][pos] = dataBlockHeader['Unit'] eventpositions[chan]+= 1 elif dataBlockHeader['Type'] == 5: #signal data = np.fromstring( fid.read(n2*2) , dtype = 'i2').astype('f4') sigarrays[chan][sample_positions[chan] : sample_positions[chan]+data.size] = data sample_positions[chan] += data.size ## Step 4: create neo object for chan, h in iteritems(eventHeaders): if lazy: times = [] labels = None else: times = evarrays[chan]['times'] labels = evarrays[chan]['labels'] ea = EventArray( times*pq.s, labels=labels, channel_name=eventHeaders[chan]['Name'], channel_index=chan ) if lazy: ea.lazy_shape = nb_events[chan] seg.eventarrays.append(ea) for chan, h in iteritems(slowChannelHeaders): if lazy: signal = [ ] else: if globalHeader['Version'] ==100 or globalHeader['Version'] ==101 : gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*1000.) elif globalHeader['Version'] ==102 : gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain']) elif globalHeader['Version'] >= 103: gain = globalHeader['SlowMaxMagnitudeMV']/(.5*(2**globalHeader['BitsPerSpikeSample'])*\ slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain']) signal = sigarrays[chan]*gain anasig = AnalogSignal(signal*pq.V, sampling_rate = float(slowChannelHeaders[chan]['ADFreq'])*pq.Hz, t_start = t_starts[chan]*pq.s, channel_index = slowChannelHeaders[chan]['Channel'], channel_name = slowChannelHeaders[chan]['Name'], ) if lazy: anasig.lazy_shape = nb_samples[chan] seg.analogsignals.append(anasig) for (chan, unit), value in np.ndenumerate(nb_spikes): if nb_spikes[chan, unit] == 0: continue if lazy: times = [ ] waveforms = None t_stop = 0 else: times = stimearrays[chan,unit] t_stop = times.max() if load_spike_waveform: if globalHeader['Version'] <103: gain = 3000./(2048*dspChannelHeaders[chan]['Gain']*1000.) elif globalHeader['Version'] >=103 and globalHeader['Version'] <105: gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*1000.) elif globalHeader['Version'] >105: gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*globalHeader['SpikePreAmpGain']) waveforms = swfarrays[chan, unit] * gain * pq.V else: waveforms = None sptr = SpikeTrain( times, units='s', t_stop=t_stop*pq.s, waveforms=waveforms ) sptr.annotate(unit_name = dspChannelHeaders[chan]['Name']) sptr.annotate(channel_index = chan) for key, val in dspChannelHeaders[chan].iteritems(): sptr.annotate(**{key: val}) if lazy: sptr.lazy_shape = nb_spikes[chan,unit] seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg
def read_block( self, # the 2 first keyword arguments are imposed by neo.io API lazy=False, cascade=True): """ Return a Block. """ def count_samples(m_length): """ Count the number of signal samples available in a type 5 data block of length m_length """ # for information about type 5 data block, see [1] count = int((m_length - 6) / 2 - 2) # -6 corresponds to the header of block 5, and the -2 take into # account the fact that last 2 values are not available as the 4 # corresponding bytes are coding the time stamp of the beginning # of the block return count # create the neo Block that will be returned at the end blck = Block(file_origin=os.path.basename(self.filename)) blck.file_origin = os.path.basename(self.filename) fid = open(self.filename, 'rb') # NOTE: in the following, the word "block" is used in the sense used in # the alpha-omega specifications (ie a data chunk in the file), rather # than in the sense of the usual Block object in neo # step 1: read the headers of all the data blocks to load the file # structure pos_block = 0 # position of the current block in the file file_blocks = [] # list of data blocks available in the file if not cascade: # we read only the main header m_length, m_TypeBlock = struct.unpack('Hcx', fid.read(4)) # m_TypeBlock should be 'h', as we read the first block block = HeaderReader( fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({ 'm_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block }) file_blocks.append(block) else: # cascade == True seg = Segment(file_origin=os.path.basename(self.filename)) seg.file_origin = os.path.basename(self.filename) blck.segments.append(seg) while True: first_4_bytes = fid.read(4) if len(first_4_bytes) < 4: # we have reached the end of the file break else: m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes) block = HeaderReader( fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({ 'm_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block }) if m_TypeBlock == '2': # The beggining of the block of type '2' is identical for # all types of channels, but the following part depends on # the type of channel. So we need a special case here. # WARNING: How to check the type of channel is not # described in the documentation. So here I use what is # proposed in the C code [2]. # According to this C code, it seems that the 'm_isAnalog' # is used to distinguished analog and digital channels, and # 'm_Mode' encodes the type of analog channel: # 0 for continuous, 1 for level, 2 for external trigger. # But in some files, I found channels that seemed to be # continuous channels with 'm_Modes' = 128 or 192. So I # decided to consider every channel with 'm_Modes' # different from 1 or 2 as continuous. I also couldn't # check that values of 1 and 2 are really for level and # external trigger as I had no test files containing data # of this types. type_subblock = 'unknown_channel_type(m_Mode=' \ + str(block['m_Mode'])+ ')' description = Type2_SubBlockUnknownChannels block.update({'m_Name': 'unknown_name'}) if block['m_isAnalog'] == 0: # digital channel type_subblock = 'digital' description = Type2_SubBlockDigitalChannels elif block['m_isAnalog'] == 1: # analog channel if block['m_Mode'] == 1: # level channel type_subblock = 'level' description = Type2_SubBlockLevelChannels elif block['m_Mode'] == 2: # external trigger channel type_subblock = 'external_trigger' description = Type2_SubBlockExtTriggerChannels else: # continuous channel type_subblock = 'continuous(Mode' \ + str(block['m_Mode']) +')' description = Type2_SubBlockContinuousChannels subblock = HeaderReader(fid, description).read_f() block.update(subblock) block.update({'type_subblock': type_subblock}) file_blocks.append(block) pos_block += m_length fid.seek(pos_block) # step 2: find the available channels list_chan = [] # list containing indexes of channel blocks for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '2': list_chan.append(ind_block) # step 3: find blocks containing data for the available channels list_data = [] # list of lists of indexes of data blocks # corresponding to each channel for ind_chan, chan in enumerate(list_chan): list_data.append([]) num_chan = file_blocks[chan]['m_numChannel'] for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '5': if block['m_numChannel'] == num_chan: list_data[ind_chan].append(ind_block) # step 4: compute the length (number of samples) of the channels chan_len = np.zeros(len(list_data), dtype=np.int) for ind_chan, list_blocks in enumerate(list_data): for ind_block in list_blocks: chan_len[ind_chan] += count_samples( file_blocks[ind_block]['m_length']) # step 5: find channels for which data are available ind_valid_chan = np.nonzero(chan_len)[0] # step 6: load the data # TODO give the possibility to load data as AnalogSignalArrays for ind_chan in ind_valid_chan: list_blocks = list_data[ind_chan] ind = 0 # index in the data vector # read time stamp for the beginning of the signal form = '<l' # reading format ind_block = list_blocks[0] count = count_samples(file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos'] + 6 + count * 2) buf = fid.read(struct.calcsize(form)) val = struct.unpack(form, buf) start_index = val[0] # WARNING: in the following blocks are read supposing taht they # are all contiguous and sorted in time. I don't know if it's # always the case. Maybe we should use the time stamp of each # data block to choose where to put the read data in the array. if not lazy: temp_array = np.empty(chan_len[ind_chan], dtype=np.int16) # NOTE: we could directly create an empty AnalogSignal and # load the data in it, but it is much faster to load data # in a temporary numpy array and create the AnalogSignals # from this temporary array for ind_block in list_blocks: count = count_samples( file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos'] + 6) temp_array[ind:ind+count] = \ np.fromfile(fid, dtype = np.int16, count = count) ind += count sampling_rate = \ file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz t_start = (start_index / sampling_rate).simplified if lazy: ana_sig = AnalogSignal([], sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) ana_sig.lazy_shape = chan_len[ind_chan] else: ana_sig = AnalogSignal(temp_array, sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) ana_sig.channel_index = \ file_blocks[list_chan[ind_chan]]['m_numChannel'] ana_sig.annotate(channel_name = \ file_blocks[list_chan[ind_chan]]['m_Name']) ana_sig.annotate(channel_type = \ file_blocks[list_chan[ind_chan]]['type_subblock']) seg.analogsignals.append(ana_sig) fid.close() if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true blck.rec_datetime = datetime.datetime(\ file_blocks[0]['m_date_year'], file_blocks[0]['m_date_month'], file_blocks[0]['m_date_day'], file_blocks[0]['m_time_hour'], file_blocks[0]['m_time_minute'], file_blocks[0]['m_time_second'], 10000 * file_blocks[0]['m_time_hsecond']) # the 10000 is here to convert m_time_hsecond from centisecond # to microsecond version = file_blocks[0]['m_version'] blck.annotate(alphamap_version=version) if cascade: seg.rec_datetime = blck.rec_datetime.replace() # I couldn't find a simple copy function for datetime, # using replace without arguments is a twisted way to make a # copy seg.annotate(alphamap_version=version) if cascade: populate_RecordingChannel(blck, remove_from_annotation=True) blck.create_many_to_one_relationship() return blck
def read_segment(self, lazy=False, cascade=True): fid = open(self.filename, 'rb') global_header = HeaderReader(fid, GlobalHeader).read_f(offset=0) # ~ print globalHeader #~ print 'version' , globalHeader['version'] seg = Segment() seg.file_origin = os.path.basename(self.filename) seg.annotate(neuroexplorer_version=global_header['version']) seg.annotate(comment=global_header['comment']) if not cascade: return seg offset = 544 for i in range(global_header['nvar']): entity_header = HeaderReader(fid, EntityHeader).read_f( offset=offset + i * 208) entity_header['name'] = entity_header['name'].replace('\x00', '') #print 'i',i, entityHeader['type'] if entity_header['type'] == 0: # neuron if lazy: spike_times = [] * pq.s else: spike_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) spike_times = spike_times.astype('f8') / global_header[ 'freq'] * pq.s sptr = SpikeTrain( times=spike_times, t_start=global_header['tbeg'] / global_header['freq'] * pq.s, t_stop=global_header['tend'] / global_header['freq'] * pq.s, name=entity_header['name']) if lazy: sptr.lazy_shape = entity_header['n'] sptr.annotate(channel_index=entity_header['WireNumber']) seg.spiketrains.append(sptr) if entity_header['type'] == 1: # event if lazy: event_times = [] * pq.s else: event_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) event_times = event_times.astype('f8') / global_header[ 'freq'] * pq.s labels = np.array([''] * event_times.size, dtype='S') evar = Event(times=event_times, labels=labels, channel_name=entity_header['name']) if lazy: evar.lazy_shape = entity_header['n'] seg.events.append(evar) if entity_header['type'] == 2: # interval if lazy: start_times = [] * pq.s stop_times = [] * pq.s else: start_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) start_times = start_times.astype('f8') / global_header[ 'freq'] * pq.s stop_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset'] + entity_header['n'] * 4) stop_times = stop_times.astype('f') / global_header[ 'freq'] * pq.s epar = Epoch(times=start_times, durations=stop_times - start_times, labels=np.array([''] * start_times.size, dtype='S'), channel_name=entity_header['name']) if lazy: epar.lazy_shape = entity_header['n'] seg.epochs.append(epar) if entity_header['type'] == 3: # spiketrain and wavefoms if lazy: spike_times = [] * pq.s waveforms = None else: spike_times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) spike_times = spike_times.astype('f8') / global_header[ 'freq'] * pq.s waveforms = np.memmap(self.filename, np.dtype('i2'), 'r', shape=(entity_header['n'], 1, entity_header['NPointsWave']), offset=entity_header['offset'] + entity_header['n'] * 4) waveforms = (waveforms.astype('f') * entity_header['ADtoMV'] + entity_header['MVOffset']) * pq.mV t_stop = global_header['tend'] / global_header['freq'] * pq.s if spike_times.size > 0: t_stop = max(t_stop, max(spike_times)) sptr = SpikeTrain( times=spike_times, t_start=global_header['tbeg'] / global_header['freq'] * pq.s, #~ t_stop = max(globalHeader['tend']/ #~ globalHeader['freq']*pq.s,max(spike_times)), t_stop=t_stop, name=entity_header['name'], waveforms=waveforms, sampling_rate=entity_header['WFrequency'] * pq.Hz, left_sweep=0 * pq.ms) if lazy: sptr.lazy_shape = entity_header['n'] sptr.annotate(channel_index=entity_header['WireNumber']) seg.spiketrains.append(sptr) if entity_header['type'] == 4: # popvectors pass if entity_header['type'] == 5: # analog timestamps = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) timestamps = timestamps.astype('f8') / global_header['freq'] fragment_starts = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) fragment_starts = fragment_starts.astype('f8') / global_header[ 'freq'] t_start = timestamps[0] - fragment_starts[0] / float( entity_header['WFrequency']) del timestamps, fragment_starts if lazy: signal = [] * pq.mV else: signal = np.memmap(self.filename, np.dtype('i2'), 'r', shape=(entity_header['NPointsWave']), offset=entity_header['offset']) signal = signal.astype('f') signal *= entity_header['ADtoMV'] signal += entity_header['MVOffset'] signal = signal * pq.mV ana_sig = AnalogSignal( signal=signal, t_start=t_start * pq.s, sampling_rate=entity_header['WFrequency'] * pq.Hz, name=entity_header['name'], channel_index=entity_header['WireNumber']) if lazy: ana_sig.lazy_shape = entity_header['NPointsWave'] seg.analogsignals.append(ana_sig) if entity_header['type'] == 6: # markers : TO TEST if lazy: times = [] * pq.s labels = np.array([], dtype='S') markertype = None else: times = np.memmap(self.filename, np.dtype('i4'), 'r', shape=(entity_header['n']), offset=entity_header['offset']) times = times.astype('f8') / global_header['freq'] * pq.s fid.seek(entity_header['offset'] + entity_header['n'] * 4) markertype = fid.read(64).replace('\x00', '') labels = np.memmap( self.filename, np.dtype( 'S' + str(entity_header['MarkerLength'])), 'r', shape=(entity_header['n']), offset=entity_header['offset'] + entity_header['n'] * 4 + 64) ea = Event(times=times, labels=labels.view(np.ndarray), name=entity_header['name'], channel_index=entity_header['WireNumber'], marker_type=markertype) if lazy: ea.lazy_shape = entity_header['n'] seg.events.append(ea) seg.create_many_to_one_relationship() return seg
def read_block(self, # the 2 first keyword arguments are imposed by neo.io API lazy = False, cascade = True): """ Return a Block. """ def count_samples(m_length): """ Count the number of signal samples available in a type 5 data block of length m_length """ # for information about type 5 data block, see [1] count = int((m_length-6)/2-2) # -6 corresponds to the header of block 5, and the -2 take into # account the fact that last 2 values are not available as the 4 # corresponding bytes are coding the time stamp of the beginning # of the block return count # create the neo Block that will be returned at the end blck = Block(file_origin = os.path.basename(self.filename)) blck.file_origin = os.path.basename(self.filename) fid = open(self.filename, 'rb') # NOTE: in the following, the word "block" is used in the sense used in # the alpha-omega specifications (ie a data chunk in the file), rather # than in the sense of the usual Block object in neo # step 1: read the headers of all the data blocks to load the file # structure pos_block = 0 # position of the current block in the file file_blocks = [] # list of data blocks available in the file if not cascade: # we read only the main header m_length, m_TypeBlock = struct.unpack('Hcx' , fid.read(4)) # m_TypeBlock should be 'h', as we read the first block block = HeaderReader(fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({'m_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block}) file_blocks.append(block) else: # cascade == True seg = Segment(file_origin = os.path.basename(self.filename)) seg.file_origin = os.path.basename(self.filename) blck.segments.append(seg) while True: first_4_bytes = fid.read(4) if len(first_4_bytes) < 4: # we have reached the end of the file break else: m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes) block = HeaderReader(fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({'m_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block}) if m_TypeBlock == '2': # The beginning of the block of type '2' is identical for # all types of channels, but the following part depends on # the type of channel. So we need a special case here. # WARNING: How to check the type of channel is not # described in the documentation. So here I use what is # proposed in the C code [2]. # According to this C code, it seems that the 'm_isAnalog' # is used to distinguished analog and digital channels, and # 'm_Mode' encodes the type of analog channel: # 0 for continuous, 1 for level, 2 for external trigger. # But in some files, I found channels that seemed to be # continuous channels with 'm_Modes' = 128 or 192. So I # decided to consider every channel with 'm_Modes' # different from 1 or 2 as continuous. I also couldn't # check that values of 1 and 2 are really for level and # external trigger as I had no test files containing data # of this types. type_subblock = 'unknown_channel_type(m_Mode=' \ + str(block['m_Mode'])+ ')' description = Type2_SubBlockUnknownChannels block.update({'m_Name': 'unknown_name'}) if block['m_isAnalog'] == 0: # digital channel type_subblock = 'digital' description = Type2_SubBlockDigitalChannels elif block['m_isAnalog'] == 1: # analog channel if block['m_Mode'] == 1: # level channel type_subblock = 'level' description = Type2_SubBlockLevelChannels elif block['m_Mode'] == 2: # external trigger channel type_subblock = 'external_trigger' description = Type2_SubBlockExtTriggerChannels else: # continuous channel type_subblock = 'continuous(Mode' \ + str(block['m_Mode']) +')' description = Type2_SubBlockContinuousChannels subblock = HeaderReader(fid, description).read_f() block.update(subblock) block.update({'type_subblock': type_subblock}) file_blocks.append(block) pos_block += m_length fid.seek(pos_block) # step 2: find the available channels list_chan = [] # list containing indexes of channel blocks for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '2': list_chan.append(ind_block) # step 3: find blocks containing data for the available channels list_data = [] # list of lists of indexes of data blocks # corresponding to each channel for ind_chan, chan in enumerate(list_chan): list_data.append([]) num_chan = file_blocks[chan]['m_numChannel'] for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '5': if block['m_numChannel'] == num_chan: list_data[ind_chan].append(ind_block) # step 4: compute the length (number of samples) of the channels chan_len = np.zeros(len(list_data), dtype = np.int) for ind_chan, list_blocks in enumerate(list_data): for ind_block in list_blocks: chan_len[ind_chan] += count_samples( file_blocks[ind_block]['m_length']) # step 5: find channels for which data are available ind_valid_chan = np.nonzero(chan_len)[0] # step 6: load the data # TODO give the possibility to load data as AnalogSignalArrays for ind_chan in ind_valid_chan: list_blocks = list_data[ind_chan] ind = 0 # index in the data vector # read time stamp for the beginning of the signal form = '<l' # reading format ind_block = list_blocks[0] count = count_samples(file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos']+6+count*2) buf = fid.read(struct.calcsize(form)) val = struct.unpack(form , buf) start_index = val[0] # WARNING: in the following blocks are read supposing taht they # are all contiguous and sorted in time. I don't know if it's # always the case. Maybe we should use the time stamp of each # data block to choose where to put the read data in the array. if not lazy: temp_array = np.empty(chan_len[ind_chan], dtype = np.int16) # NOTE: we could directly create an empty AnalogSignal and # load the data in it, but it is much faster to load data # in a temporary numpy array and create the AnalogSignals # from this temporary array for ind_block in list_blocks: count = count_samples( file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos']+6) temp_array[ind:ind+count] = \ np.fromfile(fid, dtype = np.int16, count = count) ind += count sampling_rate = \ file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz t_start = (start_index / sampling_rate).simplified if lazy: ana_sig = AnalogSignal([], sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) ana_sig.lazy_shape = chan_len[ind_chan] else: ana_sig = AnalogSignal(temp_array, sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) # todo apibreak: create ChannelIndex for each signals # ana_sig.channel_index = \ # file_blocks[list_chan[ind_chan]]['m_numChannel'] ana_sig.annotate(channel_name = \ file_blocks[list_chan[ind_chan]]['m_Name']) ana_sig.annotate(channel_type = \ file_blocks[list_chan[ind_chan]]['type_subblock']) seg.analogsignals.append(ana_sig) fid.close() if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true blck.rec_datetime = datetime.datetime(\ file_blocks[0]['m_date_year'], file_blocks[0]['m_date_month'], file_blocks[0]['m_date_day'], file_blocks[0]['m_time_hour'], file_blocks[0]['m_time_minute'], file_blocks[0]['m_time_second'], 10000 * file_blocks[0]['m_time_hsecond']) # the 10000 is here to convert m_time_hsecond from centisecond # to microsecond version = file_blocks[0]['m_version'] blck.annotate(alphamap_version = version) if cascade: seg.rec_datetime = blck.rec_datetime.replace() # I couldn't find a simple copy function for datetime, # using replace without arguments is a twisted way to make a # copy seg.annotate(alphamap_version = version) if cascade: blck.create_many_to_one_relationship() return blck
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True): """ """ fid = open(self.filename, "rb") globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0) # metadatas seg = Segment() seg.rec_datetime = datetime.datetime( globalHeader["Year"], globalHeader["Month"], globalHeader["Day"], globalHeader["Hour"], globalHeader["Minute"], globalHeader["Second"], ) seg.file_origin = os.path.basename(self.filename) seg.annotate(plexon_version=globalHeader["Version"]) if not cascade: return seg ## Step 1 : read headers # dsp channels header = sipkes and waveforms dspChannelHeaders = {} maxunit = 0 maxchan = 0 for _ in range(globalHeader["NumDSPChannels"]): # channel is 1 based channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None) channelHeader["Template"] = np.array(channelHeader["Template"]).reshape((5, 64)) channelHeader["Boxes"] = np.array(channelHeader["Boxes"]).reshape((5, 2, 4)) dspChannelHeaders[channelHeader["Channel"]] = channelHeader maxunit = max(channelHeader["NUnits"], maxunit) maxchan = max(channelHeader["Channel"], maxchan) # event channel header eventHeaders = {} for _ in range(globalHeader["NumEventChannels"]): eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None) eventHeaders[eventHeader["Channel"]] = eventHeader # slow channel header = signal slowChannelHeaders = {} for _ in range(globalHeader["NumSlowChannels"]): slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None) slowChannelHeaders[slowChannelHeader["Channel"]] = slowChannelHeader ## Step 2 : a first loop for counting size # signal nb_samples = np.zeros(len(slowChannelHeaders)) sample_positions = np.zeros(len(slowChannelHeaders)) t_starts = np.zeros(len(slowChannelHeaders), dtype="f") # spiketimes and waveform nb_spikes = np.zeros((maxchan + 1, maxunit + 1), dtype="i") wf_sizes = np.zeros((maxchan + 1, maxunit + 1, 2), dtype="i") # eventarrays nb_events = {} # maxstrsizeperchannel = { } for chan, h in iteritems(eventHeaders): nb_events[chan] = 0 # maxstrsizeperchannel[chan] = 0 start = fid.tell() while fid.tell() != -1: # read block header dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None) if dataBlockHeader is None: break chan = dataBlockHeader["Channel"] unit = dataBlockHeader["Unit"] n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"] time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"] if dataBlockHeader["Type"] == 1: nb_spikes[chan, unit] += 1 wf_sizes[chan, unit, :] = [n1, n2] fid.seek(n1 * n2 * 2, 1) elif dataBlockHeader["Type"] == 4: # event nb_events[chan] += 1 elif dataBlockHeader["Type"] == 5: # continuous signal fid.seek(n2 * 2, 1) if n2 > 0: nb_samples[chan] += n2 if nb_samples[chan] == 0: t_starts[chan] = time ## Step 3: allocating memory and 2 loop for reading if not lazy if not lazy: # allocating mem for signal sigarrays = {} for chan, h in iteritems(slowChannelHeaders): sigarrays[chan] = np.zeros(nb_samples[chan]) # allocating mem for SpikeTrain stimearrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object) swfarrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object) for (chan, unit), _ in np.ndenumerate(nb_spikes): stimearrays[chan, unit] = np.zeros(nb_spikes[chan, unit], dtype="f") if load_spike_waveform: n1, n2 = wf_sizes[chan, unit, :] swfarrays[chan, unit] = np.zeros((nb_spikes[chan, unit], n1, n2), dtype="f4") pos_spikes = np.zeros(nb_spikes.shape, dtype="i") # allocating mem for event eventpositions = {} evarrays = {} for chan, nb in iteritems(nb_events): evarrays[chan] = np.zeros(nb, dtype="f") eventpositions[chan] = 0 fid.seek(start) while fid.tell() != -1: dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None) if dataBlockHeader is None: break chan = dataBlockHeader["Channel"] n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"] time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"] time /= globalHeader["ADFrequency"] if n2 < 0: break if dataBlockHeader["Type"] == 1: # spike unit = dataBlockHeader["Unit"] pos = pos_spikes[chan, unit] stimearrays[chan, unit][pos] = time if load_spike_waveform and n1 * n2 != 0: swfarrays[chan, unit][pos, :, :] = ( np.fromstring(fid.read(n1 * n2 * 2), dtype="i2").reshape(n1, n2).astype("f4") ) else: fid.seek(n1 * n2 * 2, 1) pos_spikes[chan, unit] += 1 elif dataBlockHeader["Type"] == 4: # event pos = eventpositions[chan] evarrays[chan][pos] = time eventpositions[chan] += 1 elif dataBlockHeader["Type"] == 5: # signal data = np.fromstring(fid.read(n2 * 2), dtype="i2").astype("f4") sigarrays[chan][sample_positions[chan] : sample_positions[chan] + data.size] = data sample_positions[chan] += data.size ## Step 3: create neo object for chan, h in iteritems(eventHeaders): if lazy: times = [] else: times = evarrays[chan] ea = EventArray(times * pq.s, channel_name=eventHeaders[chan]["Name"], channel_index=chan) if lazy: ea.lazy_shape = nb_events[chan] seg.eventarrays.append(ea) for chan, h in iteritems(slowChannelHeaders): if lazy: signal = [] else: if globalHeader["Version"] == 100 or globalHeader["Version"] == 101: gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * 1000.0) elif globalHeader["Version"] == 102: gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"]) elif globalHeader["Version"] >= 103: gain = globalHeader["SlowMaxMagnitudeMV"] / ( 0.5 * (2 ** globalHeader["BitsPerSpikeSample"]) * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"] ) signal = sigarrays[chan] * gain anasig = AnalogSignal( signal * pq.V, sampling_rate=float(slowChannelHeaders[chan]["ADFreq"]) * pq.Hz, t_start=t_starts[chan] * pq.s, channel_index=slowChannelHeaders[chan]["Channel"], channel_name=slowChannelHeaders[chan]["Name"], ) if lazy: anasig.lazy_shape = nb_samples[chan] seg.analogsignals.append(anasig) for (chan, unit), value in np.ndenumerate(nb_spikes): if nb_spikes[chan, unit] == 0: continue if lazy: times = [] waveforms = None t_stop = 0 else: times = stimearrays[chan, unit] t_stop = times.max() if load_spike_waveform: if globalHeader["Version"] < 103: gain = 3000.0 / (2048 * dspChannelHeaders[chan]["Gain"] * 1000.0) elif globalHeader["Version"] >= 103 and globalHeader["Version"] < 105: gain = globalHeader["SpikeMaxMagnitudeMV"] / ( 0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * 1000.0 ) elif globalHeader["Version"] > 105: gain = globalHeader["SpikeMaxMagnitudeMV"] / ( 0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * globalHeader["SpikePreAmpGain"] ) waveforms = swfarrays[chan, unit] * gain * pq.V else: waveforms = None sptr = SpikeTrain(times, units="s", t_stop=t_stop * pq.s, waveforms=waveforms) sptr.annotate(unit_name=dspChannelHeaders[chan]["Name"]) sptr.annotate(channel_index=chan) if lazy: sptr.lazy_shape = nb_spikes[chan, unit] seg.spiketrains.append(sptr) seg.create_many_to_one_relationship() return seg