def _group_to_neo(self, nix_group):
neo_attrs = self._nix_attr_to_neo(nix_group)
neo_segment = Segment(**neo_attrs)
neo_segment.rec_datetime = datetime.fromtimestamp(
nix_group.created_at
)
self._neo_map[nix_group.name] = neo_segment
return neo_segment
def proc_src_comments(srcfile, filename):
'''Get the comments in an src file that has been#!N
processed by the official
matlab function. See proc_src for details'''
comm_seg = Segment(name='Comments', file_origin=filename)
commentarray = srcfile['comments'].flatten()[0]
senders = [res[0] for res in commentarray['sender'].flatten()]
texts = [res[0] for res in commentarray['text'].flatten()]
timeStamps = [res[0, 0] for res in commentarray['timeStamp'].flatten()]
timeStamps = np.array(timeStamps, dtype=np.float32)
t_start = timeStamps.min()
timeStamps = pq.Quantity(timeStamps - t_start, units=pq.d).rescale(pq.s)
texts = np.array(texts, dtype='S')
senders = np.array(senders, dtype='S')
t_start = brainwaresrcio.convert_brainwaresrc_timestamp(t_start.tolist())
comments = Event(times=timeStamps, labels=texts, senders=senders)
comm_seg.events = [comments]
comm_seg.rec_datetime = t_start
return comm_seg
def proc_src_comments(srcfile, filename):
'''Get the comments in an src file that has been#!N
processed by the official
matlab function. See proc_src for details'''
comm_seg = Segment(name='Comments', file_origin=filename)
commentarray = srcfile['comments'].flatten()[0]
senders = [res[0] for res in commentarray['sender'].flatten()]
texts = [res[0] for res in commentarray['text'].flatten()]
timeStamps = [res[0, 0] for res in commentarray['timeStamp'].flatten()]
timeStamps = np.array(timeStamps, dtype=np.float32)
t_start = timeStamps.min()
timeStamps = pq.Quantity(timeStamps - t_start, units=pq.d).rescale(pq.s)
texts = np.array(texts, dtype='S')
senders = np.array(senders, dtype='S')
t_start = brainwaresrcio.convert_brainwaresrc_timestamp(t_start.tolist())
comments = EventArray(times=timeStamps, labels=texts, senders=senders)
comm_seg.eventarrays = [comments]
comm_seg.rec_datetime = t_start
return comm_seg
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True):
"""
Read in a segment.
Arguments:
load_spike_waveform : load or not waveform of spikes (default True)
"""
fid = open(self.filename, 'rb')
globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# metadatas
seg = Segment()
seg.rec_datetime = datetime.datetime(
globalHeader.pop('Year'),
globalHeader.pop('Month'),
globalHeader.pop('Day'),
globalHeader.pop('Hour'),
globalHeader.pop('Minute'),
globalHeader.pop('Second')
)
seg.file_origin = os.path.basename(self.filename)
for key, val in globalHeader.iteritems():
seg.annotate(**{key: val})
if not cascade:
return seg
## Step 1 : read headers
# dsp channels header = spikes and waveforms
dspChannelHeaders = {}
maxunit = 0
maxchan = 0
for _ in range(globalHeader['NumDSPChannels']):
# channel is 1 based
channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None)
channelHeader['Template'] = np.array(channelHeader['Template']).reshape((5,64))
channelHeader['Boxes'] = np.array(channelHeader['Boxes']).reshape((5,2,4))
dspChannelHeaders[channelHeader['Channel']] = channelHeader
maxunit = max(channelHeader['NUnits'], maxunit)
maxchan = max(channelHeader['Channel'], maxchan)
# event channel header
eventHeaders = { }
for _ in range(globalHeader['NumEventChannels']):
eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None)
eventHeaders[eventHeader['Channel']] = eventHeader
# slow channel header = signal
slowChannelHeaders = {}
for _ in range(globalHeader['NumSlowChannels']):
slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None)
slowChannelHeaders[slowChannelHeader['Channel']] = slowChannelHeader
## Step 2 : a first loop for counting size
# signal
nb_samples = np.zeros(len(slowChannelHeaders))
sample_positions = np.zeros(len(slowChannelHeaders))
t_starts = np.zeros(len(slowChannelHeaders), dtype='f')
#spiketimes and waveform
nb_spikes = np.zeros((maxchan+1, maxunit+1) ,dtype='i')
wf_sizes = np.zeros((maxchan+1, maxunit+1, 2) ,dtype='i')
# eventarrays
nb_events = { }
#maxstrsizeperchannel = { }
for chan, h in iteritems(eventHeaders):
nb_events[chan] = 0
#maxstrsizeperchannel[chan] = 0
start = fid.tell()
while fid.tell() !=-1 :
# read block header
dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None)
if dataBlockHeader is None : break
chan = dataBlockHeader['Channel']
unit = dataBlockHeader['Unit']
n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform']
time = (dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 +
dataBlockHeader['TimeStamp'])
if dataBlockHeader['Type'] == 1:
nb_spikes[chan,unit] +=1
wf_sizes[chan,unit,:] = [n1,n2]
fid.seek(n1*n2*2,1)
elif dataBlockHeader['Type'] ==4:
#event
nb_events[chan] += 1
elif dataBlockHeader['Type'] == 5:
#continuous signal
fid.seek(n2*2, 1)
if n2> 0:
nb_samples[chan] += n2
if nb_samples[chan] ==0:
t_starts[chan] = time
## Step 3: allocating memory and 2 loop for reading if not lazy
if not lazy:
# allocating mem for signal
sigarrays = { }
for chan, h in iteritems(slowChannelHeaders):
sigarrays[chan] = np.zeros(nb_samples[chan])
# allocating mem for SpikeTrain
stimearrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object)
swfarrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object)
for (chan, unit), _ in np.ndenumerate(nb_spikes):
stimearrays[chan,unit] = np.zeros(nb_spikes[chan,unit], dtype = 'f')
if load_spike_waveform:
n1,n2 = wf_sizes[chan, unit,:]
swfarrays[chan, unit] = np.zeros( (nb_spikes[chan, unit], n1, n2 ) , dtype = 'f4' )
pos_spikes = np.zeros(nb_spikes.shape, dtype = 'i')
# allocating mem for event
eventpositions = { }
evarrays = { }
for chan, nb in iteritems(nb_events):
evarrays[chan] = {
'times': np.zeros(nb, dtype='f'),
'labels': np.zeros(nb, dtype='S4')
}
eventpositions[chan]=0
fid.seek(start)
while fid.tell() !=-1 :
dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None)
if dataBlockHeader is None : break
chan = dataBlockHeader['Channel']
n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform']
time = dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp']
time/= globalHeader['ADFrequency']
if n2 <0: break
if dataBlockHeader['Type'] == 1:
#spike
unit = dataBlockHeader['Unit']
pos = pos_spikes[chan,unit]
stimearrays[chan, unit][pos] = time
if load_spike_waveform and n1*n2 != 0 :
swfarrays[chan,unit][pos,:,:] = np.fromstring( fid.read(n1*n2*2) , dtype = 'i2').reshape(n1,n2).astype('f4')
else:
fid.seek(n1*n2*2,1)
pos_spikes[chan,unit] +=1
elif dataBlockHeader['Type'] == 4:
# event
pos = eventpositions[chan]
evarrays[chan]['times'][pos] = time
evarrays[chan]['labels'][pos] = dataBlockHeader['Unit']
eventpositions[chan]+= 1
elif dataBlockHeader['Type'] == 5:
#signal
data = np.fromstring( fid.read(n2*2) , dtype = 'i2').astype('f4')
sigarrays[chan][sample_positions[chan] : sample_positions[chan]+data.size] = data
sample_positions[chan] += data.size
## Step 4: create neo object
for chan, h in iteritems(eventHeaders):
if lazy:
times = []
labels = None
else:
times = evarrays[chan]['times']
labels = evarrays[chan]['labels']
ea = EventArray(
times*pq.s,
labels=labels,
channel_name=eventHeaders[chan]['Name'],
channel_index=chan
)
if lazy:
ea.lazy_shape = nb_events[chan]
seg.eventarrays.append(ea)
for chan, h in iteritems(slowChannelHeaders):
if lazy:
signal = [ ]
else:
if globalHeader['Version'] ==100 or globalHeader['Version'] ==101 :
gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*1000.)
elif globalHeader['Version'] ==102 :
gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain'])
elif globalHeader['Version'] >= 103:
gain = globalHeader['SlowMaxMagnitudeMV']/(.5*(2**globalHeader['BitsPerSpikeSample'])*\
slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain'])
signal = sigarrays[chan]*gain
anasig = AnalogSignal(signal*pq.V,
sampling_rate = float(slowChannelHeaders[chan]['ADFreq'])*pq.Hz,
t_start = t_starts[chan]*pq.s,
channel_index = slowChannelHeaders[chan]['Channel'],
channel_name = slowChannelHeaders[chan]['Name'],
)
if lazy:
anasig.lazy_shape = nb_samples[chan]
seg.analogsignals.append(anasig)
for (chan, unit), value in np.ndenumerate(nb_spikes):
if nb_spikes[chan, unit] == 0: continue
if lazy:
times = [ ]
waveforms = None
t_stop = 0
else:
times = stimearrays[chan,unit]
t_stop = times.max()
if load_spike_waveform:
if globalHeader['Version'] <103:
gain = 3000./(2048*dspChannelHeaders[chan]['Gain']*1000.)
elif globalHeader['Version'] >=103 and globalHeader['Version'] <105:
gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*1000.)
elif globalHeader['Version'] >105:
gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*globalHeader['SpikePreAmpGain'])
waveforms = swfarrays[chan, unit] * gain * pq.V
else:
waveforms = None
sptr = SpikeTrain(
times,
units='s',
t_stop=t_stop*pq.s,
waveforms=waveforms
)
sptr.annotate(unit_name = dspChannelHeaders[chan]['Name'])
sptr.annotate(channel_index = chan)
for key, val in dspChannelHeaders[chan].iteritems():
sptr.annotate(**{key: val})
if lazy:
sptr.lazy_shape = nb_spikes[chan,unit]
seg.spiketrains.append(sptr)
seg.create_many_to_one_relationship()
return seg
def read_block(
self,
# the 2 first keyword arguments are imposed by neo.io API
lazy=False,
cascade=True):
"""
Return a Block.
"""
def count_samples(m_length):
"""
Count the number of signal samples available in a type 5 data block
of length m_length
"""
# for information about type 5 data block, see [1]
count = int((m_length - 6) / 2 - 2)
# -6 corresponds to the header of block 5, and the -2 take into
# account the fact that last 2 values are not available as the 4
# corresponding bytes are coding the time stamp of the beginning
# of the block
return count
# create the neo Block that will be returned at the end
blck = Block(file_origin=os.path.basename(self.filename))
blck.file_origin = os.path.basename(self.filename)
fid = open(self.filename, 'rb')
# NOTE: in the following, the word "block" is used in the sense used in
# the alpha-omega specifications (ie a data chunk in the file), rather
# than in the sense of the usual Block object in neo
# step 1: read the headers of all the data blocks to load the file
# structure
pos_block = 0 # position of the current block in the file
file_blocks = [] # list of data blocks available in the file
if not cascade:
# we read only the main header
m_length, m_TypeBlock = struct.unpack('Hcx', fid.read(4))
# m_TypeBlock should be 'h', as we read the first block
block = HeaderReader(
fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f()
block.update({
'm_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block
})
file_blocks.append(block)
else: # cascade == True
seg = Segment(file_origin=os.path.basename(self.filename))
seg.file_origin = os.path.basename(self.filename)
blck.segments.append(seg)
while True:
first_4_bytes = fid.read(4)
if len(first_4_bytes) < 4:
# we have reached the end of the file
break
else:
m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes)
block = HeaderReader(
fid, dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({
'm_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block
})
if m_TypeBlock == '2':
# The beggining of the block of type '2' is identical for
# all types of channels, but the following part depends on
# the type of channel. So we need a special case here.
# WARNING: How to check the type of channel is not
# described in the documentation. So here I use what is
# proposed in the C code [2].
# According to this C code, it seems that the 'm_isAnalog'
# is used to distinguished analog and digital channels, and
# 'm_Mode' encodes the type of analog channel:
# 0 for continuous, 1 for level, 2 for external trigger.
# But in some files, I found channels that seemed to be
# continuous channels with 'm_Modes' = 128 or 192. So I
# decided to consider every channel with 'm_Modes'
# different from 1 or 2 as continuous. I also couldn't
# check that values of 1 and 2 are really for level and
# external trigger as I had no test files containing data
# of this types.
type_subblock = 'unknown_channel_type(m_Mode=' \
+ str(block['m_Mode'])+ ')'
description = Type2_SubBlockUnknownChannels
block.update({'m_Name': 'unknown_name'})
if block['m_isAnalog'] == 0:
# digital channel
type_subblock = 'digital'
description = Type2_SubBlockDigitalChannels
elif block['m_isAnalog'] == 1:
# analog channel
if block['m_Mode'] == 1:
# level channel
type_subblock = 'level'
description = Type2_SubBlockLevelChannels
elif block['m_Mode'] == 2:
# external trigger channel
type_subblock = 'external_trigger'
description = Type2_SubBlockExtTriggerChannels
else:
# continuous channel
type_subblock = 'continuous(Mode' \
+ str(block['m_Mode']) +')'
description = Type2_SubBlockContinuousChannels
subblock = HeaderReader(fid, description).read_f()
block.update(subblock)
block.update({'type_subblock': type_subblock})
file_blocks.append(block)
pos_block += m_length
fid.seek(pos_block)
# step 2: find the available channels
list_chan = [] # list containing indexes of channel blocks
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '2':
list_chan.append(ind_block)
# step 3: find blocks containing data for the available channels
list_data = [] # list of lists of indexes of data blocks
# corresponding to each channel
for ind_chan, chan in enumerate(list_chan):
list_data.append([])
num_chan = file_blocks[chan]['m_numChannel']
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '5':
if block['m_numChannel'] == num_chan:
list_data[ind_chan].append(ind_block)
# step 4: compute the length (number of samples) of the channels
chan_len = np.zeros(len(list_data), dtype=np.int)
for ind_chan, list_blocks in enumerate(list_data):
for ind_block in list_blocks:
chan_len[ind_chan] += count_samples(
file_blocks[ind_block]['m_length'])
# step 5: find channels for which data are available
ind_valid_chan = np.nonzero(chan_len)[0]
# step 6: load the data
# TODO give the possibility to load data as AnalogSignalArrays
for ind_chan in ind_valid_chan:
list_blocks = list_data[ind_chan]
ind = 0 # index in the data vector
# read time stamp for the beginning of the signal
form = '<l' # reading format
ind_block = list_blocks[0]
count = count_samples(file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos'] + 6 + count * 2)
buf = fid.read(struct.calcsize(form))
val = struct.unpack(form, buf)
start_index = val[0]
# WARNING: in the following blocks are read supposing taht they
# are all contiguous and sorted in time. I don't know if it's
# always the case. Maybe we should use the time stamp of each
# data block to choose where to put the read data in the array.
if not lazy:
temp_array = np.empty(chan_len[ind_chan], dtype=np.int16)
# NOTE: we could directly create an empty AnalogSignal and
# load the data in it, but it is much faster to load data
# in a temporary numpy array and create the AnalogSignals
# from this temporary array
for ind_block in list_blocks:
count = count_samples(
file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos'] + 6)
temp_array[ind:ind+count] = \
np.fromfile(fid, dtype = np.int16, count = count)
ind += count
sampling_rate = \
file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz
t_start = (start_index / sampling_rate).simplified
if lazy:
ana_sig = AnalogSignal([],
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.lazy_shape = chan_len[ind_chan]
else:
ana_sig = AnalogSignal(temp_array,
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.channel_index = \
file_blocks[list_chan[ind_chan]]['m_numChannel']
ana_sig.annotate(channel_name = \
file_blocks[list_chan[ind_chan]]['m_Name'])
ana_sig.annotate(channel_type = \
file_blocks[list_chan[ind_chan]]['type_subblock'])
seg.analogsignals.append(ana_sig)
fid.close()
if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true
blck.rec_datetime = datetime.datetime(\
file_blocks[0]['m_date_year'],
file_blocks[0]['m_date_month'],
file_blocks[0]['m_date_day'],
file_blocks[0]['m_time_hour'],
file_blocks[0]['m_time_minute'],
file_blocks[0]['m_time_second'],
10000 * file_blocks[0]['m_time_hsecond'])
# the 10000 is here to convert m_time_hsecond from centisecond
# to microsecond
version = file_blocks[0]['m_version']
blck.annotate(alphamap_version=version)
if cascade:
seg.rec_datetime = blck.rec_datetime.replace()
# I couldn't find a simple copy function for datetime,
# using replace without arguments is a twisted way to make a
# copy
seg.annotate(alphamap_version=version)
if cascade:
populate_RecordingChannel(blck, remove_from_annotation=True)
blck.create_many_to_one_relationship()
return blck
def read_block(self,
# the 2 first keyword arguments are imposed by neo.io API
lazy = False,
cascade = True):
"""
Return a Block.
"""
def count_samples(m_length):
"""
Count the number of signal samples available in a type 5 data block
of length m_length
"""
# for information about type 5 data block, see [1]
count = int((m_length-6)/2-2)
# -6 corresponds to the header of block 5, and the -2 take into
# account the fact that last 2 values are not available as the 4
# corresponding bytes are coding the time stamp of the beginning
# of the block
return count
# create the neo Block that will be returned at the end
blck = Block(file_origin = os.path.basename(self.filename))
blck.file_origin = os.path.basename(self.filename)
fid = open(self.filename, 'rb')
# NOTE: in the following, the word "block" is used in the sense used in
# the alpha-omega specifications (ie a data chunk in the file), rather
# than in the sense of the usual Block object in neo
# step 1: read the headers of all the data blocks to load the file
# structure
pos_block = 0 # position of the current block in the file
file_blocks = [] # list of data blocks available in the file
if not cascade:
# we read only the main header
m_length, m_TypeBlock = struct.unpack('Hcx' , fid.read(4))
# m_TypeBlock should be 'h', as we read the first block
block = HeaderReader(fid,
dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({'m_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block})
file_blocks.append(block)
else: # cascade == True
seg = Segment(file_origin = os.path.basename(self.filename))
seg.file_origin = os.path.basename(self.filename)
blck.segments.append(seg)
while True:
first_4_bytes = fid.read(4)
if len(first_4_bytes) < 4:
# we have reached the end of the file
break
else:
m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes)
block = HeaderReader(fid,
dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({'m_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block})
if m_TypeBlock == '2':
# The beginning of the block of type '2' is identical for
# all types of channels, but the following part depends on
# the type of channel. So we need a special case here.
# WARNING: How to check the type of channel is not
# described in the documentation. So here I use what is
# proposed in the C code [2].
# According to this C code, it seems that the 'm_isAnalog'
# is used to distinguished analog and digital channels, and
# 'm_Mode' encodes the type of analog channel:
# 0 for continuous, 1 for level, 2 for external trigger.
# But in some files, I found channels that seemed to be
# continuous channels with 'm_Modes' = 128 or 192. So I
# decided to consider every channel with 'm_Modes'
# different from 1 or 2 as continuous. I also couldn't
# check that values of 1 and 2 are really for level and
# external trigger as I had no test files containing data
# of this types.
type_subblock = 'unknown_channel_type(m_Mode=' \
+ str(block['m_Mode'])+ ')'
description = Type2_SubBlockUnknownChannels
block.update({'m_Name': 'unknown_name'})
if block['m_isAnalog'] == 0:
# digital channel
type_subblock = 'digital'
description = Type2_SubBlockDigitalChannels
elif block['m_isAnalog'] == 1:
# analog channel
if block['m_Mode'] == 1:
# level channel
type_subblock = 'level'
description = Type2_SubBlockLevelChannels
elif block['m_Mode'] == 2:
# external trigger channel
type_subblock = 'external_trigger'
description = Type2_SubBlockExtTriggerChannels
else:
# continuous channel
type_subblock = 'continuous(Mode' \
+ str(block['m_Mode']) +')'
description = Type2_SubBlockContinuousChannels
subblock = HeaderReader(fid, description).read_f()
block.update(subblock)
block.update({'type_subblock': type_subblock})
file_blocks.append(block)
pos_block += m_length
fid.seek(pos_block)
# step 2: find the available channels
list_chan = [] # list containing indexes of channel blocks
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '2':
list_chan.append(ind_block)
# step 3: find blocks containing data for the available channels
list_data = [] # list of lists of indexes of data blocks
# corresponding to each channel
for ind_chan, chan in enumerate(list_chan):
list_data.append([])
num_chan = file_blocks[chan]['m_numChannel']
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '5':
if block['m_numChannel'] == num_chan:
list_data[ind_chan].append(ind_block)
# step 4: compute the length (number of samples) of the channels
chan_len = np.zeros(len(list_data), dtype = np.int)
for ind_chan, list_blocks in enumerate(list_data):
for ind_block in list_blocks:
chan_len[ind_chan] += count_samples(
file_blocks[ind_block]['m_length'])
# step 5: find channels for which data are available
ind_valid_chan = np.nonzero(chan_len)[0]
# step 6: load the data
# TODO give the possibility to load data as AnalogSignalArrays
for ind_chan in ind_valid_chan:
list_blocks = list_data[ind_chan]
ind = 0 # index in the data vector
# read time stamp for the beginning of the signal
form = '<l' # reading format
ind_block = list_blocks[0]
count = count_samples(file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos']+6+count*2)
buf = fid.read(struct.calcsize(form))
val = struct.unpack(form , buf)
start_index = val[0]
# WARNING: in the following blocks are read supposing taht they
# are all contiguous and sorted in time. I don't know if it's
# always the case. Maybe we should use the time stamp of each
# data block to choose where to put the read data in the array.
if not lazy:
temp_array = np.empty(chan_len[ind_chan], dtype = np.int16)
# NOTE: we could directly create an empty AnalogSignal and
# load the data in it, but it is much faster to load data
# in a temporary numpy array and create the AnalogSignals
# from this temporary array
for ind_block in list_blocks:
count = count_samples(
file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos']+6)
temp_array[ind:ind+count] = \
np.fromfile(fid, dtype = np.int16, count = count)
ind += count
sampling_rate = \
file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz
t_start = (start_index / sampling_rate).simplified
if lazy:
ana_sig = AnalogSignal([],
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.lazy_shape = chan_len[ind_chan]
else:
ana_sig = AnalogSignal(temp_array,
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
# todo apibreak: create ChannelIndex for each signals
# ana_sig.channel_index = \
# file_blocks[list_chan[ind_chan]]['m_numChannel']
ana_sig.annotate(channel_name = \
file_blocks[list_chan[ind_chan]]['m_Name'])
ana_sig.annotate(channel_type = \
file_blocks[list_chan[ind_chan]]['type_subblock'])
seg.analogsignals.append(ana_sig)
fid.close()
if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true
blck.rec_datetime = datetime.datetime(\
file_blocks[0]['m_date_year'],
file_blocks[0]['m_date_month'],
file_blocks[0]['m_date_day'],
file_blocks[0]['m_time_hour'],
file_blocks[0]['m_time_minute'],
file_blocks[0]['m_time_second'],
10000 * file_blocks[0]['m_time_hsecond'])
# the 10000 is here to convert m_time_hsecond from centisecond
# to microsecond
version = file_blocks[0]['m_version']
blck.annotate(alphamap_version = version)
if cascade:
seg.rec_datetime = blck.rec_datetime.replace()
# I couldn't find a simple copy function for datetime,
# using replace without arguments is a twisted way to make a
# copy
seg.annotate(alphamap_version = version)
if cascade:
blck.create_many_to_one_relationship()
return blck
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True):
"""
"""
fid = open(self.filename, "rb")
globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# metadatas
seg = Segment()
seg.rec_datetime = datetime.datetime(
globalHeader["Year"],
globalHeader["Month"],
globalHeader["Day"],
globalHeader["Hour"],
globalHeader["Minute"],
globalHeader["Second"],
)
seg.file_origin = os.path.basename(self.filename)
seg.annotate(plexon_version=globalHeader["Version"])
if not cascade:
return seg
## Step 1 : read headers
# dsp channels header = sipkes and waveforms
dspChannelHeaders = {}
maxunit = 0
maxchan = 0
for _ in range(globalHeader["NumDSPChannels"]):
# channel is 1 based
channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None)
channelHeader["Template"] = np.array(channelHeader["Template"]).reshape((5, 64))
channelHeader["Boxes"] = np.array(channelHeader["Boxes"]).reshape((5, 2, 4))
dspChannelHeaders[channelHeader["Channel"]] = channelHeader
maxunit = max(channelHeader["NUnits"], maxunit)
maxchan = max(channelHeader["Channel"], maxchan)
# event channel header
eventHeaders = {}
for _ in range(globalHeader["NumEventChannels"]):
eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None)
eventHeaders[eventHeader["Channel"]] = eventHeader
# slow channel header = signal
slowChannelHeaders = {}
for _ in range(globalHeader["NumSlowChannels"]):
slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None)
slowChannelHeaders[slowChannelHeader["Channel"]] = slowChannelHeader
## Step 2 : a first loop for counting size
# signal
nb_samples = np.zeros(len(slowChannelHeaders))
sample_positions = np.zeros(len(slowChannelHeaders))
t_starts = np.zeros(len(slowChannelHeaders), dtype="f")
# spiketimes and waveform
nb_spikes = np.zeros((maxchan + 1, maxunit + 1), dtype="i")
wf_sizes = np.zeros((maxchan + 1, maxunit + 1, 2), dtype="i")
# eventarrays
nb_events = {}
# maxstrsizeperchannel = { }
for chan, h in iteritems(eventHeaders):
nb_events[chan] = 0
# maxstrsizeperchannel[chan] = 0
start = fid.tell()
while fid.tell() != -1:
# read block header
dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None)
if dataBlockHeader is None:
break
chan = dataBlockHeader["Channel"]
unit = dataBlockHeader["Unit"]
n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"]
time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"]
if dataBlockHeader["Type"] == 1:
nb_spikes[chan, unit] += 1
wf_sizes[chan, unit, :] = [n1, n2]
fid.seek(n1 * n2 * 2, 1)
elif dataBlockHeader["Type"] == 4:
# event
nb_events[chan] += 1
elif dataBlockHeader["Type"] == 5:
# continuous signal
fid.seek(n2 * 2, 1)
if n2 > 0:
nb_samples[chan] += n2
if nb_samples[chan] == 0:
t_starts[chan] = time
## Step 3: allocating memory and 2 loop for reading if not lazy
if not lazy:
# allocating mem for signal
sigarrays = {}
for chan, h in iteritems(slowChannelHeaders):
sigarrays[chan] = np.zeros(nb_samples[chan])
# allocating mem for SpikeTrain
stimearrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object)
swfarrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object)
for (chan, unit), _ in np.ndenumerate(nb_spikes):
stimearrays[chan, unit] = np.zeros(nb_spikes[chan, unit], dtype="f")
if load_spike_waveform:
n1, n2 = wf_sizes[chan, unit, :]
swfarrays[chan, unit] = np.zeros((nb_spikes[chan, unit], n1, n2), dtype="f4")
pos_spikes = np.zeros(nb_spikes.shape, dtype="i")
# allocating mem for event
eventpositions = {}
evarrays = {}
for chan, nb in iteritems(nb_events):
evarrays[chan] = np.zeros(nb, dtype="f")
eventpositions[chan] = 0
fid.seek(start)
while fid.tell() != -1:
dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None)
if dataBlockHeader is None:
break
chan = dataBlockHeader["Channel"]
n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"]
time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"]
time /= globalHeader["ADFrequency"]
if n2 < 0:
break
if dataBlockHeader["Type"] == 1:
# spike
unit = dataBlockHeader["Unit"]
pos = pos_spikes[chan, unit]
stimearrays[chan, unit][pos] = time
if load_spike_waveform and n1 * n2 != 0:
swfarrays[chan, unit][pos, :, :] = (
np.fromstring(fid.read(n1 * n2 * 2), dtype="i2").reshape(n1, n2).astype("f4")
)
else:
fid.seek(n1 * n2 * 2, 1)
pos_spikes[chan, unit] += 1
elif dataBlockHeader["Type"] == 4:
# event
pos = eventpositions[chan]
evarrays[chan][pos] = time
eventpositions[chan] += 1
elif dataBlockHeader["Type"] == 5:
# signal
data = np.fromstring(fid.read(n2 * 2), dtype="i2").astype("f4")
sigarrays[chan][sample_positions[chan] : sample_positions[chan] + data.size] = data
sample_positions[chan] += data.size
## Step 3: create neo object
for chan, h in iteritems(eventHeaders):
if lazy:
times = []
else:
times = evarrays[chan]
ea = EventArray(times * pq.s, channel_name=eventHeaders[chan]["Name"], channel_index=chan)
if lazy:
ea.lazy_shape = nb_events[chan]
seg.eventarrays.append(ea)
for chan, h in iteritems(slowChannelHeaders):
if lazy:
signal = []
else:
if globalHeader["Version"] == 100 or globalHeader["Version"] == 101:
gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * 1000.0)
elif globalHeader["Version"] == 102:
gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"])
elif globalHeader["Version"] >= 103:
gain = globalHeader["SlowMaxMagnitudeMV"] / (
0.5
* (2 ** globalHeader["BitsPerSpikeSample"])
* slowChannelHeaders[chan]["Gain"]
* slowChannelHeaders[chan]["PreampGain"]
)
signal = sigarrays[chan] * gain
anasig = AnalogSignal(
signal * pq.V,
sampling_rate=float(slowChannelHeaders[chan]["ADFreq"]) * pq.Hz,
t_start=t_starts[chan] * pq.s,
channel_index=slowChannelHeaders[chan]["Channel"],
channel_name=slowChannelHeaders[chan]["Name"],
)
if lazy:
anasig.lazy_shape = nb_samples[chan]
seg.analogsignals.append(anasig)
for (chan, unit), value in np.ndenumerate(nb_spikes):
if nb_spikes[chan, unit] == 0:
continue
if lazy:
times = []
waveforms = None
t_stop = 0
else:
times = stimearrays[chan, unit]
t_stop = times.max()
if load_spike_waveform:
if globalHeader["Version"] < 103:
gain = 3000.0 / (2048 * dspChannelHeaders[chan]["Gain"] * 1000.0)
elif globalHeader["Version"] >= 103 and globalHeader["Version"] < 105:
gain = globalHeader["SpikeMaxMagnitudeMV"] / (
0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * 1000.0
)
elif globalHeader["Version"] > 105:
gain = globalHeader["SpikeMaxMagnitudeMV"] / (
0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * globalHeader["SpikePreAmpGain"]
)
waveforms = swfarrays[chan, unit] * gain * pq.V
else:
waveforms = None
sptr = SpikeTrain(times, units="s", t_stop=t_stop * pq.s, waveforms=waveforms)
sptr.annotate(unit_name=dspChannelHeaders[chan]["Name"])
sptr.annotate(channel_index=chan)
if lazy:
sptr.lazy_shape = nb_spikes[chan, unit]
seg.spiketrains.append(sptr)
seg.create_many_to_one_relationship()
return seg
