def read_block(self, **kargs):
if self.filename is not None:
self.axo_obj = axographio.read(self.filename)
# Build up the block
blk = Block()
blk.rec_datetime = None
if self.filename is not None:
# modified time is not ideal but less prone to
# cross-platform issues than created time (ctime)
blk.file_datetime = datetime.fromtimestamp(os.path.getmtime(self.filename))
# store the filename if it is available
blk.file_origin = self.filename
# determine the channel names and counts
_, channel_ordering = np.unique(self.axo_obj.names[1:], return_index=True)
channel_names = np.array(self.axo_obj.names[1:])[np.sort(channel_ordering)]
channel_count = len(channel_names)
# determine the time signal and sample period
sample_period = self.axo_obj.data[0].step * pq.s
start_time = self.axo_obj.data[0].start * pq.s
# Attempt to read units from the channel names
channel_unit_names = [x.split()[-1].strip('()') for x in channel_names]
channel_units = []
for unit in channel_unit_names:
try:
channel_units.append(pq.Quantity(1, unit))
except LookupError:
channel_units.append(None)
# Strip units from channel names
channel_names = [' '.join(x.split()[:-1]) for x in channel_names]
# build up segments by grouping axograph columns
for seg_idx in range(1, len(self.axo_obj.data), channel_count):
seg = Segment(index=seg_idx)
# add in the channels
for chan_idx in range(0, channel_count):
signal = pq.Quantity(
self.axo_obj.data[seg_idx + chan_idx], channel_units[chan_idx])
analog = AnalogSignal(signal,
sampling_period=sample_period, t_start=start_time,
name=channel_names[chan_idx], channel_index=chan_idx)
seg.analogsignals.append(analog)
blk.segments.append(seg)
blk.create_many_to_one_relationship()
return blk
def read_block(self, lazy=False, cascade=True):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
seg = self.read_segment(blockname, lazy, cascade)
bl.segments.append(seg)
bl.create_many_to_one_relationship()
return bl
def read_block(self, lazy=False, cascade=True, sortname=''):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade: return bl
for blockname in os.listdir(self.dirname):
if self.is_tdtblock(blockname): # if the folder is a tdt block
seg = self.read_segment(blockname, lazy, cascade, sortname)
bl.segments.append(seg)
bl.create_many_to_one_relationship()
return bl
def read_block(self, lazy=False, cascade=True):
header = self.read_header()
version = header['fFileVersionNumber']
bl = Block()
bl.file_origin = os.path.basename(self.filename)
bl.annotate(abf_version=str(version))
# date and time
if version < 2.:
YY = 1900
MM = 1
DD = 1
hh = int(header['lFileStartTime'] / 3600.)
mm = int((header['lFileStartTime'] - hh * 3600) / 60)
ss = header['lFileStartTime'] - hh * 3600 - mm * 60
ms = int(np.mod(ss, 1) * 1e6)
ss = int(ss)
elif version >= 2.:
YY = int(header['uFileStartDate'] / 10000)
MM = int((header['uFileStartDate'] - YY * 10000) / 100)
DD = int(header['uFileStartDate'] - YY * 10000 - MM * 100)
hh = int(header['uFileStartTimeMS'] / 1000. / 3600.)
mm = int((header['uFileStartTimeMS'] / 1000. - hh * 3600) / 60)
ss = header['uFileStartTimeMS'] / 1000. - hh * 3600 - mm * 60
ms = int(np.mod(ss, 1) * 1e6)
ss = int(ss)
bl.rec_datetime = datetime.datetime(YY, MM, DD, hh, mm, ss, ms)
if not cascade:
return bl
# file format
if header['nDataFormat'] == 0:
dt = np.dtype('i2')
elif header['nDataFormat'] == 1:
dt = np.dtype('f4')
if version < 2.:
nbchannel = header['nADCNumChannels']
head_offset = header['lDataSectionPtr'] * BLOCKSIZE + header[
'nNumPointsIgnored'] * dt.itemsize
totalsize = header['lActualAcqLength']
elif version >= 2.:
nbchannel = header['sections']['ADCSection']['llNumEntries']
head_offset = header['sections']['DataSection'][
'uBlockIndex'] * BLOCKSIZE
totalsize = header['sections']['DataSection']['llNumEntries']
data = np.memmap(self.filename, dt, 'r',
shape=(totalsize,), offset=head_offset)
# 3 possible modes
if version < 2.:
mode = header['nOperationMode']
elif version >= 2.:
mode = header['protocol']['nOperationMode']
if (mode == 1) or (mode == 2) or (mode == 5) or (mode == 3):
# event-driven variable-length mode (mode 1)
# event-driven fixed-length mode (mode 2 or 5)
# gap free mode (mode 3) can be in several episodes
# read sweep pos
if version < 2.:
nbepisod = header['lSynchArraySize']
offset_episode = header['lSynchArrayPtr'] * BLOCKSIZE
elif version >= 2.:
nbepisod = header['sections']['SynchArraySection'][
'llNumEntries']
offset_episode = header['sections']['SynchArraySection'][
'uBlockIndex'] * BLOCKSIZE
if nbepisod > 0:
episode_array = np.memmap(
self.filename, [('offset', 'i4'), ('len', 'i4')], 'r',
shape=nbepisod, offset=offset_episode)
else:
episode_array = np.empty(1, [('offset', 'i4'), ('len', 'i4')])
episode_array[0]['len'] = data.size
episode_array[0]['offset'] = 0
# sampling_rate
if version < 2.:
sampling_rate = 1. / (header['fADCSampleInterval'] *
nbchannel * 1.e-6) * pq.Hz
elif version >= 2.:
sampling_rate = 1.e6 / \
header['protocol']['fADCSequenceInterval'] * pq.Hz
# construct block
# one sweep = one segment in a block
pos = 0
for j in range(episode_array.size):
seg = Segment(index=j)
length = episode_array[j]['len']
if version < 2.:
fSynchTimeUnit = header['fSynchTimeUnit']
elif version >= 2.:
fSynchTimeUnit = header['protocol']['fSynchTimeUnit']
if (fSynchTimeUnit != 0) and (mode == 1):
length /= fSynchTimeUnit
if not lazy:
subdata = data[pos:pos+length]
subdata = subdata.reshape((int(subdata.size/nbchannel),
nbchannel)).astype('f')
if dt == np.dtype('i2'):
if version < 2.:
reformat_integer_v1(subdata, nbchannel, header)
elif version >= 2.:
reformat_integer_v2(subdata, nbchannel, header)
pos += length
if version < 2.:
chans = [chan_num for chan_num in
header['nADCSamplingSeq'] if chan_num >= 0]
else:
chans = range(nbchannel)
for n, i in enumerate(chans[:nbchannel]): # fix SamplingSeq
if version < 2.:
name = header['sADCChannelName'][i].replace(b' ', b'')
unit = header['sADCUnits'][i].replace(b'\xb5', b'u').\
replace(b' ', b'').decode('utf-8') # \xb5 is µ
num = header['nADCPtoLChannelMap'][i]
elif version >= 2.:
lADCIi = header['listADCInfo'][i]
name = lADCIi['ADCChNames'].replace(b' ', b'')
unit = lADCIi['ADCChUnits'].replace(b'\xb5', b'u').\
replace(b' ', b'').decode('utf-8')
num = header['listADCInfo'][i]['nADCNum']
if (fSynchTimeUnit == 0):
t_start = float(episode_array[j]['offset']) / sampling_rate
else:
t_start = float(episode_array[j]['offset']) * fSynchTimeUnit *1e-6* pq.s
t_start = t_start.rescale('s')
try:
pq.Quantity(1, unit)
except:
unit = ''
if lazy:
signal = [] * pq.Quantity(1, unit)
else:
signal = pq.Quantity(subdata[:, n], unit)
anaSig = AnalogSignal(signal, sampling_rate=sampling_rate,
t_start=t_start,
name=str(name),
channel_index=int(num))
if lazy:
anaSig.lazy_shape = length / nbchannel
seg.analogsignals.append(anaSig)
bl.segments.append(seg)
if mode in [3, 5]: # TODO check if tags exits in other mode
# tag is EventArray that should be attached to Block
# It is attched to the first Segment
times = []
labels = []
comments = []
for i, tag in enumerate(header['listTag']):
times.append(tag['lTagTime']/sampling_rate)
labels.append(str(tag['nTagType']))
comments.append(clean_string(tag['sComment']))
times = np.array(times)
labels = np.array(labels, dtype='S')
comments = np.array(comments, dtype='S')
# attach all tags to the first segment.
seg = bl.segments[0]
if lazy:
ea = Event(times=[] * pq.s, labels=np.array([], dtype='S'))
ea.lazy_shape = len(times)
else:
ea = Event(times=times * pq.s, labels=labels,
comments=comments)
seg.events.append(ea)
bl.create_many_to_one_relationship()
return bl
def read_block(self, lazy=False, cascade=True):
header = self.read_header()
version = header['fFileVersionNumber']
bl = Block()
bl.file_origin = os.path.basename(self.filename)
bl.annotate(abf_version=str(version))
# date and time
if version < 2.:
YY = 1900
MM = 1
DD = 1
hh = int(header['lFileStartTime'] / 3600.)
mm = int((header['lFileStartTime'] - hh * 3600) / 60)
ss = header['lFileStartTime'] - hh * 3600 - mm * 60
ms = int(np.mod(ss, 1) * 1e6)
ss = int(ss)
elif version >= 2.:
YY = int(header['uFileStartDate'] / 10000)
MM = int((header['uFileStartDate'] - YY * 10000) / 100)
DD = int(header['uFileStartDate'] - YY * 10000 - MM * 100)
hh = int(header['uFileStartTimeMS'] / 1000. / 3600.)
mm = int((header['uFileStartTimeMS'] / 1000. - hh * 3600) / 60)
ss = header['uFileStartTimeMS'] / 1000. - hh * 3600 - mm * 60
ms = int(np.mod(ss, 1) * 1e6)
ss = int(ss)
bl.rec_datetime = datetime.datetime(YY, MM, DD, hh, mm, ss, ms)
if not cascade:
return bl
# file format
if header['nDataFormat'] == 0:
dt = np.dtype('i2')
elif header['nDataFormat'] == 1:
dt = np.dtype('f4')
if version < 2.:
nbchannel = header['nADCNumChannels']
head_offset = header['lDataSectionPtr'] * BLOCKSIZE + header[
'nNumPointsIgnored'] * dt.itemsize
totalsize = header['lActualAcqLength']
elif version >= 2.:
nbchannel = header['sections']['ADCSection']['llNumEntries']
head_offset = header['sections']['DataSection'][
'uBlockIndex'] * BLOCKSIZE
totalsize = header['sections']['DataSection']['llNumEntries']
data = np.memmap(self.filename,
dt,
'r',
shape=(totalsize, ),
offset=head_offset)
# 3 possible modes
if version < 2.:
mode = header['nOperationMode']
elif version >= 2.:
mode = header['protocol']['nOperationMode']
if (mode == 1) or (mode == 2) or (mode == 5) or (mode == 3):
# event-driven variable-length mode (mode 1)
# event-driven fixed-length mode (mode 2 or 5)
# gap free mode (mode 3) can be in several episodes
# read sweep pos
if version < 2.:
nbepisod = header['lSynchArraySize']
offset_episode = header['lSynchArrayPtr'] * BLOCKSIZE
elif version >= 2.:
nbepisod = header['sections']['SynchArraySection'][
'llNumEntries']
offset_episode = header['sections']['SynchArraySection'][
'uBlockIndex'] * BLOCKSIZE
if nbepisod > 0:
episode_array = np.memmap(self.filename, [('offset', 'i4'),
('len', 'i4')],
'r',
shape=nbepisod,
offset=offset_episode)
else:
episode_array = np.empty(1, [('offset', 'i4'), ('len', 'i4')])
episode_array[0]['len'] = data.size
episode_array[0]['offset'] = 0
# sampling_rate
if version < 2.:
sampling_rate = 1. / (header['fADCSampleInterval'] *
nbchannel * 1.e-6) * pq.Hz
elif version >= 2.:
sampling_rate = 1.e6 / \
header['protocol']['fADCSequenceInterval'] * pq.Hz
# construct block
# one sweep = one segment in a block
pos = 0
for j in range(episode_array.size):
seg = Segment(index=j)
length = episode_array[j]['len']
if version < 2.:
fSynchTimeUnit = header['fSynchTimeUnit']
elif version >= 2.:
fSynchTimeUnit = header['protocol']['fSynchTimeUnit']
if (fSynchTimeUnit != 0) and (mode == 1):
length /= fSynchTimeUnit
if not lazy:
subdata = data[pos:pos + length]
subdata = subdata.reshape(
(int(subdata.size / nbchannel), nbchannel)).astype('f')
if dt == np.dtype('i2'):
if version < 2.:
reformat_integer_v1(subdata, nbchannel, header)
elif version >= 2.:
reformat_integer_v2(subdata, nbchannel, header)
pos += length
if version < 2.:
chans = [
chan_num for chan_num in header['nADCSamplingSeq']
if chan_num >= 0
]
else:
chans = range(nbchannel)
for n, i in enumerate(chans[:nbchannel]): # fix SamplingSeq
if version < 2.:
name = header['sADCChannelName'][i].replace(b' ', b'')
unit = header['sADCUnits'][i].replace(b'\xb5', b'u').\
replace(b' ', b'').decode('utf-8') # \xb5 is µ
num = header['nADCPtoLChannelMap'][i]
elif version >= 2.:
lADCIi = header['listADCInfo'][i]
name = lADCIi['ADCChNames'].replace(b' ', b'')
unit = lADCIi['ADCChUnits'].replace(b'\xb5', b'u').\
replace(b' ', b'').decode('utf-8')
num = header['listADCInfo'][i]['nADCNum']
if (fSynchTimeUnit == 0):
t_start = float(
episode_array[j]['offset']) / sampling_rate
else:
t_start = float(episode_array[j]['offset']
) * fSynchTimeUnit * 1e-6 * pq.s
t_start = t_start.rescale('s')
try:
pq.Quantity(1, unit)
except:
unit = ''
if lazy:
signal = [] * pq.Quantity(1, unit)
else:
signal = pq.Quantity(subdata[:, n], unit)
anaSig = AnalogSignal(signal,
sampling_rate=sampling_rate,
t_start=t_start,
name=str(name),
channel_index=int(num))
if lazy:
anaSig.lazy_shape = length / nbchannel
seg.analogsignals.append(anaSig)
bl.segments.append(seg)
if mode in [3, 5]: # TODO check if tags exits in other mode
# tag is EventArray that should be attached to Block
# It is attched to the first Segment
times = []
labels = []
comments = []
for i, tag in enumerate(header['listTag']):
times.append(tag['lTagTime'] / sampling_rate)
labels.append(str(tag['nTagType']))
comments.append(clean_string(tag['sComment']))
times = np.array(times)
labels = np.array(labels, dtype='S')
comments = np.array(comments, dtype='S')
# attach all tags to the first segment.
seg = bl.segments[0]
if lazy:
ea = Event(times=[] * pq.s, labels=np.array([], dtype='S'))
ea.lazy_shape = len(times)
else:
ea = Event(times=times * pq.s,
labels=labels,
comments=comments)
seg.events.append(ea)
bl.create_many_to_one_relationship()
return bl
def read_block(
self,
# the 2 first keyword arguments are imposed by neo.io API
lazy=False,
cascade=True):
"""
Return a Block.
"""
def count_samples(m_length):
"""
Count the number of signal samples available in a type 5 data block
of length m_length
"""
# for information about type 5 data block, see [1]
count = int((m_length - 6) / 2 - 2)
# -6 corresponds to the header of block 5, and the -2 take into
# account the fact that last 2 values are not available as the 4
# corresponding bytes are coding the time stamp of the beginning
# of the block
return count
# create the neo Block that will be returned at the end
blck = Block(file_origin=os.path.basename(self.filename))
blck.file_origin = os.path.basename(self.filename)
fid = open(self.filename, 'rb')
# NOTE: in the following, the word "block" is used in the sense used in
# the alpha-omega specifications (ie a data chunk in the file), rather
# than in the sense of the usual Block object in neo
# step 1: read the headers of all the data blocks to load the file
# structure
pos_block = 0 # position of the current block in the file
file_blocks = [] # list of data blocks available in the file
if not cascade:
# we read only the main header
m_length, m_TypeBlock = struct.unpack('Hcx', fid.read(4))
# m_TypeBlock should be 'h', as we read the first block
block = HeaderReader(
fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f()
block.update({
'm_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block
})
file_blocks.append(block)
else: # cascade == True
seg = Segment(file_origin=os.path.basename(self.filename))
seg.file_origin = os.path.basename(self.filename)
blck.segments.append(seg)
while True:
first_4_bytes = fid.read(4)
if len(first_4_bytes) < 4:
# we have reached the end of the file
break
else:
m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes)
block = HeaderReader(
fid, dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({
'm_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block
})
if m_TypeBlock == '2':
# The beggining of the block of type '2' is identical for
# all types of channels, but the following part depends on
# the type of channel. So we need a special case here.
# WARNING: How to check the type of channel is not
# described in the documentation. So here I use what is
# proposed in the C code [2].
# According to this C code, it seems that the 'm_isAnalog'
# is used to distinguished analog and digital channels, and
# 'm_Mode' encodes the type of analog channel:
# 0 for continuous, 1 for level, 2 for external trigger.
# But in some files, I found channels that seemed to be
# continuous channels with 'm_Modes' = 128 or 192. So I
# decided to consider every channel with 'm_Modes'
# different from 1 or 2 as continuous. I also couldn't
# check that values of 1 and 2 are really for level and
# external trigger as I had no test files containing data
# of this types.
type_subblock = 'unknown_channel_type(m_Mode=' \
+ str(block['m_Mode'])+ ')'
description = Type2_SubBlockUnknownChannels
block.update({'m_Name': 'unknown_name'})
if block['m_isAnalog'] == 0:
# digital channel
type_subblock = 'digital'
description = Type2_SubBlockDigitalChannels
elif block['m_isAnalog'] == 1:
# analog channel
if block['m_Mode'] == 1:
# level channel
type_subblock = 'level'
description = Type2_SubBlockLevelChannels
elif block['m_Mode'] == 2:
# external trigger channel
type_subblock = 'external_trigger'
description = Type2_SubBlockExtTriggerChannels
else:
# continuous channel
type_subblock = 'continuous(Mode' \
+ str(block['m_Mode']) +')'
description = Type2_SubBlockContinuousChannels
subblock = HeaderReader(fid, description).read_f()
block.update(subblock)
block.update({'type_subblock': type_subblock})
file_blocks.append(block)
pos_block += m_length
fid.seek(pos_block)
# step 2: find the available channels
list_chan = [] # list containing indexes of channel blocks
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '2':
list_chan.append(ind_block)
# step 3: find blocks containing data for the available channels
list_data = [] # list of lists of indexes of data blocks
# corresponding to each channel
for ind_chan, chan in enumerate(list_chan):
list_data.append([])
num_chan = file_blocks[chan]['m_numChannel']
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '5':
if block['m_numChannel'] == num_chan:
list_data[ind_chan].append(ind_block)
# step 4: compute the length (number of samples) of the channels
chan_len = np.zeros(len(list_data), dtype=np.int)
for ind_chan, list_blocks in enumerate(list_data):
for ind_block in list_blocks:
chan_len[ind_chan] += count_samples(
file_blocks[ind_block]['m_length'])
# step 5: find channels for which data are available
ind_valid_chan = np.nonzero(chan_len)[0]
# step 6: load the data
# TODO give the possibility to load data as AnalogSignalArrays
for ind_chan in ind_valid_chan:
list_blocks = list_data[ind_chan]
ind = 0 # index in the data vector
# read time stamp for the beginning of the signal
form = '<l' # reading format
ind_block = list_blocks[0]
count = count_samples(file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos'] + 6 + count * 2)
buf = fid.read(struct.calcsize(form))
val = struct.unpack(form, buf)
start_index = val[0]
# WARNING: in the following blocks are read supposing taht they
# are all contiguous and sorted in time. I don't know if it's
# always the case. Maybe we should use the time stamp of each
# data block to choose where to put the read data in the array.
if not lazy:
temp_array = np.empty(chan_len[ind_chan], dtype=np.int16)
# NOTE: we could directly create an empty AnalogSignal and
# load the data in it, but it is much faster to load data
# in a temporary numpy array and create the AnalogSignals
# from this temporary array
for ind_block in list_blocks:
count = count_samples(
file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos'] + 6)
temp_array[ind:ind+count] = \
np.fromfile(fid, dtype = np.int16, count = count)
ind += count
sampling_rate = \
file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz
t_start = (start_index / sampling_rate).simplified
if lazy:
ana_sig = AnalogSignal([],
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.lazy_shape = chan_len[ind_chan]
else:
ana_sig = AnalogSignal(temp_array,
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.channel_index = \
file_blocks[list_chan[ind_chan]]['m_numChannel']
ana_sig.annotate(channel_name = \
file_blocks[list_chan[ind_chan]]['m_Name'])
ana_sig.annotate(channel_type = \
file_blocks[list_chan[ind_chan]]['type_subblock'])
seg.analogsignals.append(ana_sig)
fid.close()
if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true
blck.rec_datetime = datetime.datetime(\
file_blocks[0]['m_date_year'],
file_blocks[0]['m_date_month'],
file_blocks[0]['m_date_day'],
file_blocks[0]['m_time_hour'],
file_blocks[0]['m_time_minute'],
file_blocks[0]['m_time_second'],
10000 * file_blocks[0]['m_time_hsecond'])
# the 10000 is here to convert m_time_hsecond from centisecond
# to microsecond
version = file_blocks[0]['m_version']
blck.annotate(alphamap_version=version)
if cascade:
seg.rec_datetime = blck.rec_datetime.replace()
# I couldn't find a simple copy function for datetime,
# using replace without arguments is a twisted way to make a
# copy
seg.annotate(alphamap_version=version)
if cascade:
populate_RecordingChannel(blck, remove_from_annotation=True)
blck.create_many_to_one_relationship()
return blck
def read_block(self,
# the 2 first keyword arguments are imposed by neo.io API
lazy = False,
cascade = True):
"""
Return a Block.
"""
def count_samples(m_length):
"""
Count the number of signal samples available in a type 5 data block
of length m_length
"""
# for information about type 5 data block, see [1]
count = int((m_length-6)/2-2)
# -6 corresponds to the header of block 5, and the -2 take into
# account the fact that last 2 values are not available as the 4
# corresponding bytes are coding the time stamp of the beginning
# of the block
return count
# create the neo Block that will be returned at the end
blck = Block(file_origin = os.path.basename(self.filename))
blck.file_origin = os.path.basename(self.filename)
fid = open(self.filename, 'rb')
# NOTE: in the following, the word "block" is used in the sense used in
# the alpha-omega specifications (ie a data chunk in the file), rather
# than in the sense of the usual Block object in neo
# step 1: read the headers of all the data blocks to load the file
# structure
pos_block = 0 # position of the current block in the file
file_blocks = [] # list of data blocks available in the file
if not cascade:
# we read only the main header
m_length, m_TypeBlock = struct.unpack('Hcx' , fid.read(4))
# m_TypeBlock should be 'h', as we read the first block
block = HeaderReader(fid,
dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({'m_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block})
file_blocks.append(block)
else: # cascade == True
seg = Segment(file_origin = os.path.basename(self.filename))
seg.file_origin = os.path.basename(self.filename)
blck.segments.append(seg)
while True:
first_4_bytes = fid.read(4)
if len(first_4_bytes) < 4:
# we have reached the end of the file
break
else:
m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes)
block = HeaderReader(fid,
dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({'m_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block})
if m_TypeBlock == '2':
# The beginning of the block of type '2' is identical for
# all types of channels, but the following part depends on
# the type of channel. So we need a special case here.
# WARNING: How to check the type of channel is not
# described in the documentation. So here I use what is
# proposed in the C code [2].
# According to this C code, it seems that the 'm_isAnalog'
# is used to distinguished analog and digital channels, and
# 'm_Mode' encodes the type of analog channel:
# 0 for continuous, 1 for level, 2 for external trigger.
# But in some files, I found channels that seemed to be
# continuous channels with 'm_Modes' = 128 or 192. So I
# decided to consider every channel with 'm_Modes'
# different from 1 or 2 as continuous. I also couldn't
# check that values of 1 and 2 are really for level and
# external trigger as I had no test files containing data
# of this types.
type_subblock = 'unknown_channel_type(m_Mode=' \
+ str(block['m_Mode'])+ ')'
description = Type2_SubBlockUnknownChannels
block.update({'m_Name': 'unknown_name'})
if block['m_isAnalog'] == 0:
# digital channel
type_subblock = 'digital'
description = Type2_SubBlockDigitalChannels
elif block['m_isAnalog'] == 1:
# analog channel
if block['m_Mode'] == 1:
# level channel
type_subblock = 'level'
description = Type2_SubBlockLevelChannels
elif block['m_Mode'] == 2:
# external trigger channel
type_subblock = 'external_trigger'
description = Type2_SubBlockExtTriggerChannels
else:
# continuous channel
type_subblock = 'continuous(Mode' \
+ str(block['m_Mode']) +')'
description = Type2_SubBlockContinuousChannels
subblock = HeaderReader(fid, description).read_f()
block.update(subblock)
block.update({'type_subblock': type_subblock})
file_blocks.append(block)
pos_block += m_length
fid.seek(pos_block)
# step 2: find the available channels
list_chan = [] # list containing indexes of channel blocks
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '2':
list_chan.append(ind_block)
# step 3: find blocks containing data for the available channels
list_data = [] # list of lists of indexes of data blocks
# corresponding to each channel
for ind_chan, chan in enumerate(list_chan):
list_data.append([])
num_chan = file_blocks[chan]['m_numChannel']
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '5':
if block['m_numChannel'] == num_chan:
list_data[ind_chan].append(ind_block)
# step 4: compute the length (number of samples) of the channels
chan_len = np.zeros(len(list_data), dtype = np.int)
for ind_chan, list_blocks in enumerate(list_data):
for ind_block in list_blocks:
chan_len[ind_chan] += count_samples(
file_blocks[ind_block]['m_length'])
# step 5: find channels for which data are available
ind_valid_chan = np.nonzero(chan_len)[0]
# step 6: load the data
# TODO give the possibility to load data as AnalogSignalArrays
for ind_chan in ind_valid_chan:
list_blocks = list_data[ind_chan]
ind = 0 # index in the data vector
# read time stamp for the beginning of the signal
form = '<l' # reading format
ind_block = list_blocks[0]
count = count_samples(file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos']+6+count*2)
buf = fid.read(struct.calcsize(form))
val = struct.unpack(form , buf)
start_index = val[0]
# WARNING: in the following blocks are read supposing taht they
# are all contiguous and sorted in time. I don't know if it's
# always the case. Maybe we should use the time stamp of each
# data block to choose where to put the read data in the array.
if not lazy:
temp_array = np.empty(chan_len[ind_chan], dtype = np.int16)
# NOTE: we could directly create an empty AnalogSignal and
# load the data in it, but it is much faster to load data
# in a temporary numpy array and create the AnalogSignals
# from this temporary array
for ind_block in list_blocks:
count = count_samples(
file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos']+6)
temp_array[ind:ind+count] = \
np.fromfile(fid, dtype = np.int16, count = count)
ind += count
sampling_rate = \
file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz
t_start = (start_index / sampling_rate).simplified
if lazy:
ana_sig = AnalogSignal([],
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.lazy_shape = chan_len[ind_chan]
else:
ana_sig = AnalogSignal(temp_array,
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
# todo apibreak: create ChannelIndex for each signals
# ana_sig.channel_index = \
# file_blocks[list_chan[ind_chan]]['m_numChannel']
ana_sig.annotate(channel_name = \
file_blocks[list_chan[ind_chan]]['m_Name'])
ana_sig.annotate(channel_type = \
file_blocks[list_chan[ind_chan]]['type_subblock'])
seg.analogsignals.append(ana_sig)
fid.close()
if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true
blck.rec_datetime = datetime.datetime(\
file_blocks[0]['m_date_year'],
file_blocks[0]['m_date_month'],
file_blocks[0]['m_date_day'],
file_blocks[0]['m_time_hour'],
file_blocks[0]['m_time_minute'],
file_blocks[0]['m_time_second'],
10000 * file_blocks[0]['m_time_hsecond'])
# the 10000 is here to convert m_time_hsecond from centisecond
# to microsecond
version = file_blocks[0]['m_version']
blck.annotate(alphamap_version = version)
if cascade:
seg.rec_datetime = blck.rec_datetime.replace()
# I couldn't find a simple copy function for datetime,
# using replace without arguments is a twisted way to make a
# copy
seg.annotate(alphamap_version = version)
if cascade:
blck.create_many_to_one_relationship()
return blck
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
#TSQ is the global index
tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq')
dt = [('size','int32'),
('evtype','int32'),
('code','S4'),
('channel','uint16'),
('sortcode','uint16'),
('timestamp','float64'),
('eventoffset','int64'),
('dataformat','int32'),
('frequency','float32'),
]
tsq = np.fromfile(tsq_filename, dtype = dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0]
#TEV is the old data file
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype = 'uint8')
else:
tev_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype']==type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code']==code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel']==channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [ ]*pq.s
labels = np.array([ ], dtype = str)
else:
times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view('float64').astype('S')
ea = EventArray(times = times, name = code , channel_index = int(channel), labels = labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.eventarrays.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode']==sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0]-10
if lazy:
times = [ ]*pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt)
waveforms = waveforms.reshape(nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike>0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 *pq.s
t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s
else:
t_start = 0 *pq.s
t_stop = 0 *pq.s
st = SpikeTrain(times = times,
name = 'Chan{} Code{}'.format(channel,sortcode),
t_start = t_start,
t_stop = t_stop,
waveforms = waveforms,
left_sweep = waveformsize/2./sr * pq.s,
sampling_rate = sr * pq.Hz,
)
st.annotate(channel_index = channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size']-10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = [ ]
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev')
if os.path.exists(sev_filename):
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename, dtype = 'uint8')
else:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt)
anasig = AnalogSignal(signal = signal* pq.V,
name = '{} {}'.format(code, channel),
sampling_rate= sr * pq.Hz,
t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s,
channel_index = int(channel))
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
bl.create_many_to_one_relationship()
return bl
def read_block(self, **kargs):
if self.filename is not None:
self.axo_obj = axographio.read(self.filename)
# Build up the block
blk = Block()
blk.rec_datetime = None
if self.filename is not None:
# modified time is not ideal but less prone to
# cross-platform issues than created time (ctime)
blk.file_datetime = datetime.fromtimestamp(
os.path.getmtime(self.filename))
# store the filename if it is available
blk.file_origin = self.filename
# determine the channel names and counts
_, channel_ordering = np.unique(self.axo_obj.names[1:],
return_index=True)
channel_names = np.array(
self.axo_obj.names[1:])[np.sort(channel_ordering)]
channel_count = len(channel_names)
# determine the time signal and sample period
sample_period = self.axo_obj.data[0].step * pq.s
start_time = self.axo_obj.data[0].start * pq.s
# Attempt to read units from the channel names
channel_unit_names = [x.split()[-1].strip('()') for x in channel_names]
channel_units = []
for unit in channel_unit_names:
try:
channel_units.append(pq.Quantity(1, unit))
except LookupError:
channel_units.append(None)
# Strip units from channel names
channel_names = [' '.join(x.split()[:-1]) for x in channel_names]
# build up segments by grouping axograph columns
for seg_idx in range(1, len(self.axo_obj.data), channel_count):
seg = Segment(index=seg_idx)
# add in the channels
for chan_idx in range(0, channel_count):
signal = pq.Quantity(self.axo_obj.data[seg_idx + chan_idx],
channel_units[chan_idx])
analog = AnalogSignal(signal,
sampling_period=sample_period,
t_start=start_time,
name=channel_names[chan_idx],
channel_index=chan_idx)
seg.analogsignals.append(analog)
blk.segments.append(seg)
blk.create_many_to_one_relationship()
return blk
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
global_t_start = None
# Step 1 : first loop for counting - tsq file
tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb')
hr = HeaderReader(tsq, TsqDescription)
allsig = { }
allspiketr = { }
allevent = { }
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_UNKNOWN':
pass
elif Types[evtype] == 'EVTYPE_MARK' :
if global_t_start is None:
global_t_start = h['timestamp']
elif Types[evtype] == 'EVTYPE_SCALER' :
# TODO
pass
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
# EVENTS
if code not in allevent:
allevent[code] = { }
if channel not in allevent[code]:
ea = EventArray(name = code , channel_index = channel)
# for counting:
ea.lazy_shape = 0
ea.maxlabelsize = 0
allevent[code][channel] = ea
allevent[code][channel].lazy_shape += 1
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
strobe = str(strobe)
if len(strobe)>= allevent[code][channel].maxlabelsize:
allevent[code][channel].maxlabelsize = len(strobe)
#~ ev = Event()
#~ ev.time = h['timestamp'] - global_t_start
#~ ev.name = code
#~ # it the strobe attribute masked with eventoffset
#~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
#~ ev.label = str(strobe)
#~ seg._events.append( ev )
elif Types[evtype] == 'EVTYPE_SNIP' :
if code not in allspiketr:
allspiketr[code] = { }
if channel not in allspiketr[code]:
allspiketr[code][channel] = { }
if h['sortcode'] not in allspiketr[code][channel]:
sptr = SpikeTrain([ ], units = 's',
name = str(h['sortcode']),
#t_start = global_t_start,
t_start = 0.*pq.s,
t_stop = 0.*pq.s, # temporary
left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s,
sampling_rate = h['frequency'] * pq.Hz,
)
#~ sptr.channel = channel
#sptr.annotations['channel_index'] = channel
sptr.annotate(channel_index = channel)
# for counting:
sptr.lazy_shape = 0
sptr.pos = 0
sptr.waveformsize = h['size']-10
#~ sptr.name = str(h['sortcode'])
#~ sptr.t_start = global_t_start
#~ sptr.sampling_rate = h['frequency']
#~ sptr.left_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.right_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.waveformsize = h['size']-10
allspiketr[code][channel][h['sortcode']] = sptr
allspiketr[code][channel][h['sortcode']].lazy_shape += 1
elif Types[evtype] == 'EVTYPE_STREAM':
if code not in allsig:
allsig[code] = { }
if channel not in allsig[code]:
#~ print 'code', code, 'channel', channel
anaSig = AnalogSignal([] * pq.V,
name=code,
sampling_rate=
h['frequency'] * pq.Hz,
t_start=(h['timestamp'] -
global_t_start) * pq.s,
channel_index=channel)
anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']])
anaSig.pos = 0
# for counting:
anaSig.lazy_shape = 0
#~ anaSig.pos = 0
allsig[code][channel] = anaSig
allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize
if not lazy:
# Step 2 : allocate memory
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V )
v[channel].pos = 0
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
ea.times = np.empty( (ea.lazy_shape) ) * pq.s
ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) )
ea.pos = 0
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s ,
name = sptr.name,
t_start = sptr.t_start,
t_stop = sptr.t_stop,
left_sweep = sptr.left_sweep,
sampling_rate = sptr.sampling_rate,
waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV ,
)
new.annotations.update(sptr.annotations)
new.pos = 0
new.waveformsize = sptr.waveformsize
allsorted[sortcode] = new
# Step 3 : searh sev (individual data files) or tev (common data file)
# sev is for version > 70
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb')
else:
tev = None
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
if PY3K:
signame = anaSig.name.decode('ascii')
else:
signame = anaSig.name
filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev')
if os.path.exists(filename):
anaSig.fid = open(filename, 'rb')
else:
anaSig.fid = tev
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
sptr.fid = tev
# Step 4 : second loop for copyin chunk of data
tsq.seek(0)
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_STREAM':
a = allsig[code][channel]
dt = a.dtype
s = int((h['size']*4-40)/dt.itemsize)
a.fid.seek(h['eventoffset'])
a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype)
a.pos += s
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
ea = allevent[code][channel]
ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
ea.labels[ea.pos] = str(strobe)
ea.pos += 1
elif Types[evtype] == 'EVTYPE_SNIP':
sptr = allspiketr[code][channel][h['sortcode']]
sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s
sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s
sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V
sptr.pos += 1
# Step 5 : populating segment
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
seg.analogsignals.append( anaSig )
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
seg.eventarrays.append( ea )
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
seg.spiketrains.append( sptr )
create_many_to_one_relationship(bl)
return bl
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
global_t_start = None
# Step 1 : first loop for counting - tsq file
tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb')
hr = HeaderReader(tsq, TsqDescription)
allsig = { }
allspiketr = { }
allevent = { }
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_UNKNOWN':
pass
elif Types[evtype] == 'EVTYPE_MARK' :
if global_t_start is None:
global_t_start = h['timestamp']
elif Types[evtype] == 'EVTYPE_SCALER' :
# TODO
pass
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
# EVENTS
if code not in allevent:
allevent[code] = { }
if channel not in allevent[code]:
ea = EventArray(name = code , channel_index = channel)
# for counting:
ea.lazy_shape = 0
ea.maxlabelsize = 0
allevent[code][channel] = ea
allevent[code][channel].lazy_shape += 1
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
strobe = str(strobe)
if len(strobe)>= allevent[code][channel].maxlabelsize:
allevent[code][channel].maxlabelsize = len(strobe)
#~ ev = Event()
#~ ev.time = h['timestamp'] - global_t_start
#~ ev.name = code
#~ # it the strobe attribute masked with eventoffset
#~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
#~ ev.label = str(strobe)
#~ seg._events.append( ev )
elif Types[evtype] == 'EVTYPE_SNIP' :
if code not in allspiketr:
allspiketr[code] = { }
if channel not in allspiketr[code]:
allspiketr[code][channel] = { }
if h['sortcode'] not in allspiketr[code][channel]:
sptr = SpikeTrain([ ], units = 's',
name = str(h['sortcode']),
#t_start = global_t_start,
t_start = 0.*pq.s,
t_stop = 0.*pq.s, # temporary
left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s,
sampling_rate = h['frequency'] * pq.Hz,
)
#~ sptr.channel = channel
#sptr.annotations['channel_index'] = channel
sptr.annotate(channel_index = channel)
# for counting:
sptr.lazy_shape = 0
sptr.pos = 0
sptr.waveformsize = h['size']-10
#~ sptr.name = str(h['sortcode'])
#~ sptr.t_start = global_t_start
#~ sptr.sampling_rate = h['frequency']
#~ sptr.left_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.right_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.waveformsize = h['size']-10
allspiketr[code][channel][h['sortcode']] = sptr
allspiketr[code][channel][h['sortcode']].lazy_shape += 1
elif Types[evtype] == 'EVTYPE_STREAM':
if code not in allsig:
allsig[code] = { }
if channel not in allsig[code]:
#~ print 'code', code, 'channel', channel
anaSig = AnalogSignal([] * pq.V,
name=code,
sampling_rate=
h['frequency'] * pq.Hz,
t_start=(h['timestamp'] -
global_t_start) * pq.s,
channel_index=channel)
anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']])
anaSig.pos = 0
# for counting:
anaSig.lazy_shape = 0
#~ anaSig.pos = 0
allsig[code][channel] = anaSig
allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize
if not lazy:
# Step 2 : allocate memory
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V )
v[channel].pos = 0
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
ea.times = np.empty( (ea.lazy_shape) ) * pq.s
ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) )
ea.pos = 0
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s ,
name = sptr.name,
t_start = sptr.t_start,
t_stop = sptr.t_stop,
left_sweep = sptr.left_sweep,
sampling_rate = sptr.sampling_rate,
waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV ,
)
new.annotations.update(sptr.annotations)
new.pos = 0
new.waveformsize = sptr.waveformsize
allsorted[sortcode] = new
# Step 3 : searh sev (individual data files) or tev (common data file)
# sev is for version > 70
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb')
else:
tev = None
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
if PY3K:
signame = anaSig.name.decode('ascii')
else:
signame = anaSig.name
filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev')
if os.path.exists(filename):
anaSig.fid = open(filename, 'rb')
else:
anaSig.fid = tev
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
sptr.fid = tev
# Step 4 : second loop for copyin chunk of data
tsq.seek(0)
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_STREAM':
a = allsig[code][channel]
dt = a.dtype
s = int((h['size']*4-40)/dt.itemsize)
a.fid.seek(h['eventoffset'])
a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype)
a.pos += s
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
ea = allevent[code][channel]
ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
ea.labels[ea.pos] = str(strobe)
ea.pos += 1
elif Types[evtype] == 'EVTYPE_SNIP':
sptr = allspiketr[code][channel][h['sortcode']]
sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s
sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s
sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V
sptr.pos += 1
# Step 5 : populating segment
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
seg.analogsignals.append( anaSig )
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
seg.eventarrays.append( ea )
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
seg.spiketrains.append( sptr )
bl.create_many_to_one_relationship()
return bl
def read_block(
self,
lazy=False,
cascade=True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade: return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname, blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name=blockname)
bl.segments.append(seg)
#TSQ is the global index
tsq_filename = os.path.join(subdir,
tankname + '_' + blockname + '.tsq')
dt = [
('size', 'int32'),
('evtype', 'int32'),
('code', 'S4'),
('channel', 'uint16'),
('sortcode', 'uint16'),
('timestamp', 'float64'),
('eventoffset', 'int64'),
('dataformat', 'int32'),
('frequency', 'float32'),
]
tsq = np.fromfile(tsq_filename, dtype=dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype'] == 0x8801]['timestamp'][0]
#TEV is the old data file
if os.path.exists(
os.path.join(subdir, tankname + '_' + blockname + '.tev')):
tev_filename = os.path.join(
subdir, tankname + '_' + blockname + '.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype='uint8')
else:
tev_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype'] == type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code'] == code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel'] == channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [] * pq.s
labels = np.array([], dtype=str)
else:
times = (tsq[mask3]['timestamp'] -
global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view(
'float64').astype('S')
ea = EventArray(times=times,
name=code,
channel_index=int(channel),
labels=labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.eventarrays.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode'] == sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0] - 10
if lazy:
times = [] * pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] -
global_t_start) * pq.s
dt = np.dtype(data_formats[
tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(
tsq[mask4]['size'],
tsq[mask4]['eventoffset'],
tev_array).view(dt)
waveforms = waveforms.reshape(
nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike > 0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 * pq.s
t_stop = (tsq['timestamp'][-1] -
global_t_start) * pq.s
else:
t_start = 0 * pq.s
t_stop = 0 * pq.s
st = SpikeTrain(
times=times,
name='Chan{} Code{}'.format(
channel, sortcode),
t_start=t_start,
t_stop=t_stop,
waveforms=waveforms,
left_sweep=waveformsize / 2. / sr * pq.s,
sampling_rate=sr * pq.Hz,
)
st.annotate(channel_index=channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(
data_formats[tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size'] - 10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = []
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(
subdir, tankname + '_' + blockname + '_' +
signame + '_ch' + str(channel) + '.sev')
if os.path.exists(sev_filename):
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename,
dtype='uint8')
else:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],
tsq[mask3]['eventoffset'],
sig_array).view(dt)
anasig = AnalogSignal(
signal=signal * pq.V,
name='{} {}'.format(code, channel),
sampling_rate=sr * pq.Hz,
t_start=(tsq[mask3]['timestamp'][0] -
global_t_start) * pq.s,
channel_index=int(channel))
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
bl.create_many_to_one_relationship()
return bl
