def read_block(self, **kargs): if self.filename is not None: self.axo_obj = axographio.read(self.filename) # Build up the block blk = Block() blk.rec_datetime = None if self.filename is not None: # modified time is not ideal but less prone to # cross-platform issues than created time (ctime) blk.file_datetime = datetime.fromtimestamp(os.path.getmtime(self.filename)) # store the filename if it is available blk.file_origin = self.filename # determine the channel names and counts _, channel_ordering = np.unique(self.axo_obj.names[1:], return_index=True) channel_names = np.array(self.axo_obj.names[1:])[np.sort(channel_ordering)] channel_count = len(channel_names) # determine the time signal and sample period sample_period = self.axo_obj.data[0].step * pq.s start_time = self.axo_obj.data[0].start * pq.s # Attempt to read units from the channel names channel_unit_names = [x.split()[-1].strip('()') for x in channel_names] channel_units = [] for unit in channel_unit_names: try: channel_units.append(pq.Quantity(1, unit)) except LookupError: channel_units.append(None) # Strip units from channel names channel_names = [' '.join(x.split()[:-1]) for x in channel_names] # build up segments by grouping axograph columns for seg_idx in range(1, len(self.axo_obj.data), channel_count): seg = Segment(index=seg_idx) # add in the channels for chan_idx in range(0, channel_count): signal = pq.Quantity( self.axo_obj.data[seg_idx + chan_idx], channel_units[chan_idx]) analog = AnalogSignal(signal, sampling_period=sample_period, t_start=start_time, name=channel_names[chan_idx], channel_index=chan_idx) seg.analogsignals.append(analog) blk.segments.append(seg) blk.create_many_to_one_relationship() return blk
def read_block(self, lazy=False, cascade=True): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): seg = self.read_segment(blockname, lazy, cascade) bl.segments.append(seg) bl.create_many_to_one_relationship() return bl
def read_block(self, lazy=False, cascade=True, sortname=''): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade: return bl for blockname in os.listdir(self.dirname): if self.is_tdtblock(blockname): # if the folder is a tdt block seg = self.read_segment(blockname, lazy, cascade, sortname) bl.segments.append(seg) bl.create_many_to_one_relationship() return bl
def read_block(self, lazy=False, cascade=True): header = self.read_header() version = header['fFileVersionNumber'] bl = Block() bl.file_origin = os.path.basename(self.filename) bl.annotate(abf_version=str(version)) # date and time if version < 2.: YY = 1900 MM = 1 DD = 1 hh = int(header['lFileStartTime'] / 3600.) mm = int((header['lFileStartTime'] - hh * 3600) / 60) ss = header['lFileStartTime'] - hh * 3600 - mm * 60 ms = int(np.mod(ss, 1) * 1e6) ss = int(ss) elif version >= 2.: YY = int(header['uFileStartDate'] / 10000) MM = int((header['uFileStartDate'] - YY * 10000) / 100) DD = int(header['uFileStartDate'] - YY * 10000 - MM * 100) hh = int(header['uFileStartTimeMS'] / 1000. / 3600.) mm = int((header['uFileStartTimeMS'] / 1000. - hh * 3600) / 60) ss = header['uFileStartTimeMS'] / 1000. - hh * 3600 - mm * 60 ms = int(np.mod(ss, 1) * 1e6) ss = int(ss) bl.rec_datetime = datetime.datetime(YY, MM, DD, hh, mm, ss, ms) if not cascade: return bl # file format if header['nDataFormat'] == 0: dt = np.dtype('i2') elif header['nDataFormat'] == 1: dt = np.dtype('f4') if version < 2.: nbchannel = header['nADCNumChannels'] head_offset = header['lDataSectionPtr'] * BLOCKSIZE + header[ 'nNumPointsIgnored'] * dt.itemsize totalsize = header['lActualAcqLength'] elif version >= 2.: nbchannel = header['sections']['ADCSection']['llNumEntries'] head_offset = header['sections']['DataSection'][ 'uBlockIndex'] * BLOCKSIZE totalsize = header['sections']['DataSection']['llNumEntries'] data = np.memmap(self.filename, dt, 'r', shape=(totalsize,), offset=head_offset) # 3 possible modes if version < 2.: mode = header['nOperationMode'] elif version >= 2.: mode = header['protocol']['nOperationMode'] if (mode == 1) or (mode == 2) or (mode == 5) or (mode == 3): # event-driven variable-length mode (mode 1) # event-driven fixed-length mode (mode 2 or 5) # gap free mode (mode 3) can be in several episodes # read sweep pos if version < 2.: nbepisod = header['lSynchArraySize'] offset_episode = header['lSynchArrayPtr'] * BLOCKSIZE elif version >= 2.: nbepisod = header['sections']['SynchArraySection'][ 'llNumEntries'] offset_episode = header['sections']['SynchArraySection'][ 'uBlockIndex'] * BLOCKSIZE if nbepisod > 0: episode_array = np.memmap( self.filename, [('offset', 'i4'), ('len', 'i4')], 'r', shape=nbepisod, offset=offset_episode) else: episode_array = np.empty(1, [('offset', 'i4'), ('len', 'i4')]) episode_array[0]['len'] = data.size episode_array[0]['offset'] = 0 # sampling_rate if version < 2.: sampling_rate = 1. / (header['fADCSampleInterval'] * nbchannel * 1.e-6) * pq.Hz elif version >= 2.: sampling_rate = 1.e6 / \ header['protocol']['fADCSequenceInterval'] * pq.Hz # construct block # one sweep = one segment in a block pos = 0 for j in range(episode_array.size): seg = Segment(index=j) length = episode_array[j]['len'] if version < 2.: fSynchTimeUnit = header['fSynchTimeUnit'] elif version >= 2.: fSynchTimeUnit = header['protocol']['fSynchTimeUnit'] if (fSynchTimeUnit != 0) and (mode == 1): length /= fSynchTimeUnit if not lazy: subdata = data[pos:pos+length] subdata = subdata.reshape((int(subdata.size/nbchannel), nbchannel)).astype('f') if dt == np.dtype('i2'): if version < 2.: reformat_integer_v1(subdata, nbchannel, header) elif version >= 2.: reformat_integer_v2(subdata, nbchannel, header) pos += length if version < 2.: chans = [chan_num for chan_num in header['nADCSamplingSeq'] if chan_num >= 0] else: chans = range(nbchannel) for n, i in enumerate(chans[:nbchannel]): # fix SamplingSeq if version < 2.: name = header['sADCChannelName'][i].replace(b' ', b'') unit = header['sADCUnits'][i].replace(b'\xb5', b'u').\ replace(b' ', b'').decode('utf-8') # \xb5 is µ num = header['nADCPtoLChannelMap'][i] elif version >= 2.: lADCIi = header['listADCInfo'][i] name = lADCIi['ADCChNames'].replace(b' ', b'') unit = lADCIi['ADCChUnits'].replace(b'\xb5', b'u').\ replace(b' ', b'').decode('utf-8') num = header['listADCInfo'][i]['nADCNum'] if (fSynchTimeUnit == 0): t_start = float(episode_array[j]['offset']) / sampling_rate else: t_start = float(episode_array[j]['offset']) * fSynchTimeUnit *1e-6* pq.s t_start = t_start.rescale('s') try: pq.Quantity(1, unit) except: unit = '' if lazy: signal = [] * pq.Quantity(1, unit) else: signal = pq.Quantity(subdata[:, n], unit) anaSig = AnalogSignal(signal, sampling_rate=sampling_rate, t_start=t_start, name=str(name), channel_index=int(num)) if lazy: anaSig.lazy_shape = length / nbchannel seg.analogsignals.append(anaSig) bl.segments.append(seg) if mode in [3, 5]: # TODO check if tags exits in other mode # tag is EventArray that should be attached to Block # It is attched to the first Segment times = [] labels = [] comments = [] for i, tag in enumerate(header['listTag']): times.append(tag['lTagTime']/sampling_rate) labels.append(str(tag['nTagType'])) comments.append(clean_string(tag['sComment'])) times = np.array(times) labels = np.array(labels, dtype='S') comments = np.array(comments, dtype='S') # attach all tags to the first segment. seg = bl.segments[0] if lazy: ea = Event(times=[] * pq.s, labels=np.array([], dtype='S')) ea.lazy_shape = len(times) else: ea = Event(times=times * pq.s, labels=labels, comments=comments) seg.events.append(ea) bl.create_many_to_one_relationship() return bl
def read_block(self, lazy=False, cascade=True): header = self.read_header() version = header['fFileVersionNumber'] bl = Block() bl.file_origin = os.path.basename(self.filename) bl.annotate(abf_version=str(version)) # date and time if version < 2.: YY = 1900 MM = 1 DD = 1 hh = int(header['lFileStartTime'] / 3600.) mm = int((header['lFileStartTime'] - hh * 3600) / 60) ss = header['lFileStartTime'] - hh * 3600 - mm * 60 ms = int(np.mod(ss, 1) * 1e6) ss = int(ss) elif version >= 2.: YY = int(header['uFileStartDate'] / 10000) MM = int((header['uFileStartDate'] - YY * 10000) / 100) DD = int(header['uFileStartDate'] - YY * 10000 - MM * 100) hh = int(header['uFileStartTimeMS'] / 1000. / 3600.) mm = int((header['uFileStartTimeMS'] / 1000. - hh * 3600) / 60) ss = header['uFileStartTimeMS'] / 1000. - hh * 3600 - mm * 60 ms = int(np.mod(ss, 1) * 1e6) ss = int(ss) bl.rec_datetime = datetime.datetime(YY, MM, DD, hh, mm, ss, ms) if not cascade: return bl # file format if header['nDataFormat'] == 0: dt = np.dtype('i2') elif header['nDataFormat'] == 1: dt = np.dtype('f4') if version < 2.: nbchannel = header['nADCNumChannels'] head_offset = header['lDataSectionPtr'] * BLOCKSIZE + header[ 'nNumPointsIgnored'] * dt.itemsize totalsize = header['lActualAcqLength'] elif version >= 2.: nbchannel = header['sections']['ADCSection']['llNumEntries'] head_offset = header['sections']['DataSection'][ 'uBlockIndex'] * BLOCKSIZE totalsize = header['sections']['DataSection']['llNumEntries'] data = np.memmap(self.filename, dt, 'r', shape=(totalsize, ), offset=head_offset) # 3 possible modes if version < 2.: mode = header['nOperationMode'] elif version >= 2.: mode = header['protocol']['nOperationMode'] if (mode == 1) or (mode == 2) or (mode == 5) or (mode == 3): # event-driven variable-length mode (mode 1) # event-driven fixed-length mode (mode 2 or 5) # gap free mode (mode 3) can be in several episodes # read sweep pos if version < 2.: nbepisod = header['lSynchArraySize'] offset_episode = header['lSynchArrayPtr'] * BLOCKSIZE elif version >= 2.: nbepisod = header['sections']['SynchArraySection'][ 'llNumEntries'] offset_episode = header['sections']['SynchArraySection'][ 'uBlockIndex'] * BLOCKSIZE if nbepisod > 0: episode_array = np.memmap(self.filename, [('offset', 'i4'), ('len', 'i4')], 'r', shape=nbepisod, offset=offset_episode) else: episode_array = np.empty(1, [('offset', 'i4'), ('len', 'i4')]) episode_array[0]['len'] = data.size episode_array[0]['offset'] = 0 # sampling_rate if version < 2.: sampling_rate = 1. / (header['fADCSampleInterval'] * nbchannel * 1.e-6) * pq.Hz elif version >= 2.: sampling_rate = 1.e6 / \ header['protocol']['fADCSequenceInterval'] * pq.Hz # construct block # one sweep = one segment in a block pos = 0 for j in range(episode_array.size): seg = Segment(index=j) length = episode_array[j]['len'] if version < 2.: fSynchTimeUnit = header['fSynchTimeUnit'] elif version >= 2.: fSynchTimeUnit = header['protocol']['fSynchTimeUnit'] if (fSynchTimeUnit != 0) and (mode == 1): length /= fSynchTimeUnit if not lazy: subdata = data[pos:pos + length] subdata = subdata.reshape( (int(subdata.size / nbchannel), nbchannel)).astype('f') if dt == np.dtype('i2'): if version < 2.: reformat_integer_v1(subdata, nbchannel, header) elif version >= 2.: reformat_integer_v2(subdata, nbchannel, header) pos += length if version < 2.: chans = [ chan_num for chan_num in header['nADCSamplingSeq'] if chan_num >= 0 ] else: chans = range(nbchannel) for n, i in enumerate(chans[:nbchannel]): # fix SamplingSeq if version < 2.: name = header['sADCChannelName'][i].replace(b' ', b'') unit = header['sADCUnits'][i].replace(b'\xb5', b'u').\ replace(b' ', b'').decode('utf-8') # \xb5 is µ num = header['nADCPtoLChannelMap'][i] elif version >= 2.: lADCIi = header['listADCInfo'][i] name = lADCIi['ADCChNames'].replace(b' ', b'') unit = lADCIi['ADCChUnits'].replace(b'\xb5', b'u').\ replace(b' ', b'').decode('utf-8') num = header['listADCInfo'][i]['nADCNum'] if (fSynchTimeUnit == 0): t_start = float( episode_array[j]['offset']) / sampling_rate else: t_start = float(episode_array[j]['offset'] ) * fSynchTimeUnit * 1e-6 * pq.s t_start = t_start.rescale('s') try: pq.Quantity(1, unit) except: unit = '' if lazy: signal = [] * pq.Quantity(1, unit) else: signal = pq.Quantity(subdata[:, n], unit) anaSig = AnalogSignal(signal, sampling_rate=sampling_rate, t_start=t_start, name=str(name), channel_index=int(num)) if lazy: anaSig.lazy_shape = length / nbchannel seg.analogsignals.append(anaSig) bl.segments.append(seg) if mode in [3, 5]: # TODO check if tags exits in other mode # tag is EventArray that should be attached to Block # It is attched to the first Segment times = [] labels = [] comments = [] for i, tag in enumerate(header['listTag']): times.append(tag['lTagTime'] / sampling_rate) labels.append(str(tag['nTagType'])) comments.append(clean_string(tag['sComment'])) times = np.array(times) labels = np.array(labels, dtype='S') comments = np.array(comments, dtype='S') # attach all tags to the first segment. seg = bl.segments[0] if lazy: ea = Event(times=[] * pq.s, labels=np.array([], dtype='S')) ea.lazy_shape = len(times) else: ea = Event(times=times * pq.s, labels=labels, comments=comments) seg.events.append(ea) bl.create_many_to_one_relationship() return bl
def read_block( self, # the 2 first keyword arguments are imposed by neo.io API lazy=False, cascade=True): """ Return a Block. """ def count_samples(m_length): """ Count the number of signal samples available in a type 5 data block of length m_length """ # for information about type 5 data block, see [1] count = int((m_length - 6) / 2 - 2) # -6 corresponds to the header of block 5, and the -2 take into # account the fact that last 2 values are not available as the 4 # corresponding bytes are coding the time stamp of the beginning # of the block return count # create the neo Block that will be returned at the end blck = Block(file_origin=os.path.basename(self.filename)) blck.file_origin = os.path.basename(self.filename) fid = open(self.filename, 'rb') # NOTE: in the following, the word "block" is used in the sense used in # the alpha-omega specifications (ie a data chunk in the file), rather # than in the sense of the usual Block object in neo # step 1: read the headers of all the data blocks to load the file # structure pos_block = 0 # position of the current block in the file file_blocks = [] # list of data blocks available in the file if not cascade: # we read only the main header m_length, m_TypeBlock = struct.unpack('Hcx', fid.read(4)) # m_TypeBlock should be 'h', as we read the first block block = HeaderReader( fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({ 'm_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block }) file_blocks.append(block) else: # cascade == True seg = Segment(file_origin=os.path.basename(self.filename)) seg.file_origin = os.path.basename(self.filename) blck.segments.append(seg) while True: first_4_bytes = fid.read(4) if len(first_4_bytes) < 4: # we have reached the end of the file break else: m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes) block = HeaderReader( fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({ 'm_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block }) if m_TypeBlock == '2': # The beggining of the block of type '2' is identical for # all types of channels, but the following part depends on # the type of channel. So we need a special case here. # WARNING: How to check the type of channel is not # described in the documentation. So here I use what is # proposed in the C code [2]. # According to this C code, it seems that the 'm_isAnalog' # is used to distinguished analog and digital channels, and # 'm_Mode' encodes the type of analog channel: # 0 for continuous, 1 for level, 2 for external trigger. # But in some files, I found channels that seemed to be # continuous channels with 'm_Modes' = 128 or 192. So I # decided to consider every channel with 'm_Modes' # different from 1 or 2 as continuous. I also couldn't # check that values of 1 and 2 are really for level and # external trigger as I had no test files containing data # of this types. type_subblock = 'unknown_channel_type(m_Mode=' \ + str(block['m_Mode'])+ ')' description = Type2_SubBlockUnknownChannels block.update({'m_Name': 'unknown_name'}) if block['m_isAnalog'] == 0: # digital channel type_subblock = 'digital' description = Type2_SubBlockDigitalChannels elif block['m_isAnalog'] == 1: # analog channel if block['m_Mode'] == 1: # level channel type_subblock = 'level' description = Type2_SubBlockLevelChannels elif block['m_Mode'] == 2: # external trigger channel type_subblock = 'external_trigger' description = Type2_SubBlockExtTriggerChannels else: # continuous channel type_subblock = 'continuous(Mode' \ + str(block['m_Mode']) +')' description = Type2_SubBlockContinuousChannels subblock = HeaderReader(fid, description).read_f() block.update(subblock) block.update({'type_subblock': type_subblock}) file_blocks.append(block) pos_block += m_length fid.seek(pos_block) # step 2: find the available channels list_chan = [] # list containing indexes of channel blocks for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '2': list_chan.append(ind_block) # step 3: find blocks containing data for the available channels list_data = [] # list of lists of indexes of data blocks # corresponding to each channel for ind_chan, chan in enumerate(list_chan): list_data.append([]) num_chan = file_blocks[chan]['m_numChannel'] for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '5': if block['m_numChannel'] == num_chan: list_data[ind_chan].append(ind_block) # step 4: compute the length (number of samples) of the channels chan_len = np.zeros(len(list_data), dtype=np.int) for ind_chan, list_blocks in enumerate(list_data): for ind_block in list_blocks: chan_len[ind_chan] += count_samples( file_blocks[ind_block]['m_length']) # step 5: find channels for which data are available ind_valid_chan = np.nonzero(chan_len)[0] # step 6: load the data # TODO give the possibility to load data as AnalogSignalArrays for ind_chan in ind_valid_chan: list_blocks = list_data[ind_chan] ind = 0 # index in the data vector # read time stamp for the beginning of the signal form = '<l' # reading format ind_block = list_blocks[0] count = count_samples(file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos'] + 6 + count * 2) buf = fid.read(struct.calcsize(form)) val = struct.unpack(form, buf) start_index = val[0] # WARNING: in the following blocks are read supposing taht they # are all contiguous and sorted in time. I don't know if it's # always the case. Maybe we should use the time stamp of each # data block to choose where to put the read data in the array. if not lazy: temp_array = np.empty(chan_len[ind_chan], dtype=np.int16) # NOTE: we could directly create an empty AnalogSignal and # load the data in it, but it is much faster to load data # in a temporary numpy array and create the AnalogSignals # from this temporary array for ind_block in list_blocks: count = count_samples( file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos'] + 6) temp_array[ind:ind+count] = \ np.fromfile(fid, dtype = np.int16, count = count) ind += count sampling_rate = \ file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz t_start = (start_index / sampling_rate).simplified if lazy: ana_sig = AnalogSignal([], sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) ana_sig.lazy_shape = chan_len[ind_chan] else: ana_sig = AnalogSignal(temp_array, sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) ana_sig.channel_index = \ file_blocks[list_chan[ind_chan]]['m_numChannel'] ana_sig.annotate(channel_name = \ file_blocks[list_chan[ind_chan]]['m_Name']) ana_sig.annotate(channel_type = \ file_blocks[list_chan[ind_chan]]['type_subblock']) seg.analogsignals.append(ana_sig) fid.close() if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true blck.rec_datetime = datetime.datetime(\ file_blocks[0]['m_date_year'], file_blocks[0]['m_date_month'], file_blocks[0]['m_date_day'], file_blocks[0]['m_time_hour'], file_blocks[0]['m_time_minute'], file_blocks[0]['m_time_second'], 10000 * file_blocks[0]['m_time_hsecond']) # the 10000 is here to convert m_time_hsecond from centisecond # to microsecond version = file_blocks[0]['m_version'] blck.annotate(alphamap_version=version) if cascade: seg.rec_datetime = blck.rec_datetime.replace() # I couldn't find a simple copy function for datetime, # using replace without arguments is a twisted way to make a # copy seg.annotate(alphamap_version=version) if cascade: populate_RecordingChannel(blck, remove_from_annotation=True) blck.create_many_to_one_relationship() return blck
def read_block(self, # the 2 first keyword arguments are imposed by neo.io API lazy = False, cascade = True): """ Return a Block. """ def count_samples(m_length): """ Count the number of signal samples available in a type 5 data block of length m_length """ # for information about type 5 data block, see [1] count = int((m_length-6)/2-2) # -6 corresponds to the header of block 5, and the -2 take into # account the fact that last 2 values are not available as the 4 # corresponding bytes are coding the time stamp of the beginning # of the block return count # create the neo Block that will be returned at the end blck = Block(file_origin = os.path.basename(self.filename)) blck.file_origin = os.path.basename(self.filename) fid = open(self.filename, 'rb') # NOTE: in the following, the word "block" is used in the sense used in # the alpha-omega specifications (ie a data chunk in the file), rather # than in the sense of the usual Block object in neo # step 1: read the headers of all the data blocks to load the file # structure pos_block = 0 # position of the current block in the file file_blocks = [] # list of data blocks available in the file if not cascade: # we read only the main header m_length, m_TypeBlock = struct.unpack('Hcx' , fid.read(4)) # m_TypeBlock should be 'h', as we read the first block block = HeaderReader(fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({'m_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block}) file_blocks.append(block) else: # cascade == True seg = Segment(file_origin = os.path.basename(self.filename)) seg.file_origin = os.path.basename(self.filename) blck.segments.append(seg) while True: first_4_bytes = fid.read(4) if len(first_4_bytes) < 4: # we have reached the end of the file break else: m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes) block = HeaderReader(fid, dict_header_type.get(m_TypeBlock, Type_Unknown)).read_f() block.update({'m_length': m_length, 'm_TypeBlock': m_TypeBlock, 'pos': pos_block}) if m_TypeBlock == '2': # The beginning of the block of type '2' is identical for # all types of channels, but the following part depends on # the type of channel. So we need a special case here. # WARNING: How to check the type of channel is not # described in the documentation. So here I use what is # proposed in the C code [2]. # According to this C code, it seems that the 'm_isAnalog' # is used to distinguished analog and digital channels, and # 'm_Mode' encodes the type of analog channel: # 0 for continuous, 1 for level, 2 for external trigger. # But in some files, I found channels that seemed to be # continuous channels with 'm_Modes' = 128 or 192. So I # decided to consider every channel with 'm_Modes' # different from 1 or 2 as continuous. I also couldn't # check that values of 1 and 2 are really for level and # external trigger as I had no test files containing data # of this types. type_subblock = 'unknown_channel_type(m_Mode=' \ + str(block['m_Mode'])+ ')' description = Type2_SubBlockUnknownChannels block.update({'m_Name': 'unknown_name'}) if block['m_isAnalog'] == 0: # digital channel type_subblock = 'digital' description = Type2_SubBlockDigitalChannels elif block['m_isAnalog'] == 1: # analog channel if block['m_Mode'] == 1: # level channel type_subblock = 'level' description = Type2_SubBlockLevelChannels elif block['m_Mode'] == 2: # external trigger channel type_subblock = 'external_trigger' description = Type2_SubBlockExtTriggerChannels else: # continuous channel type_subblock = 'continuous(Mode' \ + str(block['m_Mode']) +')' description = Type2_SubBlockContinuousChannels subblock = HeaderReader(fid, description).read_f() block.update(subblock) block.update({'type_subblock': type_subblock}) file_blocks.append(block) pos_block += m_length fid.seek(pos_block) # step 2: find the available channels list_chan = [] # list containing indexes of channel blocks for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '2': list_chan.append(ind_block) # step 3: find blocks containing data for the available channels list_data = [] # list of lists of indexes of data blocks # corresponding to each channel for ind_chan, chan in enumerate(list_chan): list_data.append([]) num_chan = file_blocks[chan]['m_numChannel'] for ind_block, block in enumerate(file_blocks): if block['m_TypeBlock'] == '5': if block['m_numChannel'] == num_chan: list_data[ind_chan].append(ind_block) # step 4: compute the length (number of samples) of the channels chan_len = np.zeros(len(list_data), dtype = np.int) for ind_chan, list_blocks in enumerate(list_data): for ind_block in list_blocks: chan_len[ind_chan] += count_samples( file_blocks[ind_block]['m_length']) # step 5: find channels for which data are available ind_valid_chan = np.nonzero(chan_len)[0] # step 6: load the data # TODO give the possibility to load data as AnalogSignalArrays for ind_chan in ind_valid_chan: list_blocks = list_data[ind_chan] ind = 0 # index in the data vector # read time stamp for the beginning of the signal form = '<l' # reading format ind_block = list_blocks[0] count = count_samples(file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos']+6+count*2) buf = fid.read(struct.calcsize(form)) val = struct.unpack(form , buf) start_index = val[0] # WARNING: in the following blocks are read supposing taht they # are all contiguous and sorted in time. I don't know if it's # always the case. Maybe we should use the time stamp of each # data block to choose where to put the read data in the array. if not lazy: temp_array = np.empty(chan_len[ind_chan], dtype = np.int16) # NOTE: we could directly create an empty AnalogSignal and # load the data in it, but it is much faster to load data # in a temporary numpy array and create the AnalogSignals # from this temporary array for ind_block in list_blocks: count = count_samples( file_blocks[ind_block]['m_length']) fid.seek(file_blocks[ind_block]['pos']+6) temp_array[ind:ind+count] = \ np.fromfile(fid, dtype = np.int16, count = count) ind += count sampling_rate = \ file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz t_start = (start_index / sampling_rate).simplified if lazy: ana_sig = AnalogSignal([], sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) ana_sig.lazy_shape = chan_len[ind_chan] else: ana_sig = AnalogSignal(temp_array, sampling_rate = sampling_rate, t_start = t_start, name = file_blocks\ [list_chan[ind_chan]]['m_Name'], file_origin = \ os.path.basename(self.filename), units = pq.dimensionless) # todo apibreak: create ChannelIndex for each signals # ana_sig.channel_index = \ # file_blocks[list_chan[ind_chan]]['m_numChannel'] ana_sig.annotate(channel_name = \ file_blocks[list_chan[ind_chan]]['m_Name']) ana_sig.annotate(channel_type = \ file_blocks[list_chan[ind_chan]]['type_subblock']) seg.analogsignals.append(ana_sig) fid.close() if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true blck.rec_datetime = datetime.datetime(\ file_blocks[0]['m_date_year'], file_blocks[0]['m_date_month'], file_blocks[0]['m_date_day'], file_blocks[0]['m_time_hour'], file_blocks[0]['m_time_minute'], file_blocks[0]['m_time_second'], 10000 * file_blocks[0]['m_time_hsecond']) # the 10000 is here to convert m_time_hsecond from centisecond # to microsecond version = file_blocks[0]['m_version'] blck.annotate(alphamap_version = version) if cascade: seg.rec_datetime = blck.rec_datetime.replace() # I couldn't find a simple copy function for datetime, # using replace without arguments is a twisted way to make a # copy seg.annotate(alphamap_version = version) if cascade: blck.create_many_to_one_relationship() return blck
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) #TSQ is the global index tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq') dt = [('size','int32'), ('evtype','int32'), ('code','S4'), ('channel','uint16'), ('sortcode','uint16'), ('timestamp','float64'), ('eventoffset','int64'), ('dataformat','int32'), ('frequency','float32'), ] tsq = np.fromfile(tsq_filename, dtype = dt) #0x8801: 'EVTYPE_MARK' give the global_start global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0] #TEV is the old data file if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev') #tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead tev_array = np.fromfile(tev_filename, dtype = 'uint8') else: tev_filename = None for type_code, type_label in tdt_event_type: mask1 = tsq['evtype']==type_code codes = np.unique(tsq[mask1]['code']) for code in codes: mask2 = mask1 & (tsq['code']==code) channels = np.unique(tsq[mask2]['channel']) for channel in channels: mask3 = mask2 & (tsq['channel']==channel) if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']: if lazy: times = [ ]*pq.s labels = np.array([ ], dtype = str) else: times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s labels = tsq[mask3]['eventoffset'].view('float64').astype('S') ea = EventArray(times = times, name = code , channel_index = int(channel), labels = labels) if lazy: ea.lazy_shape = np.sum(mask3) seg.eventarrays.append(ea) elif type_label == 'EVTYPE_SNIP': sortcodes = np.unique(tsq[mask3]['sortcode']) for sortcode in sortcodes: mask4 = mask3 & (tsq['sortcode']==sortcode) nb_spike = np.sum(mask4) sr = tsq[mask4]['frequency'][0] waveformsize = tsq[mask4]['size'][0]-10 if lazy: times = [ ]*pq.s waveforms = None else: times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt) waveforms = waveforms.reshape(nb_spike, -1, waveformsize) waveforms = waveforms * pq.mV if nb_spike>0: # t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not t_start = 0 *pq.s t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s else: t_start = 0 *pq.s t_stop = 0 *pq.s st = SpikeTrain(times = times, name = 'Chan{} Code{}'.format(channel,sortcode), t_start = t_start, t_stop = t_stop, waveforms = waveforms, left_sweep = waveformsize/2./sr * pq.s, sampling_rate = sr * pq.Hz, ) st.annotate(channel_index = channel) if lazy: st.lazy_shape = nb_spike seg.spiketrains.append(st) elif type_label == 'EVTYPE_STREAM': dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) shape = np.sum(tsq[mask3]['size']-10) sr = tsq[mask3]['frequency'][0] if lazy: signal = [ ] else: if PY3K: signame = code.decode('ascii') else: signame = code sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev') if os.path.exists(sev_filename): #sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead sig_array = np.fromfile(sev_filename, dtype = 'uint8') else: sig_array = tev_array signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt) anasig = AnalogSignal(signal = signal* pq.V, name = '{} {}'.format(code, channel), sampling_rate= sr * pq.Hz, t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s, channel_index = int(channel)) if lazy: anasig.lazy_shape = shape seg.analogsignals.append(anasig) bl.create_many_to_one_relationship() return bl
def read_block(self, **kargs): if self.filename is not None: self.axo_obj = axographio.read(self.filename) # Build up the block blk = Block() blk.rec_datetime = None if self.filename is not None: # modified time is not ideal but less prone to # cross-platform issues than created time (ctime) blk.file_datetime = datetime.fromtimestamp( os.path.getmtime(self.filename)) # store the filename if it is available blk.file_origin = self.filename # determine the channel names and counts _, channel_ordering = np.unique(self.axo_obj.names[1:], return_index=True) channel_names = np.array( self.axo_obj.names[1:])[np.sort(channel_ordering)] channel_count = len(channel_names) # determine the time signal and sample period sample_period = self.axo_obj.data[0].step * pq.s start_time = self.axo_obj.data[0].start * pq.s # Attempt to read units from the channel names channel_unit_names = [x.split()[-1].strip('()') for x in channel_names] channel_units = [] for unit in channel_unit_names: try: channel_units.append(pq.Quantity(1, unit)) except LookupError: channel_units.append(None) # Strip units from channel names channel_names = [' '.join(x.split()[:-1]) for x in channel_names] # build up segments by grouping axograph columns for seg_idx in range(1, len(self.axo_obj.data), channel_count): seg = Segment(index=seg_idx) # add in the channels for chan_idx in range(0, channel_count): signal = pq.Quantity(self.axo_obj.data[seg_idx + chan_idx], channel_units[chan_idx]) analog = AnalogSignal(signal, sampling_period=sample_period, t_start=start_time, name=channel_names[chan_idx], channel_index=chan_idx) seg.analogsignals.append(analog) blk.segments.append(seg) blk.create_many_to_one_relationship() return blk
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) create_many_to_one_relationship(bl) return bl
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) bl.create_many_to_one_relationship() return bl
def read_block( self, lazy=False, cascade=True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade: return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname, blockname) if not os.path.isdir(subdir): continue seg = Segment(name=blockname) bl.segments.append(seg) #TSQ is the global index tsq_filename = os.path.join(subdir, tankname + '_' + blockname + '.tsq') dt = [ ('size', 'int32'), ('evtype', 'int32'), ('code', 'S4'), ('channel', 'uint16'), ('sortcode', 'uint16'), ('timestamp', 'float64'), ('eventoffset', 'int64'), ('dataformat', 'int32'), ('frequency', 'float32'), ] tsq = np.fromfile(tsq_filename, dtype=dt) #0x8801: 'EVTYPE_MARK' give the global_start global_t_start = tsq[tsq['evtype'] == 0x8801]['timestamp'][0] #TEV is the old data file if os.path.exists( os.path.join(subdir, tankname + '_' + blockname + '.tev')): tev_filename = os.path.join( subdir, tankname + '_' + blockname + '.tev') #tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead tev_array = np.fromfile(tev_filename, dtype='uint8') else: tev_filename = None for type_code, type_label in tdt_event_type: mask1 = tsq['evtype'] == type_code codes = np.unique(tsq[mask1]['code']) for code in codes: mask2 = mask1 & (tsq['code'] == code) channels = np.unique(tsq[mask2]['channel']) for channel in channels: mask3 = mask2 & (tsq['channel'] == channel) if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']: if lazy: times = [] * pq.s labels = np.array([], dtype=str) else: times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s labels = tsq[mask3]['eventoffset'].view( 'float64').astype('S') ea = EventArray(times=times, name=code, channel_index=int(channel), labels=labels) if lazy: ea.lazy_shape = np.sum(mask3) seg.eventarrays.append(ea) elif type_label == 'EVTYPE_SNIP': sortcodes = np.unique(tsq[mask3]['sortcode']) for sortcode in sortcodes: mask4 = mask3 & (tsq['sortcode'] == sortcode) nb_spike = np.sum(mask4) sr = tsq[mask4]['frequency'][0] waveformsize = tsq[mask4]['size'][0] - 10 if lazy: times = [] * pq.s waveforms = None else: times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]]) waveforms = get_chunks( tsq[mask4]['size'], tsq[mask4]['eventoffset'], tev_array).view(dt) waveforms = waveforms.reshape( nb_spike, -1, waveformsize) waveforms = waveforms * pq.mV if nb_spike > 0: # t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not t_start = 0 * pq.s t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s else: t_start = 0 * pq.s t_stop = 0 * pq.s st = SpikeTrain( times=times, name='Chan{} Code{}'.format( channel, sortcode), t_start=t_start, t_stop=t_stop, waveforms=waveforms, left_sweep=waveformsize / 2. / sr * pq.s, sampling_rate=sr * pq.Hz, ) st.annotate(channel_index=channel) if lazy: st.lazy_shape = nb_spike seg.spiketrains.append(st) elif type_label == 'EVTYPE_STREAM': dt = np.dtype( data_formats[tsq[mask3]['dataformat'][0]]) shape = np.sum(tsq[mask3]['size'] - 10) sr = tsq[mask3]['frequency'][0] if lazy: signal = [] else: if PY3K: signame = code.decode('ascii') else: signame = code sev_filename = os.path.join( subdir, tankname + '_' + blockname + '_' + signame + '_ch' + str(channel) + '.sev') if os.path.exists(sev_filename): #sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead sig_array = np.fromfile(sev_filename, dtype='uint8') else: sig_array = tev_array signal = get_chunks(tsq[mask3]['size'], tsq[mask3]['eventoffset'], sig_array).view(dt) anasig = AnalogSignal( signal=signal * pq.V, name='{} {}'.format(code, channel), sampling_rate=sr * pq.Hz, t_start=(tsq[mask3]['timestamp'][0] - global_t_start) * pq.s, channel_index=int(channel)) if lazy: anasig.lazy_shape = shape seg.analogsignals.append(anasig) bl.create_many_to_one_relationship() return bl