def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) create_many_to_one_relationship(bl) return bl
def read_block(self, lazy = False, cascade = True, ): bl = Block() tankname = os.path.basename(self.dirname) bl.file_origin = tankname if not cascade : return bl for blockname in os.listdir(self.dirname): if blockname == 'TempBlk': continue subdir = os.path.join(self.dirname,blockname) if not os.path.isdir(subdir): continue seg = Segment(name = blockname) bl.segments.append( seg) global_t_start = None # Step 1 : first loop for counting - tsq file tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb') hr = HeaderReader(tsq, TsqDescription) allsig = { } allspiketr = { } allevent = { } while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_UNKNOWN': pass elif Types[evtype] == 'EVTYPE_MARK' : if global_t_start is None: global_t_start = h['timestamp'] elif Types[evtype] == 'EVTYPE_SCALER' : # TODO pass elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': # EVENTS if code not in allevent: allevent[code] = { } if channel not in allevent[code]: ea = EventArray(name = code , channel_index = channel) # for counting: ea.lazy_shape = 0 ea.maxlabelsize = 0 allevent[code][channel] = ea allevent[code][channel].lazy_shape += 1 strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) strobe = str(strobe) if len(strobe)>= allevent[code][channel].maxlabelsize: allevent[code][channel].maxlabelsize = len(strobe) #~ ev = Event() #~ ev.time = h['timestamp'] - global_t_start #~ ev.name = code #~ # it the strobe attribute masked with eventoffset #~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) #~ ev.label = str(strobe) #~ seg._events.append( ev ) elif Types[evtype] == 'EVTYPE_SNIP' : if code not in allspiketr: allspiketr[code] = { } if channel not in allspiketr[code]: allspiketr[code][channel] = { } if h['sortcode'] not in allspiketr[code][channel]: sptr = SpikeTrain([ ], units = 's', name = str(h['sortcode']), #t_start = global_t_start, t_start = 0.*pq.s, t_stop = 0.*pq.s, # temporary left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s, sampling_rate = h['frequency'] * pq.Hz, ) #~ sptr.channel = channel #sptr.annotations['channel_index'] = channel sptr.annotate(channel_index = channel) # for counting: sptr.lazy_shape = 0 sptr.pos = 0 sptr.waveformsize = h['size']-10 #~ sptr.name = str(h['sortcode']) #~ sptr.t_start = global_t_start #~ sptr.sampling_rate = h['frequency'] #~ sptr.left_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.right_sweep = (h['size']-10.)/2./h['frequency'] #~ sptr.waveformsize = h['size']-10 allspiketr[code][channel][h['sortcode']] = sptr allspiketr[code][channel][h['sortcode']].lazy_shape += 1 elif Types[evtype] == 'EVTYPE_STREAM': if code not in allsig: allsig[code] = { } if channel not in allsig[code]: #~ print 'code', code, 'channel', channel anaSig = AnalogSignal([] * pq.V, name=code, sampling_rate= h['frequency'] * pq.Hz, t_start=(h['timestamp'] - global_t_start) * pq.s, channel_index=channel) anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']]) anaSig.pos = 0 # for counting: anaSig.lazy_shape = 0 #~ anaSig.pos = 0 allsig[code][channel] = anaSig allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize if not lazy: # Step 2 : allocate memory for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V ) v[channel].pos = 0 for code, v in iteritems(allevent): for channel, ea in iteritems(v): ea.times = np.empty( (ea.lazy_shape) ) * pq.s ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) ) ea.pos = 0 for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s , name = sptr.name, t_start = sptr.t_start, t_stop = sptr.t_stop, left_sweep = sptr.left_sweep, sampling_rate = sptr.sampling_rate, waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV , ) new.annotations.update(sptr.annotations) new.pos = 0 new.waveformsize = sptr.waveformsize allsorted[sortcode] = new # Step 3 : searh sev (individual data files) or tev (common data file) # sev is for version > 70 if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')): tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb') else: tev = None for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): if PY3K: signame = anaSig.name.decode('ascii') else: signame = anaSig.name filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev') if os.path.exists(filename): anaSig.fid = open(filename, 'rb') else: anaSig.fid = tev for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): sptr.fid = tev # Step 4 : second loop for copyin chunk of data tsq.seek(0) while 1: h= hr.read_f() if h==None:break channel, code , evtype = h['channel'], h['code'], h['evtype'] if Types[evtype] == 'EVTYPE_STREAM': a = allsig[code][channel] dt = a.dtype s = int((h['size']*4-40)/dt.itemsize) a.fid.seek(h['eventoffset']) a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype) a.pos += s elif Types[evtype] == 'EVTYPE_STRON' or \ Types[evtype] == 'EVTYPE_STROFF': ea = allevent[code][channel] ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset'])) ea.labels[ea.pos] = str(strobe) ea.pos += 1 elif Types[evtype] == 'EVTYPE_SNIP': sptr = allspiketr[code][channel][h['sortcode']] sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V sptr.pos += 1 # Step 5 : populating segment for code, v in iteritems(allsig): for channel, anaSig in iteritems(v): seg.analogsignals.append( anaSig ) for code, v in iteritems(allevent): for channel, ea in iteritems(v): seg.eventarrays.append( ea ) for code, v in iteritems(allspiketr): for channel, allsorted in iteritems(v): for sortcode, sptr in iteritems(allsorted): seg.spiketrains.append( sptr ) bl.create_many_to_one_relationship() return bl