def read_segment(
self,
lazy=False,
delimiter="\t",
t_start=0. * pq.s,
unit=pq.s,
):
"""
delimiter is the columns delimiter in file "\t" or one space or two space or "," or ";"
t_start is the time start of all spiketrain 0 by default. unit is the unit of spike times,
can be a str or directly a quantity.
"""
assert not lazy, "Do not support lazy"
unit = pq.Quantity(1, unit)
seg = Segment(file_origin=os.path.basename(self.filename))
f = open(self.filename, "Ur")
for i, line in enumerate(f):
alldata = line[:-1].split(delimiter)
if alldata[-1] == "":
alldata = alldata[:-1]
if alldata[0] == "":
alldata = alldata[1:]
spike_times = np.array(alldata).astype("f")
t_stop = spike_times.max() * unit
sptr = SpikeTrain(spike_times * unit,
t_start=t_start,
t_stop=t_stop)
sptr.annotate(channel_index=i)
seg.spiketrains.append(sptr)
f.close()
seg.create_many_to_one_relationship()
return seg
def read_spiketrain(self ,
# the 2 first key arguments are imposed by neo.io API
lazy = False,
cascade = True,
segment_duration = 15.,
t_start = -1,
channel_index = 0,
):
"""
With this IO SpikeTrain can e acces directly with its channel number
"""
# There are 2 possibles behaviour for a SpikeTrain
# holding many Spike instance or directly holding spike times
# we choose here the first :
if not HAVE_SCIPY:
raise SCIPY_ERR
num_spike_by_spiketrain = 40
sr = 10000.
if lazy:
times = [ ]
else:
times = (np.random.rand(num_spike_by_spiketrain)*segment_duration +
t_start)
# create a spiketrain
spiketr = SpikeTrain(times, t_start = t_start*pq.s, t_stop = (t_start+segment_duration)*pq.s ,
units = pq.s,
name = 'it is a spiketrain from exampleio',
)
if lazy:
# we add the attribute lazy_shape with the size if loaded
spiketr.lazy_shape = (num_spike_by_spiketrain,)
# ours spiketrains also hold the waveforms:
# 1 generate a fake spike shape (2d array if trodness >1)
w1 = -stats.nct.pdf(np.arange(11,60,4), 5,20)[::-1]/3.
w2 = stats.nct.pdf(np.arange(11,60,2), 5,20)
w = np.r_[ w1 , w2 ]
w = -w/max(w)
if not lazy:
# in the neo API the waveforms attr is 3 D in case tetrode
# in our case it is mono electrode so dim 1 is size 1
waveforms = np.tile( w[np.newaxis,np.newaxis,:], ( num_spike_by_spiketrain ,1, 1) )
waveforms *= np.random.randn(*waveforms.shape)/6+1
spiketr.waveforms = waveforms*pq.mV
spiketr.sampling_rate = sr * pq.Hz
spiketr.left_sweep = 1.5* pq.s
# for attributes out of neo you can annotate
spiketr.annotate(channel_index = channel_index)
return spiketr
def read_spiketrain(self ,
# the 2 first key arguments are imposed by neo.io API
lazy = False,
cascade = True,
segment_duration = 15.,
t_start = -1,
channel_index = 0,
):
"""
With this IO SpikeTrain can e acces directly with its channel number
"""
# There are 2 possibles behaviour for a SpikeTrain
# holding many Spike instance or directly holding spike times
# we choose here the first :
if not HAVE_SCIPY:
raise SCIPY_ERR
num_spike_by_spiketrain = 40
sr = 10000.
if lazy:
times = [ ]
else:
times = (np.random.rand(num_spike_by_spiketrain)*segment_duration +
t_start)
# create a spiketrain
spiketr = SpikeTrain(times, t_start = t_start*pq.s, t_stop = (t_start+segment_duration)*pq.s ,
units = pq.s,
name = 'it is a spiketrain from exampleio',
)
if lazy:
# we add the attribute lazy_shape with the size if loaded
spiketr.lazy_shape = (num_spike_by_spiketrain,)
# ours spiketrains also hold the waveforms:
# 1 generate a fake spike shape (2d array if trodness >1)
w1 = -stats.nct.pdf(np.arange(11,60,4), 5,20)[::-1]/3.
w2 = stats.nct.pdf(np.arange(11,60,2), 5,20)
w = np.r_[ w1 , w2 ]
w = -w/max(w)
if not lazy:
# in the neo API the waveforms attr is 3 D in case tetrode
# in our case it is mono electrode so dim 1 is size 1
waveforms = np.tile( w[np.newaxis,np.newaxis,:], ( num_spike_by_spiketrain ,1, 1) )
waveforms *= np.random.randn(*waveforms.shape)/6+1
spiketr.waveforms = waveforms*pq.mV
spiketr.sampling_rate = sr * pq.Hz
spiketr.left_sweep = 1.5* pq.s
# for attributes out of neo you can annotate
spiketr.annotate(channel_index = channel_index)
return spiketr
def _extract_spikes(self, data, metadata, channel_index):
spiketrain = None
spike_times = self._extract_array(data, channel_index)
if len(spike_times) > 0:
spiketrain = SpikeTrain(spike_times,
units=pq.ms,
t_stop=spike_times.max())
spiketrain.annotate(label=metadata["label"],
channel_index=channel_index,
dt=metadata["dt"])
return spiketrain
def spiketrain_from_raw(channel: AnalogSignal, threshold: Quantity) -> SpikeTrain:
assert channel.signal.shape[1] == 1
spikes = find_spikes(channel, threshold)
signal = channel.signal.squeeze()
times = np.array([channel_index_to_time(channel, start) for start, _ in spikes]) * channel.sampling_period.units
waveforms = np.array([[signal[start:stop]] for start, stop in spikes]) * channel.units
name = f"{channel.name}#{threshold}"
result = SpikeTrain(name=name, times=times, units=channel.units, t_start=channel.t_start, t_stop=channel.t_stop,
waveforms=waveforms, sampling_rate=channel.sampling_rate, sort=True)
result.annotate(from_channel=channel.name, threshold=threshold)
return result
def _extract_spikes(self, data, metadata, channel_index, lazy):
spiketrain = None
if lazy:
if channel_index in data[:, 1]:
spiketrain = SpikeTrain([], units=pq.ms, t_stop=0.0)
spiketrain.lazy_shape = None
else:
spike_times = self._extract_array(data, channel_index)
if len(spike_times) > 0:
spiketrain = SpikeTrain(spike_times, units=pq.ms, t_stop=spike_times.max())
if spiketrain is not None:
spiketrain.annotate(label=metadata["label"],
channel_index=channel_index,
dt=metadata["dt"])
return spiketrain
def _extract_spikes(self, data, metadata, channel_index, lazy):
spiketrain = None
if lazy:
if channel_index in data[:, 1]:
spiketrain = SpikeTrain([], units=pq.ms, t_stop=0.0)
spiketrain.lazy_shape = None
else:
spike_times = self._extract_array(data, channel_index)
if len(spike_times) > 0:
spiketrain = SpikeTrain(spike_times, units=pq.ms, t_stop=spike_times.max())
if spiketrain is not None:
spiketrain.annotate(label=metadata["label"],
channel_index=channel_index,
dt=metadata["dt"])
return spiketrain
def create_all_annotated(cls):
times = cls.rquant(1, pq.s)
signal = cls.rquant(1, pq.V)
blk = Block()
blk.annotate(**cls.rdict(3))
cls.populate_dates(blk)
seg = Segment()
seg.annotate(**cls.rdict(4))
cls.populate_dates(seg)
blk.segments.append(seg)
asig = AnalogSignal(signal=signal, sampling_rate=pq.Hz)
asig.annotate(**cls.rdict(2))
seg.analogsignals.append(asig)
isig = IrregularlySampledSignal(times=times,
signal=signal,
time_units=pq.s)
isig.annotate(**cls.rdict(2))
seg.irregularlysampledsignals.append(isig)
epoch = Epoch(times=times, durations=times)
epoch.annotate(**cls.rdict(4))
seg.epochs.append(epoch)
event = Event(times=times)
event.annotate(**cls.rdict(4))
seg.events.append(event)
spiketrain = SpikeTrain(times=times, t_stop=pq.s, units=pq.s)
d = cls.rdict(6)
d["quantity"] = pq.Quantity(10, "mV")
d["qarray"] = pq.Quantity(range(10), "mA")
spiketrain.annotate(**d)
seg.spiketrains.append(spiketrain)
chx = ChannelIndex(name="achx", index=[1, 2], channel_ids=[0, 10])
chx.annotate(**cls.rdict(5))
blk.channel_indexes.append(chx)
unit = Unit()
unit.annotate(**cls.rdict(2))
chx.units.append(unit)
return blk
def read_segment(
self,
lazy=False,
cascade=True,
delimiter='\t',
t_start=0. * pq.s,
unit=pq.s,
):
"""
Arguments:
delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';'
t_start : time start of all spiketrain 0 by default
unit : unit of spike times, can be a str or directly a Quantities
"""
unit = pq.Quantity(1, unit)
seg = Segment(file_origin=os.path.basename(self.filename))
if not cascade:
return seg
f = open(self.filename, 'Ur')
for i, line in enumerate(f):
alldata = line[:-1].split(delimiter)
if alldata[-1] == '': alldata = alldata[:-1]
if alldata[0] == '': alldata = alldata[1:]
if lazy:
spike_times = []
t_stop = t_start
else:
spike_times = np.array(alldata).astype('f')
t_stop = spike_times.max() * unit
sptr = SpikeTrain(spike_times * unit,
t_start=t_start,
t_stop=t_stop)
if lazy:
sptr.lazy_shape = len(alldata)
sptr.annotate(channel_index=i)
seg.spiketrains.append(sptr)
f.close()
seg.create_many_to_one_relationship()
return seg
def create_all_annotated(cls):
times = cls.rquant(1, pq.s)
signal = cls.rquant(1, pq.V)
blk = Block()
blk.annotate(**cls.rdict(3))
seg = Segment()
seg.annotate(**cls.rdict(4))
blk.segments.append(seg)
asig = AnalogSignal(signal=signal, sampling_rate=pq.Hz)
asig.annotate(**cls.rdict(2))
seg.analogsignals.append(asig)
isig = IrregularlySampledSignal(times=times, signal=signal,
time_units=pq.s)
isig.annotate(**cls.rdict(2))
seg.irregularlysampledsignals.append(isig)
epoch = Epoch(times=times, durations=times)
epoch.annotate(**cls.rdict(4))
seg.epochs.append(epoch)
event = Event(times=times)
event.annotate(**cls.rdict(4))
seg.events.append(event)
spiketrain = SpikeTrain(times=times, t_stop=pq.s, units=pq.s)
d = cls.rdict(6)
d["quantity"] = pq.Quantity(10, "mV")
d["qarray"] = pq.Quantity(range(10), "mA")
spiketrain.annotate(**d)
seg.spiketrains.append(spiketrain)
chx = ChannelIndex(name="achx", index=[1, 2], channel_ids=[0, 10])
chx.annotate(**cls.rdict(5))
blk.channel_indexes.append(chx)
unit = Unit()
unit.annotate(**cls.rdict(2))
chx.units.append(unit)
return blk
def read_segment(self,
lazy = False,
cascade = True,
delimiter = '\t',
t_start = 0.*pq.s,
unit = pq.s,
):
"""
Arguments:
delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';'
t_start : time start of all spiketrain 0 by default
unit : unit of spike times, can be a str or directly a Quantities
"""
unit = pq.Quantity(1, unit)
seg = Segment(file_origin = os.path.basename(self.filename))
if not cascade:
return seg
f = open(self.filename, 'Ur')
for i,line in enumerate(f) :
alldata = line[:-1].split(delimiter)
if alldata[-1] == '': alldata = alldata[:-1]
if alldata[0] == '': alldata = alldata[1:]
if lazy:
spike_times = [ ]
t_stop = t_start
else:
spike_times = np.array(alldata).astype('f')
t_stop = spike_times.max()*unit
sptr = SpikeTrain(spike_times*unit, t_start=t_start, t_stop=t_stop)
if lazy:
sptr.lazy_shape = len(alldata)
sptr.annotate(channel_index = i)
seg.spiketrains.append(sptr)
f.close()
seg.create_many_to_one_relationship()
return seg
def read_segment(
self,
lazy=False,
delimiter='\t',
t_start=0. * pq.s,
unit=pq.s,
):
"""
Arguments:
delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';'
t_start : time start of all spiketrain 0 by default
unit : unit of spike times, can be a str or directly a Quantities
"""
assert not lazy, 'Do not support lazy'
unit = pq.Quantity(1, unit)
seg = Segment(file_origin=os.path.basename(self.filename))
with open(self.filename, 'r', newline=None) as f:
for i, line in enumerate(f):
alldata = line[:-1].split(delimiter)
if alldata[-1] == '':
alldata = alldata[:-1]
if alldata[0] == '':
alldata = alldata[1:]
spike_times = np.array(alldata).astype('f')
t_stop = spike_times.max() * unit
sptr = SpikeTrain(spike_times * unit,
t_start=t_start,
t_stop=t_stop)
sptr.annotate(channel_index=i)
seg.spiketrains.append(sptr)
seg.create_many_to_one_relationship()
return seg
def read_segment(self,
blockname=None,
lazy=False,
cascade=True,
sortname=''):
"""
Read a single segment from the tank. Note that TDT blocks are Neo
segments, and TDT tanks are Neo blocks, so here the 'blockname' argument
refers to the TDT block's name, which will be the Neo segment name.
'sortname' is used to specify the external sortcode generated by offline spike sorting.
if sortname=='PLX', there should be a ./sort/PLX/*.SortResult file in the tdt block,
which stores the sortcode for every spike; defaults to '', which uses the original online sort
"""
if not blockname:
blockname = os.listdir(self.dirname)[0]
if blockname == 'TempBlk': return None
if not self.is_tdtblock(blockname): return None # if not a tdt block
subdir = os.path.join(self.dirname, blockname)
if not os.path.isdir(subdir): return None
seg = Segment(name=blockname)
tankname = os.path.basename(self.dirname)
#TSQ is the global index
tsq_filename = os.path.join(subdir,
tankname + '_' + blockname + '.tsq')
dt = [
('size', 'int32'),
('evtype', 'int32'),
('code', 'S4'),
('channel', 'uint16'),
('sortcode', 'uint16'),
('timestamp', 'float64'),
('eventoffset', 'int64'),
('dataformat', 'int32'),
('frequency', 'float32'),
]
tsq = np.fromfile(tsq_filename, dtype=dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype'] == 0x8801]['timestamp'][0]
#TEV is the old data file
try:
tev_filename = os.path.join(subdir,
tankname + '_' + blockname + '.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype='uint8')
except IOError:
tev_filename = None
#if there exists an external sortcode in ./sort/[sortname]/*.SortResult (generated after offline sortting)
sortresult_filename = None
if sortname is not '':
try:
for file in os.listdir(os.path.join(subdir, 'sort', sortname)):
if file.endswith(".SortResult"):
sortresult_filename = os.path.join(
subdir, 'sort', sortname, file)
# get new sortcode
newsorcode = np.fromfile(sortresult_filename, 'int8')[
1024:] # the first 1024 byte is file header
# update the sort code with the info from this file
tsq['sortcode'][1:-1] = newsorcode
# print('sortcode updated')
break
except OSError:
sortresult_filename = None
except IOError:
sortresult_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype'] == type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code'] == code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel'] == channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [] * pq.s
labels = np.array([], dtype=str)
else:
times = (tsq[mask3]['timestamp'] -
global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view(
'float64').astype('S')
ea = Event(times=times,
name=code,
channel_index=int(channel),
labels=labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.events.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode'] == sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0] - 10
if lazy:
times = [] * pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] -
global_t_start) * pq.s
dt = np.dtype(
data_formats[tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(
tsq[mask4]['size'],
tsq[mask4]['eventoffset'],
tev_array).view(dt)
waveforms = waveforms.reshape(
nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike > 0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 * pq.s
t_stop = (tsq['timestamp'][-1] -
global_t_start) * pq.s
else:
t_start = 0 * pq.s
t_stop = 0 * pq.s
st = SpikeTrain(
times=times,
name='Chan{0} Code{1}'.format(
channel, sortcode),
t_start=t_start,
t_stop=t_stop,
waveforms=waveforms,
left_sweep=waveformsize / 2. / sr * pq.s,
sampling_rate=sr * pq.Hz,
)
st.annotate(channel_index=channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(
data_formats[tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size'] - 10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = []
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(
subdir, tankname + '_' + blockname + '_' +
signame + '_ch' + str(channel) + '.sev')
try:
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename,
dtype='uint8')
except IOError:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],
tsq[mask3]['eventoffset'],
sig_array).view(dt)
anasig = AnalogSignal(
signal=signal * pq.V,
name='{0} {1}'.format(code, channel),
sampling_rate=sr * pq.Hz,
t_start=(tsq[mask3]['timestamp'][0] -
global_t_start) * pq.s,
channel_index=int(channel))
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
return seg
def read_block(
self,
lazy=False,
cascade=True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade: return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname, blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name=blockname)
bl.segments.append(seg)
#TSQ is the global index
tsq_filename = os.path.join(subdir,
tankname + '_' + blockname + '.tsq')
dt = [
('size', 'int32'),
('evtype', 'int32'),
('code', 'S4'),
('channel', 'uint16'),
('sortcode', 'uint16'),
('timestamp', 'float64'),
('eventoffset', 'int64'),
('dataformat', 'int32'),
('frequency', 'float32'),
]
tsq = np.fromfile(tsq_filename, dtype=dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype'] == 0x8801]['timestamp'][0]
#TEV is the old data file
if os.path.exists(
os.path.join(subdir, tankname + '_' + blockname + '.tev')):
tev_filename = os.path.join(
subdir, tankname + '_' + blockname + '.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype='uint8')
else:
tev_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype'] == type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code'] == code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel'] == channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [] * pq.s
labels = np.array([], dtype=str)
else:
times = (tsq[mask3]['timestamp'] -
global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view(
'float64').astype('S')
ea = EventArray(times=times,
name=code,
channel_index=int(channel),
labels=labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.eventarrays.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode'] == sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0] - 10
if lazy:
times = [] * pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] -
global_t_start) * pq.s
dt = np.dtype(data_formats[
tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(
tsq[mask4]['size'],
tsq[mask4]['eventoffset'],
tev_array).view(dt)
waveforms = waveforms.reshape(
nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike > 0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 * pq.s
t_stop = (tsq['timestamp'][-1] -
global_t_start) * pq.s
else:
t_start = 0 * pq.s
t_stop = 0 * pq.s
st = SpikeTrain(
times=times,
name='Chan{} Code{}'.format(
channel, sortcode),
t_start=t_start,
t_stop=t_stop,
waveforms=waveforms,
left_sweep=waveformsize / 2. / sr * pq.s,
sampling_rate=sr * pq.Hz,
)
st.annotate(channel_index=channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(
data_formats[tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size'] - 10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = []
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(
subdir, tankname + '_' + blockname + '_' +
signame + '_ch' + str(channel) + '.sev')
if os.path.exists(sev_filename):
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename,
dtype='uint8')
else:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],
tsq[mask3]['eventoffset'],
sig_array).view(dt)
anasig = AnalogSignal(
signal=signal * pq.V,
name='{} {}'.format(code, channel),
sampling_rate=sr * pq.Hz,
t_start=(tsq[mask3]['timestamp'][0] -
global_t_start) * pq.s,
channel_index=int(channel))
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
bl.create_many_to_one_relationship()
return bl
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True):
"""
Read in a segment.
Arguments:
load_spike_waveform : load or not waveform of spikes (default True)
"""
fid = open(self.filename, 'rb')
globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# metadatas
seg = Segment()
seg.rec_datetime = datetime.datetime(
globalHeader.pop('Year'),
globalHeader.pop('Month'),
globalHeader.pop('Day'),
globalHeader.pop('Hour'),
globalHeader.pop('Minute'),
globalHeader.pop('Second')
)
seg.file_origin = os.path.basename(self.filename)
for key, val in globalHeader.iteritems():
seg.annotate(**{key: val})
if not cascade:
return seg
## Step 1 : read headers
# dsp channels header = spikes and waveforms
dspChannelHeaders = {}
maxunit = 0
maxchan = 0
for _ in range(globalHeader['NumDSPChannels']):
# channel is 1 based
channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None)
channelHeader['Template'] = np.array(channelHeader['Template']).reshape((5,64))
channelHeader['Boxes'] = np.array(channelHeader['Boxes']).reshape((5,2,4))
dspChannelHeaders[channelHeader['Channel']] = channelHeader
maxunit = max(channelHeader['NUnits'], maxunit)
maxchan = max(channelHeader['Channel'], maxchan)
# event channel header
eventHeaders = { }
for _ in range(globalHeader['NumEventChannels']):
eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None)
eventHeaders[eventHeader['Channel']] = eventHeader
# slow channel header = signal
slowChannelHeaders = {}
for _ in range(globalHeader['NumSlowChannels']):
slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None)
slowChannelHeaders[slowChannelHeader['Channel']] = slowChannelHeader
## Step 2 : a first loop for counting size
# signal
nb_samples = np.zeros(len(slowChannelHeaders))
sample_positions = np.zeros(len(slowChannelHeaders))
t_starts = np.zeros(len(slowChannelHeaders), dtype='f')
#spiketimes and waveform
nb_spikes = np.zeros((maxchan+1, maxunit+1) ,dtype='i')
wf_sizes = np.zeros((maxchan+1, maxunit+1, 2) ,dtype='i')
# eventarrays
nb_events = { }
#maxstrsizeperchannel = { }
for chan, h in iteritems(eventHeaders):
nb_events[chan] = 0
#maxstrsizeperchannel[chan] = 0
start = fid.tell()
while fid.tell() !=-1 :
# read block header
dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None)
if dataBlockHeader is None : break
chan = dataBlockHeader['Channel']
unit = dataBlockHeader['Unit']
n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform']
time = (dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 +
dataBlockHeader['TimeStamp'])
if dataBlockHeader['Type'] == 1:
nb_spikes[chan,unit] +=1
wf_sizes[chan,unit,:] = [n1,n2]
fid.seek(n1*n2*2,1)
elif dataBlockHeader['Type'] ==4:
#event
nb_events[chan] += 1
elif dataBlockHeader['Type'] == 5:
#continuous signal
fid.seek(n2*2, 1)
if n2> 0:
nb_samples[chan] += n2
if nb_samples[chan] ==0:
t_starts[chan] = time
## Step 3: allocating memory and 2 loop for reading if not lazy
if not lazy:
# allocating mem for signal
sigarrays = { }
for chan, h in iteritems(slowChannelHeaders):
sigarrays[chan] = np.zeros(nb_samples[chan])
# allocating mem for SpikeTrain
stimearrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object)
swfarrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object)
for (chan, unit), _ in np.ndenumerate(nb_spikes):
stimearrays[chan,unit] = np.zeros(nb_spikes[chan,unit], dtype = 'f')
if load_spike_waveform:
n1,n2 = wf_sizes[chan, unit,:]
swfarrays[chan, unit] = np.zeros( (nb_spikes[chan, unit], n1, n2 ) , dtype = 'f4' )
pos_spikes = np.zeros(nb_spikes.shape, dtype = 'i')
# allocating mem for event
eventpositions = { }
evarrays = { }
for chan, nb in iteritems(nb_events):
evarrays[chan] = {
'times': np.zeros(nb, dtype='f'),
'labels': np.zeros(nb, dtype='S4')
}
eventpositions[chan]=0
fid.seek(start)
while fid.tell() !=-1 :
dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None)
if dataBlockHeader is None : break
chan = dataBlockHeader['Channel']
n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform']
time = dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp']
time/= globalHeader['ADFrequency']
if n2 <0: break
if dataBlockHeader['Type'] == 1:
#spike
unit = dataBlockHeader['Unit']
pos = pos_spikes[chan,unit]
stimearrays[chan, unit][pos] = time
if load_spike_waveform and n1*n2 != 0 :
swfarrays[chan,unit][pos,:,:] = np.fromstring( fid.read(n1*n2*2) , dtype = 'i2').reshape(n1,n2).astype('f4')
else:
fid.seek(n1*n2*2,1)
pos_spikes[chan,unit] +=1
elif dataBlockHeader['Type'] == 4:
# event
pos = eventpositions[chan]
evarrays[chan]['times'][pos] = time
evarrays[chan]['labels'][pos] = dataBlockHeader['Unit']
eventpositions[chan]+= 1
elif dataBlockHeader['Type'] == 5:
#signal
data = np.fromstring( fid.read(n2*2) , dtype = 'i2').astype('f4')
sigarrays[chan][sample_positions[chan] : sample_positions[chan]+data.size] = data
sample_positions[chan] += data.size
## Step 4: create neo object
for chan, h in iteritems(eventHeaders):
if lazy:
times = []
labels = None
else:
times = evarrays[chan]['times']
labels = evarrays[chan]['labels']
ea = EventArray(
times*pq.s,
labels=labels,
channel_name=eventHeaders[chan]['Name'],
channel_index=chan
)
if lazy:
ea.lazy_shape = nb_events[chan]
seg.eventarrays.append(ea)
for chan, h in iteritems(slowChannelHeaders):
if lazy:
signal = [ ]
else:
if globalHeader['Version'] ==100 or globalHeader['Version'] ==101 :
gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*1000.)
elif globalHeader['Version'] ==102 :
gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain'])
elif globalHeader['Version'] >= 103:
gain = globalHeader['SlowMaxMagnitudeMV']/(.5*(2**globalHeader['BitsPerSpikeSample'])*\
slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain'])
signal = sigarrays[chan]*gain
anasig = AnalogSignal(signal*pq.V,
sampling_rate = float(slowChannelHeaders[chan]['ADFreq'])*pq.Hz,
t_start = t_starts[chan]*pq.s,
channel_index = slowChannelHeaders[chan]['Channel'],
channel_name = slowChannelHeaders[chan]['Name'],
)
if lazy:
anasig.lazy_shape = nb_samples[chan]
seg.analogsignals.append(anasig)
for (chan, unit), value in np.ndenumerate(nb_spikes):
if nb_spikes[chan, unit] == 0: continue
if lazy:
times = [ ]
waveforms = None
t_stop = 0
else:
times = stimearrays[chan,unit]
t_stop = times.max()
if load_spike_waveform:
if globalHeader['Version'] <103:
gain = 3000./(2048*dspChannelHeaders[chan]['Gain']*1000.)
elif globalHeader['Version'] >=103 and globalHeader['Version'] <105:
gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*1000.)
elif globalHeader['Version'] >105:
gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*globalHeader['SpikePreAmpGain'])
waveforms = swfarrays[chan, unit] * gain * pq.V
else:
waveforms = None
sptr = SpikeTrain(
times,
units='s',
t_stop=t_stop*pq.s,
waveforms=waveforms
)
sptr.annotate(unit_name = dspChannelHeaders[chan]['Name'])
sptr.annotate(channel_index = chan)
for key, val in dspChannelHeaders[chan].iteritems():
sptr.annotate(**{key: val})
if lazy:
sptr.lazy_shape = nb_spikes[chan,unit]
seg.spiketrains.append(sptr)
seg.create_many_to_one_relationship()
return seg
def read_one_channel_event_or_spike(self, fid, channel_num, header,
lazy=True):
# return SPikeTrain or Event
channelHeader = header.channelHeaders[channel_num]
if channelHeader.firstblock < 0:
return
if channelHeader.kind not in [2, 3, 4, 5, 6, 7, 8]:
return
# # Step 1 : type of blocks
if channelHeader.kind in [2, 3, 4]:
# Event data
fmt = [('tick', 'i4')]
elif channelHeader.kind in [5]:
# Marker data
fmt = [('tick', 'i4'), ('marker', 'i4')]
elif channelHeader.kind in [6]:
# AdcMark data
fmt = [('tick', 'i4'), ('marker', 'i4'),
('adc', 'S%d' % channelHeader.n_extra)]
elif channelHeader.kind in [7]:
# RealMark data
fmt = [('tick', 'i4'), ('marker', 'i4'),
('real', 'S%d' % channelHeader.n_extra)]
elif channelHeader.kind in [8]:
# TextMark data
fmt = [('tick', 'i4'), ('marker', 'i4'),
('label', 'S%d' % channelHeader.n_extra)]
dt = np.dtype(fmt)
## Step 2 : first read for allocating mem
fid.seek(channelHeader.firstblock)
totalitems = 0
for _ in range(channelHeader.blocks):
blockHeader = HeaderReader(fid, np.dtype(blockHeaderDesciption))
totalitems += blockHeader.items
if blockHeader.succ_block > 0:
fid.seek(blockHeader.succ_block)
#~ print 'totalitems' , totalitems
if lazy:
if channelHeader.kind in [2, 3, 4, 5, 8]:
ea = Event()
ea.annotate(channel_index=channel_num)
ea.lazy_shape = totalitems
return ea
elif channelHeader.kind in [6, 7]:
# correct value for t_stop to be put in later
sptr = SpikeTrain([] * pq.s, t_stop=1e99)
sptr.annotate(channel_index=channel_num, ced_unit = 0)
sptr.lazy_shape = totalitems
return sptr
else:
alltrigs = np.zeros(totalitems, dtype=dt)
## Step 3 : read
fid.seek(channelHeader.firstblock)
pos = 0
for _ in range(channelHeader.blocks):
blockHeader = HeaderReader(
fid, np.dtype(blockHeaderDesciption))
# read all events in block
trigs = np.fromstring(
fid.read(blockHeader.items * dt.itemsize), dtype=dt)
alltrigs[pos:pos + trigs.size] = trigs
pos += trigs.size
if blockHeader.succ_block > 0:
fid.seek(blockHeader.succ_block)
## Step 3 convert in neo standard class: eventarrays or spiketrains
alltimes = alltrigs['tick'].astype(
'f') * header.us_per_time * header.dtime_base * pq.s
if channelHeader.kind in [2, 3, 4, 5, 8]:
#events
ea = Event(alltimes)
ea.annotate(channel_index=channel_num)
if channelHeader.kind >= 5:
# Spike2 marker is closer to label sens of neo
ea.labels = alltrigs['marker'].astype('S32')
if channelHeader.kind == 8:
ea.annotate(extra_labels=alltrigs['label'])
return ea
elif channelHeader.kind in [6, 7]:
# spiketrains
# waveforms
if channelHeader.kind == 6:
waveforms = np.fromstring(alltrigs['adc'].tostring(),
dtype='i2')
waveforms = waveforms.astype(
'f4') * channelHeader.scale / 6553.6 + \
channelHeader.offset
elif channelHeader.kind == 7:
waveforms = np.fromstring(alltrigs['real'].tostring(),
dtype='f4')
if header.system_id >= 6 and channelHeader.interleave > 1:
waveforms = waveforms.reshape(
(alltimes.size, -1, channelHeader.interleave))
waveforms = waveforms.swapaxes(1, 2)
else:
waveforms = waveforms.reshape((alltimes.size, 1, -1))
if header.system_id in [1, 2, 3, 4, 5]:
sample_interval = (channelHeader.divide *
header.us_per_time *
header.time_per_adc) * 1e-6
else:
sample_interval = (channelHeader.l_chan_dvd *
header.us_per_time *
header.dtime_base)
if channelHeader.unit in unit_convert:
unit = pq.Quantity(1, unit_convert[channelHeader.unit])
else:
#print channelHeader.unit
try:
unit = pq.Quantity(1, channelHeader.unit)
except:
unit = pq.Quantity(1, '')
if len(alltimes) > 0:
# can get better value from associated AnalogSignal(s) ?
t_stop = alltimes.max()
else:
t_stop = 0.0
if not self.ced_units:
sptr = SpikeTrain(alltimes,
waveforms = waveforms*unit,
sampling_rate = (1./sample_interval)*pq.Hz,
t_stop = t_stop
)
sptr.annotate(channel_index = channel_num, ced_unit = 0)
return [sptr]
sptrs = []
for i in set(alltrigs['marker'] & 255):
sptr = SpikeTrain(alltimes[alltrigs['marker'] == i],
waveforms = waveforms[alltrigs['marker'] == i]*unit,
sampling_rate = (1./sample_interval)*pq.Hz,
t_stop = t_stop
)
sptr.annotate(channel_index = channel_num, ced_unit = i)
sptrs.append(sptr)
return sptrs
def read_nev(self, filename_nev, seg, lazy, cascade, load_waveforms = False):
# basic header
dt = [('header_id','S8'),
('ver_major','uint8'),
('ver_minor','uint8'),
('additionnal_flag', 'uint16'), # Read flags, currently basically unused
('header_size', 'uint32'), #i.e. index of first data
('packet_size', 'uint32'),# Read number of packet bytes, i.e. byte per sample
('sampling_rate', 'uint32'),# Read time resolution in Hz of time stamps, i.e. data packets
('waveform_sampling_rate', 'uint32'),# Read sampling frequency of waveforms in Hz
('window_datetime', 'S16'),
('application', 'S32'), #
('comments', 'S256'), # comments
('num_ext_header', 'uint32') #Read number of extended headers
]
nev_header = h = np.fromfile(filename_nev, count = 1, dtype = dt)[0]
version = '{0}.{1}'.format(h['ver_major'], h['ver_minor'])
assert h['header_id'].decode('ascii') == 'NEURALEV' or version == '2.1', 'Unsupported version {0}'.format(version)
version = '{0}.{1}'.format(h['ver_major'], h['ver_minor'])
seg.annotate(blackrock_version = version)
seg.rec_datetime = get_window_datetime(nev_header['window_datetime'])
sr = float(h['sampling_rate'])
wsr = float(h['waveform_sampling_rate'])
if not cascade:
return
# extented header
# this consist in N block with code 8bytes + 24 data bytes
# the data bytes depend on the code and need to be converted cafilename_nsx, segse by case
raw_ext_header = np.memmap(filename_nev, offset = np.dtype(dt).itemsize,
dtype = [('code', 'S8'), ('data', 'S24')], shape = h['num_ext_header'])
# this is for debuging
ext_header = { }
for code, dt_ext in ext_nev_header_codes.items():
sel = raw_ext_header['code']==code
ext_header[code] = raw_ext_header[sel].view(dt_ext)
# channel label
neuelbl_header = ext_header['NEUEVLBL']
# Sometimes when making the channel labels we have only one channel and so must address it differently.
try:
channel_labels = dict(zip(neuelbl_header['channel_id'], neuelbl_header['channel_label']))
except TypeError:
channel_labels = dict([(neuelbl_header['channel_id'], neuelbl_header['channel_label'])])
# TODO ext_header['DIGLABEL'] is there only one label ???? because no id in that case
# TODO ECOMMENT + CCOMMENT for annotations
# TODO NEUEVFLT for annotations
# read data packet and markers
dt0 = [('samplepos', 'uint32'),
('id', 'uint16'),
('value', 'S{0}'.format(h['packet_size']-6)),
]
data = np.memmap( filename_nev, offset = h['header_size'], dtype = dt0)
all_ids = np.unique(data['id'])
t_start = 0*pq.s
t_stop = data['samplepos'][-1]/sr*pq.s
# read event (digital 9+ analog+comment)
def create_event_array_trig_or_analog(selection, name, labelmode = None):
if lazy:
times = [ ]
labels = np.array([ ], dtype = 'S')
else:
times = data_trigger['samplepos'][selection].astype(float)/sr
if labelmode == 'digital_port':
labels = data_trigger['digital_port'][selection].astype('S2')
elif labelmode is None:
label = None
ev = EventArray(times= times*pq.s,
labels= labels,
name=name)
if lazy:
ev.lazy_shape = np.sum(is_digital)
seg.eventarrays.append(ev)
mask = (data['id']==0)
dt_trig = [('samplepos', 'uint32'),
('id', 'uint16'),
('reason', 'uint8'),
('reserved0', 'uint8'),
('digital_port', 'uint16'),
('reserved1', 'S{0}'.format(h['packet_size']-10)),
]
data_trigger = data.view(dt_trig)[mask]
# Digital Triggers (PaquetID 0)
is_digital = (data_trigger ['reason']&1)>0
create_event_array_trig_or_analog(is_digital, 'Digital trigger', labelmode = 'digital_port' )
# Analog Triggers (PaquetID 0)
if version in ['2.1', '2.2' ]:
for i in range(5):
is_analog = (data_trigger ['reason']&(2**(i+1)))>0
create_event_array_trig_or_analog(is_analog, 'Analog trigger {0}'.format(i), labelmode = None)
# Comments
mask = (data['id']==0xFFF)
dt_comments = [('samplepos', 'uint32'),
('id', 'uint16'),
('charset', 'uint8'),
('reserved0', 'uint8'),
('color', 'uint32'),
('comment', 'S{0}'.format(h['packet_size']-12)),
]
data_comments = data.view(dt_comments)[mask]
if data_comments.size>0:
if lazy:
times = [ ]
labels = [ ]
else:
times = data_comments['samplepos'].astype(float)/sr
labels = data_comments['comment'].astype('S')
ev = EventArray(times= times*pq.s,
labels= labels,
name='Comments')
if lazy:
ev.lazy_shape = np.sum(is_digital)
seg.eventarrays.append(ev)
# READ Spike channel
channel_ids = all_ids[(all_ids>0) & (all_ids<=2048)]
# get the dtype of waveform (this is stupidly complicated)
if nev_header['additionnal_flag']&0x1:
#dtype_waveforms = { k:'int16' for k in channel_ids }
dtype_waveforms = dict( (k,'int16') for k in channel_ids)
else:
# there is a code electrodes by electrodes given the approiate dtype
neuewav_header = ext_header['NEUEVWAV']
dtype_waveform = dict(zip(neuewav_header['channel_id'], neuewav_header['num_bytes_per_waveform']))
dtypes_conv = { 0: 'int8', 1 : 'int8', 2: 'int16', 4 : 'int32' }
#dtype_waveforms = { k:dtypes_conv[v] for k,v in dtype_waveform.items() }
dtype_waveforms = dict( (k,dtypes_conv[v]) for k,v in dtype_waveform.items() )
dt2 = [('samplepos', 'uint32'),
('id', 'uint16'),
('cluster', 'uint8'),
('reserved0', 'uint8'),
('waveform','uint8',(h['packet_size']-8, )),
]
data_spike = data.view(dt2)
for channel_id in channel_ids:
data_spike_chan = data_spike[data['id']==channel_id]
cluster_ids = np.unique(data_spike_chan['cluster'])
for cluster_id in cluster_ids:
if cluster_id==0:
name = 'unclassified'
elif cluster_id==255:
name = 'noise'
else:
name = 'Cluster {0}'.format(cluster_id)
name = 'Channel {0} '.format(channel_id)+name
data_spike_chan_clus = data_spike_chan[data_spike_chan['cluster']==cluster_id]
n_spike = data_spike_chan_clus.size
waveforms, w_sampling_rate, left_sweep = None, None, None
if lazy:
times = [ ]
else:
times = data_spike_chan_clus['samplepos'].astype(float)/sr
if load_waveforms:
dtype_waveform = dtype_waveforms[channel_id]
waveform_size = (h['packet_size']-8)/np.dtype(dtype_waveform).itemsize
waveforms = data_spike_chan_clus['waveform'].flatten().view(dtype_waveform)
waveforms = waveforms.reshape(n_spike,1, waveform_size)
waveforms =waveforms*pq.uV
w_sampling_rate = wsr*pq.Hz
left_sweep = waveform_size//2/sr*pq.s
st = SpikeTrain(times = times*pq.s, name = name,
t_start = t_start, t_stop =t_stop,
waveforms = waveforms, sampling_rate = w_sampling_rate, left_sweep = left_sweep)
st.annotate(channel_index = int(channel_id))
if lazy:
st.lazy_shape = n_spike
seg.spiketrains.append(st)
def _find_action_potentials_on_tracks(ap_tracks: Iterable[APTrack],
el_stimuli: Event,
signal: Union[IrregularlySampledSignal,
AnalogSignal],
window_size: Quantity = Quantity(
0.003, "s"),
sampling_rate=None) -> SpikeTrain:
# TODO implement this function for analog signals and make it reusable
if isinstance(signal, IrregularlySampledSignal) and sampling_rate is None:
raise ValueError(
"If an irregularly sampled signal is passed, you need to set the sampling rate!"
)
elif isinstance(signal, AnalogSignal):
sampling_rate = signal.sampling_rate
# initialize our list of action_potentials
ap_times = []
ap_waveforms = []
# iterate over all the tracks
for track_idx, ap_track in enumerate(ap_tracks):
# first, get the template of our current AP track
if ap_track.ap_template == None:
warnings.warn(
f"""No AP template for AP track no. {track_idx}! Cannot extract APs for this track."""
)
continue
else:
ap_template = ap_track.ap_template
try:
# then, slide the template over the window, calculating cross correlation for all the datapoints
for sweep_idx, ap_latency in tqdm(zip(ap_track.sweep_idcs, ap_track.latencies), total = len(ap_track), \
desc = f"""Processing AP Track {track_idx} with {len(ap_track)} latencies."""):
# we need the time of the main pulse and add the latency to define the point around which we want to search
window_center_time = el_stimuli.times[sweep_idx] + ap_latency
# now, we define the indices of the first and last data points that we consider for our windowing
if isinstance(signal, IrregularlySampledSignal):
signal: IrregularlySampledSignal
# find first signal index
first_signal_idx = bisect.bisect_left(
signal.times, (window_center_time - window_size -
ap_template.duration / 2))
# find last signal index
last_signal_idx = bisect.bisect_left(
signal.times, (window_center_time + window_size +
ap_template.duration / 2))
elif isinstance(signal, AnalogSignal):
# TODO Check why this function is much slower for analog signals
first_signal_idx = floor(
(window_center_time - window_size -
(ap_template.duration / 2.0)) * signal.sampling_rate)
last_signal_idx = floor(
(window_center_time + window_size +
(ap_template.duration / 2.0)) * signal.sampling_rate)
# slide the template over the window
correlations = sliding_window_normalized_cross_correlation(
signal[first_signal_idx:last_signal_idx],
ap_template.signal_template)
# then, retrieve the index for which we had the maximum correlation
max_correlation_idx = np.argmax(
correlations) + first_signal_idx
# finally, append the starting time and
ap_times.append(signal.times[max_correlation_idx])
ap_waveforms.append(
signal[max_correlation_idx:max_correlation_idx +
len(ap_template)])
except Exception as ex:
traceback.print_exc()
# sort the APs and return spiketrain object
ap_times = sorted(ap_times)
# build the waveforms array
num_aps = len(ap_waveforms)
max_len = max([len(ap) for ap in ap_waveforms])
waveforms = np.zeros(shape=(num_aps, 1, max_len),
dtype=np.float64) * signal.units
for ap_idx, ap_waveform in enumerate(ap_waveforms):
waveforms[ap_idx, 0, 0:len(ap_waveform)] = ap_waveform.ravel()
result = SpikeTrain(times=Quantity(ap_times, "s"),
t_start=signal.t_start,
t_stop=signal.t_stop,
name="APs from tracks",
waveforms=waveforms,
sampling_rate=sampling_rate)
result.annotate(id=f"{TypeID.ACTION_POTENTIAL.value}.0",
type_id=TypeID.ACTION_POTENTIAL.value)
return result
def read_segment(self, lazy=False, cascade=True):
fid = open(self.filename, 'rb')
global_header = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# ~ print globalHeader
#~ print 'version' , globalHeader['version']
seg = Segment()
seg.file_origin = os.path.basename(self.filename)
seg.annotate(neuroexplorer_version=global_header['version'])
seg.annotate(comment=global_header['comment'])
if not cascade:
return seg
offset = 544
for i in range(global_header['nvar']):
entity_header = HeaderReader(fid, EntityHeader).read_f(
offset=offset + i * 208)
entity_header['name'] = entity_header['name'].replace('\x00', '')
#print 'i',i, entityHeader['type']
if entity_header['type'] == 0:
# neuron
if lazy:
spike_times = [] * pq.s
else:
spike_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
spike_times = spike_times.astype('f8') / global_header[
'freq'] * pq.s
sptr = SpikeTrain(
times=spike_times,
t_start=global_header['tbeg'] /
global_header['freq'] * pq.s,
t_stop=global_header['tend'] /
global_header['freq'] * pq.s,
name=entity_header['name'])
if lazy:
sptr.lazy_shape = entity_header['n']
sptr.annotate(channel_index=entity_header['WireNumber'])
seg.spiketrains.append(sptr)
if entity_header['type'] == 1:
# event
if lazy:
event_times = [] * pq.s
else:
event_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
event_times = event_times.astype('f8') / global_header[
'freq'] * pq.s
labels = np.array([''] * event_times.size, dtype='S')
evar = Event(times=event_times, labels=labels,
channel_name=entity_header['name'])
if lazy:
evar.lazy_shape = entity_header['n']
seg.events.append(evar)
if entity_header['type'] == 2:
# interval
if lazy:
start_times = [] * pq.s
stop_times = [] * pq.s
else:
start_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
start_times = start_times.astype('f8') / global_header[
'freq'] * pq.s
stop_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'] +
entity_header['n'] * 4)
stop_times = stop_times.astype('f') / global_header[
'freq'] * pq.s
epar = Epoch(times=start_times,
durations=stop_times - start_times,
labels=np.array([''] * start_times.size,
dtype='S'),
channel_name=entity_header['name'])
if lazy:
epar.lazy_shape = entity_header['n']
seg.epochs.append(epar)
if entity_header['type'] == 3:
# spiketrain and wavefoms
if lazy:
spike_times = [] * pq.s
waveforms = None
else:
spike_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
spike_times = spike_times.astype('f8') / global_header[
'freq'] * pq.s
waveforms = np.memmap(self.filename, np.dtype('i2'), 'r',
shape=(entity_header['n'], 1,
entity_header['NPointsWave']),
offset=entity_header['offset'] +
entity_header['n'] * 4)
waveforms = (waveforms.astype('f') *
entity_header['ADtoMV'] +
entity_header['MVOffset']) * pq.mV
t_stop = global_header['tend'] / global_header['freq'] * pq.s
if spike_times.size > 0:
t_stop = max(t_stop, max(spike_times))
sptr = SpikeTrain(
times=spike_times,
t_start=global_header['tbeg'] /
global_header['freq'] * pq.s,
#~ t_stop = max(globalHeader['tend']/
#~ globalHeader['freq']*pq.s,max(spike_times)),
t_stop=t_stop, name=entity_header['name'],
waveforms=waveforms,
sampling_rate=entity_header['WFrequency'] * pq.Hz,
left_sweep=0 * pq.ms)
if lazy:
sptr.lazy_shape = entity_header['n']
sptr.annotate(channel_index=entity_header['WireNumber'])
seg.spiketrains.append(sptr)
if entity_header['type'] == 4:
# popvectors
pass
if entity_header['type'] == 5:
# analog
timestamps = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
timestamps = timestamps.astype('f8') / global_header['freq']
fragment_starts = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
fragment_starts = fragment_starts.astype('f8') / global_header[
'freq']
t_start = timestamps[0] - fragment_starts[0] / float(
entity_header['WFrequency'])
del timestamps, fragment_starts
if lazy:
signal = [] * pq.mV
else:
signal = np.memmap(self.filename, np.dtype('i2'), 'r',
shape=(entity_header['NPointsWave']),
offset=entity_header['offset'])
signal = signal.astype('f')
signal *= entity_header['ADtoMV']
signal += entity_header['MVOffset']
signal = signal * pq.mV
ana_sig = AnalogSignal(
signal=signal, t_start=t_start * pq.s,
sampling_rate=entity_header['WFrequency'] * pq.Hz,
name=entity_header['name'],
channel_index=entity_header['WireNumber'])
if lazy:
ana_sig.lazy_shape = entity_header['NPointsWave']
seg.analogsignals.append(ana_sig)
if entity_header['type'] == 6:
# markers : TO TEST
if lazy:
times = [] * pq.s
labels = np.array([], dtype='S')
markertype = None
else:
times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
times = times.astype('f8') / global_header['freq'] * pq.s
fid.seek(entity_header['offset'] + entity_header['n'] * 4)
markertype = fid.read(64).replace('\x00', '')
labels = np.memmap(
self.filename, np.dtype(
'S' + str(entity_header['MarkerLength'])),
'r', shape=(entity_header['n']),
offset=entity_header['offset'] +
entity_header['n'] * 4 + 64)
ea = Event(times=times,
labels=labels.view(np.ndarray),
name=entity_header['name'],
channel_index=entity_header['WireNumber'],
marker_type=markertype)
if lazy:
ea.lazy_shape = entity_header['n']
seg.events.append(ea)
seg.create_many_to_one_relationship()
return seg
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True):
"""
"""
fid = open(self.filename, "rb")
globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# metadatas
seg = Segment()
seg.rec_datetime = datetime.datetime(
globalHeader["Year"],
globalHeader["Month"],
globalHeader["Day"],
globalHeader["Hour"],
globalHeader["Minute"],
globalHeader["Second"],
)
seg.file_origin = os.path.basename(self.filename)
seg.annotate(plexon_version=globalHeader["Version"])
if not cascade:
return seg
## Step 1 : read headers
# dsp channels header = sipkes and waveforms
dspChannelHeaders = {}
maxunit = 0
maxchan = 0
for _ in range(globalHeader["NumDSPChannels"]):
# channel is 1 based
channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None)
channelHeader["Template"] = np.array(channelHeader["Template"]).reshape((5, 64))
channelHeader["Boxes"] = np.array(channelHeader["Boxes"]).reshape((5, 2, 4))
dspChannelHeaders[channelHeader["Channel"]] = channelHeader
maxunit = max(channelHeader["NUnits"], maxunit)
maxchan = max(channelHeader["Channel"], maxchan)
# event channel header
eventHeaders = {}
for _ in range(globalHeader["NumEventChannels"]):
eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None)
eventHeaders[eventHeader["Channel"]] = eventHeader
# slow channel header = signal
slowChannelHeaders = {}
for _ in range(globalHeader["NumSlowChannels"]):
slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None)
slowChannelHeaders[slowChannelHeader["Channel"]] = slowChannelHeader
## Step 2 : a first loop for counting size
# signal
nb_samples = np.zeros(len(slowChannelHeaders))
sample_positions = np.zeros(len(slowChannelHeaders))
t_starts = np.zeros(len(slowChannelHeaders), dtype="f")
# spiketimes and waveform
nb_spikes = np.zeros((maxchan + 1, maxunit + 1), dtype="i")
wf_sizes = np.zeros((maxchan + 1, maxunit + 1, 2), dtype="i")
# eventarrays
nb_events = {}
# maxstrsizeperchannel = { }
for chan, h in iteritems(eventHeaders):
nb_events[chan] = 0
# maxstrsizeperchannel[chan] = 0
start = fid.tell()
while fid.tell() != -1:
# read block header
dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None)
if dataBlockHeader is None:
break
chan = dataBlockHeader["Channel"]
unit = dataBlockHeader["Unit"]
n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"]
time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"]
if dataBlockHeader["Type"] == 1:
nb_spikes[chan, unit] += 1
wf_sizes[chan, unit, :] = [n1, n2]
fid.seek(n1 * n2 * 2, 1)
elif dataBlockHeader["Type"] == 4:
# event
nb_events[chan] += 1
elif dataBlockHeader["Type"] == 5:
# continuous signal
fid.seek(n2 * 2, 1)
if n2 > 0:
nb_samples[chan] += n2
if nb_samples[chan] == 0:
t_starts[chan] = time
## Step 3: allocating memory and 2 loop for reading if not lazy
if not lazy:
# allocating mem for signal
sigarrays = {}
for chan, h in iteritems(slowChannelHeaders):
sigarrays[chan] = np.zeros(nb_samples[chan])
# allocating mem for SpikeTrain
stimearrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object)
swfarrays = np.zeros((maxchan + 1, maxunit + 1), dtype=object)
for (chan, unit), _ in np.ndenumerate(nb_spikes):
stimearrays[chan, unit] = np.zeros(nb_spikes[chan, unit], dtype="f")
if load_spike_waveform:
n1, n2 = wf_sizes[chan, unit, :]
swfarrays[chan, unit] = np.zeros((nb_spikes[chan, unit], n1, n2), dtype="f4")
pos_spikes = np.zeros(nb_spikes.shape, dtype="i")
# allocating mem for event
eventpositions = {}
evarrays = {}
for chan, nb in iteritems(nb_events):
evarrays[chan] = np.zeros(nb, dtype="f")
eventpositions[chan] = 0
fid.seek(start)
while fid.tell() != -1:
dataBlockHeader = HeaderReader(fid, DataBlockHeader).read_f(offset=None)
if dataBlockHeader is None:
break
chan = dataBlockHeader["Channel"]
n1, n2 = dataBlockHeader["NumberOfWaveforms"], dataBlockHeader["NumberOfWordsInWaveform"]
time = dataBlockHeader["UpperByteOf5ByteTimestamp"] * 2.0 ** 32 + dataBlockHeader["TimeStamp"]
time /= globalHeader["ADFrequency"]
if n2 < 0:
break
if dataBlockHeader["Type"] == 1:
# spike
unit = dataBlockHeader["Unit"]
pos = pos_spikes[chan, unit]
stimearrays[chan, unit][pos] = time
if load_spike_waveform and n1 * n2 != 0:
swfarrays[chan, unit][pos, :, :] = (
np.fromstring(fid.read(n1 * n2 * 2), dtype="i2").reshape(n1, n2).astype("f4")
)
else:
fid.seek(n1 * n2 * 2, 1)
pos_spikes[chan, unit] += 1
elif dataBlockHeader["Type"] == 4:
# event
pos = eventpositions[chan]
evarrays[chan][pos] = time
eventpositions[chan] += 1
elif dataBlockHeader["Type"] == 5:
# signal
data = np.fromstring(fid.read(n2 * 2), dtype="i2").astype("f4")
sigarrays[chan][sample_positions[chan] : sample_positions[chan] + data.size] = data
sample_positions[chan] += data.size
## Step 3: create neo object
for chan, h in iteritems(eventHeaders):
if lazy:
times = []
else:
times = evarrays[chan]
ea = EventArray(times * pq.s, channel_name=eventHeaders[chan]["Name"], channel_index=chan)
if lazy:
ea.lazy_shape = nb_events[chan]
seg.eventarrays.append(ea)
for chan, h in iteritems(slowChannelHeaders):
if lazy:
signal = []
else:
if globalHeader["Version"] == 100 or globalHeader["Version"] == 101:
gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * 1000.0)
elif globalHeader["Version"] == 102:
gain = 5000.0 / (2048 * slowChannelHeaders[chan]["Gain"] * slowChannelHeaders[chan]["PreampGain"])
elif globalHeader["Version"] >= 103:
gain = globalHeader["SlowMaxMagnitudeMV"] / (
0.5
* (2 ** globalHeader["BitsPerSpikeSample"])
* slowChannelHeaders[chan]["Gain"]
* slowChannelHeaders[chan]["PreampGain"]
)
signal = sigarrays[chan] * gain
anasig = AnalogSignal(
signal * pq.V,
sampling_rate=float(slowChannelHeaders[chan]["ADFreq"]) * pq.Hz,
t_start=t_starts[chan] * pq.s,
channel_index=slowChannelHeaders[chan]["Channel"],
channel_name=slowChannelHeaders[chan]["Name"],
)
if lazy:
anasig.lazy_shape = nb_samples[chan]
seg.analogsignals.append(anasig)
for (chan, unit), value in np.ndenumerate(nb_spikes):
if nb_spikes[chan, unit] == 0:
continue
if lazy:
times = []
waveforms = None
t_stop = 0
else:
times = stimearrays[chan, unit]
t_stop = times.max()
if load_spike_waveform:
if globalHeader["Version"] < 103:
gain = 3000.0 / (2048 * dspChannelHeaders[chan]["Gain"] * 1000.0)
elif globalHeader["Version"] >= 103 and globalHeader["Version"] < 105:
gain = globalHeader["SpikeMaxMagnitudeMV"] / (
0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * 1000.0
)
elif globalHeader["Version"] > 105:
gain = globalHeader["SpikeMaxMagnitudeMV"] / (
0.5 * 2.0 ** (globalHeader["BitsPerSpikeSample"]) * globalHeader["SpikePreAmpGain"]
)
waveforms = swfarrays[chan, unit] * gain * pq.V
else:
waveforms = None
sptr = SpikeTrain(times, units="s", t_stop=t_stop * pq.s, waveforms=waveforms)
sptr.annotate(unit_name=dspChannelHeaders[chan]["Name"])
sptr.annotate(channel_index=chan)
if lazy:
sptr.lazy_shape = nb_spikes[chan, unit]
seg.spiketrains.append(sptr)
seg.create_many_to_one_relationship()
return seg
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
global_t_start = None
# Step 1 : first loop for counting - tsq file
tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb')
hr = HeaderReader(tsq, TsqDescription)
allsig = { }
allspiketr = { }
allevent = { }
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_UNKNOWN':
pass
elif Types[evtype] == 'EVTYPE_MARK' :
if global_t_start is None:
global_t_start = h['timestamp']
elif Types[evtype] == 'EVTYPE_SCALER' :
# TODO
pass
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
# EVENTS
if code not in allevent:
allevent[code] = { }
if channel not in allevent[code]:
ea = EventArray(name = code , channel_index = channel)
# for counting:
ea.lazy_shape = 0
ea.maxlabelsize = 0
allevent[code][channel] = ea
allevent[code][channel].lazy_shape += 1
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
strobe = str(strobe)
if len(strobe)>= allevent[code][channel].maxlabelsize:
allevent[code][channel].maxlabelsize = len(strobe)
#~ ev = Event()
#~ ev.time = h['timestamp'] - global_t_start
#~ ev.name = code
#~ # it the strobe attribute masked with eventoffset
#~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
#~ ev.label = str(strobe)
#~ seg._events.append( ev )
elif Types[evtype] == 'EVTYPE_SNIP' :
if code not in allspiketr:
allspiketr[code] = { }
if channel not in allspiketr[code]:
allspiketr[code][channel] = { }
if h['sortcode'] not in allspiketr[code][channel]:
sptr = SpikeTrain([ ], units = 's',
name = str(h['sortcode']),
#t_start = global_t_start,
t_start = 0.*pq.s,
t_stop = 0.*pq.s, # temporary
left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s,
sampling_rate = h['frequency'] * pq.Hz,
)
#~ sptr.channel = channel
#sptr.annotations['channel_index'] = channel
sptr.annotate(channel_index = channel)
# for counting:
sptr.lazy_shape = 0
sptr.pos = 0
sptr.waveformsize = h['size']-10
#~ sptr.name = str(h['sortcode'])
#~ sptr.t_start = global_t_start
#~ sptr.sampling_rate = h['frequency']
#~ sptr.left_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.right_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.waveformsize = h['size']-10
allspiketr[code][channel][h['sortcode']] = sptr
allspiketr[code][channel][h['sortcode']].lazy_shape += 1
elif Types[evtype] == 'EVTYPE_STREAM':
if code not in allsig:
allsig[code] = { }
if channel not in allsig[code]:
#~ print 'code', code, 'channel', channel
anaSig = AnalogSignal([] * pq.V,
name=code,
sampling_rate=
h['frequency'] * pq.Hz,
t_start=(h['timestamp'] -
global_t_start) * pq.s,
channel_index=channel)
anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']])
anaSig.pos = 0
# for counting:
anaSig.lazy_shape = 0
#~ anaSig.pos = 0
allsig[code][channel] = anaSig
allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize
if not lazy:
# Step 2 : allocate memory
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V )
v[channel].pos = 0
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
ea.times = np.empty( (ea.lazy_shape) ) * pq.s
ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) )
ea.pos = 0
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s ,
name = sptr.name,
t_start = sptr.t_start,
t_stop = sptr.t_stop,
left_sweep = sptr.left_sweep,
sampling_rate = sptr.sampling_rate,
waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV ,
)
new.annotations.update(sptr.annotations)
new.pos = 0
new.waveformsize = sptr.waveformsize
allsorted[sortcode] = new
# Step 3 : searh sev (individual data files) or tev (common data file)
# sev is for version > 70
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb')
else:
tev = None
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
if PY3K:
signame = anaSig.name.decode('ascii')
else:
signame = anaSig.name
filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev')
if os.path.exists(filename):
anaSig.fid = open(filename, 'rb')
else:
anaSig.fid = tev
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
sptr.fid = tev
# Step 4 : second loop for copyin chunk of data
tsq.seek(0)
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_STREAM':
a = allsig[code][channel]
dt = a.dtype
s = int((h['size']*4-40)/dt.itemsize)
a.fid.seek(h['eventoffset'])
a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype)
a.pos += s
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
ea = allevent[code][channel]
ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
ea.labels[ea.pos] = str(strobe)
ea.pos += 1
elif Types[evtype] == 'EVTYPE_SNIP':
sptr = allspiketr[code][channel][h['sortcode']]
sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s
sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s
sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V
sptr.pos += 1
# Step 5 : populating segment
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
seg.analogsignals.append( anaSig )
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
seg.eventarrays.append( ea )
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
seg.spiketrains.append( sptr )
bl.create_many_to_one_relationship()
return bl
def read_segment(self, blockname=None, lazy=False, cascade=True, sortname=''):
"""
Read a single segment from the tank. Note that TDT blocks are Neo
segments, and TDT tanks are Neo blocks, so here the 'blockname' argument
refers to the TDT block's name, which will be the Neo segment name.
sortname is used to specify the external sortcode generated by offline spike sorting, if sortname=='PLX',
there should be a ./sort/PLX/*.SortResult file in the tdt block, which stores the sortcode for every spike,
default to '', which uses the original online sort
"""
if not blockname:
blockname = os.listdir(self.dirname)[0]
if blockname == 'TempBlk': return None
if not self.is_tdtblock(blockname): return None # if not a tdt block
subdir = os.path.join(self.dirname, blockname)
if not os.path.isdir(subdir): return None
seg = Segment(name=blockname)
tankname = os.path.basename(self.dirname)
#TSQ is the global index
tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq')
dt = [('size','int32'),
('evtype','int32'),
('code','S4'),
('channel','uint16'),
('sortcode','uint16'),
('timestamp','float64'),
('eventoffset','int64'),
('dataformat','int32'),
('frequency','float32'),
]
tsq = np.fromfile(tsq_filename, dtype=dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0]
#TEV is the old data file
try:
tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype='uint8')
except IOError:
tev_filename = None
#if exists an external sortcode in ./sort/[sortname]/*.SortResult (generated after offline sortting)
sortresult_filename = None
if sortname is not '':
try:
for file in os.listdir(os.path.join(subdir, 'sort', sortname)):
if file.endswith(".SortResult"):
sortresult_filename = os.path.join(subdir, 'sort', sortname, file)
# get new sortcode
newsorcode = np.fromfile(sortresult_filename,'int8')[1024:] # the first 1024 byte is file header
# update the sort code with the info from this file
tsq['sortcode'][1:-1]=newsorcode
# print('sortcode updated')
break
except OSError:
sortresult_filename = None
except IOError:
sortresult_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype']==type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code']==code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel']==channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [ ]*pq.s
labels = np.array([ ], dtype=str)
else:
times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view('float64').astype('S')
ea = Event(times=times,
name=code,
channel_index=int(channel),
labels=labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.events.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode']==sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0]-10
if lazy:
times = [ ]*pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt)
waveforms = waveforms.reshape(nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike > 0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 *pq.s
t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s
else:
t_start = 0 *pq.s
t_stop = 0 *pq.s
st = SpikeTrain(times = times,
name = 'Chan{0} Code{1}'.format(channel,sortcode),
t_start = t_start,
t_stop = t_stop,
waveforms = waveforms,
left_sweep = waveformsize/2./sr * pq.s,
sampling_rate = sr * pq.Hz,
)
st.annotate(channel_index=channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size']-10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = [ ]
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev')
try:
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename, dtype='uint8')
except IOError:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt)
anasig = AnalogSignal(signal = signal* pq.V,
name = '{0} {1}'.format(code, channel),
sampling_rate = sr * pq.Hz,
t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s,
channel_index = int(channel)
)
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
return seg
def read_nev(self, filename_nev, seg, lazy, cascade, load_waveforms=False):
# basic hedaer
dt = [
('header_id', 'S8'),
('ver_major', 'uint8'),
('ver_minor', 'uint8'),
('additionnal_flag',
'uint16'), # Read flags, currently basically unused
('header_size', 'uint32'), #i.e. index of first data
('packet_size',
'uint32'), # Read number of packet bytes, i.e. byte per sample
('sampling_rate', 'uint32'
), # Read time resolution in Hz of time stamps, i.e. data packets
('waveform_sampling_rate',
'uint32'), # Read sampling frequency of waveforms in Hz
('window_datetime', 'S16'),
('application', 'S32'), #
('comments', 'S256'), # comments
('num_ext_header', 'uint32') #Read number of extended headers
]
nev_header = h = np.fromfile(filename_nev, count=1, dtype=dt)[0]
version = '{}.{}'.format(h['ver_major'], h['ver_minor'])
assert h['header_id'].decode(
'ascii'
) == 'NEURALEV' or version == '2.1', 'Unsupported version {}'.format(
version)
version = '{}.{}'.format(h['ver_major'], h['ver_minor'])
seg.annotate(blackrock_version=version)
seg.rec_datetime = get_window_datetime(nev_header['window_datetime'])
sr = float(h['sampling_rate'])
wsr = float(h['waveform_sampling_rate'])
if not cascade:
return
# extented header
# this consist in N block with code 8bytes + 24 data bytes
# the data bytes depend on the code and need to be converted cafilename_nsx, segse by case
raw_ext_header = np.memmap(filename_nev,
offset=np.dtype(dt).itemsize,
dtype=[('code', 'S8'), ('data', 'S24')],
shape=h['num_ext_header'])
# this is for debuging
ext_header = {}
for code, dt_ext in ext_nev_header_codes.items():
sel = raw_ext_header['code'] == code
ext_header[code] = raw_ext_header[sel].view(dt_ext)
# channel label
neuelbl_header = ext_header['NEUEVLBL']
channel_labels = dict(
zip(neuelbl_header['channel_id'], neuelbl_header['channel_label']))
# TODO ext_header['DIGLABEL'] is there only one label ???? because no id in that case
# TODO ECOMMENT + CCOMMENT for annotations
# TODO NEUEVFLT for annotations
# read data packet and markers
dt0 = [
('samplepos', 'uint32'),
('id', 'uint16'),
('value', 'S{}'.format(h['packet_size'] - 6)),
]
data = np.memmap(filename_nev, offset=h['header_size'], dtype=dt0)
all_ids = np.unique(data['id'])
t_start = 0 * pq.s
t_stop = data['samplepos'][-1] / sr * pq.s
# read event (digital 9+ analog+comment)
def create_event_array_trig_or_analog(selection, name, labelmode=None):
if lazy:
times = []
labels = np.array([], dtype='S')
else:
times = data_trigger['samplepos'][selection].astype(float) / sr
if labelmode == 'digital_port':
labels = data_trigger['digital_port'][selection].astype(
'S2')
elif labelmode is None:
label = None
ev = EventArray(times=times * pq.s, labels=labels, name=name)
if lazy:
ev.lazy_shape = np.sum(is_digital)
seg.eventarrays.append(ev)
mask = (data['id'] == 0)
dt_trig = [
('samplepos', 'uint32'),
('id', 'uint16'),
('reason', 'uint8'),
('reserved0', 'uint8'),
('digital_port', 'uint16'),
('reserved1', 'S{}'.format(h['packet_size'] - 10)),
]
data_trigger = data.view(dt_trig)[mask]
# Digital Triggers (PaquetID 0)
is_digital = (data_trigger['reason'] & 1) > 0
create_event_array_trig_or_analog(is_digital,
'Digital trigger',
labelmode='digital_port')
# Analog Triggers (PaquetID 0)
if version in ['2.1', '2.2']:
for i in range(5):
is_analog = (data_trigger['reason'] & (2**(i + 1))) > 0
create_event_array_trig_or_analog(
is_analog, 'Analog trigger {}'.format(i), labelmode=None)
# Comments
mask = (data['id'] == 0xFFF)
dt_comments = [
('samplepos', 'uint32'),
('id', 'uint16'),
('charset', 'uint8'),
('reserved0', 'uint8'),
('color', 'uint32'),
('comment', 'S{}'.format(h['packet_size'] - 12)),
]
data_comments = data.view(dt_comments)[mask]
if data_comments.size > 0:
if lazy:
times = []
labels = []
else:
times = data_comments['samplepos'].astype(float) / sr
labels = data_comments['comment'].astype('S')
ev = EventArray(times=times * pq.s, labels=labels, name='Comments')
if lazy:
ev.lazy_shape = np.sum(is_digital)
seg.eventarrays.append(ev)
# READ Spike channel
channel_ids = all_ids[(all_ids > 0) & (all_ids <= 2048)]
# get the dtype of waveform (this is stupidly complicated)
if nev_header['additionnal_flag'] & 0x1:
#dtype_waveforms = { k:'int16' for k in channel_ids }
dtype_waveforms = dict((k, 'int16') for k in channel_ids)
else:
# there is a code electrodes by electrodes given the approiate dtype
neuewav_header = ext_header['NEUEVWAV']
dtype_waveform = dict(
zip(neuewav_header['channel_id'],
neuewav_header['num_bytes_per_waveform']))
dtypes_conv = {0: 'int8', 1: 'int8', 2: 'int16', 4: 'int32'}
#dtype_waveforms = { k:dtypes_conv[v] for k,v in dtype_waveform.items() }
dtype_waveforms = dict(
(k, dtypes_conv[v]) for k, v in dtype_waveform.items())
dt2 = [
('samplepos', 'uint32'),
('id', 'uint16'),
('cluster', 'uint8'),
('reserved0', 'uint8'),
('waveform', 'uint8', (h['packet_size'] - 8, )),
]
data_spike = data.view(dt2)
for channel_id in channel_ids:
data_spike_chan = data_spike[data['id'] == channel_id]
cluster_ids = np.unique(data_spike_chan['cluster'])
for cluster_id in cluster_ids:
if cluster_id == 0:
name = 'unclassified'
elif cluster_id == 255:
name = 'noise'
else:
name = 'Cluster {}'.format(cluster_id)
name = 'Channel {} '.format(channel_id) + name
data_spike_chan_clus = data_spike_chan[
data_spike_chan['cluster'] == cluster_id]
n_spike = data_spike_chan_clus.size
waveforms, w_sampling_rate, left_sweep = None, None, None
if lazy:
times = []
else:
times = data_spike_chan_clus['samplepos'].astype(
float) / sr
if load_waveforms:
dtype_waveform = dtype_waveforms[channel_id]
waveform_size = (h['packet_size'] -
8) / np.dtype(dtype_waveform).itemsize
waveforms = data_spike_chan_clus['waveform'].flatten(
).view(dtype_waveform)
waveforms = waveforms.reshape(n_spike, 1,
waveform_size)
waveforms = waveforms * pq.uV
w_sampling_rate = wsr * pq.Hz
left_sweep = waveform_size // 2 / sr * pq.s
st = SpikeTrain(times=times * pq.s,
name=name,
t_start=t_start,
t_stop=t_stop,
waveforms=waveforms,
sampling_rate=w_sampling_rate,
left_sweep=left_sweep)
st.annotate(channel_index=int(channel_id))
if lazy:
st.lazy_shape = n_spike
seg.spiketrains.append(st)
def readOneChannelEventOrSpike(self, fid, channel_num, header, lazy=True):
# return SPikeTrain or EventArray
channelHeader = header.channelHeaders[channel_num]
if channelHeader.firstblock < 0: return
if channelHeader.kind not in [2, 3, 4, 5, 6, 7, 8]: return
## Step 1 : type of blocks
if channelHeader.kind in [2, 3, 4]:
# Event data
fmt = [('tick', 'i4')]
elif channelHeader.kind in [5]:
# Marker data
fmt = [('tick', 'i4'), ('marker', 'i4')]
elif channelHeader.kind in [6]:
# AdcMark data
fmt = [('tick', 'i4'), ('marker', 'i4'),
('adc', 'S%d' % channelHeader.n_extra)]
elif channelHeader.kind in [7]:
# RealMark data
fmt = [('tick', 'i4'), ('marker', 'i4'),
('real', 'S%d' % channelHeader.n_extra)]
elif channelHeader.kind in [8]:
# TextMark data
fmt = [('tick', 'i4'), ('marker', 'i4'),
('label', 'S%d' % channelHeader.n_extra)]
dt = np.dtype(fmt)
## Step 2 : first read for allocating mem
fid.seek(channelHeader.firstblock)
totalitems = 0
for _ in range(channelHeader.blocks):
blockHeader = HeaderReader(fid, np.dtype(blockHeaderDesciption))
totalitems += blockHeader.items
if blockHeader.succ_block > 0:
fid.seek(blockHeader.succ_block)
#~ print 'totalitems' , totalitems
if lazy:
if channelHeader.kind in [2, 3, 4, 5, 8]:
ea = EventArray()
ea.annotate(channel_index=channel_num)
ea.lazy_shape = totalitems
return [ea]
elif channelHeader.kind in [6, 7]:
sptr = SpikeTrain(
[] * pq.s,
t_stop=1e99) # correct value for t_stop to be put in later
sptr.annotate(channel_index=channel_num, ced_unit=0)
sptr.lazy_shape = totalitems
return [sptr]
else:
alltrigs = np.zeros(totalitems, dtype=dt)
## Step 3 : read
fid.seek(channelHeader.firstblock)
pos = 0
for _ in range(channelHeader.blocks):
blockHeader = HeaderReader(fid,
np.dtype(blockHeaderDesciption))
# read all events in block
trigs = np.fromstring(fid.read(blockHeader.items *
dt.itemsize),
dtype=dt)
alltrigs[pos:pos + trigs.size] = trigs
pos += trigs.size
if blockHeader.succ_block > 0:
fid.seek(blockHeader.succ_block)
## Step 3 convert in neo standard class : eventarrays or spiketrains
alltimes = alltrigs['tick'].astype(
'f') * header.us_per_time * header.dtime_base * pq.s
if channelHeader.kind in [2, 3, 4, 5, 8]:
#events
ea = EventArray()
ea.annotate(channel_index=channel_num)
ea.times = alltimes
if channelHeader.kind >= 5:
# Spike2 marker is closer to label sens of neo
ea.labels = alltrigs['marker'].astype('S32')
if channelHeader.kind == 8:
ea.annotate(extra_labels=alltrigs['label'])
return [ea]
elif channelHeader.kind in [6, 7]:
# spiketrains
# waveforms
if channelHeader.kind == 6:
waveforms = np.fromstring(alltrigs['adc'].tostring(),
dtype='i2')
waveforms = waveforms.astype(
'f4'
) * channelHeader.scale / 6553.6 + channelHeader.offset
elif channelHeader.kind == 7:
waveforms = np.fromstring(alltrigs['real'].tostring(),
dtype='f4')
if header.system_id >= 6 and channelHeader.interleave > 1:
waveforms = waveforms.reshape(
(alltimes.size, -1, channelHeader.interleave))
waveforms = waveforms.swapaxes(1, 2)
else:
waveforms = waveforms.reshape((alltimes.size, 1, -1))
if header.system_id in [1, 2, 3, 4, 5]:
sample_interval = (channelHeader.divide *
header.us_per_time *
header.time_per_adc) * 1e-6
else:
sample_interval = (channelHeader.l_chan_dvd *
header.us_per_time * header.dtime_base)
if channelHeader.unit in unit_convert:
unit = pq.Quantity(1, unit_convert[channelHeader.unit])
else:
#print channelHeader.unit
try:
unit = pq.Quantity(1, channelHeader.unit)
except:
unit = pq.Quantity(1, '')
if len(alltimes) > 0:
t_stop = alltimes.max(
) # can get better value from associated AnalogSignal(s) ?
else:
t_stop = 0.0
if not self.ced_units:
sptr = SpikeTrain(alltimes,
waveforms=waveforms * unit,
sampling_rate=(1. / sample_interval) *
pq.Hz,
t_stop=t_stop)
sptr.annotate(channel_index=channel_num, ced_unit=0)
return [sptr]
sptrs = []
for i in set(alltrigs['marker'] & 255):
sptr = SpikeTrain(
alltimes[alltrigs['marker'] == i],
waveforms=waveforms[alltrigs['marker'] == i] * unit,
sampling_rate=(1. / sample_interval) * pq.Hz,
t_stop=t_stop)
sptr.annotate(channel_index=channel_num, ced_unit=i)
sptrs.append(sptr)
return sptrs
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
global_t_start = None
# Step 1 : first loop for counting - tsq file
tsq = open(os.path.join(subdir, tankname+'_'+blockname+'.tsq'), 'rb')
hr = HeaderReader(tsq, TsqDescription)
allsig = { }
allspiketr = { }
allevent = { }
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_UNKNOWN':
pass
elif Types[evtype] == 'EVTYPE_MARK' :
if global_t_start is None:
global_t_start = h['timestamp']
elif Types[evtype] == 'EVTYPE_SCALER' :
# TODO
pass
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
# EVENTS
if code not in allevent:
allevent[code] = { }
if channel not in allevent[code]:
ea = EventArray(name = code , channel_index = channel)
# for counting:
ea.lazy_shape = 0
ea.maxlabelsize = 0
allevent[code][channel] = ea
allevent[code][channel].lazy_shape += 1
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
strobe = str(strobe)
if len(strobe)>= allevent[code][channel].maxlabelsize:
allevent[code][channel].maxlabelsize = len(strobe)
#~ ev = Event()
#~ ev.time = h['timestamp'] - global_t_start
#~ ev.name = code
#~ # it the strobe attribute masked with eventoffset
#~ strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
#~ ev.label = str(strobe)
#~ seg._events.append( ev )
elif Types[evtype] == 'EVTYPE_SNIP' :
if code not in allspiketr:
allspiketr[code] = { }
if channel not in allspiketr[code]:
allspiketr[code][channel] = { }
if h['sortcode'] not in allspiketr[code][channel]:
sptr = SpikeTrain([ ], units = 's',
name = str(h['sortcode']),
#t_start = global_t_start,
t_start = 0.*pq.s,
t_stop = 0.*pq.s, # temporary
left_sweep = (h['size']-10.)/2./h['frequency'] * pq.s,
sampling_rate = h['frequency'] * pq.Hz,
)
#~ sptr.channel = channel
#sptr.annotations['channel_index'] = channel
sptr.annotate(channel_index = channel)
# for counting:
sptr.lazy_shape = 0
sptr.pos = 0
sptr.waveformsize = h['size']-10
#~ sptr.name = str(h['sortcode'])
#~ sptr.t_start = global_t_start
#~ sptr.sampling_rate = h['frequency']
#~ sptr.left_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.right_sweep = (h['size']-10.)/2./h['frequency']
#~ sptr.waveformsize = h['size']-10
allspiketr[code][channel][h['sortcode']] = sptr
allspiketr[code][channel][h['sortcode']].lazy_shape += 1
elif Types[evtype] == 'EVTYPE_STREAM':
if code not in allsig:
allsig[code] = { }
if channel not in allsig[code]:
#~ print 'code', code, 'channel', channel
anaSig = AnalogSignal([] * pq.V,
name=code,
sampling_rate=
h['frequency'] * pq.Hz,
t_start=(h['timestamp'] -
global_t_start) * pq.s,
channel_index=channel)
anaSig.lazy_dtype = np.dtype(DataFormats[h['dataformat']])
anaSig.pos = 0
# for counting:
anaSig.lazy_shape = 0
#~ anaSig.pos = 0
allsig[code][channel] = anaSig
allsig[code][channel].lazy_shape += (h['size']*4-40)/anaSig.dtype.itemsize
if not lazy:
# Step 2 : allocate memory
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
v[channel] = anaSig.duplicate_with_new_array(np.zeros((anaSig.lazy_shape) , dtype = anaSig.lazy_dtype)*pq.V )
v[channel].pos = 0
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
ea.times = np.empty( (ea.lazy_shape) ) * pq.s
ea.labels = np.empty( (ea.lazy_shape), dtype = 'S'+str(ea.maxlabelsize) )
ea.pos = 0
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
new = SpikeTrain(np.zeros( (sptr.lazy_shape), dtype = 'f8' ) *pq.s ,
name = sptr.name,
t_start = sptr.t_start,
t_stop = sptr.t_stop,
left_sweep = sptr.left_sweep,
sampling_rate = sptr.sampling_rate,
waveforms = np.ones( (sptr.lazy_shape, 1, sptr.waveformsize) , dtype = 'f') * pq.mV ,
)
new.annotations.update(sptr.annotations)
new.pos = 0
new.waveformsize = sptr.waveformsize
allsorted[sortcode] = new
# Step 3 : searh sev (individual data files) or tev (common data file)
# sev is for version > 70
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev = open(os.path.join(subdir, tankname+'_'+blockname+'.tev'), 'rb')
else:
tev = None
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
if PY3K:
signame = anaSig.name.decode('ascii')
else:
signame = anaSig.name
filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(anaSig.channel_index)+'.sev')
if os.path.exists(filename):
anaSig.fid = open(filename, 'rb')
else:
anaSig.fid = tev
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
sptr.fid = tev
# Step 4 : second loop for copyin chunk of data
tsq.seek(0)
while 1:
h= hr.read_f()
if h==None:break
channel, code , evtype = h['channel'], h['code'], h['evtype']
if Types[evtype] == 'EVTYPE_STREAM':
a = allsig[code][channel]
dt = a.dtype
s = int((h['size']*4-40)/dt.itemsize)
a.fid.seek(h['eventoffset'])
a[ a.pos:a.pos+s ] = np.fromstring( a.fid.read( s*dt.itemsize ), dtype = a.dtype)
a.pos += s
elif Types[evtype] == 'EVTYPE_STRON' or \
Types[evtype] == 'EVTYPE_STROFF':
ea = allevent[code][channel]
ea.times[ea.pos] = (h['timestamp'] - global_t_start) * pq.s
strobe, = struct.unpack('d' , struct.pack('q' , h['eventoffset']))
ea.labels[ea.pos] = str(strobe)
ea.pos += 1
elif Types[evtype] == 'EVTYPE_SNIP':
sptr = allspiketr[code][channel][h['sortcode']]
sptr.t_stop = (h['timestamp'] - global_t_start) * pq.s
sptr[sptr.pos] = (h['timestamp'] - global_t_start) * pq.s
sptr.waveforms[sptr.pos, 0, :] = np.fromstring( sptr.fid.read( sptr.waveformsize*4 ), dtype = 'f4') * pq.V
sptr.pos += 1
# Step 5 : populating segment
for code, v in iteritems(allsig):
for channel, anaSig in iteritems(v):
seg.analogsignals.append( anaSig )
for code, v in iteritems(allevent):
for channel, ea in iteritems(v):
seg.eventarrays.append( ea )
for code, v in iteritems(allspiketr):
for channel, allsorted in iteritems(v):
for sortcode, sptr in iteritems(allsorted):
seg.spiketrains.append( sptr )
create_many_to_one_relationship(bl)
return bl
def read_segment(
self,
lazy=False,
cascade=True,
):
fid = open(self.filename, 'rb')
globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0)
#~ print globalHeader
#~ print 'version' , globalHeader['version']
seg = Segment()
seg.file_origin = os.path.basename(self.filename)
seg.annotate(neuroexplorer_version=globalHeader['version'])
seg.annotate(comment=globalHeader['comment'])
if not cascade:
return seg
offset = 544
for i in range(globalHeader['nvar']):
entityHeader = HeaderReader(
fid, EntityHeader).read_f(offset=offset + i * 208)
entityHeader['name'] = entityHeader['name'].replace('\x00', '')
#print 'i',i, entityHeader['type']
if entityHeader['type'] == 0:
# neuron
if lazy:
spike_times = [] * pq.s
else:
spike_times = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
spike_times = spike_times.astype(
'f8') / globalHeader['freq'] * pq.s
sptr = SpikeTrain(
times=spike_times,
t_start=globalHeader['tbeg'] / globalHeader['freq'] * pq.s,
t_stop=globalHeader['tend'] / globalHeader['freq'] * pq.s,
name=entityHeader['name'],
)
if lazy:
sptr.lazy_shape = entityHeader['n']
sptr.annotate(channel_index=entityHeader['WireNumber'])
seg.spiketrains.append(sptr)
if entityHeader['type'] == 1:
# event
if lazy:
event_times = [] * pq.s
else:
event_times = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
event_times = event_times.astype(
'f8') / globalHeader['freq'] * pq.s
labels = np.array([''] * event_times.size, dtype='S')
evar = EventArray(times=event_times,
labels=labels,
channel_name=entityHeader['name'])
if lazy:
evar.lazy_shape = entityHeader['n']
seg.eventarrays.append(evar)
if entityHeader['type'] == 2:
# interval
if lazy:
start_times = [] * pq.s
stop_times = [] * pq.s
else:
start_times = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
start_times = start_times.astype(
'f8') / globalHeader['freq'] * pq.s
stop_times = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'] + entityHeader['n'] * 4,
)
stop_times = stop_times.astype(
'f') / globalHeader['freq'] * pq.s
epar = EpochArray(times=start_times,
durations=stop_times - start_times,
labels=np.array([''] * start_times.size,
dtype='S'),
channel_name=entityHeader['name'])
if lazy:
epar.lazy_shape = entityHeader['n']
seg.epocharrays.append(epar)
if entityHeader['type'] == 3:
# spiketrain and wavefoms
if lazy:
spike_times = [] * pq.s
waveforms = None
else:
spike_times = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
spike_times = spike_times.astype(
'f8') / globalHeader['freq'] * pq.s
waveforms = np.memmap(
self.filename,
np.dtype('i2'),
'r',
shape=(entityHeader['n'], 1,
entityHeader['NPointsWave']),
offset=entityHeader['offset'] + entityHeader['n'] * 4,
)
waveforms = (waveforms.astype('f') * entityHeader['ADtoMV']
+ entityHeader['MVOffset']) * pq.mV
t_stop = globalHeader['tend'] / globalHeader['freq'] * pq.s
if spike_times.size > 0:
t_stop = max(t_stop, max(spike_times))
sptr = SpikeTrain(
times=spike_times,
t_start=globalHeader['tbeg'] / globalHeader['freq'] * pq.s,
#~ t_stop = max(globalHeader['tend']/globalHeader['freq']*pq.s,max(spike_times)),
t_stop=t_stop,
name=entityHeader['name'],
waveforms=waveforms,
sampling_rate=entityHeader['WFrequency'] * pq.Hz,
left_sweep=0 * pq.ms,
)
if lazy:
sptr.lazy_shape = entityHeader['n']
sptr.annotate(channel_index=entityHeader['WireNumber'])
seg.spiketrains.append(sptr)
if entityHeader['type'] == 4:
# popvectors
pass
if entityHeader['type'] == 5:
# analog
timestamps = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
timestamps = timestamps.astype('f8') / globalHeader['freq']
fragmentStarts = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
fragmentStarts = fragmentStarts.astype(
'f8') / globalHeader['freq']
t_start = timestamps[0] - fragmentStarts[0] / float(
entityHeader['WFrequency'])
del timestamps, fragmentStarts
if lazy:
signal = [] * pq.mV
else:
signal = np.memmap(
self.filename,
np.dtype('i2'),
'r',
shape=(entityHeader['NPointsWave']),
offset=entityHeader['offset'],
)
signal = signal.astype('f')
signal *= entityHeader['ADtoMV']
signal += entityHeader['MVOffset']
signal = signal * pq.mV
anaSig = AnalogSignal(
signal=signal,
t_start=t_start * pq.s,
sampling_rate=entityHeader['WFrequency'] * pq.Hz,
name=entityHeader['name'],
channel_index=entityHeader['WireNumber'])
if lazy:
anaSig.lazy_shape = entityHeader['NPointsWave']
seg.analogsignals.append(anaSig)
if entityHeader['type'] == 6:
# markers : TO TEST
if lazy:
times = [] * pq.s
labels = np.array([], dtype='S')
markertype = None
else:
times = np.memmap(
self.filename,
np.dtype('i4'),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'],
)
times = times.astype('f8') / globalHeader['freq'] * pq.s
fid.seek(entityHeader['offset'] + entityHeader['n'] * 4)
markertype = fid.read(64).replace('\x00', '')
labels = np.memmap(
self.filename,
np.dtype('S' + str(entityHeader['MarkerLength'])),
'r',
shape=(entityHeader['n']),
offset=entityHeader['offset'] + entityHeader['n'] * 4 +
64)
ea = EventArray(times=times,
labels=labels.view(np.ndarray),
name=entityHeader['name'],
channel_index=entityHeader['WireNumber'],
marker_type=markertype)
if lazy:
ea.lazy_shape = entityHeader['n']
seg.eventarrays.append(ea)
create_many_to_one_relationship(seg)
return seg
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
#TSQ is the global index
tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq')
dt = [('size','int32'),
('evtype','int32'),
('code','S4'),
('channel','uint16'),
('sortcode','uint16'),
('timestamp','float64'),
('eventoffset','int64'),
('dataformat','int32'),
('frequency','float32'),
]
tsq = np.fromfile(tsq_filename, dtype = dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0]
#TEV is the old data file
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype = 'uint8')
else:
tev_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype']==type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code']==code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel']==channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [ ]*pq.s
labels = np.array([ ], dtype = str)
else:
times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view('float64').astype('S')
ea = EventArray(times = times, name = code , channel_index = int(channel), labels = labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.eventarrays.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode']==sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0]-10
if lazy:
times = [ ]*pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt)
waveforms = waveforms.reshape(nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike>0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 *pq.s
t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s
else:
t_start = 0 *pq.s
t_stop = 0 *pq.s
st = SpikeTrain(times = times,
name = 'Chan{} Code{}'.format(channel,sortcode),
t_start = t_start,
t_stop = t_stop,
waveforms = waveforms,
left_sweep = waveformsize/2./sr * pq.s,
sampling_rate = sr * pq.Hz,
)
st.annotate(channel_index = channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size']-10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = [ ]
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev')
if os.path.exists(sev_filename):
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename, dtype = 'uint8')
else:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt)
anasig = AnalogSignal(signal = signal* pq.V,
name = '{} {}'.format(code, channel),
sampling_rate= sr * pq.Hz,
t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s,
channel_index = int(channel))
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
bl.create_many_to_one_relationship()
return bl
