def read_analogsignal(self, channel_index=None, lazy=False, cascade=True):
"""
Read raw traces
Arguments:
channel_index: must be integer array
"""
if self._attrs["app_data"]:
bit_volts = self._attrs["app_data"]["channel_bit_volts"]
sig_unit = "uV"
else:
bit_volts = np.ones((self._attrs["shape"][1])) # TODO: find conversion in phy generated files
sig_unit = "bit"
if lazy:
anasig = AnalogSignal(
[],
units=sig_unit,
sampling_rate=self._attrs["kwik"]["sample_rate"] * pq.Hz,
t_start=self._attrs["kwik"]["start_time"] * pq.s,
)
# we add the attribute lazy_shape with the size if loaded
anasig.lazy_shape = self._attrs["shape"][0]
else:
data = self._kwd["recordings"][str(self._dataset)]["data"].value[:, channel_index]
data = data * bit_volts[channel_index]
anasig = AnalogSignal(
data,
units=sig_unit,
sampling_rate=self._attrs["kwik"]["sample_rate"] * pq.Hz,
t_start=self._attrs["kwik"]["start_time"] * pq.s,
)
data = [] # delete from memory
# for attributes out of neo you can annotate
anasig.annotate(info="raw traces")
return anasig
def read_analogsignal(self ,
# the 2 first key arguments are imposed by neo.io API
lazy = False,
cascade = True,
channel_index = 0,
segment_duration = 15.,
t_start = -1,
):
"""
With this IO AnalogSignal can e acces directly with its channel number
"""
sr = 10000.
sinus_freq = 3. # Hz
#time vector for generated signal:
tvect = np.arange(t_start, t_start+ segment_duration , 1./sr)
if lazy:
anasig = AnalogSignal([], units='V', sampling_rate=sr * pq.Hz,
t_start=t_start * pq.s,
channel_index=channel_index)
# we add the attribute lazy_shape with the size if loaded
anasig.lazy_shape = tvect.shape
else:
# create analogsignal (sinus of 3 Hz)
sig = np.sin(2*np.pi*tvect*sinus_freq + channel_index/5.*2*np.pi)+np.random.rand(tvect.size)
anasig = AnalogSignal(sig, units= 'V', sampling_rate=sr * pq.Hz,
t_start=t_start * pq.s,
channel_index=channel_index)
# for attributes out of neo you can annotate
anasig.annotate(info = 'it is a sinus of %f Hz' %sinus_freq )
return anasig
def create_analogsignal2(self, parent=None, name='AnalogSignal2'):
signal = AnalogSignal([[1], [2], [3], [4], [5]], units='mA',
sampling_period=0.5 * pq.ms)
signal.segment = parent
self._assign_annotations(signal)
return signal
def create_analogsignal3(self, parent=None, name='AnalogSignal3'):
signal = AnalogSignal([[1, 2, 3], [4, 5, 6]], units='mV',
sampling_rate=2 * pq.kHz, t_start=100 * pq.s)
signal.segment = parent
self._assign_basic_attributes(signal, name=name)
return signal
def create_analogsignal(self, parent=None, name='AnalogSignal1'):
signal = AnalogSignal([[1.0, 2.5], [2.2, 3.1], [3.2, 4.4]], units='mV',
sampling_rate=100 * pq.Hz, t_start=2 * pq.min)
signal.segment = parent
self._assign_basic_attributes(signal, name=name)
self._assign_annotations(signal)
return signal
def read_analogsignal(self,
# the 2 first key arguments are imposed by neo.io
lazy = False,
cascade = True,
#channel index as given by the neuroshare API
channel_index = 0,
#time in seconds to be read
segment_duration = 0.,
#time in seconds to start reading from
t_start = 0.,
):
#some controls:
#if no segment duration is given, use the complete file
if segment_duration ==0.:
segment_duration=float(self.metadata["TimeSpan"])
#if the segment duration is bigger than file, use the complete file
if segment_duration >=float(self.metadata["TimeSpan"]):
segment_duration=float(self.metadata["TimeSpan"])
if lazy:
anasig = AnalogSignal([], units="V", sampling_rate = self.metadata["sampRate"] * pq.Hz,
t_start=t_start * pq.s,
)
#create a dummie time vector
tvect = np.arange(t_start, t_start+ segment_duration , 1./self.metadata["sampRate"])
# we add the attribute lazy_shape with the size if loaded
anasig.lazy_shape = tvect.shape
else:
#get the analog object
sig = self.fd.get_entity(channel_index)
#get the units (V, mV etc)
sigUnits = sig.units
#get the electrode number
chanName = sig.label[-4:]
#transform t_start into index (reading will start from this index)
startat = int(t_start*self.metadata["sampRate"])
#get the number of bins to read in
bins = int((segment_duration+t_start) * self.metadata["sampRate"])
#if the number of bins to read is bigger than
#the total number of bins, read only till the end of analog object
if startat+bins > sig.item_count:
bins = sig.item_count-startat
#read the data from the sig object
sig,_,_ = sig.get_data(index = startat, count = bins)
#store it to the 'AnalogSignal' object
anasig = AnalogSignal(sig, units = sigUnits, sampling_rate=self.metadata["sampRate"] * pq.Hz,
t_start=t_start * pq.s,
t_stop = (t_start+segment_duration)*pq.s,
channel_index=channel_index)
# annotate from which electrode the signal comes from
anasig.annotate(info = "signal from channel %s" %chanName )
return anasig
def read_segment(self, n_start, n_stop, chlist=None, lazy=False, cascade=True):
"""Reads a Segment from the file and stores in database.
The Segment will contain one AnalogSignal for each channel
and will go from n_start to n_stop (in samples).
Arguments:
n_start : time in samples that the Segment begins
n_stop : time in samples that the Segment ends
Python indexing is used, so n_stop is not inclusive.
Returns a Segment object containing the data.
"""
# If no channel numbers provided, get all of them
if chlist is None:
chlist = self.loader.get_neural_channel_numbers()
# Conversion from bits to full_range units
conversion = self.full_range / 2**(8*self.header.sample_width)
# Create the Segment
seg = Segment(file_origin=self.filename)
t_start = float(n_start) / self.header.f_samp
t_stop = float(n_stop) / self.header.f_samp
seg.annotate(t_start=t_start)
seg.annotate(t_stop=t_stop)
# Load data from each channel and store
for ch in chlist:
if lazy:
sig = np.array([]) * conversion
else:
# Get the data from the loader
sig = np.array(\
self.loader._get_channel(ch)[n_start:n_stop]) * conversion
# Create an AnalogSignal with the data in it
anasig = AnalogSignal(signal=sig,
sampling_rate=self.header.f_samp*pq.Hz,
t_start=t_start*pq.s, file_origin=self.filename,
description='Channel %d from %f to %f' % (ch, t_start, t_stop),
channel_index=int(ch))
if lazy:
anasig.lazy_shape = n_stop-n_start
# Link the signal to the segment
seg.analogsignals.append(anasig)
# Link the signal to the recording channel from which it came
#rc = self.channel_number_to_recording_channel[ch]
#rc.analogsignals.append(anasig)
return seg
def _read_analogsignalarray(self, node, parent):
attributes = self._get_standard_attributes(node)
# todo: handle channel_index
sampling_rate = self._get_quantity(node["sampling_rate"])
t_start = self._get_quantity(node["t_start"])
signal = AnalogSignal(self._get_quantity(node["signal"]),
sampling_rate=sampling_rate, t_start=t_start,
**attributes)
signal.segment = parent
self.object_refs[node.attrs["object_ref"]] = signal
return signal
def read_block(self, lazy=False, cascade=True):
if self.filename is not None:
self.stfio_rec = stfio.read(self.filename)
bl = Block()
bl.description = self.stfio_rec.file_description
bl.annotate(comment=self.stfio_rec.comment)
try:
bl.rec_datetime = self.stfio_rec.datetime
except:
bl.rec_datetime = None
if not cascade:
return bl
dt = np.round(self.stfio_rec.dt * 1e-3, 9) * pq.s # ms to s
sampling_rate = 1.0 / dt
t_start = 0 * pq.s
# iterate over sections first:
for j, recseg in enumerate(self.stfio_rec[0]):
seg = Segment(index=j)
length = len(recseg)
# iterate over channels:
for i, recsig in enumerate(self.stfio_rec):
name = recsig.name
unit = recsig.yunits
try:
pq.Quantity(1, unit)
except:
unit = ''
if lazy:
signal = pq.Quantity([], unit)
else:
signal = pq.Quantity(recsig[j], unit)
anaSig = AnalogSignal(signal,
sampling_rate=sampling_rate,
t_start=t_start,
name=str(name),
channel_index=i)
if lazy:
anaSig.lazy_shape = length
seg.analogsignals.append(anaSig)
bl.segments.append(seg)
t_start = t_start + length * dt
bl.create_many_to_one_relationship()
return bl
def _read_analogsignalarray(self, node, parent):
attributes = self._get_standard_attributes(node)
# todo: handle channel_index
sampling_rate = self._get_quantity(node["sampling_rate"])
t_start = self._get_quantity(node["t_start"])
signal = AnalogSignal(self._get_quantity(node["signal"]),
sampling_rate=sampling_rate,
t_start=t_start,
**attributes)
signal.segment = parent
self.object_refs[node.attrs["object_ref"]] = signal
return signal
def get_model_parts(data_set, sweep_numbers, specimen_id):
sweep_numbers = sweep_numbers['Square - 2s Suprathreshold']
rheobase = -1
above_threshold_sn = []
currents = {}
for sn in sweep_numbers:
sweep_data = data_set.get_sweep(sn)
spike_times = data_set.get_spike_times(sn)
# stimulus is a numpy array in amps
stimulus = sweep_data['stimulus']
if len(spike_times) == 1:
if np.max(stimulus) > rheobase and rheobase == -1:
rheobase = np.max(stimulus)
stim = rheobase
currents['rh'] = stim
sampling_rate = sweep_data['sampling_rate']
vmrh = AnalogSignal([v * 1000 for v in sweep_data['response']],
sampling_rate=sampling_rate * qt.Hz,
units=qt.mV)
vmrh = vmrh[0:int(len(vmrh) / 2.1)]
if len(spike_times) >= 1:
reponse = sweep_data['response']
sampling_rate = sweep_data['sampling_rate']
vmm = AnalogSignal([v * 1000 for v in sweep_data['response']],
sampling_rate=sampling_rate * qt.Hz,
units=qt.mV)
vmm = vmm[0:int(len(vmm) / 2.1)]
above_threshold_sn.append((np.max(stimulus), sn, vmm))
if rheobase == -1:
rheobase = above_threshold_sn[0][0]
vmrh = above_threshold_sn[0][2]
print(len(spike_times))
myNumber = 3.0 * rheobase
currents_ = [t[0] for t in above_threshold_sn]
indexvm30 = closest(currents_, myNumber)
stim = above_threshold_sn[indexvm30][0]
currents['30'] = stim
vm30 = above_threshold_sn[indexvm30][2]
myNumber = 1.5 * rheobase
currents_ = [t[0] for t in above_threshold_sn]
indexvm15 = closest(currents_, myNumber)
stim = above_threshold_sn[indexvm15][0]
currents['15'] = stim
vm15 = above_threshold_sn[indexvm15][2]
vm15.sn = None
vm15.sn = above_threshold_sn[0][1]
vm15.specimen_id = None
vm15.specimen_id = specimen_id
del sweep_numbers
del data_set
return vm15, vm30, rheobase, currents, vmrh
def create_analogsignal(self, parent=None, name='AnalogSignal1'):
signal = AnalogSignal([[1.0, 2.5], [2.2, 3.1], [3.2, 4.4]],
units='mV',
sampling_rate=100 * pq.Hz,
t_start=2 * pq.min)
signal.segment = parent
self._assign_basic_attributes(signal, name=name)
self._assign_annotations(signal)
return signal
def read_block(self, lazy=False, cascade=True):
if self.filename is not None:
self.stfio_rec = stfio.read(self.filename)
bl = Block()
bl.description = self.stfio_rec.file_description
bl.annotate(comment=self.stfio_rec.comment)
try:
bl.rec_datetime = self.stfio_rec.datetime
except:
bl.rec_datetime = None
if not cascade:
return bl
dt = np.round(self.stfio_rec.dt * 1e-3, 9) * pq.s # ms to s
sampling_rate = 1.0/dt
t_start = 0 * pq.s
# iterate over sections first:
for j, recseg in enumerate(self.stfio_rec[0]):
seg = Segment(index=j)
length = len(recseg)
# iterate over channels:
for i, recsig in enumerate(self.stfio_rec):
name = recsig.name
unit = recsig.yunits
try:
pq.Quantity(1, unit)
except:
unit = ''
if lazy:
signal = pq.Quantity([], unit)
else:
signal = pq.Quantity(recsig[j], unit)
anaSig = AnalogSignal(signal, sampling_rate=sampling_rate,
t_start=t_start, name=str(name),
channel_index=i)
if lazy:
anaSig.lazy_shape = length
seg.analogsignals.append(anaSig)
bl.segments.append(seg)
t_start = t_start + length * dt
bl.create_many_to_one_relationship()
return bl
def read_analogsignal(
self,
lazy=False,
# channel index as given by the neuroshare API
channel_index=0,
# time in seconds to be read
segment_duration=0.,
# time in seconds to start reading from
t_start=0.,
):
assert not lazy, 'Do not support lazy'
# some controls:
# if no segment duration is given, use the complete file
if segment_duration == 0.:
segment_duration = float(self.metadata["TimeSpan"])
# if the segment duration is bigger than file, use the complete file
if segment_duration >= float(self.metadata["TimeSpan"]):
segment_duration = float(self.metadata["TimeSpan"])
# get the analog object
sig = self.fd.get_entity(channel_index)
# get the units (V, mV etc)
sigUnits = sig.units
# get the electrode number
chanName = sig.label[-4:]
# transform t_start into index (reading will start from this index)
startat = int(t_start * self.metadata["sampRate"])
# get the number of bins to read in
bins = int(segment_duration * self.metadata["sampRate"])
# if the number of bins to read is bigger than
# the total number of bins, read only till the end of analog object
if startat + bins > sig.item_count:
bins = sig.item_count - startat
# read the data from the sig object
sig, _, _ = sig.get_data(index=startat, count=bins)
# store it to the 'AnalogSignal' object
anasig = AnalogSignal(sig,
units=sigUnits,
sampling_rate=self.metadata["sampRate"] * pq.Hz,
t_start=t_start * pq.s,
t_stop=(t_start + segment_duration) * pq.s,
channel_index=channel_index)
# annotate from which electrode the signal comes from
anasig.annotate(info="signal from channel %s" % chanName)
return anasig
def read_analogsignal(self, lazy=False, cascade=True):
if not HAVE_IGOR:
raise Exception(
"igor package not installed. Try `pip install igor`")
data = bw.load(self.filename)
version = data['version']
if version > 3:
raise IOError(
"Igor binary wave file format version {0} is not supported.".
format(version))
content = data['wave']
if "padding" in content:
assert content[
'padding'].size == 0, "Cannot handle non-empty padding"
if lazy:
# not really lazy, since the `igor` module loads the data anyway
signal = np.array((), dtype=content['wData'].dtype)
else:
signal = content['wData']
note = content['note']
header = content['wave_header']
name = header['bname']
assert header['botFullScale'] == 0
assert header['topFullScale'] == 0
units = "".join(header['dataUnits'])
time_units = "".join(header['xUnits']) or "s"
t_start = pq.Quantity(header['hsB'], time_units)
sampling_period = pq.Quantity(header['hsA'], time_units)
if self.parse_notes:
try:
annotations = self.parse_notes(note)
except ValueError:
warn("Couldn't parse notes field.")
annotations = {'note': note}
else:
annotations = {'note': note}
signal = AnalogSignal(signal,
units=units,
copy=False,
t_start=t_start,
sampling_period=sampling_period,
name=name,
file_origin=self.filename,
**annotations)
if lazy:
signal.lazy_shape = content['wData'].shape
return signal
def inhom_poiss_2(arr, dt=0.025, speed_cm=20, field_size_cm=100):
#length of the trajectory that mouse went
t_sec = field_size_cm / speed_cm
arr_len = arr.shape[1]
t_arr = np.linspace(0, t_sec, arr_len)
default_dt = t_sec / arr_len
new_len = int(t_sec / dt)
new_t_arr = np.linspace(0, t_sec, new_len)
# arr=signal.resample(arr, new_len, axis=1)
if dt != default_dt:
new_len = int(t_sec / dt)
new_t_arr = np.linspace(0, t_sec, new_len)
f = interpolate.interp1d(t_arr, arr, axis=1)
arr = f(new_t_arr)
n_traj = dev_deg.shape[0] #8
n_cells = arr.shape[0]
spi_arr = np.empty((n_cells, n_traj), dtype=np.ndarray)
for grid_idc in range(n_cells):
for i in range(dev_deg.shape[0]):
rate_profile = arr[grid_idc, :, i]
asig = AnalogSignal(rate_profile,
units=1 * pq.Hz,
t_start=0 * pq.s,
t_stop=t_sec * pq.s,
sampling_period=dt * pq.s,
sampling_interval=dt * pq.s)
curr_train = stg.inhomogeneous_poisson_process(asig)
spi_arr[grid_idc, i] = np.around(curr_train.times * 1000,
decimals=1)
return spi_arr
def fill_irregularly_sampled_signal_with_zeros(
irregular_sig: IrregularlySampledSignal,
sampling_rate: Quantity(10000, "Hz")
) -> AnalogSignal:
# allocate array for the regular signal
num_regular_samples = ceil(
Quantity(irregular_sig.duration * sampling_rate).magnitude)
regular_sig = Quantity(np.zeros(num_regular_samples, dtype=np.float64),
irregular_sig.dimensionality)
# calculate the indices of the samples
idcs: Quantity = (irregular_sig.times -
irregular_sig.times[0]) * sampling_rate
idcs = idcs.magnitude
to_int = np.vectorize(np.int)
idcs = to_int(idcs)
# conversion step
regular_sig[idcs] = irregular_sig[:].ravel()
result: AnalogSignal = AnalogSignal(regular_sig,
t_start=irregular_sig.times[0],
sampling_rate=sampling_rate)
return result
def ProcessSignalData(seg=None, sig_ch='LMag 1', ref_ch='LMag 2',
name='deltaf_f', **kwargs):
"""Given a segment object, it will extract the analog signal channels specified as
signal (sig_ch) and reference (ref_ch), and will perform NormalizeSignal on them.
Will append the new signal to the segment object as 'name'."""
# Gets any keyword arguments
options = kwargs
# Check that segment object was passed
if not isinstance(seg, neo.core.Segment):
raise TypeError('%s must be a Segment object' % seg)
# Retrieves signal and reference
if sig_ch:
try:
signal = filter(lambda x: x.name == sig_ch, seg.analogsignals)[0]
except IndexError:
raise ValueError('There is no signal channel named %s' % sig_ch)
if ref_ch:
try:
reference = filter(lambda x: x.name == ref_ch, seg.analogsignals)[0]
except IndexError:
raise ValueError('There is no reference channel named %s' % ref_ch)
else:
reference = None
# Build a new AnalogSignal based on the other ones
new_signal = NormalizeSignal(signal=signal, reference=reference, **options)
units = pq.percent # new units are in %
t_start = signal.t_start # has the same start time as all other analog signal objects in segment
fs = signal.sampling_rate # has the same sampling rate as all other analog signal objects in segment
# Creates new AnalogSignal object
deltaf_f = AnalogSignal(new_signal, units=units, t_start=t_start, sampling_rate=fs, name=name)
# Adds processed signal back to segment
seg.analogsignals.append(deltaf_f)
def setUp(self):
self.signal = AnalogSignal(np.random.randn(1000, 1),
units='V',
sampling_rate=1 * pq.Hz)
self.not_analog = np.random.randn(1000, 1)
self.signal_start = 10
self.signal_end = 10
def make_analog_trials(ana, epoch, t_start, t_stop):
'''
Makes trials based on an Epoch and given temporal bound
Parameters
----------
epoch : neo.Epoch
t_start : quantities.Quantity
time before epochs
t_stop : quantities.Quantity
time after epochs
ana : neo.AnalogSignal
Returns
-------
out : list of neo.AnalogSignal
'''
assert t_start != t_stop, 't_start cannot be equal to t_stop'
from neo.core import AnalogSignal
dim = 's'
t_start = t_start.rescale(dim)
t_stop = t_stop.rescale(dim)
times = epoch.times.rescale(dim)
trials = []
nsamp = int(abs(t_start - t_stop) * ana.sampling_rate) - 1
for j, t in enumerate(times):
sig = ana.magnitude[(t + t_start <= ana.times) &
(ana.times <= t + t_stop), :]
trials.append(
AnalogSignal(signal=sig[:nsamp, :] * ana.units,
sampling_rate=ana.sampling_rate,
t_start=t_start,
t_stop=t_stop))
return trials
def get_membrane_potential(self,
signal=[[1, 2, 3], [4, 5, 6]],
units='V'):
import quantities as pq
result = AnalogSignal(signal, units, sampling_rate=1 * pq.Hz)
return result
def _create_normal_analogsignal(self, data, dataobj, uid, t_start):
return AnalogSignal(np.swapaxes(data, 0, 1),
dtype=dataobj.dtype,
units=dataobj.unit,
t_start=t_start,
sampling_period=pq.Quantity(
dataobj.dt, dataobj.tunit))
def _read_template_file(filepaths: List[str], track_idx: int,
sampling_rate: Quantity) -> ActionPotentialTemplate:
try:
# find the right file
template_files = [
file for file in filepaths
if "template" in os.path.basename(file).lower()
and "track" in os.path.basename(file).lower()
]
template_file = [
file for file in template_files
if "track" + str(track_idx) + ".csv" in file.lower()
][0]
# read and return the AP template
template_df = pd.read_csv(filepath_or_buffer=template_file,
header=None,
names=["signal_value"])
signal_template = AnalogSignal(
signal=template_df["signal_value"].values,
units="uV",
sampling_rate=sampling_rate)
return ActionPotentialTemplate(signal_template=signal_template)
except Exception as ex:
warnings.warn(
f"""Cannot load AP template for track no. {track_idx}!""")
def test_time_slice_None(self):
time_slices = [(None, 5.0 * pq.s), (5.0 * pq.s, None), (None, None)]
anasig = AnalogSignal(np.arange(50.0) * pq.mV,
sampling_rate=1.0 * pq.Hz)
seg = Segment()
seg.analogsignals = [anasig]
block = Block()
block.segments = [seg]
block.create_many_to_one_relationship()
# test without resetting the time
for t_start, t_stop in time_slices:
sliced = seg.time_slice(t_start, t_stop)
assert_neo_object_is_compliant(sliced)
self.assertEqual(len(sliced.analogsignals), 1)
exp_t_start, exp_t_stop = t_start, t_stop
if exp_t_start is None:
exp_t_start = seg.t_start
if exp_t_stop is None:
exp_t_stop = seg.t_stop
self.assertEqual(exp_t_start, sliced.t_start)
self.assertEqual(exp_t_stop, sliced.t_stop)
def test_roundtrip_with_json_metadata(self):
sample_data = np.random.uniform(size=(200, 3))
filename = "test_roundtrip_with_json_metadata.txt"
metadata_filename = "test_roundtrip_with_json_metadata_about.json"
signal1 = AnalogSignal(sample_data,
units="pA",
sampling_rate=2 * pq.kHz)
seg1 = Segment()
block1 = Block()
seg1.analogsignals.append(signal1)
seg1.block = block1
block1.segments.append(seg1)
iow = AsciiSignalIO(filename, metadata_filename=metadata_filename)
iow.write_block(block1)
self.assert_(os.path.exists(metadata_filename))
ior = AsciiSignalIO(filename)
block2 = ior.read_block()
assert len(block2.segments[0].analogsignals) == 3
signal2 = block2.segments[0].analogsignals[1]
assert_array_almost_equal(signal1.magnitude[:, 1],
signal2.magnitude.reshape(-1),
decimal=7)
self.assertEqual(signal1.units, signal2.units)
self.assertEqual(signal1.sampling_rate, signal2.sampling_rate)
assert_array_equal(signal1.times, signal2.times)
os.remove(filename)
os.remove(metadata_filename)
def _read_segment(self, fobject):
'''
Read a single segment with a single analogsignal
Returns the segment or None if there are no more segments
'''
try:
# float64 -- start time of the AnalogSignal
t_start = np.fromfile(fobject, dtype=np.float64, count=1)[0]
except IndexError:
# if there are no more Segments, return
return False
# int16 -- index of the stimulus parameters
seg_index = np.fromfile(fobject, dtype=np.int16, count=1)[0].tolist()
# int16 -- number of stimulus parameters
numelements = np.fromfile(fobject, dtype=np.int16, count=1)[0]
# read the name strings for the stimulus parameters
paramnames = []
for _ in range(numelements):
# unit8 -- the number of characters in the string
numchars = np.fromfile(fobject, dtype=np.uint8, count=1)[0]
# char * numchars -- a single name string
name = np.fromfile(fobject, dtype=np.uint8, count=numchars)
# exclude invalid characters
name = str(name[name >= 32].view('c').tostring())
# add the name to the list of names
paramnames.append(name)
# float32 * numelements -- the values for the stimulus parameters
paramvalues = np.fromfile(fobject, dtype=np.float32, count=numelements)
# combine parameter names and the parameters as a dict
params = dict(zip(paramnames, paramvalues))
# int32 -- the number elements in the AnalogSignal
numpts = np.fromfile(fobject, dtype=np.int32, count=1)[0]
# int16 * numpts -- the AnalogSignal itself
signal = np.fromfile(fobject, dtype=np.int16, count=numpts)
sig = AnalogSignal(signal.astype(np.float) * pq.mV,
t_start=t_start * pq.d,
file_origin=self._filename,
sampling_period=1. * pq.s,
copy=False)
# Note: setting the sampling_period to 1 s is arbitrary
# load the AnalogSignal and parameters into a new Segment
seg = Segment(file_origin=self._filename, index=seg_index, **params)
seg.analogsignals = [sig]
return seg
def sweep_to_neo(sweeps, n):
sweep_name = 'Sweep %d' % n
t = sweeps[sweep_name].index
vm = AnalogSignal(sweeps[sweep_name],
times=t.values,
units=pq.mV,
sampling_period=(t[1] - t[0]) * pq.s)
return vm
def get_model_parts_sweep_from_number(sn,data_set,sweep_numbers,specimen_id):
sweep_numbers = sweep_numbers['Square - 2s Suprathreshold']
rheobase = -1
above_threshold_sn = []
currents = {}
sweep_data = data_set.get_sweep(sn)
stimulus = sweep_data['stimulus']
spike_times = data_set.get_spike_times(sn)
reponse = sweep_data['response']
sampling_rate = sweep_data['sampling_rate']
vmm = AnalogSignal([v*1000 for v in reponse],sampling_rate=sampling_rate*qt.Hz,units=qt.mV)
vmm = vmm[0:int(len(vmm)/2.1)]
vmm.sn = None
vmm.sn = sn
return vmm,stimulus,sn,spike_times
def read_analogsignal(self, path, cascade=True, lazy=False):
group = self._exdir_directory[path]
signal = group["data"]
attrs = {'exdir_path': path}
attrs.update(group.attrs.to_dict())
if lazy:
ana = AnalogSignal([],
lazy_shape=(signal.attrs["num_samples"], ),
units=signal.attrs["unit"],
sampling_rate=group.attrs['sample_rate'],
**attrs)
else:
ana = AnalogSignal(signal.data,
units=signal.attrs["unit"],
sampling_rate=group.attrs['sample_rate'],
**attrs)
return ana
def create_all_annotated(cls):
times = cls.rquant(1, pq.s)
signal = cls.rquant(1, pq.V)
blk = Block()
blk.annotate(**cls.rdict(3))
cls.populate_dates(blk)
seg = Segment()
seg.annotate(**cls.rdict(4))
cls.populate_dates(seg)
blk.segments.append(seg)
asig = AnalogSignal(signal=signal, sampling_rate=pq.Hz)
asig.annotate(**cls.rdict(2))
seg.analogsignals.append(asig)
isig = IrregularlySampledSignal(times=times,
signal=signal,
time_units=pq.s)
isig.annotate(**cls.rdict(2))
seg.irregularlysampledsignals.append(isig)
epoch = Epoch(times=times, durations=times)
epoch.annotate(**cls.rdict(4))
seg.epochs.append(epoch)
event = Event(times=times)
event.annotate(**cls.rdict(4))
seg.events.append(event)
spiketrain = SpikeTrain(times=times, t_stop=pq.s, units=pq.s)
d = cls.rdict(6)
d["quantity"] = pq.Quantity(10, "mV")
d["qarray"] = pq.Quantity(range(10), "mA")
spiketrain.annotate(**d)
seg.spiketrains.append(spiketrain)
chx = ChannelIndex(name="achx", index=[1, 2], channel_ids=[0, 10])
chx.annotate(**cls.rdict(5))
blk.channel_indexes.append(chx)
unit = Unit()
unit.annotate(**cls.rdict(2))
chx.units.append(unit)
return blk
def proc_dam(filename):
'''Load an dam file that has already been processed by the official matlab
file converter. That matlab data is saved to an m-file, which is then
converted to a numpy '.npz' file. This numpy file is the file actually
loaded. This function converts it to a neo block and returns the block.
This block can be compared to the block produced by BrainwareDamIO to
make sure BrainwareDamIO is working properly
block = proc_dam(filename)
filename: The file name of the numpy file to load. It should end with
'*_dam_py?.npz'. This will be converted to a neo 'file_origin' property
with the value '*.dam', so the filename to compare should fit that pattern.
'py?' should be 'py2' for the python 2 version of the numpy file or 'py3'
for the python 3 version of the numpy file.
example: filename = 'file1_dam_py2.npz'
dam file name = 'file1.dam'
'''
with np.load(filename) as damobj:
damfile = damobj.items()[0][1].flatten()
filename = os.path.basename(filename[:-12] + '.dam')
signals = [res.flatten() for res in damfile['signal']]
stimIndexes = [int(res[0, 0].tolist()) for res in damfile['stimIndex']]
timestamps = [res[0, 0] for res in damfile['timestamp']]
block = Block(file_origin=filename)
chx = ChannelIndex(file_origin=filename,
index=np.array([0]),
channel_ids=np.array([1]),
channel_names=np.array(['Chan1'], dtype='S'))
block.channel_indexes.append(chx)
params = [res['params'][0, 0].flatten() for res in damfile['stim']]
values = [res['values'][0, 0].flatten() for res in damfile['stim']]
params = [[res1[0] for res1 in res] for res in params]
values = [[res1 for res1 in res] for res in values]
stims = [dict(zip(param, value)) for param, value in zip(params, values)]
fulldam = zip(stimIndexes, timestamps, signals, stims)
for stimIndex, timestamp, signal, stim in fulldam:
sig = AnalogSignal(signal=signal * pq.mV,
t_start=timestamp * pq.d,
file_origin=filename,
sampling_period=1. * pq.s)
segment = Segment(file_origin=filename,
index=stimIndex,
**stim)
segment.analogsignals = [sig]
block.segments.append(segment)
block.create_many_to_one_relationship()
return block
def read_protocol(self):
"""
Read the protocol waveform of the file, if present; function works with ABF2 only.
Returns: list of segments (one for every episode)
with list of analog signls (one for every DAC).
"""
header = self.read_header()
if header['fFileVersionNumber'] < 2.:
raise IOError("Protocol is only present in ABF2 files.")
nADC = header['sections']['ADCSection'][
'llNumEntries'] # Number of ADC channels
nDAC = header['sections']['DACSection'][
'llNumEntries'] # Number of DAC channels
nSam = header['protocol'][
'lNumSamplesPerEpisode'] / nADC # Number of samples per episode
nEpi = header['lActualEpisodes'] # Actual number of episodes
sampling_rate = 1.e6 / header['protocol'][
'fADCSequenceInterval'] * pq.Hz
# Creating a list of segments with analog signals with just holding levels
# List of segments relates to number of episodes, as for recorded data
segments = []
for epiNum in range(nEpi):
seg = Segment(index=epiNum)
# One analog signal for each DAC in segment (episode)
for DACNum in range(nDAC):
t_start = 0 * pq.s # TODO: Possibly check with episode array
name = header['listDACInfo'][DACNum]['DACChNames']
unit = header['listDACInfo'][DACNum]['DACChUnits'].replace(
'\xb5', 'u') #\xb5 is µ
signal = np.ones(nSam) * header['listDACInfo'][DACNum][
'fDACHoldingLevel'] * pq.Quantity(1, unit)
anaSig = AnalogSignal(signal,
sampling_rate=sampling_rate,
t_start=t_start,
name=str(name),
channel_index=DACNum)
# If there are epoch infos for this DAC
if header['dictEpochInfoPerDAC'].has_key(DACNum):
# Save last sample index
i_last = int(nSam * 15625 /
10**6) # TODO guess for first holding
# Go over EpochInfoPerDAC and change the analog signal according to the epochs
for epochNum, epoch in iteritems(
header['dictEpochInfoPerDAC'][DACNum]):
i_begin = i_last
i_end = i_last + epoch['lEpochInitDuration'] + epoch[
'lEpochDurationInc'] * epiNum
anaSig[i_begin:i_end] = np.ones(len(range(i_end-i_begin)))*pq.Quantity(1, unit)* \
(epoch['fEpochInitLevel']+epoch['fEpochLevelInc'] * epiNum)
i_last += epoch['lEpochInitDuration']
seg.analogsignals.append(anaSig)
segments.append(seg)
return segments
def read_analogsignal(self ,
# the 2 first key arguments are imposed by neo.io API
lazy = False,
cascade = True,
channel_index = 0,
segment_duration = 15.,
t_start = -1,
):
"""
With this IO AnalogSignal can e acces directly with its channel number
"""
sr = 10000.
sinus_freq = 3. # Hz
#time vector for generated signal:
tvect = np.arange(t_start, t_start+ segment_duration , 1./sr)
if lazy:
anasig = AnalogSignal([], units='V', sampling_rate=sr * pq.Hz,
t_start=t_start * pq.s,
channel_index=channel_index)
# we add the attribute lazy_shape with the size if loaded
anasig.lazy_shape = tvect.shape
else:
# create analogsignal (sinus of 3 Hz)
sig = np.sin(2*np.pi*tvect*sinus_freq + channel_index/5.*2*np.pi)+np.random.rand(tvect.size)
anasig = AnalogSignal(sig, units= 'V', sampling_rate=sr * pq.Hz,
t_start=t_start * pq.s,
channel_index=channel_index)
# for attributes out of neo you can annotate
anasig.annotate(info = 'it is a sinus of %f Hz' %sinus_freq )
return anasig
def get_membrane_potential(self, **run_params):
self.run(**run_params)
for rkey in self.results.keys():
if 'v' in rkey or 'vm' in rkey:
v = np.array(self.results[rkey])
t = np.array(self.results['t'])
dt = (t[1] - t[0]) * pq.s # Time per sample in seconds.
vm = AnalogSignal(v, units=pq.V, sampling_rate=1.0 / dt)
return vm
def test_read_analogsignal_using_eager(self):
io = self.io_cls(self.test_file)
sig = io.read_analogsignal(lazy=False)
self.assertIsInstance(sig, AnalogSignal)
assert_array_equal(sig[:, 3],
AnalogSignal(np.arange(3, 104, dtype=float),
sampling_period=0.1*pq.ms,
t_start=0*pq.s,
units=pq.mV))
def read_analogsignal(self, path=None, lazy=False):
assert not lazy, 'Do not support lazy'
if not HAVE_IGOR:
raise Exception(("`igor` package not installed. "
"Try `pip install igor`"))
if self.extension == 'ibw':
data = bw.load(self.filename)
version = data['version']
if version > 5:
raise IOError(("Igor binary wave file format version {0} "
"is not supported.".format(version)))
elif self.extension == 'pxp':
assert type(path) is str, \
"A colon-separated Igor-style path must be provided."
_, filesystem = pxp.load(self.filename)
path = path.split(':')
location = filesystem['root']
for element in path:
if element != 'root':
location = location[element.encode('utf8')]
data = location.wave
content = data['wave']
if "padding" in content:
assert content['padding'].size == 0, \
"Cannot handle non-empty padding"
signal = content['wData']
note = content['note']
header = content['wave_header']
name = str(header['bname'].decode('utf-8'))
units = "".join([x.decode() for x in header['dataUnits']])
try:
time_units = "".join([x.decode() for x in header['xUnits']])
assert len(time_units)
except:
time_units = "s"
try:
t_start = pq.Quantity(header['hsB'], time_units)
except KeyError:
t_start = pq.Quantity(header['sfB'][0], time_units)
try:
sampling_period = pq.Quantity(header['hsA'], time_units)
except:
sampling_period = pq.Quantity(header['sfA'][0], time_units)
if self.parse_notes:
try:
annotations = self.parse_notes(note)
except ValueError:
warn("Couldn't parse notes field.")
annotations = {'note': note}
else:
annotations = {'note': note}
signal = AnalogSignal(signal, units=units, copy=False, t_start=t_start,
sampling_period=sampling_period, name=name,
file_origin=self.filename, **annotations)
return signal
def test_signals_compound_units(self):
block = Block()
seg = Segment()
block.segments.append(seg)
units = pq.CompoundUnit("1/30000*V")
srate = pq.Quantity(10, pq.CompoundUnit("1.0/10 * Hz"))
asig = AnalogSignal(signal=self.rquant((10, 3), units),
sampling_rate=srate)
seg.analogsignals.append(asig)
self.write_and_compare([block])
anotherblock = Block("ir signal block")
seg = Segment("ir signal seg")
anotherblock.segments.append(seg)
irsig = IrregularlySampledSignal(signal=np.random.random((20, 3)),
times=self.rquant(
20, pq.CompoundUnit("0.1 * ms"),
True),
units=pq.CompoundUnit("10 * V / s"))
seg.irregularlysampledsignals.append(irsig)
self.write_and_compare([block, anotherblock])
block.segments[0].analogsignals.append(
AnalogSignal(signal=[10.0, 1.0, 3.0],
units=pq.S,
sampling_period=pq.Quantity(3, "s"),
dtype=np.double,
name="signal42",
description="this is an analogsignal",
t_start=45 * pq.CompoundUnit("3.14 * s")), )
self.write_and_compare([block, anotherblock])
times = self.rquant(10, pq.CompoundUnit("3 * year"), True)
block.segments[0].irregularlysampledsignals.append(
IrregularlySampledSignal(times=times,
signal=np.random.random((10, 3)),
units="mV",
dtype=np.float,
name="some sort of signal",
description="the signal is described"))
self.write_and_compare([block, anotherblock])
def _extract_signals(self, data, metadata, lazy):
signal = None
if lazy and data.size > 0:
signal = AnalogSignal([],
units=self._determine_units(metadata),
sampling_period=metadata['dt']*pq.ms)
signal.lazy_shape = None
else:
arr = numpy.vstack(self._extract_array(data, channel_index)
for channel_index in range(metadata['first_index'], metadata['last_index'] + 1))
if len(arr) > 0:
signal = AnalogSignal(arr.T,
units=self._determine_units(metadata),
sampling_period=metadata['dt']*pq.ms)
if signal is not None:
signal.annotate(label=metadata["label"],
variable=metadata["variable"])
return signal
def create_all_annotated(cls):
times = cls.rquant(1, pq.s)
signal = cls.rquant(1, pq.V)
blk = Block()
blk.annotate(**cls.rdict(3))
seg = Segment()
seg.annotate(**cls.rdict(4))
blk.segments.append(seg)
asig = AnalogSignal(signal=signal, sampling_rate=pq.Hz)
asig.annotate(**cls.rdict(2))
seg.analogsignals.append(asig)
isig = IrregularlySampledSignal(times=times, signal=signal,
time_units=pq.s)
isig.annotate(**cls.rdict(2))
seg.irregularlysampledsignals.append(isig)
epoch = Epoch(times=times, durations=times)
epoch.annotate(**cls.rdict(4))
seg.epochs.append(epoch)
event = Event(times=times)
event.annotate(**cls.rdict(4))
seg.events.append(event)
spiketrain = SpikeTrain(times=times, t_stop=pq.s, units=pq.s)
d = cls.rdict(6)
d["quantity"] = pq.Quantity(10, "mV")
d["qarray"] = pq.Quantity(range(10), "mA")
spiketrain.annotate(**d)
seg.spiketrains.append(spiketrain)
chx = ChannelIndex(name="achx", index=[1, 2], channel_ids=[0, 10])
chx.annotate(**cls.rdict(5))
blk.channel_indexes.append(chx)
unit = Unit()
unit.annotate(**cls.rdict(2))
chx.units.append(unit)
return blk
def test_read_analogsignal_using_eager(self):
io = self.io_cls(self.test_file)
as3 = io.read_analogsignal(lazy=False, channel_index=3)
self.assertIsInstance(as3, AnalogSignal)
assert_arrays_equal(
as3,
AnalogSignal(np.arange(3, 104, dtype=float),
sampling_period=0.1 * pq.ms,
t_start=0 * pq.s,
units=pq.mV))
def read_analogsignal(self,
channel_index=None,
lazy=False,
cascade=True,
):
"""
Read raw traces
Arguments:
channel_index: must be integer
"""
try:
channel_index = int(channel_index)
except TypeError:
print('channel_index must be int, not %s' %type(channel_index))
if self._attrs['app_data']:
bit_volts = self._attrs['app_data']['channel_bit_volts']
sig_unit = 'uV'
else:
bit_volts = np.ones((self._attrs['shape'][1])) # TODO: find conversion in phy generated files
sig_unit = 'bit'
if lazy:
anasig = AnalogSignal([],
units=sig_unit,
sampling_rate=self._attrs['kwik']['sample_rate']*pq.Hz,
t_start=self._attrs['kwik']['start_time']*pq.s,
channel_index=channel_index,
)
# we add the attribute lazy_shape with the size if loaded
anasig.lazy_shape = self._attrs['shape'][0]
else:
data = self._kwd['recordings'][str(self._dataset)]['data'].value[:,channel_index]
data = data * bit_volts[channel_index]
anasig = AnalogSignal(data,
units=sig_unit,
sampling_rate=self._attrs['kwik']['sample_rate']*pq.Hz,
t_start=self._attrs['kwik']['start_time']*pq.s,
channel_index=channel_index,
)
data = [] # delete from memory
# for attributes out of neo you can annotate
anasig.annotate(info='raw traces')
return anasig
def _extract_signal(self, data, metadata, channel_index, lazy):
signal = None
if lazy:
if channel_index in data[:, 1]:
signal = AnalogSignal([],
units=self._determine_units(metadata),
sampling_period=metadata['dt']*pq.ms,
channel_index=channel_index)
signal.lazy_shape = None
else:
arr = self._extract_array(data, channel_index)
if len(arr) > 0:
signal = AnalogSignal(arr,
units=self._determine_units(metadata),
sampling_period=metadata['dt']*pq.ms,
channel_index=channel_index)
if signal is not None:
signal.annotate(label=metadata["label"],
variable=metadata["variable"])
return signal
def gettrace(trec,f):
import numpy as np
format_type_lenghts = [2,4,4,8]
format_type = [np.int16,np.int32,np.float32,np.float64]
pointsize = format_type_lenghts[int(trec.trDataFormat)]
dtype = format_type[int(trec.trDataFormat)]
f.seek(int(trec.trData))
byte_string = f.read(int(trec.trDataPoints)*pointsize)
import numpy as np
ydata = np.fromstring(byte_string,dtype = dtype)
tunit = pq.Quantity(1,str(trec.trXUnit))
yunit = pq.Quantity(1,str(trec.trYUnit))
sig = AnalogSignal(ydata*float(trec.trDataScaler)*yunit,
sampling_period=float(trec.trXInterval)*tunit,
units = trec.trYUnit[0])
annotations = trec.__dict__.keys()
annotations.remove('readlist')
for a in annotations:
d = {a:str(trec.__dict__[a])}
sig.annotate(**d)
return sig
def read_analogsignal(self, lazy=False, cascade=True):
if not HAVE_IGOR:
raise Exception("igor package not installed. Try `pip install igor`")
data = bw.load(self.filename)
version = data['version']
if version > 3:
raise IOError("Igor binary wave file format version {0} is not supported.".format(version))
content = data['wave']
if "padding" in content:
assert content['padding'].size == 0, "Cannot handle non-empty padding"
if lazy:
# not really lazy, since the `igor` module loads the data anyway
signal = np.array((), dtype=content['wData'].dtype)
else:
signal = content['wData']
note = content['note']
header = content['wave_header']
name = header['bname']
assert header['botFullScale'] == 0
assert header['topFullScale'] == 0
units = "".join(header['dataUnits'])
time_units = "".join(header['xUnits']) or "s"
t_start = pq.Quantity(header['hsB'], time_units)
sampling_period = pq.Quantity(header['hsA'], time_units)
if self.parse_notes:
try:
annotations = self.parse_notes(note)
except ValueError:
warn("Couldn't parse notes field.")
annotations = {'note': note}
else:
annotations = {'note': note}
signal = AnalogSignal(signal, units=units, copy=False, t_start=t_start,
sampling_period=sampling_period, name=name,
file_origin=self.filename, **annotations)
if lazy:
signal.lazy_shape = content['wData'].shape
return signal
def test_read_segment_containing_analogsignals_using_eager_cascade(self):
# eager == not lazy
io = self.io_cls(self.test_file)
segment = io.read_segment(lazy=False, cascade=True)
self.assertIsInstance(segment, Segment)
self.assertEqual(len(segment.analogsignals), NCELLS)
as0 = segment.analogsignals[0]
self.assertIsInstance(as0, AnalogSignal)
assert_arrays_equal(
as0,
AnalogSignal(np.arange(0, 101, dtype=float),
sampling_period=0.1 * pq.ms,
t_start=0 * pq.s,
units=pq.mV))
as4 = segment.analogsignals[4]
self.assertIsInstance(as4, AnalogSignal)
assert_arrays_equal(
as4,
AnalogSignal(np.arange(4, 105, dtype=float),
sampling_period=0.1 * pq.ms,
t_start=0 * pq.s,
units=pq.mV))
def read_block(self, lazy=False, cascade=True):
header = self.read_header()
version = header['fFileVersionNumber']
bl = Block()
bl.file_origin = os.path.basename(self.filename)
bl.annotate(abf_version=str(version))
# date and time
if version < 2.:
YY = 1900
MM = 1
DD = 1
hh = int(header['lFileStartTime'] / 3600.)
mm = int((header['lFileStartTime'] - hh * 3600) / 60)
ss = header['lFileStartTime'] - hh * 3600 - mm * 60
ms = int(np.mod(ss, 1) * 1e6)
ss = int(ss)
elif version >= 2.:
YY = int(header['uFileStartDate'] / 10000)
MM = int((header['uFileStartDate'] - YY * 10000) / 100)
DD = int(header['uFileStartDate'] - YY * 10000 - MM * 100)
hh = int(header['uFileStartTimeMS'] / 1000. / 3600.)
mm = int((header['uFileStartTimeMS'] / 1000. - hh * 3600) / 60)
ss = header['uFileStartTimeMS'] / 1000. - hh * 3600 - mm * 60
ms = int(np.mod(ss, 1) * 1e6)
ss = int(ss)
bl.rec_datetime = datetime.datetime(YY, MM, DD, hh, mm, ss, ms)
if not cascade:
return bl
# file format
if header['nDataFormat'] == 0:
dt = np.dtype('i2')
elif header['nDataFormat'] == 1:
dt = np.dtype('f4')
if version < 2.:
nbchannel = header['nADCNumChannels']
head_offset = header['lDataSectionPtr'] * BLOCKSIZE + header[
'nNumPointsIgnored'] * dt.itemsize
totalsize = header['lActualAcqLength']
elif version >= 2.:
nbchannel = header['sections']['ADCSection']['llNumEntries']
head_offset = header['sections']['DataSection'][
'uBlockIndex'] * BLOCKSIZE
totalsize = header['sections']['DataSection']['llNumEntries']
data = np.memmap(self.filename, dt, 'r',
shape=(totalsize,), offset=head_offset)
# 3 possible modes
if version < 2.:
mode = header['nOperationMode']
elif version >= 2.:
mode = header['protocol']['nOperationMode']
if (mode == 1) or (mode == 2) or (mode == 5) or (mode == 3):
# event-driven variable-length mode (mode 1)
# event-driven fixed-length mode (mode 2 or 5)
# gap free mode (mode 3) can be in several episodes
# read sweep pos
if version < 2.:
nbepisod = header['lSynchArraySize']
offset_episode = header['lSynchArrayPtr'] * BLOCKSIZE
elif version >= 2.:
nbepisod = header['sections']['SynchArraySection'][
'llNumEntries']
offset_episode = header['sections']['SynchArraySection'][
'uBlockIndex'] * BLOCKSIZE
if nbepisod > 0:
episode_array = np.memmap(
self.filename, [('offset', 'i4'), ('len', 'i4')], 'r',
shape=nbepisod, offset=offset_episode)
else:
episode_array = np.empty(1, [('offset', 'i4'), ('len', 'i4')])
episode_array[0]['len'] = data.size
episode_array[0]['offset'] = 0
# sampling_rate
if version < 2.:
sampling_rate = 1. / (header['fADCSampleInterval'] *
nbchannel * 1.e-6) * pq.Hz
elif version >= 2.:
sampling_rate = 1.e6 / \
header['protocol']['fADCSequenceInterval'] * pq.Hz
# construct block
# one sweep = one segment in a block
pos = 0
for j in range(episode_array.size):
seg = Segment(index=j)
length = episode_array[j]['len']
if version < 2.:
fSynchTimeUnit = header['fSynchTimeUnit']
elif version >= 2.:
fSynchTimeUnit = header['protocol']['fSynchTimeUnit']
if (fSynchTimeUnit != 0) and (mode == 1):
length /= fSynchTimeUnit
if not lazy:
subdata = data[pos:pos+length]
subdata = subdata.reshape((int(subdata.size/nbchannel),
nbchannel)).astype('f')
if dt == np.dtype('i2'):
if version < 2.:
reformat_integer_v1(subdata, nbchannel, header)
elif version >= 2.:
reformat_integer_v2(subdata, nbchannel, header)
pos += length
if version < 2.:
chans = [chan_num for chan_num in
header['nADCSamplingSeq'] if chan_num >= 0]
else:
chans = range(nbchannel)
for n, i in enumerate(chans[:nbchannel]): # fix SamplingSeq
if version < 2.:
name = header['sADCChannelName'][i].replace(b' ', b'')
unit = header['sADCUnits'][i].replace(b'\xb5', b'u').\
replace(b' ', b'').decode('utf-8') # \xb5 is µ
num = header['nADCPtoLChannelMap'][i]
elif version >= 2.:
lADCIi = header['listADCInfo'][i]
name = lADCIi['ADCChNames'].replace(b' ', b'')
unit = lADCIi['ADCChUnits'].replace(b'\xb5', b'u').\
replace(b' ', b'').decode('utf-8')
num = header['listADCInfo'][i]['nADCNum']
if (fSynchTimeUnit == 0):
t_start = float(episode_array[j]['offset']) / sampling_rate
else:
t_start = float(episode_array[j]['offset']) * fSynchTimeUnit *1e-6* pq.s
t_start = t_start.rescale('s')
try:
pq.Quantity(1, unit)
except:
unit = ''
if lazy:
signal = [] * pq.Quantity(1, unit)
else:
signal = pq.Quantity(subdata[:, n], unit)
anaSig = AnalogSignal(signal, sampling_rate=sampling_rate,
t_start=t_start,
name=str(name),
channel_index=int(num))
if lazy:
anaSig.lazy_shape = length / nbchannel
seg.analogsignals.append(anaSig)
bl.segments.append(seg)
if mode in [3, 5]: # TODO check if tags exits in other mode
# tag is EventArray that should be attached to Block
# It is attched to the first Segment
times = []
labels = []
comments = []
for i, tag in enumerate(header['listTag']):
times.append(tag['lTagTime']/sampling_rate)
labels.append(str(tag['nTagType']))
comments.append(clean_string(tag['sComment']))
times = np.array(times)
labels = np.array(labels, dtype='S')
comments = np.array(comments, dtype='S')
# attach all tags to the first segment.
seg = bl.segments[0]
if lazy:
ea = Event(times=[] * pq.s, labels=np.array([], dtype='S'))
ea.lazy_shape = len(times)
else:
ea = Event(times=times * pq.s, labels=labels,
comments=comments)
seg.events.append(ea)
bl.create_many_to_one_relationship()
return bl
def read_one_channel_continuous(self, fid, channel_num, header,
take_ideal_sampling_rate, lazy=True):
# read AnalogSignal
channelHeader = header.channelHeaders[channel_num]
# data type
if channelHeader.kind == 1:
dt = np.dtype('i2')
elif channelHeader.kind == 9:
dt = np.dtype('f4')
# sample rate
if take_ideal_sampling_rate:
sampling_rate = channelHeader.ideal_rate * pq.Hz
else:
if header.system_id in [1, 2, 3, 4, 5]: # Before version 5
#~ print channel_num, channelHeader.divide, \
#~ header.us_per_time, header.time_per_adc
sample_interval = (channelHeader.divide * header.us_per_time *
header.time_per_adc) * 1e-6
else:
sample_interval = (channelHeader.l_chan_dvd *
header.us_per_time * header.dtime_base)
sampling_rate = (1. / sample_interval) * pq.Hz
# read blocks header to preallocate memory by jumping block to block
if channelHeader.blocks==0:
return [ ]
fid.seek(channelHeader.firstblock)
blocksize = [0]
starttimes = []
for b in range(channelHeader.blocks):
blockHeader = HeaderReader(fid, np.dtype(blockHeaderDesciption))
if len(blocksize) > len(starttimes):
starttimes.append(blockHeader.start_time)
blocksize[-1] += blockHeader.items
if blockHeader.succ_block > 0:
# ugly but CED does not guarantee continuity in AnalogSignal
fid.seek(blockHeader.succ_block)
nextBlockHeader = HeaderReader(fid,
np.dtype(blockHeaderDesciption))
sample_interval = (blockHeader.end_time -
blockHeader.start_time) / \
(blockHeader.items - 1)
interval_with_next = nextBlockHeader.start_time - \
blockHeader.end_time
if interval_with_next > sample_interval:
blocksize.append(0)
fid.seek(blockHeader.succ_block)
ana_sigs = []
if channelHeader.unit in unit_convert:
unit = pq.Quantity(1, unit_convert[channelHeader.unit])
else:
# print channelHeader.unit
try:
unit = pq.Quantity(1, channelHeader.unit)
except:
unit = pq.Quantity(1, '')
for b, bs in enumerate(blocksize):
if lazy:
signal = [] * unit
else:
signal = pq.Quantity(np.empty(bs, dtype='f4'), units=unit)
ana_sig = AnalogSignal(
signal, sampling_rate=sampling_rate,
t_start=(starttimes[b] * header.us_per_time *
header.dtime_base * pq.s),
channel_index=channel_num)
ana_sigs.append(ana_sig)
if lazy:
for s, ana_sig in enumerate(ana_sigs):
ana_sig.lazy_shape = blocksize[s]
else:
# read data by jumping block to block
fid.seek(channelHeader.firstblock)
pos = 0
numblock = 0
for b in range(channelHeader.blocks):
blockHeader = HeaderReader(
fid, np.dtype(blockHeaderDesciption))
# read data
sig = np.fromstring(fid.read(blockHeader.items * dt.itemsize),
dtype=dt)
ana_sigs[numblock][pos:pos + sig.size] = \
sig.reshape(-1, 1).astype('f4') * unit
pos += sig.size
if pos >= blocksize[numblock]:
numblock += 1
pos = 0
# jump to next block
if blockHeader.succ_block > 0:
fid.seek(blockHeader.succ_block)
# convert for int16
if dt.kind == 'i':
for ana_sig in ana_sigs:
ana_sig *= channelHeader.scale / 6553.6
ana_sig += channelHeader.offset * unit
return ana_sigs
def read_segment(self, lazy = False, cascade = True):
## Read header file (vhdr)
header = readBrainSoup(self.filename)
assert header['Common Infos']['DataFormat'] == 'BINARY', NotImplementedError
assert header['Common Infos']['DataOrientation'] == 'MULTIPLEXED', NotImplementedError
nb_channel = int(header['Common Infos']['NumberOfChannels'])
sampling_rate = 1.e6/float(header['Common Infos']['SamplingInterval']) * pq.Hz
fmt = header['Binary Infos']['BinaryFormat']
fmts = { 'INT_16':np.int16, 'IEEE_FLOAT_32':np.float32,}
assert fmt in fmts, NotImplementedError
dt = fmts[fmt]
seg = Segment(file_origin = os.path.basename(self.filename), )
if not cascade : return seg
# read binary
if not lazy:
binary_file = os.path.splitext(self.filename)[0]+'.eeg'
sigs = np.memmap(binary_file , dt, 'r', ).astype('f')
n = int(sigs.size/nb_channel)
sigs = sigs[:n*nb_channel]
sigs = sigs.reshape(n, nb_channel)
for c in range(nb_channel):
name, ref, res, units = header['Channel Infos']['Ch%d' % (c+1,)].split(',')
units = pq.Quantity(1, units.replace('µ', 'u') )
if lazy:
signal = [ ]*units
else:
signal = sigs[:,c]*units
anasig = AnalogSignal(signal = signal,
channel_index = c,
name = name,
sampling_rate = sampling_rate,
)
if lazy:
anasig.lazy_shape = -1
seg.analogsignals.append(anasig)
# read marker
marker_file = os.path.splitext(self.filename)[0]+'.vmrk'
all_info = readBrainSoup(marker_file)['Marker Infos']
all_types = [ ]
times = [ ]
labels = [ ]
for i in range(len(all_info)):
type_, label, pos, size, channel = all_info['Mk%d' % (i+1,)].split(',')[:5]
all_types.append(type_)
times.append(float(pos)/sampling_rate.magnitude)
labels.append(label)
all_types = np.array(all_types)
times = np.array(times) * pq.s
labels = np.array(labels, dtype = 'S')
for type_ in np.unique(all_types):
ind = type_ == all_types
if lazy:
ea = EventArray(name = str(type_))
ea.lazy_shape = -1
else:
ea = EventArray( times = times[ind],
labels = labels[ind],
name = str(type_),
)
seg.eventarrays.append(ea)
seg.create_many_to_one_relationship()
return seg
def read_nsx(self, filename_nsx, seg, lazy, cascade):
# basic header
dt0 = [('header_id','S8'),
('ver_major','uint8'),
('ver_minor','uint8'),
('header_size', 'uint32'), #i.e. index of first data
('group_label', 'S16'),# Read number of packet bytes, i.e. byte per samplepos
('comments', 'S256'),
('period_ratio', 'uint32'),
('sampling_rate', 'uint32'),
('window_datetime', 'S16'),
('nb_channel', 'uint32'),
]
nsx_header = h = np.fromfile(filename_nsx, count = 1, dtype = dt0)[0]
version = '{0}.{1}'.format(h['ver_major'], h['ver_minor'])
seg.annotate(blackrock_version = version)
seg.rec_datetime = get_window_datetime(nsx_header['window_datetime'])
nb_channel = h['nb_channel']
sr = float(h['sampling_rate'])/h['period_ratio']
if not cascade:
return
# extended header = channel information
dt1 = [('header_id','S2'),
('channel_id', 'uint16'),
('label', 'S16'),
('connector_id', 'uint8'),
('connector_pin', 'uint8'),
('min_digital_val', 'int16'),
('max_digital_val', 'int16'),
('min_analog_val', 'int16'),
('max_analog_val', 'int16'),
('units', 'S16'),
('hi_freq_corner', 'uint32'),
('hi_freq_order', 'uint32'),
('hi_freq_type', 'uint16'), #0=None 1=Butterworth
('lo_freq_corner', 'uint32'),
('lo_freq_order', 'uint32'),
('lo_freq_type', 'uint16'), #0=None 1=Butterworth
]
channels_header = ch= np.memmap(filename_nsx, shape = nb_channel,
offset = np.dtype(dt0).itemsize, dtype = dt1)
# read data
dt2 = [('header_id','uint8'),
('n_start','uint32'),
('nb_sample','uint32'),
]
sample_header = sh = np.memmap(filename_nsx, dtype = dt2, shape = 1,
offset = nsx_header['header_size'])[0]
nb_sample = sample_header['nb_sample']
data = np.memmap(filename_nsx, dtype = 'int16', shape = (nb_sample, nb_channel),
offset = nsx_header['header_size'] +np.dtype(dt2).itemsize )
# create new objects
for i in range(nb_channel):
unit = channels_header['units'][i].decode()
if lazy:
sig = [ ]
else:
sig = data[:,i].astype(float)
# dig value to physical value
if ch['max_analog_val'][i] == -ch['min_analog_val'][i] and\
ch['max_digital_val'][i] == -ch['min_digital_val'][i]:
# when symmetric it is simple
sig *= float(ch['max_analog_val'][i])/float(ch['max_digital_val'][i])
else:
# general case
sig -= ch['min_digital_val'][i]
sig *= float(ch['max_analog_val'][i] - ch['min_analog_val'])/\
float(ch['max_digital_val'][i] - ch['min_digital_val'])
sig += float(ch['min_analog_val'][i])
anasig = AnalogSignal(signal = pq.Quantity(sig,unit, copy = False),
sampling_rate = sr*pq.Hz,
t_start = sample_header['n_start']/sr*pq.s,
name = str(ch['label'][i]),
channel_index = int(ch['channel_id'][i]))
if lazy:
anasig.lazy_shape = nb_sample
seg.analogsignals.append(anasig)
def read_analogsignal(self, path=None, lazy=False, cascade=True):
if not HAVE_IGOR:
raise Exception(("`igor` package not installed. "
"Try `pip install igor`"))
if self.extension == 'ibw':
data = bw.load(self.filename)
version = data['version']
if version > 5:
raise IOError(("Igor binary wave file format version {0} "
"is not supported.".format(version)))
elif self.extension == 'pxp':
assert type(path) is str, \
"A colon-separated Igor-style path must be provided."
_,filesystem = pxp.load(self.filename)
path = path.split(':')
location = filesystem['root']
for element in path:
if element != 'root':
location = location[element.encode('utf8')]
data = location.wave
content = data['wave']
if "padding" in content:
assert content['padding'].size == 0, \
"Cannot handle non-empty padding"
if lazy:
# not really lazy, since the `igor` module loads the data anyway
signal = np.array((), dtype=content['wData'].dtype)
else:
signal = content['wData']
note = content['note']
header = content['wave_header']
name = header['bname']
units = "".join([x.decode() for x in header['dataUnits']])
try:
time_units = "".join([x.decode() for x in header['xUnits']])
assert len(time_units)
except:
time_units = "s"
try:
t_start = pq.Quantity(header['hsB'], time_units)
except KeyError:
t_start = pq.Quantity(header['sfB'][0], time_units)
try:
sampling_period = pq.Quantity(header['hsA'], time_units)
except:
sampling_period = pq.Quantity(header['sfA'][0], time_units)
if self.parse_notes:
try:
annotations = self.parse_notes(note)
except ValueError:
warn("Couldn't parse notes field.")
annotations = {'note': note}
else:
annotations = {'note': note}
signal = AnalogSignal(signal, units=units, copy=False, t_start=t_start,
sampling_period=sampling_period, name=name,
file_origin=self.filename, **annotations)
if lazy:
signal.lazy_shape = content['wData'].shape
return signal
def read_segment(self, blockname=None, lazy=False, cascade=True, sortname=''):
"""
Read a single segment from the tank. Note that TDT blocks are Neo
segments, and TDT tanks are Neo blocks, so here the 'blockname' argument
refers to the TDT block's name, which will be the Neo segment name.
sortname is used to specify the external sortcode generated by offline spike sorting, if sortname=='PLX',
there should be a ./sort/PLX/*.SortResult file in the tdt block, which stores the sortcode for every spike,
default to '', which uses the original online sort
"""
if not blockname:
blockname = os.listdir(self.dirname)[0]
if blockname == 'TempBlk': return None
if not self.is_tdtblock(blockname): return None # if not a tdt block
subdir = os.path.join(self.dirname, blockname)
if not os.path.isdir(subdir): return None
seg = Segment(name=blockname)
tankname = os.path.basename(self.dirname)
#TSQ is the global index
tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq')
dt = [('size','int32'),
('evtype','int32'),
('code','S4'),
('channel','uint16'),
('sortcode','uint16'),
('timestamp','float64'),
('eventoffset','int64'),
('dataformat','int32'),
('frequency','float32'),
]
tsq = np.fromfile(tsq_filename, dtype=dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0]
#TEV is the old data file
try:
tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype='uint8')
except IOError:
tev_filename = None
#if exists an external sortcode in ./sort/[sortname]/*.SortResult (generated after offline sortting)
sortresult_filename = None
if sortname is not '':
try:
for file in os.listdir(os.path.join(subdir, 'sort', sortname)):
if file.endswith(".SortResult"):
sortresult_filename = os.path.join(subdir, 'sort', sortname, file)
# get new sortcode
newsorcode = np.fromfile(sortresult_filename,'int8')[1024:] # the first 1024 byte is file header
# update the sort code with the info from this file
tsq['sortcode'][1:-1]=newsorcode
# print('sortcode updated')
break
except OSError:
sortresult_filename = None
except IOError:
sortresult_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype']==type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code']==code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel']==channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [ ]*pq.s
labels = np.array([ ], dtype=str)
else:
times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view('float64').astype('S')
ea = Event(times=times,
name=code,
channel_index=int(channel),
labels=labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.events.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode']==sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0]-10
if lazy:
times = [ ]*pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt)
waveforms = waveforms.reshape(nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike > 0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 *pq.s
t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s
else:
t_start = 0 *pq.s
t_stop = 0 *pq.s
st = SpikeTrain(times = times,
name = 'Chan{0} Code{1}'.format(channel,sortcode),
t_start = t_start,
t_stop = t_stop,
waveforms = waveforms,
left_sweep = waveformsize/2./sr * pq.s,
sampling_rate = sr * pq.Hz,
)
st.annotate(channel_index=channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size']-10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = [ ]
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev')
try:
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename, dtype='uint8')
except IOError:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt)
anasig = AnalogSignal(signal = signal* pq.V,
name = '{0} {1}'.format(code, channel),
sampling_rate = sr * pq.Hz,
t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s,
channel_index = int(channel)
)
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
return seg
def read_segment(self, lazy=False, cascade=True):
fid = open(self.filename, 'rb')
global_header = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# ~ print globalHeader
#~ print 'version' , globalHeader['version']
seg = Segment()
seg.file_origin = os.path.basename(self.filename)
seg.annotate(neuroexplorer_version=global_header['version'])
seg.annotate(comment=global_header['comment'])
if not cascade:
return seg
offset = 544
for i in range(global_header['nvar']):
entity_header = HeaderReader(fid, EntityHeader).read_f(
offset=offset + i * 208)
entity_header['name'] = entity_header['name'].replace('\x00', '')
#print 'i',i, entityHeader['type']
if entity_header['type'] == 0:
# neuron
if lazy:
spike_times = [] * pq.s
else:
spike_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
spike_times = spike_times.astype('f8') / global_header[
'freq'] * pq.s
sptr = SpikeTrain(
times=spike_times,
t_start=global_header['tbeg'] /
global_header['freq'] * pq.s,
t_stop=global_header['tend'] /
global_header['freq'] * pq.s,
name=entity_header['name'])
if lazy:
sptr.lazy_shape = entity_header['n']
sptr.annotate(channel_index=entity_header['WireNumber'])
seg.spiketrains.append(sptr)
if entity_header['type'] == 1:
# event
if lazy:
event_times = [] * pq.s
else:
event_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
event_times = event_times.astype('f8') / global_header[
'freq'] * pq.s
labels = np.array([''] * event_times.size, dtype='S')
evar = Event(times=event_times, labels=labels,
channel_name=entity_header['name'])
if lazy:
evar.lazy_shape = entity_header['n']
seg.events.append(evar)
if entity_header['type'] == 2:
# interval
if lazy:
start_times = [] * pq.s
stop_times = [] * pq.s
else:
start_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
start_times = start_times.astype('f8') / global_header[
'freq'] * pq.s
stop_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'] +
entity_header['n'] * 4)
stop_times = stop_times.astype('f') / global_header[
'freq'] * pq.s
epar = Epoch(times=start_times,
durations=stop_times - start_times,
labels=np.array([''] * start_times.size,
dtype='S'),
channel_name=entity_header['name'])
if lazy:
epar.lazy_shape = entity_header['n']
seg.epochs.append(epar)
if entity_header['type'] == 3:
# spiketrain and wavefoms
if lazy:
spike_times = [] * pq.s
waveforms = None
else:
spike_times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
spike_times = spike_times.astype('f8') / global_header[
'freq'] * pq.s
waveforms = np.memmap(self.filename, np.dtype('i2'), 'r',
shape=(entity_header['n'], 1,
entity_header['NPointsWave']),
offset=entity_header['offset'] +
entity_header['n'] * 4)
waveforms = (waveforms.astype('f') *
entity_header['ADtoMV'] +
entity_header['MVOffset']) * pq.mV
t_stop = global_header['tend'] / global_header['freq'] * pq.s
if spike_times.size > 0:
t_stop = max(t_stop, max(spike_times))
sptr = SpikeTrain(
times=spike_times,
t_start=global_header['tbeg'] /
global_header['freq'] * pq.s,
#~ t_stop = max(globalHeader['tend']/
#~ globalHeader['freq']*pq.s,max(spike_times)),
t_stop=t_stop, name=entity_header['name'],
waveforms=waveforms,
sampling_rate=entity_header['WFrequency'] * pq.Hz,
left_sweep=0 * pq.ms)
if lazy:
sptr.lazy_shape = entity_header['n']
sptr.annotate(channel_index=entity_header['WireNumber'])
seg.spiketrains.append(sptr)
if entity_header['type'] == 4:
# popvectors
pass
if entity_header['type'] == 5:
# analog
timestamps = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
timestamps = timestamps.astype('f8') / global_header['freq']
fragment_starts = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
fragment_starts = fragment_starts.astype('f8') / global_header[
'freq']
t_start = timestamps[0] - fragment_starts[0] / float(
entity_header['WFrequency'])
del timestamps, fragment_starts
if lazy:
signal = [] * pq.mV
else:
signal = np.memmap(self.filename, np.dtype('i2'), 'r',
shape=(entity_header['NPointsWave']),
offset=entity_header['offset'])
signal = signal.astype('f')
signal *= entity_header['ADtoMV']
signal += entity_header['MVOffset']
signal = signal * pq.mV
ana_sig = AnalogSignal(
signal=signal, t_start=t_start * pq.s,
sampling_rate=entity_header['WFrequency'] * pq.Hz,
name=entity_header['name'],
channel_index=entity_header['WireNumber'])
if lazy:
ana_sig.lazy_shape = entity_header['NPointsWave']
seg.analogsignals.append(ana_sig)
if entity_header['type'] == 6:
# markers : TO TEST
if lazy:
times = [] * pq.s
labels = np.array([], dtype='S')
markertype = None
else:
times = np.memmap(self.filename, np.dtype('i4'), 'r',
shape=(entity_header['n']),
offset=entity_header['offset'])
times = times.astype('f8') / global_header['freq'] * pq.s
fid.seek(entity_header['offset'] + entity_header['n'] * 4)
markertype = fid.read(64).replace('\x00', '')
labels = np.memmap(
self.filename, np.dtype(
'S' + str(entity_header['MarkerLength'])),
'r', shape=(entity_header['n']),
offset=entity_header['offset'] +
entity_header['n'] * 4 + 64)
ea = Event(times=times,
labels=labels.view(np.ndarray),
name=entity_header['name'],
channel_index=entity_header['WireNumber'],
marker_type=markertype)
if lazy:
ea.lazy_shape = entity_header['n']
seg.events.append(ea)
seg.create_many_to_one_relationship()
return seg
def read_segment(self,
lazy = False,
cascade = True,
delimiter = '\t',
usecols = None,
skiprows =0,
timecolumn = None,
sampling_rate = 1.*pq.Hz,
t_start = 0.*pq.s,
unit = pq.V,
method = 'genfromtxt',
):
"""
Arguments:
delimiter : columns delimiter in file '\t' or one space or two space or ',' or ';'
usecols : if None take all columns otherwise a list for selected columns
skiprows : skip n first lines in case they contains header informations
timecolumn : None or a valid int that point the time vector
samplerate : the samplerate of signals if timecolumn is not None this is not take in account
t_start : time of the first sample
unit : unit of AnalogSignal can be a str or directly a Quantities
method : 'genfromtxt' or 'csv' or 'homemade'
in case of bugs you can try one of this methods
'genfromtxt' use numpy.genfromtxt
'csv' use cvs module
'homemade' use a intuitive more robust but slow method
"""
seg = Segment(file_origin = os.path.basename(self.filename))
if not cascade:
return seg
if type(sampling_rate) == float or type(sampling_rate)==int:
# if not quantitities Hz by default
sampling_rate = sampling_rate*pq.Hz
if type(t_start) == float or type(t_start)==int:
# if not quantitities s by default
t_start = t_start*pq.s
unit = pq.Quantity(1, unit)
#loadtxt
if method == 'genfromtxt' :
sig = np.genfromtxt(self.filename,
delimiter = delimiter,
usecols = usecols ,
skip_header = skiprows,
dtype = 'f')
if len(sig.shape) ==1:
sig = sig[:, np.newaxis]
elif method == 'csv' :
tab = [l for l in csv.reader( file(self.filename,'rU') , delimiter = delimiter ) ]
tab = tab[skiprows:]
sig = np.array( tab , dtype = 'f')
elif method == 'homemade' :
fid = open(self.filename,'rU')
for l in range(skiprows):
fid.readline()
tab = [ ]
for line in fid.readlines():
line = line.replace('\r','')
line = line.replace('\n','')
l = line.split(delimiter)
while '' in l :
l.remove('')
tab.append(l)
sig = np.array( tab , dtype = 'f')
if timecolumn is not None:
sampling_rate = 1./np.mean(np.diff(sig[:,timecolumn])) * pq.Hz
t_start = sig[0,timecolumn] * pq.s
for i in range(sig.shape[1]) :
if timecolumn == i : continue
if usecols is not None and i not in usecols: continue
if lazy:
signal = [ ]*unit
else:
signal = sig[:,i]*unit
anaSig = AnalogSignal(signal, sampling_rate=sampling_rate,
t_start=t_start, channel_index=i,
name='Column %d'%i)
if lazy:
anaSig.lazy_shape = sig.shape
seg.analogsignals.append( anaSig )
seg.create_many_to_one_relationship()
return seg
def read_block(self,
# the 2 first keyword arguments are imposed by neo.io API
lazy = False,
cascade = True):
"""
Return a Block.
"""
def count_samples(m_length):
"""
Count the number of signal samples available in a type 5 data block
of length m_length
"""
# for information about type 5 data block, see [1]
count = int((m_length-6)/2-2)
# -6 corresponds to the header of block 5, and the -2 take into
# account the fact that last 2 values are not available as the 4
# corresponding bytes are coding the time stamp of the beginning
# of the block
return count
# create the neo Block that will be returned at the end
blck = Block(file_origin = os.path.basename(self.filename))
blck.file_origin = os.path.basename(self.filename)
fid = open(self.filename, 'rb')
# NOTE: in the following, the word "block" is used in the sense used in
# the alpha-omega specifications (ie a data chunk in the file), rather
# than in the sense of the usual Block object in neo
# step 1: read the headers of all the data blocks to load the file
# structure
pos_block = 0 # position of the current block in the file
file_blocks = [] # list of data blocks available in the file
if not cascade:
# we read only the main header
m_length, m_TypeBlock = struct.unpack('Hcx' , fid.read(4))
# m_TypeBlock should be 'h', as we read the first block
block = HeaderReader(fid,
dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({'m_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block})
file_blocks.append(block)
else: # cascade == True
seg = Segment(file_origin = os.path.basename(self.filename))
seg.file_origin = os.path.basename(self.filename)
blck.segments.append(seg)
while True:
first_4_bytes = fid.read(4)
if len(first_4_bytes) < 4:
# we have reached the end of the file
break
else:
m_length, m_TypeBlock = struct.unpack('Hcx', first_4_bytes)
block = HeaderReader(fid,
dict_header_type.get(m_TypeBlock,
Type_Unknown)).read_f()
block.update({'m_length': m_length,
'm_TypeBlock': m_TypeBlock,
'pos': pos_block})
if m_TypeBlock == '2':
# The beginning of the block of type '2' is identical for
# all types of channels, but the following part depends on
# the type of channel. So we need a special case here.
# WARNING: How to check the type of channel is not
# described in the documentation. So here I use what is
# proposed in the C code [2].
# According to this C code, it seems that the 'm_isAnalog'
# is used to distinguished analog and digital channels, and
# 'm_Mode' encodes the type of analog channel:
# 0 for continuous, 1 for level, 2 for external trigger.
# But in some files, I found channels that seemed to be
# continuous channels with 'm_Modes' = 128 or 192. So I
# decided to consider every channel with 'm_Modes'
# different from 1 or 2 as continuous. I also couldn't
# check that values of 1 and 2 are really for level and
# external trigger as I had no test files containing data
# of this types.
type_subblock = 'unknown_channel_type(m_Mode=' \
+ str(block['m_Mode'])+ ')'
description = Type2_SubBlockUnknownChannels
block.update({'m_Name': 'unknown_name'})
if block['m_isAnalog'] == 0:
# digital channel
type_subblock = 'digital'
description = Type2_SubBlockDigitalChannels
elif block['m_isAnalog'] == 1:
# analog channel
if block['m_Mode'] == 1:
# level channel
type_subblock = 'level'
description = Type2_SubBlockLevelChannels
elif block['m_Mode'] == 2:
# external trigger channel
type_subblock = 'external_trigger'
description = Type2_SubBlockExtTriggerChannels
else:
# continuous channel
type_subblock = 'continuous(Mode' \
+ str(block['m_Mode']) +')'
description = Type2_SubBlockContinuousChannels
subblock = HeaderReader(fid, description).read_f()
block.update(subblock)
block.update({'type_subblock': type_subblock})
file_blocks.append(block)
pos_block += m_length
fid.seek(pos_block)
# step 2: find the available channels
list_chan = [] # list containing indexes of channel blocks
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '2':
list_chan.append(ind_block)
# step 3: find blocks containing data for the available channels
list_data = [] # list of lists of indexes of data blocks
# corresponding to each channel
for ind_chan, chan in enumerate(list_chan):
list_data.append([])
num_chan = file_blocks[chan]['m_numChannel']
for ind_block, block in enumerate(file_blocks):
if block['m_TypeBlock'] == '5':
if block['m_numChannel'] == num_chan:
list_data[ind_chan].append(ind_block)
# step 4: compute the length (number of samples) of the channels
chan_len = np.zeros(len(list_data), dtype = np.int)
for ind_chan, list_blocks in enumerate(list_data):
for ind_block in list_blocks:
chan_len[ind_chan] += count_samples(
file_blocks[ind_block]['m_length'])
# step 5: find channels for which data are available
ind_valid_chan = np.nonzero(chan_len)[0]
# step 6: load the data
# TODO give the possibility to load data as AnalogSignalArrays
for ind_chan in ind_valid_chan:
list_blocks = list_data[ind_chan]
ind = 0 # index in the data vector
# read time stamp for the beginning of the signal
form = '<l' # reading format
ind_block = list_blocks[0]
count = count_samples(file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos']+6+count*2)
buf = fid.read(struct.calcsize(form))
val = struct.unpack(form , buf)
start_index = val[0]
# WARNING: in the following blocks are read supposing taht they
# are all contiguous and sorted in time. I don't know if it's
# always the case. Maybe we should use the time stamp of each
# data block to choose where to put the read data in the array.
if not lazy:
temp_array = np.empty(chan_len[ind_chan], dtype = np.int16)
# NOTE: we could directly create an empty AnalogSignal and
# load the data in it, but it is much faster to load data
# in a temporary numpy array and create the AnalogSignals
# from this temporary array
for ind_block in list_blocks:
count = count_samples(
file_blocks[ind_block]['m_length'])
fid.seek(file_blocks[ind_block]['pos']+6)
temp_array[ind:ind+count] = \
np.fromfile(fid, dtype = np.int16, count = count)
ind += count
sampling_rate = \
file_blocks[list_chan[ind_chan]]['m_SampleRate'] * pq.kHz
t_start = (start_index / sampling_rate).simplified
if lazy:
ana_sig = AnalogSignal([],
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
ana_sig.lazy_shape = chan_len[ind_chan]
else:
ana_sig = AnalogSignal(temp_array,
sampling_rate = sampling_rate,
t_start = t_start,
name = file_blocks\
[list_chan[ind_chan]]['m_Name'],
file_origin = \
os.path.basename(self.filename),
units = pq.dimensionless)
# todo apibreak: create ChannelIndex for each signals
# ana_sig.channel_index = \
# file_blocks[list_chan[ind_chan]]['m_numChannel']
ana_sig.annotate(channel_name = \
file_blocks[list_chan[ind_chan]]['m_Name'])
ana_sig.annotate(channel_type = \
file_blocks[list_chan[ind_chan]]['type_subblock'])
seg.analogsignals.append(ana_sig)
fid.close()
if file_blocks[0]['m_TypeBlock'] == 'h': # this should always be true
blck.rec_datetime = datetime.datetime(\
file_blocks[0]['m_date_year'],
file_blocks[0]['m_date_month'],
file_blocks[0]['m_date_day'],
file_blocks[0]['m_time_hour'],
file_blocks[0]['m_time_minute'],
file_blocks[0]['m_time_second'],
10000 * file_blocks[0]['m_time_hsecond'])
# the 10000 is here to convert m_time_hsecond from centisecond
# to microsecond
version = file_blocks[0]['m_version']
blck.annotate(alphamap_version = version)
if cascade:
seg.rec_datetime = blck.rec_datetime.replace()
# I couldn't find a simple copy function for datetime,
# using replace without arguments is a twisted way to make a
# copy
seg.annotate(alphamap_version = version)
if cascade:
blck.create_many_to_one_relationship()
return blck
def read_segment(self, lazy=False, cascade=True, load_spike_waveform=True):
"""
Read in a segment.
Arguments:
load_spike_waveform : load or not waveform of spikes (default True)
"""
fid = open(self.filename, 'rb')
globalHeader = HeaderReader(fid, GlobalHeader).read_f(offset=0)
# metadatas
seg = Segment()
seg.rec_datetime = datetime.datetime(
globalHeader.pop('Year'),
globalHeader.pop('Month'),
globalHeader.pop('Day'),
globalHeader.pop('Hour'),
globalHeader.pop('Minute'),
globalHeader.pop('Second')
)
seg.file_origin = os.path.basename(self.filename)
for key, val in globalHeader.iteritems():
seg.annotate(**{key: val})
if not cascade:
return seg
## Step 1 : read headers
# dsp channels header = spikes and waveforms
dspChannelHeaders = {}
maxunit = 0
maxchan = 0
for _ in range(globalHeader['NumDSPChannels']):
# channel is 1 based
channelHeader = HeaderReader(fid, ChannelHeader).read_f(offset=None)
channelHeader['Template'] = np.array(channelHeader['Template']).reshape((5,64))
channelHeader['Boxes'] = np.array(channelHeader['Boxes']).reshape((5,2,4))
dspChannelHeaders[channelHeader['Channel']] = channelHeader
maxunit = max(channelHeader['NUnits'], maxunit)
maxchan = max(channelHeader['Channel'], maxchan)
# event channel header
eventHeaders = { }
for _ in range(globalHeader['NumEventChannels']):
eventHeader = HeaderReader(fid, EventHeader).read_f(offset=None)
eventHeaders[eventHeader['Channel']] = eventHeader
# slow channel header = signal
slowChannelHeaders = {}
for _ in range(globalHeader['NumSlowChannels']):
slowChannelHeader = HeaderReader(fid, SlowChannelHeader).read_f(offset=None)
slowChannelHeaders[slowChannelHeader['Channel']] = slowChannelHeader
## Step 2 : a first loop for counting size
# signal
nb_samples = np.zeros(len(slowChannelHeaders))
sample_positions = np.zeros(len(slowChannelHeaders))
t_starts = np.zeros(len(slowChannelHeaders), dtype='f')
#spiketimes and waveform
nb_spikes = np.zeros((maxchan+1, maxunit+1) ,dtype='i')
wf_sizes = np.zeros((maxchan+1, maxunit+1, 2) ,dtype='i')
# eventarrays
nb_events = { }
#maxstrsizeperchannel = { }
for chan, h in iteritems(eventHeaders):
nb_events[chan] = 0
#maxstrsizeperchannel[chan] = 0
start = fid.tell()
while fid.tell() !=-1 :
# read block header
dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None)
if dataBlockHeader is None : break
chan = dataBlockHeader['Channel']
unit = dataBlockHeader['Unit']
n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform']
time = (dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 +
dataBlockHeader['TimeStamp'])
if dataBlockHeader['Type'] == 1:
nb_spikes[chan,unit] +=1
wf_sizes[chan,unit,:] = [n1,n2]
fid.seek(n1*n2*2,1)
elif dataBlockHeader['Type'] ==4:
#event
nb_events[chan] += 1
elif dataBlockHeader['Type'] == 5:
#continuous signal
fid.seek(n2*2, 1)
if n2> 0:
nb_samples[chan] += n2
if nb_samples[chan] ==0:
t_starts[chan] = time
## Step 3: allocating memory and 2 loop for reading if not lazy
if not lazy:
# allocating mem for signal
sigarrays = { }
for chan, h in iteritems(slowChannelHeaders):
sigarrays[chan] = np.zeros(nb_samples[chan])
# allocating mem for SpikeTrain
stimearrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object)
swfarrays = np.zeros((maxchan+1, maxunit+1) ,dtype=object)
for (chan, unit), _ in np.ndenumerate(nb_spikes):
stimearrays[chan,unit] = np.zeros(nb_spikes[chan,unit], dtype = 'f')
if load_spike_waveform:
n1,n2 = wf_sizes[chan, unit,:]
swfarrays[chan, unit] = np.zeros( (nb_spikes[chan, unit], n1, n2 ) , dtype = 'f4' )
pos_spikes = np.zeros(nb_spikes.shape, dtype = 'i')
# allocating mem for event
eventpositions = { }
evarrays = { }
for chan, nb in iteritems(nb_events):
evarrays[chan] = {
'times': np.zeros(nb, dtype='f'),
'labels': np.zeros(nb, dtype='S4')
}
eventpositions[chan]=0
fid.seek(start)
while fid.tell() !=-1 :
dataBlockHeader = HeaderReader(fid , DataBlockHeader ).read_f(offset = None)
if dataBlockHeader is None : break
chan = dataBlockHeader['Channel']
n1,n2 = dataBlockHeader['NumberOfWaveforms'] , dataBlockHeader['NumberOfWordsInWaveform']
time = dataBlockHeader['UpperByteOf5ByteTimestamp']*2.**32 + dataBlockHeader['TimeStamp']
time/= globalHeader['ADFrequency']
if n2 <0: break
if dataBlockHeader['Type'] == 1:
#spike
unit = dataBlockHeader['Unit']
pos = pos_spikes[chan,unit]
stimearrays[chan, unit][pos] = time
if load_spike_waveform and n1*n2 != 0 :
swfarrays[chan,unit][pos,:,:] = np.fromstring( fid.read(n1*n2*2) , dtype = 'i2').reshape(n1,n2).astype('f4')
else:
fid.seek(n1*n2*2,1)
pos_spikes[chan,unit] +=1
elif dataBlockHeader['Type'] == 4:
# event
pos = eventpositions[chan]
evarrays[chan]['times'][pos] = time
evarrays[chan]['labels'][pos] = dataBlockHeader['Unit']
eventpositions[chan]+= 1
elif dataBlockHeader['Type'] == 5:
#signal
data = np.fromstring( fid.read(n2*2) , dtype = 'i2').astype('f4')
sigarrays[chan][sample_positions[chan] : sample_positions[chan]+data.size] = data
sample_positions[chan] += data.size
## Step 4: create neo object
for chan, h in iteritems(eventHeaders):
if lazy:
times = []
labels = None
else:
times = evarrays[chan]['times']
labels = evarrays[chan]['labels']
ea = EventArray(
times*pq.s,
labels=labels,
channel_name=eventHeaders[chan]['Name'],
channel_index=chan
)
if lazy:
ea.lazy_shape = nb_events[chan]
seg.eventarrays.append(ea)
for chan, h in iteritems(slowChannelHeaders):
if lazy:
signal = [ ]
else:
if globalHeader['Version'] ==100 or globalHeader['Version'] ==101 :
gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*1000.)
elif globalHeader['Version'] ==102 :
gain = 5000./(2048*slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain'])
elif globalHeader['Version'] >= 103:
gain = globalHeader['SlowMaxMagnitudeMV']/(.5*(2**globalHeader['BitsPerSpikeSample'])*\
slowChannelHeaders[chan]['Gain']*slowChannelHeaders[chan]['PreampGain'])
signal = sigarrays[chan]*gain
anasig = AnalogSignal(signal*pq.V,
sampling_rate = float(slowChannelHeaders[chan]['ADFreq'])*pq.Hz,
t_start = t_starts[chan]*pq.s,
channel_index = slowChannelHeaders[chan]['Channel'],
channel_name = slowChannelHeaders[chan]['Name'],
)
if lazy:
anasig.lazy_shape = nb_samples[chan]
seg.analogsignals.append(anasig)
for (chan, unit), value in np.ndenumerate(nb_spikes):
if nb_spikes[chan, unit] == 0: continue
if lazy:
times = [ ]
waveforms = None
t_stop = 0
else:
times = stimearrays[chan,unit]
t_stop = times.max()
if load_spike_waveform:
if globalHeader['Version'] <103:
gain = 3000./(2048*dspChannelHeaders[chan]['Gain']*1000.)
elif globalHeader['Version'] >=103 and globalHeader['Version'] <105:
gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*1000.)
elif globalHeader['Version'] >105:
gain = globalHeader['SpikeMaxMagnitudeMV']/(.5*2.**(globalHeader['BitsPerSpikeSample'])*globalHeader['SpikePreAmpGain'])
waveforms = swfarrays[chan, unit] * gain * pq.V
else:
waveforms = None
sptr = SpikeTrain(
times,
units='s',
t_stop=t_stop*pq.s,
waveforms=waveforms
)
sptr.annotate(unit_name = dspChannelHeaders[chan]['Name'])
sptr.annotate(channel_index = chan)
for key, val in dspChannelHeaders[chan].iteritems():
sptr.annotate(**{key: val})
if lazy:
sptr.lazy_shape = nb_spikes[chan,unit]
seg.spiketrains.append(sptr)
seg.create_many_to_one_relationship()
return seg
def _read_segment(self, fobject, lazy):
"""
Read a single segment with a single analogsignal
Returns the segment or None if there are no more segments
"""
try:
# float64 -- start time of the AnalogSignal
t_start = np.fromfile(fobject, dtype=np.float64, count=1)[0]
except IndexError:
# if there are no more Segments, return
return False
# int16 -- index of the stimulus parameters
seg_index = np.fromfile(fobject, dtype=np.int16, count=1)[0].tolist()
# int16 -- number of stimulus parameters
numelements = np.fromfile(fobject, dtype=np.int16, count=1)[0]
# read the name strings for the stimulus parameters
paramnames = []
for _ in range(numelements):
# unit8 -- the number of characters in the string
numchars = np.fromfile(fobject, dtype=np.uint8, count=1)[0]
# char * numchars -- a single name string
name = np.fromfile(fobject, dtype=np.uint8, count=numchars)
# exclude invalid characters
name = str(name[name >= 32].view("c").tostring())
# add the name to the list of names
paramnames.append(name)
# float32 * numelements -- the values for the stimulus parameters
paramvalues = np.fromfile(fobject, dtype=np.float32, count=numelements)
# combine parameter names and the parameters as a dict
params = dict(zip(paramnames, paramvalues))
# int32 -- the number elements in the AnalogSignal
numpts = np.fromfile(fobject, dtype=np.int32, count=1)[0]
# int16 * numpts -- the AnalogSignal itself
signal = np.fromfile(fobject, dtype=np.int16, count=numpts)
# handle lazy loading
if lazy:
sig = AnalogSignal(
[],
t_start=t_start * pq.d,
file_origin=self._filename,
sampling_period=1.0 * pq.s,
units=pq.mV,
dtype=np.float,
)
sig.lazy_shape = len(signal)
else:
sig = AnalogSignal(
signal.astype(np.float) * pq.mV,
t_start=t_start * pq.d,
file_origin=self._filename,
sampling_period=1.0 * pq.s,
copy=False,
)
# Note: setting the sampling_period to 1 s is arbitrary
# load the AnalogSignal and parameters into a new Segment
seg = Segment(file_origin=self._filename, index=seg_index, **params)
seg.analogsignals = [sig]
return seg
def read_segment(self, cascade=True, lazy=False, ):
"""
Arguments:
"""
f = StructFile(open(self.filename, 'rb'))
# Name
f.seek(64, 0)
surname = f.read(22).decode('ascii')
while surname[-1] == ' ':
if len(surname) == 0:
break
surname = surname[:-1]
firstname = f.read(20).decode('ascii')
while firstname[-1] == ' ':
if len(firstname) == 0:
break
firstname = firstname[:-1]
#Date
f.seek(128, 0)
day, month, year, hour, minute, sec = f.read_f('bbbbbb')
rec_datetime = datetime.datetime(year + 1900, month, day, hour, minute,
sec)
f.seek(138, 0)
Data_Start_Offset, Num_Chan, Multiplexer, Rate_Min, Bytes = f.read_f(
'IHHHH')
#~ print Num_Chan, Bytes
#header version
f.seek(175, 0)
header_version, = f.read_f('b')
assert header_version == 4
seg = Segment(name=str(firstname + ' ' + surname),
file_origin=os.path.basename(self.filename))
seg.annotate(surname=surname)
seg.annotate(firstname=firstname)
seg.annotate(rec_datetime=rec_datetime)
if not cascade:
f.close()
return seg
# area
f.seek(176, 0)
zone_names = ['ORDER', 'LABCOD', 'NOTE', 'FLAGS', 'TRONCA', 'IMPED_B',
'IMPED_E', 'MONTAGE',
'COMPRESS', 'AVERAGE', 'HISTORY', 'DVIDEO', 'EVENT A',
'EVENT B', 'TRIGGER']
zones = {}
for zname in zone_names:
zname2, pos, length = f.read_f('8sII')
zones[zname] = zname2, pos, length
#~ print zname2, pos, length
# reading raw data
if not lazy:
f.seek(Data_Start_Offset, 0)
rawdata = np.fromstring(f.read(), dtype='u' + str(Bytes))
rawdata = rawdata.reshape((-1, Num_Chan))
# Reading Code Info
zname2, pos, length = zones['ORDER']
f.seek(pos, 0)
code = np.fromstring(f.read(Num_Chan*2), dtype='u2', count=Num_Chan)
units = {-1: pq.nano * pq.V, 0: pq.uV, 1: pq.mV, 2: 1, 100: pq.percent,
101: pq.dimensionless, 102: pq.dimensionless}
for c in range(Num_Chan):
zname2, pos, length = zones['LABCOD']
f.seek(pos + code[c] * 128 + 2, 0)
label = f.read(6).strip(b"\x00").decode('ascii')
ground = f.read(6).strip(b"\x00").decode('ascii')
(logical_min, logical_max, logical_ground, physical_min,
physical_max) = f.read_f('iiiii')
k, = f.read_f('h')
if k in units.keys():
unit = units[k]
else:
unit = pq.uV
f.seek(8, 1)
sampling_rate, = f.read_f('H') * pq.Hz
sampling_rate *= Rate_Min
if lazy:
signal = [] * unit
else:
factor = float(physical_max - physical_min) / float(
logical_max - logical_min + 1)
signal = (rawdata[:, c].astype(
'f') - logical_ground) * factor * unit
ana_sig = AnalogSignal(signal, sampling_rate=sampling_rate,
name=str(label), channel_index=c)
if lazy:
ana_sig.lazy_shape = None
ana_sig.annotate(ground=ground)
seg.analogsignals.append(ana_sig)
sampling_rate = np.mean(
[ana_sig.sampling_rate for ana_sig in seg.analogsignals]) * pq.Hz
# Read trigger and notes
for zname, label_dtype in [('TRIGGER', 'u2'), ('NOTE', 'S40')]:
zname2, pos, length = zones[zname]
f.seek(pos, 0)
triggers = np.fromstring(f.read(length), dtype=[('pos', 'u4'), (
'label', label_dtype)])
if not lazy:
keep = (triggers['pos'] >= triggers['pos'][0]) & (
triggers['pos'] < rawdata.shape[0]) & (
triggers['pos'] != 0)
triggers = triggers[keep]
ea = Event(name=zname[0] + zname[1:].lower(),
labels=triggers['label'].astype('S'),
times=(triggers['pos'] / sampling_rate).rescale('s'))
else:
ea = Event(name=zname[0] + zname[1:].lower())
ea.lazy_shape = triggers.size
seg.events.append(ea)
# Read Event A and B
# Not so well tested
for zname in ['EVENT A', 'EVENT B']:
zname2, pos, length = zones[zname]
f.seek(pos, 0)
epochs = np.fromstring(f.read(length),
dtype=[('label', 'u4'), ('start', 'u4'),
('stop', 'u4'), ])
ep = Epoch(name=zname[0] + zname[1:].lower())
if not lazy:
keep = (epochs['start'] > 0) & (
epochs['start'] < rawdata.shape[0]) & (
epochs['stop'] < rawdata.shape[0])
epochs = epochs[keep]
ep = Epoch(name=zname[0] + zname[1:].lower(),
labels=epochs['label'].astype('S'),
times=(epochs['start'] / sampling_rate).rescale('s'),
durations=((epochs['stop'] - epochs['start']) / sampling_rate).rescale('s'))
else:
ep = Epoch(name=zname[0] + zname[1:].lower())
ep.lazy_shape = triggers.size
seg.epochs.append(ep)
seg.create_many_to_one_relationship()
f.close()
return seg
def read_segment(self, lazy=False, cascade=True):
# # Read header file
f = open(self.filename + '.ent', 'rU')
#version
version = f.readline()
if version[:2] != 'V2' and version[:2] != 'V3':
# raise('read only V2 .eeg.ent files')
raise VersionError('Read only V2 or V3 .eeg.ent files. %s given' %
version[:2])
#info
info1 = f.readline()[:-1]
info2 = f.readline()[:-1]
# strange 2 line for datetime
#line1
l = f.readline()
r1 = re.findall('(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)', l)
r2 = re.findall('(\d+):(\d+):(\d+)', l)
r3 = re.findall('(\d+)-(\d+)-(\d+)', l)
YY, MM, DD, hh, mm, ss = (None, ) * 6
if len(r1) != 0:
DD, MM, YY, hh, mm, ss = r1[0]
elif len(r2) != 0:
hh, mm, ss = r2[0]
elif len(r3) != 0:
DD, MM, YY = r3[0]
#line2
l = f.readline()
r1 = re.findall('(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)', l)
r2 = re.findall('(\d+):(\d+):(\d+)', l)
r3 = re.findall('(\d+)-(\d+)-(\d+)', l)
if len(r1) != 0:
DD, MM, YY, hh, mm, ss = r1[0]
elif len(r2) != 0:
hh, mm, ss = r2[0]
elif len(r3) != 0:
DD, MM, YY = r3[0]
try:
fulldatetime = datetime.datetime(int(YY), int(MM), int(DD),
int(hh), int(mm), int(ss))
except:
fulldatetime = None
seg = Segment(file_origin=os.path.basename(self.filename),
elan_version=version,
info1=info1,
info2=info2,
rec_datetime=fulldatetime)
if not cascade:
return seg
l = f.readline()
l = f.readline()
l = f.readline()
# sampling rate sample
l = f.readline()
sampling_rate = 1. / float(l) * pq.Hz
# nb channel
l = f.readline()
nbchannel = int(l) - 2
#channel label
labels = []
for c in range(nbchannel + 2):
labels.append(f.readline()[:-1])
# channel type
types = []
for c in range(nbchannel + 2):
types.append(f.readline()[:-1])
# channel unit
units = []
for c in range(nbchannel + 2):
units.append(f.readline()[:-1])
#print units
#range
min_physic = []
for c in range(nbchannel + 2):
min_physic.append(float(f.readline()))
max_physic = []
for c in range(nbchannel + 2):
max_physic.append(float(f.readline()))
min_logic = []
for c in range(nbchannel + 2):
min_logic.append(float(f.readline()))
max_logic = []
for c in range(nbchannel + 2):
max_logic.append(float(f.readline()))
#info filter
info_filter = []
for c in range(nbchannel + 2):
info_filter.append(f.readline()[:-1])
f.close()
#raw data
n = int(round(np.log(max_logic[0] - min_logic[0]) / np.log(2)) / 8)
data = np.fromfile(self.filename, dtype='i' + str(n))
data = data.byteswap().reshape(
(data.size / (nbchannel + 2), nbchannel + 2)).astype('f4')
for c in range(nbchannel):
if lazy:
sig = []
else:
sig = (data[:, c] - min_logic[c]) / (
max_logic[c] - min_logic[c]) * \
(max_physic[c] - min_physic[c]) + min_physic[c]
try:
unit = pq.Quantity(1, units[c])
except:
unit = pq.Quantity(1, '')
ana_sig = AnalogSignal(
sig * unit, sampling_rate=sampling_rate,
t_start=0. * pq.s, name=labels[c], channel_index=c)
if lazy:
ana_sig.lazy_shape = data.shape[0]
ana_sig.annotate(channel_name=labels[c])
seg.analogsignals.append(ana_sig)
# triggers
f = open(self.filename + '.pos')
times = []
labels = []
reject_codes = []
for l in f.readlines():
r = re.findall(' *(\d+) *(\d+) *(\d+) *', l)
times.append(float(r[0][0]) / sampling_rate.magnitude)
labels.append(str(r[0][1]))
reject_codes.append(str(r[0][2]))
if lazy:
times = [] * pq.S
labels = np.array([], dtype='S')
reject_codes = []
else:
times = np.array(times) * pq.s
labels = np.array(labels)
reject_codes = np.array(reject_codes)
ea = Event(times=times, labels=labels, reject_codes=reject_codes)
if lazy:
ea.lazy_shape = len(times)
seg.events.append(ea)
f.close()
seg.create_many_to_one_relationship()
return seg
def read_segment(self, import_neuroshare_segment = True,
lazy=False, cascade=True):
"""
Arguments:
import_neuroshare_segment: import neuroshare segment as SpikeTrain with associated waveforms or not imported at all.
"""
seg = Segment( file_origin = os.path.basename(self.filename), )
if sys.platform.startswith('win'):
neuroshare = ctypes.windll.LoadLibrary(self.dllname)
elif sys.platform.startswith('linux'):
neuroshare = ctypes.cdll.LoadLibrary(self.dllname)
neuroshare = DllWithError(neuroshare)
#elif sys.platform.startswith('darwin'):
# API version
info = ns_LIBRARYINFO()
neuroshare.ns_GetLibraryInfo(ctypes.byref(info) , ctypes.sizeof(info))
seg.annotate(neuroshare_version = str(info.dwAPIVersionMaj)+'.'+str(info.dwAPIVersionMin))
if not cascade:
return seg
# open file
hFile = ctypes.c_uint32(0)
neuroshare.ns_OpenFile(ctypes.c_char_p(self.filename) ,ctypes.byref(hFile))
fileinfo = ns_FILEINFO()
neuroshare.ns_GetFileInfo(hFile, ctypes.byref(fileinfo) , ctypes.sizeof(fileinfo))
# read all entities
for dwEntityID in range(fileinfo.dwEntityCount):
entityInfo = ns_ENTITYINFO()
neuroshare.ns_GetEntityInfo( hFile, dwEntityID, ctypes.byref(entityInfo), ctypes.sizeof(entityInfo))
# EVENT
if entity_types[entityInfo.dwEntityType] == 'ns_ENTITY_EVENT':
pEventInfo = ns_EVENTINFO()
neuroshare.ns_GetEventInfo ( hFile, dwEntityID, ctypes.byref(pEventInfo), ctypes.sizeof(pEventInfo))
if pEventInfo.dwEventType == 0: #TEXT
pData = ctypes.create_string_buffer(pEventInfo.dwMaxDataLength)
elif pEventInfo.dwEventType == 1:#CVS
pData = ctypes.create_string_buffer(pEventInfo.dwMaxDataLength)
elif pEventInfo.dwEventType == 2:# 8bit
pData = ctypes.c_byte(0)
elif pEventInfo.dwEventType == 3:# 16bit
pData = ctypes.c_int16(0)
elif pEventInfo.dwEventType == 4:# 32bit
pData = ctypes.c_int32(0)
pdTimeStamp = ctypes.c_double(0.)
pdwDataRetSize = ctypes.c_uint32(0)
ea = Event(name = str(entityInfo.szEntityLabel),)
if not lazy:
times = [ ]
labels = [ ]
for dwIndex in range(entityInfo.dwItemCount ):
neuroshare.ns_GetEventData ( hFile, dwEntityID, dwIndex,
ctypes.byref(pdTimeStamp), ctypes.byref(pData),
ctypes.sizeof(pData), ctypes.byref(pdwDataRetSize) )
times.append(pdTimeStamp.value)
labels.append(str(pData.value))
ea.times = times*pq.s
ea.labels = np.array(labels, dtype ='S')
else :
ea.lazy_shape = entityInfo.dwItemCount
seg.eventarrays.append(ea)
# analog
if entity_types[entityInfo.dwEntityType] == 'ns_ENTITY_ANALOG':
pAnalogInfo = ns_ANALOGINFO()
neuroshare.ns_GetAnalogInfo( hFile, dwEntityID,ctypes.byref(pAnalogInfo),ctypes.sizeof(pAnalogInfo) )
dwIndexCount = entityInfo.dwItemCount
if lazy:
signal = [ ]*pq.Quantity(1, pAnalogInfo.szUnits)
else:
pdwContCount = ctypes.c_uint32(0)
pData = np.zeros( (entityInfo.dwItemCount,), dtype = 'float64')
total_read = 0
while total_read< entityInfo.dwItemCount:
dwStartIndex = ctypes.c_uint32(total_read)
dwStopIndex = ctypes.c_uint32(entityInfo.dwItemCount - total_read)
neuroshare.ns_GetAnalogData( hFile, dwEntityID, dwStartIndex,
dwStopIndex, ctypes.byref( pdwContCount) , pData[total_read:].ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
total_read += pdwContCount.value
signal = pq.Quantity(pData, units=pAnalogInfo.szUnits, copy = False)
#t_start
dwIndex = 0
pdTime = ctypes.c_double(0)
neuroshare.ns_GetTimeByIndex( hFile, dwEntityID, dwIndex, ctypes.byref(pdTime))
anaSig = AnalogSignal(signal,
sampling_rate = pAnalogInfo.dSampleRate*pq.Hz,
t_start = pdTime.value * pq.s,
name = str(entityInfo.szEntityLabel),
)
anaSig.annotate( probe_info = str(pAnalogInfo.szProbeInfo))
if lazy:
anaSig.lazy_shape = entityInfo.dwItemCount
seg.analogsignals.append( anaSig )
#segment
if entity_types[entityInfo.dwEntityType] == 'ns_ENTITY_SEGMENT' and import_neuroshare_segment:
pdwSegmentInfo = ns_SEGMENTINFO()
if not str(entityInfo.szEntityLabel).startswith('spks'):
continue
neuroshare.ns_GetSegmentInfo( hFile, dwEntityID,
ctypes.byref(pdwSegmentInfo), ctypes.sizeof(pdwSegmentInfo) )
nsource = pdwSegmentInfo.dwSourceCount
pszMsgBuffer = ctypes.create_string_buffer(" "*256)
neuroshare.ns_GetLastErrorMsg(ctypes.byref(pszMsgBuffer), 256)
for dwSourceID in range(pdwSegmentInfo.dwSourceCount) :
pSourceInfo = ns_SEGSOURCEINFO()
neuroshare.ns_GetSegmentSourceInfo( hFile, dwEntityID, dwSourceID,
ctypes.byref(pSourceInfo), ctypes.sizeof(pSourceInfo) )
if lazy:
sptr = SpikeTrain(times, name = str(entityInfo.szEntityLabel), t_stop = 0.*pq.s)
sptr.lazy_shape = entityInfo.dwItemCount
else:
pdTimeStamp = ctypes.c_double(0.)
dwDataBufferSize = pdwSegmentInfo.dwMaxSampleCount*pdwSegmentInfo.dwSourceCount
pData = np.zeros( (dwDataBufferSize), dtype = 'float64')
pdwSampleCount = ctypes.c_uint32(0)
pdwUnitID= ctypes.c_uint32(0)
nsample = int(dwDataBufferSize)
times = np.empty( (entityInfo.dwItemCount), dtype = 'f')
waveforms = np.empty( (entityInfo.dwItemCount, nsource, nsample), dtype = 'f')
for dwIndex in range(entityInfo.dwItemCount ):
neuroshare.ns_GetSegmentData ( hFile, dwEntityID, dwIndex,
ctypes.byref(pdTimeStamp), pData.ctypes.data_as(ctypes.POINTER(ctypes.c_double)),
dwDataBufferSize * 8, ctypes.byref(pdwSampleCount),
ctypes.byref(pdwUnitID ) )
times[dwIndex] = pdTimeStamp.value
waveforms[dwIndex, :,:] = pData[:nsample*nsource].reshape(nsample ,nsource).transpose()
sptr = SpikeTrain(times = pq.Quantity(times, units = 's', copy = False),
t_stop = times.max(),
waveforms = pq.Quantity(waveforms, units = str(pdwSegmentInfo.szUnits), copy = False ),
left_sweep = nsample/2./float(pdwSegmentInfo.dSampleRate)*pq.s,
sampling_rate = float(pdwSegmentInfo.dSampleRate)*pq.Hz,
name = str(entityInfo.szEntityLabel),
)
seg.spiketrains.append(sptr)
# neuralevent
if entity_types[entityInfo.dwEntityType] == 'ns_ENTITY_NEURALEVENT':
pNeuralInfo = ns_NEURALINFO()
neuroshare.ns_GetNeuralInfo ( hFile, dwEntityID,
ctypes.byref(pNeuralInfo), ctypes.sizeof(pNeuralInfo))
if lazy:
times = [ ]*pq.s
t_stop = 0*pq.s
else:
pData = np.zeros( (entityInfo.dwItemCount,), dtype = 'float64')
dwStartIndex = 0
dwIndexCount = entityInfo.dwItemCount
neuroshare.ns_GetNeuralData( hFile, dwEntityID, dwStartIndex,
dwIndexCount, pData.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
times = pData*pq.s
t_stop = times.max()
sptr = SpikeTrain(times, t_stop =t_stop,
name = str(entityInfo.szEntityLabel),)
if lazy:
sptr.lazy_shape = entityInfo.dwItemCount
seg.spiketrains.append(sptr)
# close
neuroshare.ns_CloseFile(hFile)
seg.create_many_to_one_relationship()
return seg
def read_block(self,
lazy = False,
cascade = True,
):
bl = Block()
tankname = os.path.basename(self.dirname)
bl.file_origin = tankname
if not cascade : return bl
for blockname in os.listdir(self.dirname):
if blockname == 'TempBlk': continue
subdir = os.path.join(self.dirname,blockname)
if not os.path.isdir(subdir): continue
seg = Segment(name = blockname)
bl.segments.append( seg)
#TSQ is the global index
tsq_filename = os.path.join(subdir, tankname+'_'+blockname+'.tsq')
dt = [('size','int32'),
('evtype','int32'),
('code','S4'),
('channel','uint16'),
('sortcode','uint16'),
('timestamp','float64'),
('eventoffset','int64'),
('dataformat','int32'),
('frequency','float32'),
]
tsq = np.fromfile(tsq_filename, dtype = dt)
#0x8801: 'EVTYPE_MARK' give the global_start
global_t_start = tsq[tsq['evtype']==0x8801]['timestamp'][0]
#TEV is the old data file
if os.path.exists(os.path.join(subdir, tankname+'_'+blockname+'.tev')):
tev_filename = os.path.join(subdir, tankname+'_'+blockname+'.tev')
#tev_array = np.memmap(tev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
tev_array = np.fromfile(tev_filename, dtype = 'uint8')
else:
tev_filename = None
for type_code, type_label in tdt_event_type:
mask1 = tsq['evtype']==type_code
codes = np.unique(tsq[mask1]['code'])
for code in codes:
mask2 = mask1 & (tsq['code']==code)
channels = np.unique(tsq[mask2]['channel'])
for channel in channels:
mask3 = mask2 & (tsq['channel']==channel)
if type_label in ['EVTYPE_STRON', 'EVTYPE_STROFF']:
if lazy:
times = [ ]*pq.s
labels = np.array([ ], dtype = str)
else:
times = (tsq[mask3]['timestamp'] - global_t_start) * pq.s
labels = tsq[mask3]['eventoffset'].view('float64').astype('S')
ea = EventArray(times = times, name = code , channel_index = int(channel), labels = labels)
if lazy:
ea.lazy_shape = np.sum(mask3)
seg.eventarrays.append(ea)
elif type_label == 'EVTYPE_SNIP':
sortcodes = np.unique(tsq[mask3]['sortcode'])
for sortcode in sortcodes:
mask4 = mask3 & (tsq['sortcode']==sortcode)
nb_spike = np.sum(mask4)
sr = tsq[mask4]['frequency'][0]
waveformsize = tsq[mask4]['size'][0]-10
if lazy:
times = [ ]*pq.s
waveforms = None
else:
times = (tsq[mask4]['timestamp'] - global_t_start) * pq.s
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
waveforms = get_chunks(tsq[mask4]['size'],tsq[mask4]['eventoffset'], tev_array).view(dt)
waveforms = waveforms.reshape(nb_spike, -1, waveformsize)
waveforms = waveforms * pq.mV
if nb_spike>0:
# t_start = (tsq['timestamp'][0] - global_t_start) * pq.s # this hould work but not
t_start = 0 *pq.s
t_stop = (tsq['timestamp'][-1] - global_t_start) * pq.s
else:
t_start = 0 *pq.s
t_stop = 0 *pq.s
st = SpikeTrain(times = times,
name = 'Chan{} Code{}'.format(channel,sortcode),
t_start = t_start,
t_stop = t_stop,
waveforms = waveforms,
left_sweep = waveformsize/2./sr * pq.s,
sampling_rate = sr * pq.Hz,
)
st.annotate(channel_index = channel)
if lazy:
st.lazy_shape = nb_spike
seg.spiketrains.append(st)
elif type_label == 'EVTYPE_STREAM':
dt = np.dtype(data_formats[ tsq[mask3]['dataformat'][0]])
shape = np.sum(tsq[mask3]['size']-10)
sr = tsq[mask3]['frequency'][0]
if lazy:
signal = [ ]
else:
if PY3K:
signame = code.decode('ascii')
else:
signame = code
sev_filename = os.path.join(subdir, tankname+'_'+blockname+'_'+signame+'_ch'+str(channel)+'.sev')
if os.path.exists(sev_filename):
#sig_array = np.memmap(sev_filename, mode = 'r', dtype = 'uint8') # if memory problem use this instead
sig_array = np.fromfile(sev_filename, dtype = 'uint8')
else:
sig_array = tev_array
signal = get_chunks(tsq[mask3]['size'],tsq[mask3]['eventoffset'], sig_array).view(dt)
anasig = AnalogSignal(signal = signal* pq.V,
name = '{} {}'.format(code, channel),
sampling_rate= sr * pq.Hz,
t_start = (tsq[mask3]['timestamp'][0] - global_t_start) * pq.s,
channel_index = int(channel))
if lazy:
anasig.lazy_shape = shape
seg.analogsignals.append(anasig)
bl.create_many_to_one_relationship()
return bl
