def open(self, event_callback_fn=None):
"""Open the device for use.
:param event_callback_fn: The function(event, message) to call on
asynchronous events, mostly to allow robust handling of device
errors. "event" is one of the :class:`DeviceEvent` values,
and the message is a more detailed description of the event.
:raise IOError: on failure.
The event_callback_fn may be called asynchronous and from other
threads. The event_callback_fn must implement any thread safety.
"""
if self.stream_buffer:
self.close()
self._usb.open(event_callback_fn)
sb_len = self._sampling_frequency * self._stream_buffer_duration
self.stream_buffer = StreamBuffer(sb_len, self._reductions, self._sampling_frequency)
self._current_ranging_set()
try:
info = self.info()
if info is not None:
log.info('info:\n%s', json.dumps(info, indent=2))
except Exception:
log.warning('could not fetch info record')
try:
self.calibration = self._calibration_read()
except Exception:
log.warning('could not fetch calibration')
try:
cfg = self._parameters_defaults.get(self._config, {})
for key, value in cfg.items():
self.parameter_set(key, value)
except Exception:
log.warning('could not set defaults')
return self
def test_init(self):
b = StreamBuffer(1.0, [10, 10], 1000.0)
self.assertIsNotNone(b)
self.assertEqual(1000, len(b))
self.assertEqual(1000.0, b.output_sampling_frequency)
self.assertEqual((0, 0), b.sample_id_range)
self.assertEqual((0.0, 1.0), b.limits_time)
self.assertEqual((-1000, 0), b.limits_samples)
self.assertEqual(0, b.time_to_sample_id(1.0))
self.assertEqual(1.0, b.sample_id_to_time(0))
del b
def test_calibration(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'off'
self.assertEqual('off', b.suppress_mode)
b.calibration_set([-10.0] * 7, [2.0] * 7, [-2.0, 1.0], [4.0, 1.0])
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 10, 1)
self.assertEqual((10, STATS_FIELD_COUNT), data.shape)
np.testing.assert_allclose([-20.0, -4.0, 80.0, 0, 0, 1], data[0, :]['mean'])
np.testing.assert_allclose([12., 60., 720., 0, 0, 1], data[8, :]['mean'])
def create_sinusoid_file(self, sample_rate, samples):
cal = Calibration()
cal.current_offset[:7] = -3000
cal.current_gain[:7] = [1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9]
cal.voltage_offset[:2] = -3000
cal.voltage_gain[:2] = [1e-3, 1e-4]
cal.data = cal.save(bytes([0] * 32))
fh = io.BytesIO()
d = DataRecorder(fh, calibration=cal)
stream_buffer = StreamBuffer(1.0, [100], sample_rate)
stream_buffer.calibration_set(cal.current_offset, cal.current_gain,
cal.voltage_offset, cal.voltage_gain)
d.stream_notify(stream_buffer)
data = self.create_sinusoid_data(sample_rate, samples)
chunk_size = (sample_rate // 2) * 2
for i in range(0, 2 * samples, chunk_size):
stream_buffer.insert_raw(data[i:(i + chunk_size)])
stream_buffer.process()
d.stream_notify(stream_buffer)
d.close()
fh.seek(0)
return fh
def test_get_over_reduction_direct_with_reduction0_nan_mean(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'mean_1_n_1'
frame = usb_packet_factory(0, 1)
frame[8 + 10 * 4:8 + 22 * 4:] = 0xff
b.insert(frame)
b.process()
r0 = b.get_reduction(0, 0, 126)
self.assertFalse(np.isfinite(r0[1, 0]['mean']))
r1 = b.get_reduction(1, 0, 126)
self.assertTrue(np.isfinite(r1[0, 0]['mean']))
def test_insert_raw_simple(self):
b = StreamBuffer(1.0, [100, 100, 100], 1000.0)
b.suppress_mode = 'off'
expect = np.arange(126 * 2, dtype=np.uint16).reshape((126, 2))
raw = np.left_shift(expect, 2)
raw[1::2, 1] = np.bitwise_or(raw[1::2, 1], 0x0002)
b.insert_raw(raw)
b.process()
data = b.data_get(0, 126)
np.testing.assert_allclose(expect[:, 0], data[:, 0]['mean'])
self.assertEqual(0, b.status()['skip_count']['value'])
def test_insert_raw_simple(self):
b = StreamBuffer(1000, [100, 100, 100])
b.suppress_mode = 'off'
expect = np.arange(126 * 2, dtype=np.uint16).reshape((126, 2))
b.insert_raw(np.left_shift(expect, 2))
b.process()
data = b.data_get(0, 126)
np.testing.assert_allclose(expect[:, 0], data[:, 0, 0])
def test_get_over_reduction_direct_with_reduction0_nan(self):
b = StreamBuffer(2000, [10, 10])
frame = usb_packet_factory(0, 1)
frame[8+10*4:8+22*4:] = 0xff
b.insert(frame)
b.process()
r0 = b.get_reduction(0, 0, 126)
self.assertFalse(np.isfinite(r0[1, 0, 0]))
r1 = b.get_reduction(1, 0, 126)
self.assertTrue(np.isfinite(r0[0, 0, 0]))
def stream_buffer_01(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
return b
def stream_buffer_02(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'off'
b.insert(usb_packet_factory(0))
b.insert(usb_packet_factory(2))
b.process()
return b
def test_insert_former_nan_case(self):
raw = np.array(
[
[61957, 16937],
[62317, 16935],
[62585, 16937],
[62853, 16935],
[65535, 16916], # raw_i = 0xffff (i_range=3)
[18932, 16942],
[8876, 16932],
[9788, 16938],
[10300, 16936],
[10368, 16930],
[10528, 16932],
[10584, 16938],
[10564, 16936],
[10568, 16942],
[12497, 16932],
[12733, 16946],
[12613, 16940],
[12561, 16930]
],
dtype=np.uint16)
b = StreamBuffer(0.2, [], 1000.0)
b.insert_raw(raw)
b.process()
self.assertEqual(0, b.status()['skip_count']['value'])
def test_get_over_reduction(self):
b = StreamBuffer(2000, [10, 10], 1000.0)
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 40, 20)
self.assertEqual((2, STATS_FIELDS, STATS_VALUES), data.shape)
np.testing.assert_allclose([19.0, 133.0, 0.0, 38.0], data[0, 0, :])
np.testing.assert_allclose([20.0, 133.0, 1.0, 39.0], data[0, 1, :])
np.testing.assert_allclose([59.0, 133.0, 40.0, 78.0], data[1, 0, :])
def test_get_over_reduction(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 40, 20)
self.assertEqual((2, STATS_FIELD_COUNT), data.shape)
assert_single_stat_close([20, 19.0, 2660.0, 0.0, 38.0], data[0, 0])
assert_single_stat_close([20, 20.0, 2660.0, 1.0, 39.0], data[0, 1])
assert_single_stat_close([20, 59.0, 2660.0, 40.0, 78.0], data[1, 0])
def test_wrap_aligned(self):
frame = usb_packet_factory(0, 4)
b = StreamBuffer(2.0 * SAMPLES_PER / 1000.0, [], 1000.0)
b.suppress_mode = 'off'
b.insert(frame)
b.process()
self.assertEqual((SAMPLES_PER * 2, SAMPLES_PER * 4), b.sample_id_range)
data = b.samples_get(SAMPLES_PER * 2, SAMPLES_PER * 4, 'raw')
data = np.right_shift(data, 2)
expect = np.arange(SAMPLES_PER * 4, SAMPLES_PER * 8, dtype=np.uint16).reshape((SAMPLES_PER * 2, 2))
np.testing.assert_allclose(expect, data)
np.testing.assert_allclose(expect[:, 0], b.data_buffer[::2])
data = b.data_get(SAMPLES_PER * 2, SAMPLES_PER * 4)
np.testing.assert_allclose(expect[:, 0], data[:, 0]['mean'])
np.testing.assert_allclose(expect[:, 1], data[:, 1]['mean'])
def test_get_over_reduction_direct_with_raw_nan(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
frame = usb_packet_factory(0, 1)
frame[8 + 11 * 4:8 + 15 * 4:] = 0xff
b.insert(frame)
b.process()
r = b.get_reduction(0, 0, 126)
self.assertTrue(all(np.isfinite(r[:, 0]['mean'])))
def test_get_over_samples(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 21, 5)
self.assertEqual((4, STATS_FIELD_COUNT), data.shape)
np.testing.assert_allclose(np.arange(10), b.data_buffer[0:10]) # processed correctly
np.testing.assert_allclose([4.0, 14.0, 24.0, 34.0], data[:, 0]['mean'])
np.testing.assert_allclose([5, 4.0, 40.0, 0.0, 8.0], single_stat_as_array(data[0, 0]))
np.testing.assert_allclose([5.0, 15.0, 25.0, 35.0], data[:, 1]['mean'])
def test_get_over_reduction_direct(self):
b = StreamBuffer(2000, [10, 10])
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 20, 10)
self.assertEqual((2, 3, 4), data.shape)
np.testing.assert_allclose([9.0, 33.0, 0.0, 18.0], data[0, 0, :])
np.testing.assert_allclose([29.0, 33.0, 20.0, 38.0], data[1, 0, :])
np.testing.assert_allclose([10.0, 33.0, 1.0, 19.0], data[0, 1, :])
np.testing.assert_allclose([30.0, 33.0, 21.0, 39.0], data[1, 1, :])
def test_get_over_reduction_direct(self):
b = StreamBuffer(2.0, [10, 10], 1000.0)
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 20, 10)
self.assertEqual((2, STATS_FIELD_COUNT), data.shape)
np.testing.assert_allclose([10, 9.0, 330.0, 0.0, 18.0], single_stat_as_array(data[0, 0]))
np.testing.assert_allclose([10, 29.0, 330.0, 20.0, 38.0], single_stat_as_array(data[1, 0]))
np.testing.assert_allclose([10, 10.0, 330.0, 1.0, 19.0], single_stat_as_array(data[0, 1]))
np.testing.assert_allclose([10, 30.0, 330.0, 21.0, 39.0], single_stat_as_array(data[1, 1]))
def test_get_over_samples(self):
b = StreamBuffer(2000, [10, 10])
b.suppress_mode = 'off'
frame = usb_packet_factory(0, 1)
b.insert(frame)
b.process()
data = b.data_get(0, 21, 5)
self.assertEqual((4, 3, 4), data.shape)
np.testing.assert_allclose(np.arange(10), b.data_buffer[0:10]) # processed correctly
np.testing.assert_allclose([4.0, 14.0, 24.0, 34.0], data[:, 0, 0])
np.testing.assert_allclose([4.0, 8.0, 0.0, 8.0], data[0, 0, :])
np.testing.assert_allclose([5.0, 15.0, 25.0, 35.0], data[:, 1, 0])
def run(cmd_queue, filehandle, sampling_frequency, calibration):
r = DataRecorder(filehandle, sampling_frequency, calibration)
b = StreamBuffer(int(sampling_frequency), [], sampling_frequency)
b.calibration_set(calibration.current_offset, calibration.current_gain,
calibration.voltage_offset, calibration.voltage_gain)
while True:
cmd, args = cmd_queue.get()
if cmd == 'stream_notify':
raw_data, voltage_range = args
b.voltage_range = voltage_range
b.insert_raw(raw_data)
b.process()
r.stream_notify(b)
elif cmd == 'close':
r.close()
break
def test_i_range_off(self):
raw = np.array([[0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001],
[0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003],
[0x1003, 0x1001], [0x1003, 0x1003]],
dtype=np.uint16)
b = StreamBuffer(0.2, [], 1000.0)
b.insert_raw(raw)
b.process()
np.testing.assert_allclose(0, b.samples_get(0, 8, fields='current'))
def test_i_range_off(self):
raw = np.array([[0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001],
[0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003],
[0x1003, 0x1001], [0x1003, 0x1003]],
dtype=np.uint16)
b = StreamBuffer(200, [], 1000.0)
b.insert_raw(raw)
b.process()
self.assertEqual(0, b.samples_get(0, 8, fields=['current'])[0][-1])
def test_wrap_unaligned(self):
frame = usb_packet_factory(0, 4)
b = StreamBuffer((2 * SAMPLES_PER + 2) / 1000.0, [], 1000.0)
b.insert(frame)
b.process()
self.assertEqual(SAMPLES_PER * 4, b.sample_id_range[1])
data = np.right_shift(b.samples_get(SAMPLES_PER * 2, SAMPLES_PER * 4, 'raw'), 2)
expect = np.arange(SAMPLES_PER * 4, SAMPLES_PER * 8, dtype=np.uint16).reshape((SAMPLES_PER * 2, 2))
np.testing.assert_allclose(expect, data)
class TestView(unittest.TestCase):
def setUp(self):
self.b = StreamBuffer(2.0, [10, 10], sampling_frequency=1000)
self.v = View(stream_buffer=self.b, calibration=None)
self.b.insert(usb_packet_factory(0, 2))
self.b.process()
self.v.open()
def tearDown(self):
self.v.close()
def test_sample_conversion(self):
self.assertEqual((('hello', ), {'one': 1}), self.v.ping('hello', one=1))
self.assertEqual(252, self.v.time_to_sample_id(2.0))
self.assertEqual(2.0, self.v.sample_id_to_time(252))
self.assertEqual(152, self.v.time_to_sample_id(1.9))
def test_statistics_get(self):
s1 = self.v.statistics_get(-2, -1, units='samples')
def _create_large_file(self, samples=None):
"""Create a large file.
:param samples: The total number of samples which will be rounded
to a full USB packet.
:return: (BytesIO file, samples)
"""
sample_rate = 2000000
packets_per_burst = 128
bursts = int(
np.ceil(samples / (SAMPLES_PER_PACKET * packets_per_burst)))
stream_buffer = StreamBuffer(1.0, [100], sample_rate)
samples_total = SAMPLES_PER_PACKET * packets_per_burst * bursts
fh = io.BytesIO()
d = DataRecorder(fh)
d.stream_notify(stream_buffer)
for burst_index in range(bursts):
packet_index = burst_index * packets_per_burst
frames = usb_packet_factory_signal(packet_index,
count=packets_per_burst,
samples_total=samples_total)
stream_buffer.insert(frames)
stream_buffer.process()
d.stream_notify(stream_buffer)
d.close()
fh.seek(0)
# dfr = datafile.DataFileReader(fh)
# dfr.pretty_print()
# fh.seek(0)
return fh, samples_total
def _export_jls(self):
view = self._view
cfg = self._cfg
sampling_frequency = view.sampling_frequency
sample_step_size = sampling_frequency
stream_buffer = StreamBuffer(sampling_frequency * 2, [])
data_recorder = DataRecorder(cfg['filename'],
calibration=view.calibration.data,
sampling_frequency=sampling_frequency)
data_recorder.process(stream_buffer)
try:
idx_start = cfg['sample_id_start']
idx_stop = cfg['sample_id_stop']
idx_range = idx_stop - idx_start
idx = idx_start
self.sigProgress.emit(0)
while not self._stop and idx < idx_stop:
log.info('export_jls iteration')
idx_next = idx + sample_step_size
if idx_next > idx_stop:
idx_next = idx_stop
data = view.raw_get(idx, idx_next)
log.info('export_jls (%d, %d) -> %d', idx, idx_next, len(data))
stream_buffer.insert_raw(data)
stream_buffer.process()
data_recorder.process(stream_buffer)
idx = idx_next
self.sigProgress.emit(int(1000 * (idx - idx_start) /
idx_range))
finally:
data_recorder.close()
def _export_jls(self, data):
cfg = self._cfg
sampling_frequency = data.sample_frequency
stream_buffer = StreamBuffer(sampling_frequency * 2, [],
sampling_frequency=sampling_frequency)
stream_buffer.calibration_set(data.calibration.current_offset,
data.calibration.current_gain,
data.calibration.voltage_offset,
data.calibration.voltage_gain)
stream_buffer.voltage_range = data.cmdp['Plugins/#state/voltage_range']
data_recorder = DataRecorder(cfg['filename'],
calibration=data.calibration.data,
sampling_frequency=sampling_frequency)
data_recorder.stream_notify(stream_buffer)
try:
for block in data:
log.info('export_jls iteration')
stream_buffer.insert_raw(block['signals']['raw']['value'])
stream_buffer.process()
data_recorder.stream_notify(stream_buffer)
finally:
data_recorder.close()
def test_truncated(self):
stream_buffer = StreamBuffer(400.0, [10], 1000.0)
stream_buffer.suppress_mode = 'off'
fh = io.BytesIO()
d = DataRecorder(fh)
d.stream_notify(stream_buffer)
count = 16
for idx in range(0, 160, count):
data = usb_packet_factory(idx, count)
stream_buffer.insert(data)
stream_buffer.process()
d.stream_notify(stream_buffer)
fh.seek(0)
#r = datafile.DataFileReader(fh)
#r.pretty_print()
r = DataReader().open(fh)
def _create_file(self, packet_index, count=None):
data_recorder.SAMPLES_PER_REDUCTION = 1000
data_recorder.REDUCTIONS_PER_TLV = 2
stream_buffer = StreamBuffer(60.0, [10], 1000.0)
stream_buffer.suppress_mode = 'off'
if packet_index > 0:
data = usb_packet_factory(0, packet_index - 1)
stream_buffer.insert(data)
stream_buffer.process()
fh = io.BytesIO()
d = data_recorder.DataRecorder(fh)
d.stream_notify(stream_buffer)
data = usb_packet_factory(packet_index, count)
stream_buffer.insert(data)
stream_buffer.process()
d.stream_notify(stream_buffer)
d.close()
fh.seek(0)
# dfr = datafile.DataFileReader(fh)
# dfr.pretty_print()
# fh.seek(0)
return fh
def _create_file_insert(self, packet_index, count_incr, count_total):
stream_buffer = StreamBuffer(10.0, [10], 1000.0)
stream_buffer.suppress_mode = 'off'
if packet_index > 0:
data = usb_packet_factory(0, packet_index - 1)
stream_buffer.insert(data)
stream_buffer.process()
fh = io.BytesIO()
d = DataRecorder(fh)
sample_id = stream_buffer.sample_id_range[1]
for _ in range(0, count_total, count_incr):
data = usb_packet_factory(packet_index, count_incr)
stream_buffer.insert(data)
stream_buffer.process()
sample_id_next = stream_buffer.sample_id_range[1]
d.insert(stream_buffer.samples_get(sample_id, sample_id_next))
sample_id = sample_id_next
packet_index += count_incr
d.close()
fh.seek(0)
# dfr = datafile.DataFileReader(fh)
# dfr.pretty_print()
# fh.seek(0)
return fh
