def stream_buffer_02(self): b = StreamBuffer(2.0, [10, 10], 1000.0) b.suppress_mode = 'off' b.insert(usb_packet_factory(0)) b.insert(usb_packet_factory(2)) b.process() return b
def stream_buffer_01(self): b = StreamBuffer(2.0, [10, 10], 1000.0) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() return b
def _export_jls(self): view = self._view cfg = self._cfg sampling_frequency = view.sampling_frequency sample_step_size = sampling_frequency stream_buffer = StreamBuffer(sampling_frequency * 2, []) data_recorder = DataRecorder(cfg['filename'], calibration=view.calibration.data, sampling_frequency=sampling_frequency) data_recorder.process(stream_buffer) try: idx_start = cfg['sample_id_start'] idx_stop = cfg['sample_id_stop'] idx_range = idx_stop - idx_start idx = idx_start self.sigProgress.emit(0) while not self._stop and idx < idx_stop: log.info('export_jls iteration') idx_next = idx + sample_step_size if idx_next > idx_stop: idx_next = idx_stop data = view.raw_get(idx, idx_next) log.info('export_jls (%d, %d) -> %d', idx, idx_next, len(data)) stream_buffer.insert_raw(data) stream_buffer.process() data_recorder.process(stream_buffer) idx = idx_next self.sigProgress.emit(int(1000 * (idx - idx_start) / idx_range)) finally: data_recorder.close()
def test_insert_former_nan_case(self): raw = np.array( [ [61957, 16937], [62317, 16935], [62585, 16937], [62853, 16935], [65535, 16916], # raw_i = 0xffff (i_range=3) [18932, 16942], [8876, 16932], [9788, 16938], [10300, 16936], [10368, 16930], [10528, 16932], [10584, 16938], [10564, 16936], [10568, 16942], [12497, 16932], [12733, 16946], [12613, 16940], [12561, 16930] ], dtype=np.uint16) b = StreamBuffer(0.2, [], 1000.0) b.insert_raw(raw) b.process() self.assertEqual(0, b.status()['skip_count']['value'])
def _create_large_file(self, samples=None): """Create a large file. :param samples: The total number of samples which will be rounded to a full USB packet. :return: (BytesIO file, samples) """ sample_rate = 2000000 packets_per_burst = 128 bursts = int( np.ceil(samples / (SAMPLES_PER_PACKET * packets_per_burst))) stream_buffer = StreamBuffer(1.0, [100], sample_rate) samples_total = SAMPLES_PER_PACKET * packets_per_burst * bursts fh = io.BytesIO() d = DataRecorder(fh) d.stream_notify(stream_buffer) for burst_index in range(bursts): packet_index = burst_index * packets_per_burst frames = usb_packet_factory_signal(packet_index, count=packets_per_burst, samples_total=samples_total) stream_buffer.insert(frames) stream_buffer.process() d.stream_notify(stream_buffer) d.close() fh.seek(0) # dfr = datafile.DataFileReader(fh) # dfr.pretty_print() # fh.seek(0) return fh, samples_total
def create_sinusoid_file(self, sample_rate, samples): cal = Calibration() cal.current_offset[:7] = -3000 cal.current_gain[:7] = [1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9] cal.voltage_offset[:2] = -3000 cal.voltage_gain[:2] = [1e-3, 1e-4] cal.data = cal.save(bytes([0] * 32)) fh = io.BytesIO() d = DataRecorder(fh, calibration=cal) stream_buffer = StreamBuffer(1.0, [100], sample_rate) stream_buffer.calibration_set(cal.current_offset, cal.current_gain, cal.voltage_offset, cal.voltage_gain) d.stream_notify(stream_buffer) data = self.create_sinusoid_data(sample_rate, samples) chunk_size = (sample_rate // 2) * 2 for i in range(0, 2 * samples, chunk_size): stream_buffer.insert_raw(data[i:(i + chunk_size)]) stream_buffer.process() d.stream_notify(stream_buffer) d.close() fh.seek(0) return fh
def test_insert_process(self): b = StreamBuffer(1.0, [100, 10], 1000.0) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) self.assertEqual(0, b.status()['device_sample_id']['value']) b.insert(frame) self.assertEqual(126, b.status()['device_sample_id']['value']) self.assertEqual(0, b.status()['sample_id']['value']) b.process() self.assertEqual((0, 126), b.sample_id_range) self.assertEqual((0.0, 1.0), b.limits_time) self.assertEqual((-874, 126), b.limits_samples) self.assertEqual(126, b.time_to_sample_id(1.0)) self.assertEqual(1.0, b.sample_id_to_time(126)) self.assertEqual(0.999, b.sample_id_to_time(125)) self.assertEqual(b.limits_time[0], b.sample_id_to_time(b.limits_samples[0])) self.assertEqual(b.limits_time[1], b.sample_id_to_time(b.limits_samples[1])) self.assertEqual(126, b.status()['sample_id']['value']) samples = b.samples_get(0, 126, ['raw']) raw_data = b.samples_get(0, 126, 'raw') np.testing.assert_allclose(samples['signals']['raw']['value'], raw_data) expect = np.arange(126 * 2, dtype=np.uint16).reshape((126, 2)) np.testing.assert_allclose(expect, np.right_shift(raw_data, 2)) np.testing.assert_allclose(expect, b.data_buffer[0:126 * 2].reshape( (126, 2))) data = b.data_get(0, 126) np.testing.assert_allclose(expect[:, 0], data[:, 0]['mean'])
def _create_file_insert(self, packet_index, count_incr, count_total): stream_buffer = StreamBuffer(10.0, [10], 1000.0) stream_buffer.suppress_mode = 'off' if packet_index > 0: data = usb_packet_factory(0, packet_index - 1) stream_buffer.insert(data) stream_buffer.process() fh = io.BytesIO() d = DataRecorder(fh) sample_id = stream_buffer.sample_id_range[1] for _ in range(0, count_total, count_incr): data = usb_packet_factory(packet_index, count_incr) stream_buffer.insert(data) stream_buffer.process() sample_id_next = stream_buffer.sample_id_range[1] d.insert(stream_buffer.samples_get(sample_id, sample_id_next)) sample_id = sample_id_next packet_index += count_incr d.close() fh.seek(0) # dfr = datafile.DataFileReader(fh) # dfr.pretty_print() # fh.seek(0) return fh
def test_insert_raw_simple(self): b = StreamBuffer(1000, [100, 100, 100]) b.suppress_mode = 'off' expect = np.arange(126 * 2, dtype=np.uint16).reshape((126, 2)) b.insert_raw(np.left_shift(expect, 2)) b.process() data = b.data_get(0, 126) np.testing.assert_allclose(expect[:, 0], data[:, 0, 0])
def test_get_over_reduction_direct_with_raw_nan(self): b = StreamBuffer(2.0, [10, 10], 1000.0) frame = usb_packet_factory(0, 1) frame[8 + 11 * 4:8 + 15 * 4:] = 0xff b.insert(frame) b.process() r = b.get_reduction(0, 0, 126) self.assertTrue(all(np.isfinite(r[:, 0]['mean'])))
def test_i_range_off(self): raw = np.array([[0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003]], dtype=np.uint16) b = StreamBuffer(0.2, [], 1000.0) b.insert_raw(raw) b.process() np.testing.assert_allclose(0, b.samples_get(0, 8, fields='current'))
def test_insert_raw_wrap(self): b = StreamBuffer(200, []) expect = np.arange(250 * 2, dtype=np.uint16).reshape((250, 2)) b.insert_raw(np.left_shift(expect[:100], 2)) b.process() b.insert_raw(np.left_shift(expect[100:], 2)) b.process() data = b.data_get(50, 250) np.testing.assert_allclose(expect[50:, 0], data[:, 0, 0])
def test_i_range_off(self): raw = np.array([[0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003], [0x1003, 0x1001], [0x1003, 0x1003]], dtype=np.uint16) b = StreamBuffer(200, [], 1000.0) b.insert_raw(raw) b.process() self.assertEqual(0, b.samples_get(0, 8, fields=['current'])[0][-1])
def test_wrap_unaligned(self): frame = usb_packet_factory(0, 4) b = StreamBuffer((2 * SAMPLES_PER + 2) / 1000.0, [], 1000.0) b.insert(frame) b.process() self.assertEqual(SAMPLES_PER * 4, b.sample_id_range[1]) data = np.right_shift(b.samples_get(SAMPLES_PER * 2, SAMPLES_PER * 4, 'raw'), 2) expect = np.arange(SAMPLES_PER * 4, SAMPLES_PER * 8, dtype=np.uint16).reshape((SAMPLES_PER * 2, 2)) np.testing.assert_allclose(expect, data)
def test_get_over_reduction_direct_with_reduction0_nan(self): b = StreamBuffer(2.0, [10, 10], 1000.0) frame = usb_packet_factory(0, 1) frame[8+10*4:8+22*4:] = 0xff b.insert(frame) b.process() r0 = b.get_reduction(0, 0, 126) self.assertFalse(np.isfinite(r0[1, 0]['mean'])) r1 = b.get_reduction(1, 0, 126) self.assertTrue(np.isfinite(r0[0, 0]['mean']))
def test_insert_raw_simple(self): b = StreamBuffer(1.0, [100, 100, 100], 1000.0) b.suppress_mode = 'off' expect = np.arange(126 * 2, dtype=np.uint16).reshape((126, 2)) raw = np.left_shift(expect, 2) raw[1::2, 1] = np.bitwise_or(raw[1::2, 1], 0x0002) b.insert_raw(raw) b.process() data = b.data_get(0, 126) np.testing.assert_allclose(expect[:, 0], data[:, 0]['mean']) self.assertEqual(0, b.status()['skip_count']['value'])
def test_get_over_reduction(self): b = StreamBuffer(2.0, [10, 10], 1000.0) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 40, 20) self.assertEqual((2, STATS_FIELD_COUNT), data.shape) assert_single_stat_close([20, 19.0, 2660.0, 0.0, 38.0], data[0, 0]) assert_single_stat_close([20, 20.0, 2660.0, 1.0, 39.0], data[0, 1]) assert_single_stat_close([20, 59.0, 2660.0, 40.0, 78.0], data[1, 0])
def test_get_over_reduction(self): b = StreamBuffer(2000, [10, 10], 1000.0) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 40, 20) self.assertEqual((2, STATS_FIELDS, STATS_VALUES), data.shape) np.testing.assert_allclose([19.0, 133.0, 0.0, 38.0], data[0, 0, :]) np.testing.assert_allclose([20.0, 133.0, 1.0, 39.0], data[0, 1, :]) np.testing.assert_allclose([59.0, 133.0, 40.0, 78.0], data[1, 0, :])
def test_insert_raw_wrap(self): b = StreamBuffer(0.2, [], 1000.0) expect = np.arange(250 * 2, dtype=np.uint16).reshape((250, 2)) raw = np.left_shift(expect, 2) raw[1::2, 1] = np.bitwise_or(raw[1::2, 1], 0x0002) b.insert_raw(raw[:100]) b.process() b.insert_raw(raw[100:]) b.process() data = b.data_get(50, 250) np.testing.assert_allclose(expect[50:, 0], data[:, 0]['mean']) self.assertEqual(0, b.status()['skip_count']['value'])
def test_get_over_reduction_direct(self): b = StreamBuffer(2000, [10, 10]) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 20, 10) self.assertEqual((2, 3, 4), data.shape) np.testing.assert_allclose([9.0, 33.0, 0.0, 18.0], data[0, 0, :]) np.testing.assert_allclose([29.0, 33.0, 20.0, 38.0], data[1, 0, :]) np.testing.assert_allclose([10.0, 33.0, 1.0, 19.0], data[0, 1, :]) np.testing.assert_allclose([30.0, 33.0, 21.0, 39.0], data[1, 1, :])
def test_get_over_samples(self): b = StreamBuffer(2.0, [10, 10], 1000.0) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 21, 5) self.assertEqual((4, STATS_FIELD_COUNT), data.shape) np.testing.assert_allclose(np.arange(10), b.data_buffer[0:10]) # processed correctly np.testing.assert_allclose([4.0, 14.0, 24.0, 34.0], data[:, 0]['mean']) np.testing.assert_allclose([5, 4.0, 40.0, 0.0, 8.0], single_stat_as_array(data[0, 0])) np.testing.assert_allclose([5.0, 15.0, 25.0, 35.0], data[:, 1]['mean'])
def test_calibration(self): b = StreamBuffer(2.0, [10, 10], 1000.0) b.suppress_mode = 'off' self.assertEqual('off', b.suppress_mode) b.calibration_set([-10.0] * 7, [2.0] * 7, [-2.0, 1.0], [4.0, 1.0]) frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 10, 1) self.assertEqual((10, STATS_FIELD_COUNT), data.shape) np.testing.assert_allclose([-20.0, -4.0, 80.0, 0, 0, 1], data[0, :]['mean']) np.testing.assert_allclose([12., 60., 720., 0, 0, 1], data[8, :]['mean'])
def test_get_over_samples(self): b = StreamBuffer(2000, [10, 10]) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 21, 5) self.assertEqual((4, 3, 4), data.shape) np.testing.assert_allclose(np.arange(10), b.data_buffer[0:10]) # processed correctly np.testing.assert_allclose([4.0, 14.0, 24.0, 34.0], data[:, 0, 0]) np.testing.assert_allclose([4.0, 8.0, 0.0, 8.0], data[0, 0, :]) np.testing.assert_allclose([5.0, 15.0, 25.0, 35.0], data[:, 1, 0])
def test_get_over_reduction_direct(self): b = StreamBuffer(2.0, [10, 10], 1000.0) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) b.insert(frame) b.process() data = b.data_get(0, 20, 10) self.assertEqual((2, STATS_FIELD_COUNT), data.shape) np.testing.assert_allclose([10, 9.0, 330.0, 0.0, 18.0], single_stat_as_array(data[0, 0])) np.testing.assert_allclose([10, 29.0, 330.0, 20.0, 38.0], single_stat_as_array(data[1, 0])) np.testing.assert_allclose([10, 10.0, 330.0, 1.0, 19.0], single_stat_as_array(data[0, 1])) np.testing.assert_allclose([10, 30.0, 330.0, 21.0, 39.0], single_stat_as_array(data[1, 1]))
def test_wrap_aligned(self): frame = usb_packet_factory(0, 4) b = StreamBuffer(2.0 * SAMPLES_PER / 1000.0, [], 1000.0) b.suppress_mode = 'off' b.insert(frame) b.process() self.assertEqual((SAMPLES_PER * 2, SAMPLES_PER * 4), b.sample_id_range) data = b.samples_get(SAMPLES_PER * 2, SAMPLES_PER * 4, 'raw') data = np.right_shift(data, 2) expect = np.arange(SAMPLES_PER * 4, SAMPLES_PER * 8, dtype=np.uint16).reshape((SAMPLES_PER * 2, 2)) np.testing.assert_allclose(expect, data) np.testing.assert_allclose(expect[:, 0], b.data_buffer[::2]) data = b.data_get(SAMPLES_PER * 2, SAMPLES_PER * 4) np.testing.assert_allclose(expect[:, 0], data[:, 0]['mean']) np.testing.assert_allclose(expect[:, 1], data[:, 1]['mean'])
def _create_file(self, packet_index, count=None, signals=None): d = FakeJS110() stream_buffer = StreamBuffer(60.0, [10], 1000.0) stream_buffer.suppress_mode = 'off' if packet_index > 0: data = usb_packet_factory(0, packet_index - 1) stream_buffer.insert(data) stream_buffer.process() with JlsWriter(d, self._filename1, signals=signals) as wr: wr.stream_notify(stream_buffer) data = usb_packet_factory(packet_index, count) stream_buffer.insert(data) stream_buffer.process() wr.stream_notify(stream_buffer) self.assertTrue(os.path.isfile(self._filename1))
def test_insert_process(self): b = StreamBuffer(1000000, [100, 100, 100]) b.suppress_mode = 'off' frame = usb_packet_factory(0, 1) self.assertEqual(0, b.status()['device_sample_id']['value']) b.insert(frame) self.assertEqual(126, b.status()['device_sample_id']['value']) self.assertEqual(0, b.status()['sample_id']['value']) b.process() self.assertEqual(126, b.status()['sample_id']['value']) data = b.raw_get(0, 126) expect = np.arange(126*2, dtype=np.uint16).reshape((126, 2)) np.testing.assert_allclose(expect, np.right_shift(data, 2)) np.testing.assert_allclose(expect, b.data_buffer[0:126*2].reshape((126, 2))) data = b.data_get(0, 126) np.testing.assert_allclose(expect[:, 0], data[:, 0, 0])
def run(cmd_queue, filehandle, sampling_frequency, calibration): r = DataRecorder(filehandle, sampling_frequency, calibration) b = StreamBuffer(int(sampling_frequency), [], sampling_frequency) b.calibration_set(calibration.current_offset, calibration.current_gain, calibration.voltage_offset, calibration.voltage_gain) while True: cmd, args = cmd_queue.get() if cmd == 'stream_notify': raw_data, voltage_range = args b.voltage_range = voltage_range b.insert_raw(raw_data) b.process() r.stream_notify(b) elif cmd == 'close': r.close() break
def test_truncated(self): stream_buffer = StreamBuffer(400.0, [10], 1000.0) stream_buffer.suppress_mode = 'off' fh = io.BytesIO() d = DataRecorder(fh) d.stream_notify(stream_buffer) count = 16 for idx in range(0, 160, count): data = usb_packet_factory(idx, count) stream_buffer.insert(data) stream_buffer.process() d.stream_notify(stream_buffer) fh.seek(0) #r = datafile.DataFileReader(fh) #r.pretty_print() r = DataReader().open(fh)
def _create_file(self, packet_index, count=None): stream_buffer = StreamBuffer(401.0, [200, 100], 1000.0) stream_buffer.suppress_mode = 'off' if packet_index > 0: data = usb_packet_factory(0, packet_index - 1) stream_buffer.insert(data) stream_buffer.process() fh = io.BytesIO() d = DataRecorder(fh) d.stream_notify(stream_buffer) data = usb_packet_factory(packet_index, count) stream_buffer.insert(data) stream_buffer.process() d.stream_notify(stream_buffer) d.close() fh.seek(0) return fh