def test_statistics_get(self):
#fh = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data_recording_01.jls')
fh = self.create_sinusoid_file(2000000, 400000)
r = DataReader().open(fh)
t_start, t_stop = 0.066780, 0.069004
k_start, k_stop = r.normalize_time_arguments(t_start,
t_stop,
units='seconds')
ranges = [
(0, 1000), # trivial, direct
(0, 20000), # trivial, single reduction
(100000, 101000), # offset, direct
(100000, 120000), # offset, ex
(99000, 120000),
(100000, 121000),
(99000, 121000),
(k_start, k_stop),
]
for k_start, k_stop in ranges:
# print(f'range {k_start}:{k_stop}')
s1 = r.statistics_get(k_start, k_stop, units='samples')
k = r.samples_get(k_start,
k_stop,
units='samples',
fields=['current'])
i_mean = np.mean(k['signals']['current']['value'])
np.testing.assert_allclose(s1['signals']['current']['µ']['value'],
i_mean,
rtol=0.0005)
r.close()
def on_cmd(args):
r = DataReader().open(args.filename)
print(r.summary_string())
start = args.start
stop = args.stop
if stop < 0:
stop = r.sample_id_range[1] + 1 + stop
if args.export is not None:
i, v = r.get_calibrated(start, stop)
data = np.hstack((i.reshape((-1, 1)), (v.reshape((-1, 1)))))
if args.export.endswith('npy'):
np.save(args.export, data)
else:
np.savetxt(args.export, data, fmt='%.5g', delimiter=',')
if args.plot:
import matplotlib.pyplot as plt
y = r.get_reduction(start, stop)
x = np.arange(len(y)) * (r.config['samples_per_reduction'] /
r.config['sampling_frequency'])
f = plt.figure()
for axis in range(3):
ax = f.add_subplot(3, 1, axis + 1)
ax.plot(x, y[:, axis, 0], color='blue')
ax.plot(x, y[:, axis, 2], color='red')
ax.plot(x, y[:, axis, 3], color='red')
plt.show()
plt.close(f)
r.close()
return 0
def load_current_range(filename):
r = DataReader().open(filename)
try:
print(r.summary_string())
r_start, r_stop = r.sample_id_range
if r_stop - r_start > MAX_SAMPLES:
print('file too big')
return 1
d = r.samples_get(fields=['current_range'])
return d['signals']['current_range']['value']
finally:
r.close()
def test_cache_test(self):
sample_rate = 2000000
sample_count = sample_rate * 2
fh = self.create_sinusoid_file(sample_rate, sample_count)
r = DataReader().open(fh)
for step_size in [1111, 2000, 11111, 20000]:
# print(f'step_size = {step_size}')
for i in range(0, sample_count - step_size, step_size):
r.raw_processor.reset()
s1 = r.statistics_get(i, i + step_size, units='samples')
_, _, data = r.raw(i, i + step_size)
i_mean = np.mean(data[:, 0])
np.testing.assert_allclose(
s1['signals']['current']['statistics']['μ'],
i_mean,
rtol=0.0005)
r.close()
def test_empty_file(self):
fh = io.BytesIO()
d = DataRecorder(fh)
d.close()
fh.seek(0)
r = DataReader().open(fh)
self.assertEqual([0, 0], r.sample_id_range)
self.assertEqual(1.0, r.sampling_frequency)
self.assertEqual(1.0, r.input_sampling_frequency)
self.assertEqual(1.0, r.output_sampling_frequency)
self.assertEqual(1.0, r.reduction_frequency)
self.assertEqual(0.0, r.duration)
self.assertEqual(0, r.voltage_range)
self.assertEqual(0, len(r.get_reduction(0, 0)))
self.assertEqual(0, len(r.data_get(0, 0)))
self.assertEqual(0, r.time_to_sample_id(0.0))
self.assertEqual(0.0, r.sample_id_to_time(0))
self.assertIsNone(r.samples_get(0, 0))
r.close()
def run():
args = get_parser().parse_args()
reader = DataReader()
reader.open(args.infile)
s_min, s_max = reader.sample_id_range
sample_count = s_max - s_min
writer = DataRecorder(args.outfile, reader.calibration, reader.user_data)
block_size = int(reader.sampling_frequency)
print(f'samples={sample_count}, fs={reader.sampling_frequency}')
block_count = (sample_count + block_size - 1) // block_size
for block in range(block_count):
offset = block * block_size
offset_next = offset + block_size
if offset_next > sample_count:
offset_next = sample_count
data = reader.samples_get(offset, offset_next, 'samples')
writer.insert(data)
progress(block, block_count - 1)
reader.close()
writer.close()
return 0
class RecordingViewerDevice:
"""A user-interface-compatible device that displays previous recorded data
:param filename: The filename path to the pre-recorded data.
"""
def __init__(self, filename, current_ranging_format=None):
if isinstance(filename, str) and not os.path.isfile(filename):
raise IOError('file not found')
self._filename = filename
self._current_ranging_format = current_ranging_format
self._reader = None
self._views = []
self._coalesce = {}
self._thread = None
self._cmd_queue = queue.Queue() # tuples of (command, args, callback)
self._response_queue = queue.Queue()
self._quit = False
self._log = logging.getLogger(__name__)
def __str__(self):
return os.path.basename(self._filename)
@property
def sampling_frequency(self):
if self._reader is None:
return None
return self._reader.sampling_frequency
@property
def calibration(self):
if self._reader is None:
return None
return self._reader.calibration
@property
def voltage_range(self):
return self._reader.voltage_range
def _cmd_process(self, cmd, view, args, cbk):
rv = None
try:
# self._log.debug('_cmd_process %s - start', cmd)
if cmd == 'refresh':
view._refresh_requested = True
elif cmd == 'on_x_change':
rv = view._on_x_change(*args)
elif cmd == 'samples_get':
rv = view._samples_get(**args)
elif cmd == 'statistics_get':
rv = view._statistics_get(**args)
elif cmd == 'statistics_get_multiple':
rv = view._statistics_get_multiple(**args)
elif cmd == 'view_factory':
self._views.append(args)
rv = args
elif cmd == 'view_close':
if args in self._views:
self._views.remove(args)
elif cmd == 'open':
rv = self._open()
elif cmd == 'close':
rv = self._close()
elif cmd == 'ping':
rv = args
else:
self._log.warning('unsupported command %s', cmd)
except:
self._log.exception('While running command')
if callable(cbk):
try:
cbk(rv)
except:
self._log.exception('in callback')
def run(self):
cmd_count = 0
timeout = 1.0
self._log.info('RecordingViewerDevice.start')
while not self._quit:
try:
cmd, view, args, cbk = self._cmd_queue.get(timeout=timeout)
except queue.Empty:
timeout = 1.0
for value in self._coalesce.values():
self._cmd_process(*value)
self._coalesce.clear()
for view in self._views:
if view._refresh_requested:
view._update()
cmd_count = 0
continue
cmd_count += 1
timeout = 0.0
try:
source_id = args.pop('source_id')
except:
source_id = None
if source_id is not None:
key = f'{view}_{cmd}_{source_id}' # keep most recent only
self._coalesce[key] = (cmd, view, args, cbk)
else:
self._cmd_process(cmd, view, args, cbk)
self._log.info('RecordingViewerDevice.run done')
def _post(self, command, view=None, args=None, cbk=None):
if self._thread is None:
self._log.info('RecordingViewerDevice._post(%s) when thread not running', command)
else:
self._cmd_queue.put((command, view, args, cbk))
def _post_block(self, command, view=None, args=None, timeout=None):
timeout = TIMEOUT if timeout is None else float(timeout)
# self._log.debug('_post_block %s start', command)
while not self._response_queue.empty():
self._log.warning('response queue not empty')
try:
self._response_queue.get(timeout=0.0)
except queue.Empty:
pass
if self._thread is None:
raise IOError('View thread not running')
self._post(command, view, args, lambda rv_=None: self._response_queue.put(rv_))
try:
rv = self._response_queue.get(timeout=timeout)
except queue.Empty as ex:
self._log.error('RecordingViewerDevice thread hung: %s - FORCE CLOSE', command)
self._post('close', None, None)
self._thread.join(timeout=TIMEOUT)
self._thread = None
rv = ex
except Exception as ex:
rv = ex
if isinstance(rv, Exception):
raise IOError(rv)
# self._log.debug('_post_block %s done', command) # rv
return rv
def _open(self):
self._reader = DataReader()
if self._current_ranging_format is not None:
self._reader.raw_processor.suppress_mode = self._current_ranging_format
self._reader.open(self._filename) # todo progress bar updates
self._log.info('RecordingViewerDevice.open')
def _close(self):
if self._reader is not None:
self._reader.close()
self._reader = None
self._quit = True
def view_factory(self):
view = RecordingView(self)
return self._post_block('view_factory', None, view)
def open(self, event_callback_fn=None):
self.close()
self._log.info('open')
self._thread = threading.Thread(name='view', target=self.run)
self._thread.start()
self._post_block('open')
def close(self):
if self._thread is not None:
self._log.info('close')
try:
self._post_block('close')
except Exception:
self._log.exception('while attempting to close')
self._thread.join(timeout=TIMEOUT)
self._thread = None
def on_cmd(args):
if args.plot_reduction or args.plot or args.plot_raw:
try:
import matplotlib.pyplot as plt
except:
print(MATPLOTLIB_IMPORT_ERROR)
return 1
r = DataReader().open(args.filename)
print(r.summary_string())
start = args.start
stop = args.stop
if stop < 0:
stop = r.sample_id_range[1] + 1 + stop
if args.pretty_print:
with open(args.filename, 'rb') as f:
rf = DataFileReader(f)
rf.pretty_print()
if args.export is not None:
k = r.samples_get(start, stop, units='samples', fields=['current', 'voltage'])
i = k['signals']['current']['value']
v = k['signals']['voltage']['value']
data = np.hstack((i.reshape((-1, 1)), (v.reshape((-1, 1)))))
if args.export.endswith('npy'):
np.save(args.export, data)
else:
np.savetxt(args.export, data, fmt='%.5g', delimiter=',')
if args.plot_reduction:
y = r.get_reduction(start, stop)
x = np.arange(len(y)) * (r.config['samples_per_reduction'] / r.config['sampling_frequency'])
f = plt.figure()
fields = r.config['reduction_fields']
for axis, name in enumerate(fields):
ax = f.add_subplot(len(fields), 1, axis + 1)
ax.fill_between(x, y[:, axis]['min'], y[:, axis]['max'], color=(0.5, 0.5, 1.0, 0.5))
ax.plot(x, y[:, axis]['mean'], color='blue')
ax.set_ylabel(name)
ax.grid(True)
plt.show()
plt.close(f)
if args.plot:
k = r.samples_get(start, stop, units='samples', fields=['current', 'voltage'])
i = k['signals']['current']['value']
v = k['signals']['voltage']['value']
x = np.arange(len(i)) * (1.0 / r.config['sampling_frequency'])
f = plt.figure()
ax_i = f.add_subplot(2, 1, 1)
ax_i.plot(x, i)
ax_i.set_ylabel('Current (A)')
ax_i.grid(True)
ax_v = f.add_subplot(2, 1, 2, sharex=ax_i)
ax_v.plot(x, v)
ax_v.set_ylabel('Voltage (V)')
ax_v.grid(True)
ax_v.set_xlabel('Time (s)')
plt.show()
plt.close(f)
if args.plot_raw:
if stop - start > 2000000:
print('Time range too long, cannot --plot-raw')
else:
plot_idx_total = len(args.plot_raw)
link_axis = None
plot_idx = 1
rv = r.samples_get(start=start, stop=stop, units='samples', fields=['raw', 'current_range'])
d_raw = rv['signals']['raw']['value']
i_sel = rv['signals']['current_range']['value']
i_raw = np.right_shift(d_raw[:, 0], 2)
v_raw = np.right_shift(d_raw[:, 1], 2)
x = np.arange(len(i_raw)) * (1.0 / r.config['sampling_frequency'])
f = plt.figure()
f.suptitle('Joulescope Raw Data')
for c in args.plot_raw:
if c == 'i':
ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis)
ax.plot(x, i_raw)
ax.set_ylabel('Current (LSBs)')
elif c == 'v':
ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis)
ax.plot(x, v_raw)
ax.set_ylabel('Voltage (LSBs)')
elif c == 'r':
ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis)
ax.plot(x, i_sel)
ax.set_ylabel('Current Range')
else:
raise ValueError('unsupported plot: %s' % c)
ax.grid(True)
if link_axis is None:
link_axis = ax
plot_idx += 1
# plt.tight_layout()
ax.set_xlabel('Time (s)')
plt.show()
plt.close(f)
r.close()
return 0
def on_cmd(args):
r = DataReader().open(args.filename)
print(r.summary_string())
start = args.start
stop = args.stop
if stop < 0:
stop = r.sample_id_range[1] + 1 + stop
if args.export is not None:
i, v = r.get_calibrated(start, stop, units='samples')
data = np.hstack((i.reshape((-1, 1)), (v.reshape((-1, 1)))))
if args.export.endswith('npy'):
np.save(args.export, data)
else:
np.savetxt(args.export, data, fmt='%.5g', delimiter=',')
if args.plot_reduction:
import matplotlib.pyplot as plt
y = r.get_reduction(start, stop)
x = np.arange(len(y)) * (r.config['samples_per_reduction'] / r.config['sampling_frequency'])
f = plt.figure()
fields = r.config['reduction_fields']
for axis, name in enumerate(fields):
ax = f.add_subplot(len(fields), 1, axis + 1)
ax.plot(x, y[:, axis, 0], color='blue')
ax.plot(x, y[:, axis, 2], color='red')
ax.plot(x, y[:, axis, 3], color='red')
ax.set_ylabel(name)
plt.show()
plt.close(f)
if args.plot:
import matplotlib.pyplot as plt
i, v = r.get_calibrated(start, stop)
x = np.arange(len(i)) * (1.0 / r.config['sampling_frequency'])
f = plt.figure()
ax_i = f.add_subplot(2, 1, 1)
ax_i.plot(x, i)
ax_i.set_ylabel('Current (A)')
ax_i.grid(True)
ax_v = f.add_subplot(2, 1, 2, sharex=ax_i)
ax_v.plot(x, v)
ax_v.set_ylabel('Voltage (V)')
ax_v.grid(True)
ax_v.set_xlabel('Time (s)')
plt.show()
plt.close(f)
if args.plot_raw:
import matplotlib.pyplot as plt
if stop - start > 2000000:
print('Time range too long, cannot --plot-raw')
else:
plot_idx_total = len(args.plot_raw)
link_axis = None
plot_idx = 1
d_raw, d_bits, d_cal = r.raw(start=start, stop=stop)
i_raw = np.right_shift(d_raw[:, 0], 2)
v_raw = np.right_shift(d_raw[:, 1], 2)
x = np.arange(len(i_raw)) * (1.0 / r.config['sampling_frequency'])
i_sel = np.bitwise_and(d_bits, 0x000F)
f = plt.figure()
f.suptitle('Joulescope Raw Data')
for c in args.plot_raw:
if c == 'i':
ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis)
ax.plot(x, i_raw)
ax.set_ylabel('Current (LSBs)')
elif c == 'v':
ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis)
ax.plot(x, v_raw)
ax.set_ylabel('Voltage (LSBs)')
elif c == 'r':
ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis)
ax.plot(x, i_sel)
ax.set_ylabel('Current Range')
else:
raise ValueError('unsupported plot: %s' % c)
ax.grid(True)
if link_axis is None:
link_axis = ax
plot_idx += 1
# plt.tight_layout()
ax.set_xlabel('Time (s)')
plt.show()
plt.close(f)
r.close()
return 0
class RecordingViewerDevice:
"""A user-interface-compatible device that displays previous recorded data
:param filename: The filename path to the pre-recorded data.
"""
def __init__(self, filename):
self._filename = filename
self.reader = None
self.ui_action = None
self.view = None # type: DataViewApi
self.x_range = [0.0, 1.0]
self.span = None
self.x = None
self.samples_per = 1
self._cache = None
def __str__(self):
return os.path.basename(self._filename)
def __len__(self):
if self.span is None:
return 0
return self.span.length
@property
def sampling_frequency(self):
if self.reader is None:
return None
return self.reader.sampling_frequency
@property
def calibration(self):
if self.reader is None:
return None
return self.reader.calibration
def open(self):
self.view = self
self.reader = DataReader().open(self._filename)
f = self.reader.sampling_frequency
r = self.reader.sample_id_range
x_lim = [x / f for x in r]
self.span = span.Span(x_lim, 1 / f, 100)
self.x_range, self.samples_per, self.x = self.span.conform_discrete(
x_lim)
self._cache = None # invalidate
log.info('RecordingViewerDevice.open: %s => %s, %s', r, self.x_range,
self.samples_per)
def close(self):
if self.reader is not None:
self.reader.close()
self.reader = None
if self.on_close is not None:
self.on_close()
self.view = None
def on_x_change(self, cmd, kwargs):
x_range = self.x_range
if cmd == 'resize': # {pixels: int}
length = kwargs['pixels']
if length is not None and length != self.span.length:
log.info('resize %s', length)
self.span.length = length
self._cache = None # invalidate
x_range, self.samples_per, self.x = self.span.conform_discrete(
x_range)
elif cmd == 'span_absolute': # {range: (start: float, stop: float)}]
x_range, self.samples_per, self.x = self.span.conform_discrete(
kwargs.get('range'))
elif cmd == 'span_relative': # {pivot: float, gain: float}]
x_range, self.samples_per, self.x = self.span.conform_discrete(
x_range, gain=kwargs.get('gain'), pivot=kwargs.get('pivot'))
elif cmd == 'span_pan':
delta = kwargs.get('delta', 0.0)
x_range = [x_range[0] + delta, x_range[-1] + delta]
x_range, self.samples_per, self.x = self.span.conform_discrete(
x_range)
elif cmd == 'refresh':
self._cache = None # invalidate
return
else:
log.warning('on_x_change(%s) unsupported', cmd)
return
if self.x_range != x_range:
self._cache = None # invalidate
self.x_range = x_range
log.info(
'cmd=%s, changed=%s, length=%s, span=%s, range=%s, samples_per=%s',
cmd, self._cache is None, len(self), self.x_range,
self.x_range[1] - self.x_range[0], self.samples_per)
def update(self):
if self._cache is not None:
return False, self._cache
f = self.reader.sampling_frequency
log.info('update: x_range=%r', self.x_range)
start, stop = [int(x * f) for x in self.x_range]
log.info('update: x_range=%r => (%s, %s)', self.x_range, start, stop)
data = self.reader.get(start, stop, self.samples_per)
t_start = start / self.reader.sampling_frequency
t_stop = stop / self.reader.sampling_frequency
x = np.linspace(t_start, t_stop, len(data), dtype=np.float64)
try:
log.info('update: len=%d, x_range=>(%s, %s)', len(data), x[0],
x[-1])
except:
print(x.shape)
self._cache = (x, data)
return True, self._cache
def time_to_sample_id(self, t):
if self.reader is None:
return None
return self.reader.time_to_sample_id(t)
def statistics_get(self, t1, t2):
"""Get the statistics for the collected sample data over a time range.
:param t1: The starting time in seconds relative to the streaming start time.
:param t2: The ending time in seconds.
:return: The statistics data structure.
"""
if self.reader is None:
return None
return self.reader.statistics_get(t1, t2)
def raw_get(self, start=None, stop=None):
if self.reader is None:
return None
return self.reader.raw(start=start, stop=stop)
def samples_get(self, start=None, stop=None):
if self.reader is None:
return None
i, v = self.reader.get_calibrated(start=start, stop=stop)
return {
'current': {
'value': i,
'units': 'A',
},
'voltage': {
'value': v,
'units': 'V',
}
}