def test_statistics_get(self): #fh = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data_recording_01.jls') fh = self.create_sinusoid_file(2000000, 400000) r = DataReader().open(fh) t_start, t_stop = 0.066780, 0.069004 k_start, k_stop = r.normalize_time_arguments(t_start, t_stop, units='seconds') ranges = [ (0, 1000), # trivial, direct (0, 20000), # trivial, single reduction (100000, 101000), # offset, direct (100000, 120000), # offset, ex (99000, 120000), (100000, 121000), (99000, 121000), (k_start, k_stop), ] for k_start, k_stop in ranges: # print(f'range {k_start}:{k_stop}') s1 = r.statistics_get(k_start, k_stop, units='samples') k = r.samples_get(k_start, k_stop, units='samples', fields=['current']) i_mean = np.mean(k['signals']['current']['value']) np.testing.assert_allclose(s1['signals']['current']['µ']['value'], i_mean, rtol=0.0005) r.close()
def on_cmd(args): r = DataReader().open(args.filename) print(r.summary_string()) start = args.start stop = args.stop if stop < 0: stop = r.sample_id_range[1] + 1 + stop if args.export is not None: i, v = r.get_calibrated(start, stop) data = np.hstack((i.reshape((-1, 1)), (v.reshape((-1, 1))))) if args.export.endswith('npy'): np.save(args.export, data) else: np.savetxt(args.export, data, fmt='%.5g', delimiter=',') if args.plot: import matplotlib.pyplot as plt y = r.get_reduction(start, stop) x = np.arange(len(y)) * (r.config['samples_per_reduction'] / r.config['sampling_frequency']) f = plt.figure() for axis in range(3): ax = f.add_subplot(3, 1, axis + 1) ax.plot(x, y[:, axis, 0], color='blue') ax.plot(x, y[:, axis, 2], color='red') ax.plot(x, y[:, axis, 3], color='red') plt.show() plt.close(f) r.close() return 0
def load_current_range(filename): r = DataReader().open(filename) try: print(r.summary_string()) r_start, r_stop = r.sample_id_range if r_stop - r_start > MAX_SAMPLES: print('file too big') return 1 d = r.samples_get(fields=['current_range']) return d['signals']['current_range']['value'] finally: r.close()
def test_cache_test(self): sample_rate = 2000000 sample_count = sample_rate * 2 fh = self.create_sinusoid_file(sample_rate, sample_count) r = DataReader().open(fh) for step_size in [1111, 2000, 11111, 20000]: # print(f'step_size = {step_size}') for i in range(0, sample_count - step_size, step_size): r.raw_processor.reset() s1 = r.statistics_get(i, i + step_size, units='samples') _, _, data = r.raw(i, i + step_size) i_mean = np.mean(data[:, 0]) np.testing.assert_allclose( s1['signals']['current']['statistics']['μ'], i_mean, rtol=0.0005) r.close()
def test_empty_file(self): fh = io.BytesIO() d = DataRecorder(fh) d.close() fh.seek(0) r = DataReader().open(fh) self.assertEqual([0, 0], r.sample_id_range) self.assertEqual(1.0, r.sampling_frequency) self.assertEqual(1.0, r.input_sampling_frequency) self.assertEqual(1.0, r.output_sampling_frequency) self.assertEqual(1.0, r.reduction_frequency) self.assertEqual(0.0, r.duration) self.assertEqual(0, r.voltage_range) self.assertEqual(0, len(r.get_reduction(0, 0))) self.assertEqual(0, len(r.data_get(0, 0))) self.assertEqual(0, r.time_to_sample_id(0.0)) self.assertEqual(0.0, r.sample_id_to_time(0)) self.assertIsNone(r.samples_get(0, 0)) r.close()
def run(): args = get_parser().parse_args() reader = DataReader() reader.open(args.infile) s_min, s_max = reader.sample_id_range sample_count = s_max - s_min writer = DataRecorder(args.outfile, reader.calibration, reader.user_data) block_size = int(reader.sampling_frequency) print(f'samples={sample_count}, fs={reader.sampling_frequency}') block_count = (sample_count + block_size - 1) // block_size for block in range(block_count): offset = block * block_size offset_next = offset + block_size if offset_next > sample_count: offset_next = sample_count data = reader.samples_get(offset, offset_next, 'samples') writer.insert(data) progress(block, block_count - 1) reader.close() writer.close() return 0
class RecordingViewerDevice: """A user-interface-compatible device that displays previous recorded data :param filename: The filename path to the pre-recorded data. """ def __init__(self, filename, current_ranging_format=None): if isinstance(filename, str) and not os.path.isfile(filename): raise IOError('file not found') self._filename = filename self._current_ranging_format = current_ranging_format self._reader = None self._views = [] self._coalesce = {} self._thread = None self._cmd_queue = queue.Queue() # tuples of (command, args, callback) self._response_queue = queue.Queue() self._quit = False self._log = logging.getLogger(__name__) def __str__(self): return os.path.basename(self._filename) @property def sampling_frequency(self): if self._reader is None: return None return self._reader.sampling_frequency @property def calibration(self): if self._reader is None: return None return self._reader.calibration @property def voltage_range(self): return self._reader.voltage_range def _cmd_process(self, cmd, view, args, cbk): rv = None try: # self._log.debug('_cmd_process %s - start', cmd) if cmd == 'refresh': view._refresh_requested = True elif cmd == 'on_x_change': rv = view._on_x_change(*args) elif cmd == 'samples_get': rv = view._samples_get(**args) elif cmd == 'statistics_get': rv = view._statistics_get(**args) elif cmd == 'statistics_get_multiple': rv = view._statistics_get_multiple(**args) elif cmd == 'view_factory': self._views.append(args) rv = args elif cmd == 'view_close': if args in self._views: self._views.remove(args) elif cmd == 'open': rv = self._open() elif cmd == 'close': rv = self._close() elif cmd == 'ping': rv = args else: self._log.warning('unsupported command %s', cmd) except: self._log.exception('While running command') if callable(cbk): try: cbk(rv) except: self._log.exception('in callback') def run(self): cmd_count = 0 timeout = 1.0 self._log.info('RecordingViewerDevice.start') while not self._quit: try: cmd, view, args, cbk = self._cmd_queue.get(timeout=timeout) except queue.Empty: timeout = 1.0 for value in self._coalesce.values(): self._cmd_process(*value) self._coalesce.clear() for view in self._views: if view._refresh_requested: view._update() cmd_count = 0 continue cmd_count += 1 timeout = 0.0 try: source_id = args.pop('source_id') except: source_id = None if source_id is not None: key = f'{view}_{cmd}_{source_id}' # keep most recent only self._coalesce[key] = (cmd, view, args, cbk) else: self._cmd_process(cmd, view, args, cbk) self._log.info('RecordingViewerDevice.run done') def _post(self, command, view=None, args=None, cbk=None): if self._thread is None: self._log.info('RecordingViewerDevice._post(%s) when thread not running', command) else: self._cmd_queue.put((command, view, args, cbk)) def _post_block(self, command, view=None, args=None, timeout=None): timeout = TIMEOUT if timeout is None else float(timeout) # self._log.debug('_post_block %s start', command) while not self._response_queue.empty(): self._log.warning('response queue not empty') try: self._response_queue.get(timeout=0.0) except queue.Empty: pass if self._thread is None: raise IOError('View thread not running') self._post(command, view, args, lambda rv_=None: self._response_queue.put(rv_)) try: rv = self._response_queue.get(timeout=timeout) except queue.Empty as ex: self._log.error('RecordingViewerDevice thread hung: %s - FORCE CLOSE', command) self._post('close', None, None) self._thread.join(timeout=TIMEOUT) self._thread = None rv = ex except Exception as ex: rv = ex if isinstance(rv, Exception): raise IOError(rv) # self._log.debug('_post_block %s done', command) # rv return rv def _open(self): self._reader = DataReader() if self._current_ranging_format is not None: self._reader.raw_processor.suppress_mode = self._current_ranging_format self._reader.open(self._filename) # todo progress bar updates self._log.info('RecordingViewerDevice.open') def _close(self): if self._reader is not None: self._reader.close() self._reader = None self._quit = True def view_factory(self): view = RecordingView(self) return self._post_block('view_factory', None, view) def open(self, event_callback_fn=None): self.close() self._log.info('open') self._thread = threading.Thread(name='view', target=self.run) self._thread.start() self._post_block('open') def close(self): if self._thread is not None: self._log.info('close') try: self._post_block('close') except Exception: self._log.exception('while attempting to close') self._thread.join(timeout=TIMEOUT) self._thread = None
def on_cmd(args): if args.plot_reduction or args.plot or args.plot_raw: try: import matplotlib.pyplot as plt except: print(MATPLOTLIB_IMPORT_ERROR) return 1 r = DataReader().open(args.filename) print(r.summary_string()) start = args.start stop = args.stop if stop < 0: stop = r.sample_id_range[1] + 1 + stop if args.pretty_print: with open(args.filename, 'rb') as f: rf = DataFileReader(f) rf.pretty_print() if args.export is not None: k = r.samples_get(start, stop, units='samples', fields=['current', 'voltage']) i = k['signals']['current']['value'] v = k['signals']['voltage']['value'] data = np.hstack((i.reshape((-1, 1)), (v.reshape((-1, 1))))) if args.export.endswith('npy'): np.save(args.export, data) else: np.savetxt(args.export, data, fmt='%.5g', delimiter=',') if args.plot_reduction: y = r.get_reduction(start, stop) x = np.arange(len(y)) * (r.config['samples_per_reduction'] / r.config['sampling_frequency']) f = plt.figure() fields = r.config['reduction_fields'] for axis, name in enumerate(fields): ax = f.add_subplot(len(fields), 1, axis + 1) ax.fill_between(x, y[:, axis]['min'], y[:, axis]['max'], color=(0.5, 0.5, 1.0, 0.5)) ax.plot(x, y[:, axis]['mean'], color='blue') ax.set_ylabel(name) ax.grid(True) plt.show() plt.close(f) if args.plot: k = r.samples_get(start, stop, units='samples', fields=['current', 'voltage']) i = k['signals']['current']['value'] v = k['signals']['voltage']['value'] x = np.arange(len(i)) * (1.0 / r.config['sampling_frequency']) f = plt.figure() ax_i = f.add_subplot(2, 1, 1) ax_i.plot(x, i) ax_i.set_ylabel('Current (A)') ax_i.grid(True) ax_v = f.add_subplot(2, 1, 2, sharex=ax_i) ax_v.plot(x, v) ax_v.set_ylabel('Voltage (V)') ax_v.grid(True) ax_v.set_xlabel('Time (s)') plt.show() plt.close(f) if args.plot_raw: if stop - start > 2000000: print('Time range too long, cannot --plot-raw') else: plot_idx_total = len(args.plot_raw) link_axis = None plot_idx = 1 rv = r.samples_get(start=start, stop=stop, units='samples', fields=['raw', 'current_range']) d_raw = rv['signals']['raw']['value'] i_sel = rv['signals']['current_range']['value'] i_raw = np.right_shift(d_raw[:, 0], 2) v_raw = np.right_shift(d_raw[:, 1], 2) x = np.arange(len(i_raw)) * (1.0 / r.config['sampling_frequency']) f = plt.figure() f.suptitle('Joulescope Raw Data') for c in args.plot_raw: if c == 'i': ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis) ax.plot(x, i_raw) ax.set_ylabel('Current (LSBs)') elif c == 'v': ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis) ax.plot(x, v_raw) ax.set_ylabel('Voltage (LSBs)') elif c == 'r': ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis) ax.plot(x, i_sel) ax.set_ylabel('Current Range') else: raise ValueError('unsupported plot: %s' % c) ax.grid(True) if link_axis is None: link_axis = ax plot_idx += 1 # plt.tight_layout() ax.set_xlabel('Time (s)') plt.show() plt.close(f) r.close() return 0
def on_cmd(args): r = DataReader().open(args.filename) print(r.summary_string()) start = args.start stop = args.stop if stop < 0: stop = r.sample_id_range[1] + 1 + stop if args.export is not None: i, v = r.get_calibrated(start, stop, units='samples') data = np.hstack((i.reshape((-1, 1)), (v.reshape((-1, 1))))) if args.export.endswith('npy'): np.save(args.export, data) else: np.savetxt(args.export, data, fmt='%.5g', delimiter=',') if args.plot_reduction: import matplotlib.pyplot as plt y = r.get_reduction(start, stop) x = np.arange(len(y)) * (r.config['samples_per_reduction'] / r.config['sampling_frequency']) f = plt.figure() fields = r.config['reduction_fields'] for axis, name in enumerate(fields): ax = f.add_subplot(len(fields), 1, axis + 1) ax.plot(x, y[:, axis, 0], color='blue') ax.plot(x, y[:, axis, 2], color='red') ax.plot(x, y[:, axis, 3], color='red') ax.set_ylabel(name) plt.show() plt.close(f) if args.plot: import matplotlib.pyplot as plt i, v = r.get_calibrated(start, stop) x = np.arange(len(i)) * (1.0 / r.config['sampling_frequency']) f = plt.figure() ax_i = f.add_subplot(2, 1, 1) ax_i.plot(x, i) ax_i.set_ylabel('Current (A)') ax_i.grid(True) ax_v = f.add_subplot(2, 1, 2, sharex=ax_i) ax_v.plot(x, v) ax_v.set_ylabel('Voltage (V)') ax_v.grid(True) ax_v.set_xlabel('Time (s)') plt.show() plt.close(f) if args.plot_raw: import matplotlib.pyplot as plt if stop - start > 2000000: print('Time range too long, cannot --plot-raw') else: plot_idx_total = len(args.plot_raw) link_axis = None plot_idx = 1 d_raw, d_bits, d_cal = r.raw(start=start, stop=stop) i_raw = np.right_shift(d_raw[:, 0], 2) v_raw = np.right_shift(d_raw[:, 1], 2) x = np.arange(len(i_raw)) * (1.0 / r.config['sampling_frequency']) i_sel = np.bitwise_and(d_bits, 0x000F) f = plt.figure() f.suptitle('Joulescope Raw Data') for c in args.plot_raw: if c == 'i': ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis) ax.plot(x, i_raw) ax.set_ylabel('Current (LSBs)') elif c == 'v': ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis) ax.plot(x, v_raw) ax.set_ylabel('Voltage (LSBs)') elif c == 'r': ax = f.add_subplot(plot_idx_total, 1, plot_idx, sharex=link_axis) ax.plot(x, i_sel) ax.set_ylabel('Current Range') else: raise ValueError('unsupported plot: %s' % c) ax.grid(True) if link_axis is None: link_axis = ax plot_idx += 1 # plt.tight_layout() ax.set_xlabel('Time (s)') plt.show() plt.close(f) r.close() return 0
class RecordingViewerDevice: """A user-interface-compatible device that displays previous recorded data :param filename: The filename path to the pre-recorded data. """ def __init__(self, filename): self._filename = filename self.reader = None self.ui_action = None self.view = None # type: DataViewApi self.x_range = [0.0, 1.0] self.span = None self.x = None self.samples_per = 1 self._cache = None def __str__(self): return os.path.basename(self._filename) def __len__(self): if self.span is None: return 0 return self.span.length @property def sampling_frequency(self): if self.reader is None: return None return self.reader.sampling_frequency @property def calibration(self): if self.reader is None: return None return self.reader.calibration def open(self): self.view = self self.reader = DataReader().open(self._filename) f = self.reader.sampling_frequency r = self.reader.sample_id_range x_lim = [x / f for x in r] self.span = span.Span(x_lim, 1 / f, 100) self.x_range, self.samples_per, self.x = self.span.conform_discrete( x_lim) self._cache = None # invalidate log.info('RecordingViewerDevice.open: %s => %s, %s', r, self.x_range, self.samples_per) def close(self): if self.reader is not None: self.reader.close() self.reader = None if self.on_close is not None: self.on_close() self.view = None def on_x_change(self, cmd, kwargs): x_range = self.x_range if cmd == 'resize': # {pixels: int} length = kwargs['pixels'] if length is not None and length != self.span.length: log.info('resize %s', length) self.span.length = length self._cache = None # invalidate x_range, self.samples_per, self.x = self.span.conform_discrete( x_range) elif cmd == 'span_absolute': # {range: (start: float, stop: float)}] x_range, self.samples_per, self.x = self.span.conform_discrete( kwargs.get('range')) elif cmd == 'span_relative': # {pivot: float, gain: float}] x_range, self.samples_per, self.x = self.span.conform_discrete( x_range, gain=kwargs.get('gain'), pivot=kwargs.get('pivot')) elif cmd == 'span_pan': delta = kwargs.get('delta', 0.0) x_range = [x_range[0] + delta, x_range[-1] + delta] x_range, self.samples_per, self.x = self.span.conform_discrete( x_range) elif cmd == 'refresh': self._cache = None # invalidate return else: log.warning('on_x_change(%s) unsupported', cmd) return if self.x_range != x_range: self._cache = None # invalidate self.x_range = x_range log.info( 'cmd=%s, changed=%s, length=%s, span=%s, range=%s, samples_per=%s', cmd, self._cache is None, len(self), self.x_range, self.x_range[1] - self.x_range[0], self.samples_per) def update(self): if self._cache is not None: return False, self._cache f = self.reader.sampling_frequency log.info('update: x_range=%r', self.x_range) start, stop = [int(x * f) for x in self.x_range] log.info('update: x_range=%r => (%s, %s)', self.x_range, start, stop) data = self.reader.get(start, stop, self.samples_per) t_start = start / self.reader.sampling_frequency t_stop = stop / self.reader.sampling_frequency x = np.linspace(t_start, t_stop, len(data), dtype=np.float64) try: log.info('update: len=%d, x_range=>(%s, %s)', len(data), x[0], x[-1]) except: print(x.shape) self._cache = (x, data) return True, self._cache def time_to_sample_id(self, t): if self.reader is None: return None return self.reader.time_to_sample_id(t) def statistics_get(self, t1, t2): """Get the statistics for the collected sample data over a time range. :param t1: The starting time in seconds relative to the streaming start time. :param t2: The ending time in seconds. :return: The statistics data structure. """ if self.reader is None: return None return self.reader.statistics_get(t1, t2) def raw_get(self, start=None, stop=None): if self.reader is None: return None return self.reader.raw(start=start, stop=stop) def samples_get(self, start=None, stop=None): if self.reader is None: return None i, v = self.reader.get_calibrated(start=start, stop=stop) return { 'current': { 'value': i, 'units': 'A', }, 'voltage': { 'value': v, 'units': 'V', } }