class _BatchWriter(object): #: Truncate overly big items to that many bytes for the error message. ERRMSG_DATA_TRUNCATION_LEN = 1024 def __init__(self, url, start, auth, size, interval, qsize, maxitemsize, content_encoding, uploader, callback=None): self.url = url self.offset = start self._nextid = count(start) self.auth = auth self.size = size self.interval = interval self.maxitemsize = maxitemsize self.content_encoding = content_encoding self.checkpoint = time.time() self.itemsq = Queue(size * 2 if qsize is None else qsize) self.closed = False self.flushme = False self.uploader = uploader self.callback = callback def write(self, item): assert not self.closed, 'attempting writes to a closed writer' data = jsonencode(item) if len(data) > self.maxitemsize: truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..." raise ValueTooLarge( 'Value exceeds max encoded size of {}: {!r}'.format( sizeof_fmt(self.maxitemsize), truncated_data)) self.itemsq.put(data) if self.itemsq.full(): self.uploader.interrupt() return next(self._nextid) def flush(self): self.flushme = True self._waitforq() self.flushme = False def close(self, block=True): self.closed = True if block: self._waitforq() def _waitforq(self): self.uploader.interrupt() self.itemsq.join() def __str__(self): return self.url
class _BatchWriter(object): #: Truncate overly big items to that many bytes for the error message. ERRMSG_DATA_TRUNCATION_LEN = 1024 def __init__(self, url, start, auth, size, interval, qsize, maxitemsize, content_encoding, uploader, callback=None): self.url = url self.offset = start self._nextid = count(start) self.auth = auth self.size = size self.interval = interval self.maxitemsize = maxitemsize self.content_encoding = content_encoding self.checkpoint = time.time() self.itemsq = Queue(size * 2 if qsize is None else qsize) self.closed = False self.flushme = False self.uploader = uploader self.callback = callback def write(self, item): assert not self.closed, 'attempting writes to a closed writer' data = jsonencode(item) if len(data) > self.maxitemsize: truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..." raise ValueTooLarge( 'Value exceeds max encoded size of {} bytes: {!r}' .format(self.maxitemsize, truncated_data)) self.itemsq.put(data) if self.itemsq.full(): self.uploader.interrupt() return next(self._nextid) def flush(self): self.flushme = True self._waitforq() self.flushme = False def close(self, block=True): self.closed = True if block: self._waitforq() def _waitforq(self): self.uploader.interrupt() self.itemsq.join() def __str__(self): return self.url
def run_multiple_commands_redirect_stdout( multiple_args_dict, print_commands=True, process_limit=-1, polling_freq=0.5, **kwargs): """ Run multiple shell commands in parallel, write each of their stdout output to files associated with each command. Parameters ---------- multiple_args_dict : dict A dictionary whose keys are files and values are args list. Run each args list as a subprocess and write stdout to the corresponding file. print_commands : bool Print shell commands before running them. process_limit : int Limit the number of concurrent processes to this number. 0 if there is no limit, -1 to use max number of processors polling_freq : int Number of seconds between checking for done processes, if we have a process limit """ assert len(multiple_args_dict) > 0 assert all(len(args) > 0 for args in multiple_args_dict.values()) assert all(hasattr(f, 'name') for f in multiple_args_dict.keys()) if process_limit < 0: logger.debug("Using %d processes" % cpu_count()) process_limit = cpu_count() start_time = time.time() processes = Queue(maxsize=process_limit) def add_to_queue(process): process.start() if print_commands: handler = logging.FileHandler(process.redirect_stdout_file.name) handler.setLevel(logging.DEBUG) logger.addHandler(handler) logger.debug(" ".join(process.args)) logger.removeHandler(handler) processes.put(process) for f, args in multiple_args_dict.items(): p = AsyncProcess( args, redirect_stdout_file=f, **kwargs) if not processes.full(): add_to_queue(p) else: while processes.full(): # Are there any done processes? to_remove = [] for possibly_done in processes.queue: if possibly_done.poll() is not None: possibly_done.wait() to_remove.append(possibly_done) # Remove them from the queue and stop checking if to_remove: for process_to_remove in to_remove: processes.queue.remove(process_to_remove) break # Check again in a second if there weren't time.sleep(polling_freq) add_to_queue(p) # Wait for all the rest of the processes while not processes.empty(): processes.get().wait() elapsed_time = time.time() - start_time logger.info( "Ran %d commands in %0.4f seconds", len(multiple_args_dict), elapsed_time)
def run_multiple_commands_redirect_stdout( multiple_args_dict, print_commands=True, process_limit=0, polling_freq=1, **kwargs): """ Run multiple shell commands in parallel, write each of their stdout output to files associated with each command. Parameters ---------- multiple_args_dict : dict A dictionary whose keys are files and values are args list. Run each args list as a subprocess and write stdout to the corresponding file. print_commands : bool Print shell commands before running them. process_limit : int Limit the number of concurrent processes to this number. 0 if there is no limit polling_freq : int Number of seconds between checking for done processes, if we have a process limit """ assert len(multiple_args_dict) > 0 assert all(len(args) > 0 for args in multiple_args_dict.values()) assert all(hasattr(f, 'name') for f in multiple_args_dict.keys()) start_time = time.time() processes = Queue(maxsize=process_limit) def add_to_queue(process): if print_commands: print(" ".join(process.args), ">", process.redirect_stdout_file.name) processes.put(process) for f, args in multiple_args_dict.items(): p = AsyncProcess( args, redirect_stdout_file=f, **kwargs) if not processes.full(): add_to_queue(p) else: while processes.full(): # Are there any done processes? to_remove = [] for possibly_done in processes.queue: if possibly_done.poll() is not None: possibly_done.wait() to_remove.append(possibly_done) # Remove them from the queue and stop checking if to_remove: for process_to_remove in to_remove: processes.queue.remove(process_to_remove) break # Check again in a second if there weren't time.sleep(polling_freq) add_to_queue(p) # Wait for all the rest of the processes while not processes.empty(): processes.get().wait() elapsed_time = time.time() - start_time logging.info( "Ran %d commands in %0.4f seconds", len(multiple_args_dict), elapsed_time)
class FifoReadout(object): def __init__(self, chip, readout_interval, moving_average_time_period): self.logger = logging.getLogger(self.__class__.__name__) self.logger.setLevel(loglevel) self.chip = chip self.callback = None self.errback = None self.readout_thread = None self.worker_thread = None self.watchdog_thread = None self.fill_buffer = False self.readout_interval = readout_interval self._moving_average_time_period = moving_average_time_period self._data_deque = deque() self._data_buffer = deque() self._words_per_read = deque(maxlen=int(self._moving_average_time_period / self.readout_interval)) self._result = Queue(maxsize=1) self._calculate = Event() self.stop_readout = Event() self.force_stop = Event() self.timestamp = None self.update_timestamp() self._is_running = False self.reset_rx() self.reset_sram_fifo() self._record_count = 0 @property def is_running(self): return self._is_running @property def is_alive(self): if self.worker_thread: return self.worker_thread.is_alive() else: False @property def data(self): if self.fill_buffer: return self._data_buffer else: self.logger.warning('Data requested but software data buffer not active') def data_words_per_second(self): if self._result.full(): self._result.get() self._calculate.set() try: result = self._result.get(timeout=2 * self.readout_interval) except Empty: self._calculate.clear() return None return result / float(self._moving_average_time_period) def start(self, callback=None, errback=None, reset_rx=False, reset_sram_fifo=False, reset_errors=True, clear_buffer=False, fill_buffer=False, no_data_timeout=None): if self._is_running: raise RuntimeError('Readout already running: use stop() before start()') self._is_running = True self.logger.debug('Starting FIFO readout...') self.callback = callback self.errback = errback self.fill_buffer = fill_buffer # self._record_count = 0 if reset_rx: self.reset_rx() if reset_sram_fifo: self.reset_sram_fifo() if reset_errors: self.rx_error_reset() else: fifo_size = self.chip['FIFO']['FIFO_SIZE'] if fifo_size != 0: self.logger.warning('FIFO not empty when starting FIFO readout: size = %i', fifo_size) self._words_per_read.clear() if clear_buffer: self._data_deque.clear() self._data_buffer.clear() self.stop_readout.clear() self.force_stop.clear() if self.errback: self.watchdog_thread = Thread(target=self.watchdog, name='WatchdogThread') self.watchdog_thread.daemon = True self.watchdog_thread.start() if self.callback: self.worker_thread = Thread(target=self.worker, name='WorkerThread') self.worker_thread.daemon = True self.worker_thread.start() self.readout_thread = Thread(target=self.readout, name='ReadoutThread', kwargs={'no_data_timeout': no_data_timeout}) self.readout_thread.daemon = True self.readout_thread.start() def stop(self, timeout=10.0): if not self._is_running: raise RuntimeError('Readout not running: use start() before stop()') self._is_running = False self.stop_readout.set() sleep(0.1) try: self.readout_thread.join(timeout=timeout) if self.readout_thread.is_alive(): if timeout: raise StopTimeout('FIFO stop timeout after %0.1f second(s)' % timeout) else: self.logger.warning('FIFO stop timeout') except StopTimeout as e: self.force_stop.set() if self.errback: self.errback(sys.exc_info()) else: self.logger.error(e) if self.readout_thread.is_alive(): self.readout_thread.join() if self.errback: self.watchdog_thread.join() if self.callback: self.worker_thread.join() self.callback = None self.errback = None self.logger.debug('Stopped FIFO readout') def print_readout_status(self): sync_status = self.get_rx_sync_status() en_status = self.get_rx_en_status() discard_count = self.get_rx_fifo_discard_count() decode_error_count = self.get_rx_decode_error_count() if not any(self.get_rx_sync_status()) or any(discard_count) or any(decode_error_count) : self.logger.warning('RX errors detected') self.logger.info('Recived words: %d', self._record_count) self.logger.info('Data queue size: %d', len(self._data_deque)) self.logger.info('FIFO size: %d', self.chip['FIFO']['FIFO_SIZE']) self.logger.info('Channel: %s', " | ".join([channel.name.rjust(3) for channel in self.chip.get_modules('tpx3_rx')])) self.logger.info('RX sync: %s', " | ".join(["YES".rjust(3) if status is True else "NO".rjust(3) for status in sync_status])) self.logger.info('RX enable: %s', " | ".join(["YES".rjust(3) if status is True else "NO".rjust(3) for status in en_status])) self.logger.info('RX FIFO discard counter: %s', " | ".join([repr(count).rjust(3) for count in discard_count])) self.logger.info('RX decode errors: %s', " | ".join([repr(count).rjust(3) for count in decode_error_count])) def readout(self, no_data_timeout=None): ''' Readout thread continuously reading FIFO. Uses read_data() and appends data to self._data_deque (collection.deque). ''' self.logger.debug('Starting %s', self.readout_thread.name) curr_time = self.get_float_time() time_wait = 0.0 while not self.force_stop.wait(time_wait if time_wait >= 0.0 else 0.0): try: time_read = time() if no_data_timeout and curr_time + no_data_timeout < self.get_float_time(): raise NoDataTimeout('Received no data for %0.1f second(s)' % no_data_timeout) # TODO: maybe not best solution? dlist = [] while time() - time_read < self.readout_interval: dlist.append(self.read_data()) data = np.hstack(dlist) self._record_count += len(data) except Exception: no_data_timeout = None # raise exception only once if self.errback: self.errback(sys.exc_info()) else: raise if self.stop_readout.is_set(): break else: n_words = data.shape[0] last_time, curr_time = self.update_timestamp() discard_error = int(np.sum(self.get_rx_fifo_discard_count(), dtype=np.uint32)) decode_error = int(np.sum(self.get_rx_decode_error_count(), dtype=np.uint32)) if self.callback: self._data_deque.append((data, last_time, curr_time, discard_error, decode_error)) if self.fill_buffer: self._data_buffer.append((data, last_time, curr_time, discard_error, decode_error)) self._words_per_read.append(n_words) # FIXME: busy FE prevents scan termination? To be checked if n_words == 0 and self.stop_readout.is_set(): break if discard_error > 0 or decode_error > 0: self.logger.warning('There were {} discard errors and {} decode errors - Resetting error counters'.format(discard_error, decode_error)) self.rx_error_reset() finally: time_wait = self.readout_interval - (time() - time_read) if self._calculate.is_set(): self._calculate.clear() self._result.put(sum(self._words_per_read)) if self.callback: self._data_deque.append(None) # last item, will stop worker self.logger.debug('Stopped %s', self.readout_thread.name) def worker(self): ''' Worker thread continuously calling callback function when data is available. ''' self.logger.debug('Starting %s', self.worker_thread.name) while True: try: data = self._data_deque.popleft() except IndexError: self.stop_readout.wait(self.readout_interval) # sleep a little bit, reducing CPU usage else: if data is None: # if None then exit break else: try: self.callback(data) except Exception: self.errback(sys.exc_info()) self.logger.debug('Stopped %s', self.worker_thread.name) def watchdog(self): self.logger.debug('Starting %s', self.watchdog_thread.name) while True: try: if not any(self.get_rx_sync_status()): raise RxSyncError('No RX sync') cnt = self.get_rx_fifo_discard_count() if any(cnt): raise FifoError('RX FIFO discard error(s) detected ', cnt) except Exception: self.errback(sys.exc_info()) if self.stop_readout.wait(self.readout_interval * 10): break self.logger.debug('Stopped %s', self.watchdog_thread.name) def read_data(self): ''' Read FIFO and return data array Can be used without threading. Returns ---------- data : list A list of FIFO data words. ''' return self.chip['FIFO'].get_data() def update_timestamp(self): curr_time = self.get_float_time() last_time = self.timestamp self.timestamp = curr_time return last_time, curr_time def read_status(self): raise NotImplementedError() def reset_sram_fifo(self): fifo_size = self.chip['FIFO']['FIFO_SIZE'] self.logger.debug('Resetting FIFO: size = %i', fifo_size) self.update_timestamp() self.chip['FIFO']['RESET'] sleep(0.01) # sleep here for a while fifo_size = self.chip['FIFO']['FIFO_SIZE'] if fifo_size != 0: self.logger.warning('FIFO not empty after reset: size = %i', fifo_size) def enable_rx(self, enable=True, channels=None): self.logger.debug('Enable RX') if channels is None: for ch in self.chip.get_modules('tpx3_rx'): ch.ENABLE = enable else: for ch in channels: self.chip[ch].ENABLE = enable def reset_rx(self, channels=None): self.logger.debug('Resetting RX') if channels: [channel for channel in channels if self.chip[channel].RESET] else: [channel for channel in self.chip.get_modules('tpx3_rx') if channel.RESET] sleep(0.1) # sleep here for a while def rx_error_reset(self, channels=None): self.logger.debug('Resetting RX errors') if channels: [channel for channel in channels if self.chip[channel].rx_error_reset()] else: [channel for channel in self.chip.get_modules('tpx3_rx') if channel.rx_error_reset()] def get_rx_sync_status(self, channels=None): if channels: return [True if self.chip[channel].READY else False for channel in channels] else: return [True if channel.READY else False for channel in self.chip.get_modules('tpx3_rx')] def get_rx_en_status(self, channels=None): if channels: return [True if self.chip[channel].ENABLE else False for channel in channels] else: return [True if channel.ENABLE else False for channel in self.chip.get_modules('tpx3_rx')] def get_rx_fifo_discard_count(self, channels=None): if channels: return [self.chip[channel].LOST_DATA_COUNTER for channel in channels] else: return [channel.LOST_DATA_COUNTER for channel in self.chip.get_modules('tpx3_rx')] def get_rx_decode_error_count(self, channels=None): if channels: return [self.chip[channel].DECODER_ERROR_COUNTER for channel in channels] else: return [channel.DECODER_ERROR_COUNTER for channel in self.chip.get_modules('tpx3_rx')] def get_float_time(self): ''' Returns time as double precision floats - Time64 in pytables - mapping to and from python datetime's ''' t1 = time() t2 = datetime.datetime.fromtimestamp(t1) return mktime(t2.timetuple()) + 1e-6 * t2.microsecond