def _centrallogger_worker(logging_q: mpq.Queue): """Entry for a thread providing central logging from subprocesses via a shared queue""" threadname = threading.currentThread().getName() thread_logger = LOGGER.getChild(threadname) thread_logger.info('{} started'.format(threadname)) while True: # only calling process terminates - keeping logging alive record = logging_q.get(block=True) if record == 'TER': thread_logger.info('received {}'.format(record)) break LOGGER.handle(record) qsize = logging_q.qsize() if qsize > 10: thread_logger.warning(f"logging_q has size {qsize}") if logging_q.full(): thread_logger.warning(f"logging_q is full")
def __call__(self, pid, iq: Queue, data, *args): if self.pass_each_data_item: it = chain(*(chunk_iter(self.create_iterator(dataitem, *args), chunk_size=self.chunk_size, as_list=True) for dataitem in data)) else: it = chunk_iter(self.create_iterator(data, *args), chunk_size=self.chunk_size, as_list=True) hprint_message('initialized', f'{self.name}{pid}') while True: while not iq.full(): try: obj = next(it) except StopIteration: return iq.put(obj) hprint_pairs(('full queue for', f'{self.name}{pid}'), ('wait for', self._wait_time)) sleep(self._wait_time)