def _centrallogger_worker(logging_q: mpq.Queue):
"""Entry for a thread providing central logging from subprocesses via a shared queue"""
threadname = threading.currentThread().getName()
thread_logger = LOGGER.getChild(threadname)
thread_logger.info('{} started'.format(threadname))
while True: # only calling process terminates - keeping logging alive
record = logging_q.get(block=True)
if record == 'TER':
thread_logger.info('received {}'.format(record))
break
LOGGER.handle(record)
qsize = logging_q.qsize()
if qsize > 10:
thread_logger.warning(f"logging_q has size {qsize}")
if logging_q.full():
thread_logger.warning(f"logging_q is full")
def __call__(self, pid, iq: Queue, data, *args):
if self.pass_each_data_item:
it = chain(*(chunk_iter(self.create_iterator(dataitem, *args),
chunk_size=self.chunk_size,
as_list=True) for dataitem in data))
else:
it = chunk_iter(self.create_iterator(data, *args),
chunk_size=self.chunk_size,
as_list=True)
hprint_message('initialized', f'{self.name}{pid}')
while True:
while not iq.full():
try:
obj = next(it)
except StopIteration:
return
iq.put(obj)
hprint_pairs(('full queue for', f'{self.name}{pid}'),
('wait for', self._wait_time))
sleep(self._wait_time)
