def _runner(cls, data=None, stop_event=None, target=None, buffer_tokens=None, result_tokens=None, dtype=None, shapes=None, log_queue=None, log_level=None, args=None, kwargs=None): """ Shared Memory Object runner """ # Fork inherits the queue handler, so skip registration with "fork" set_root_logger(log_level, queue=log_queue) logger.debug("FixedProducerDispatcher worker for %s started", str(target)) np_data = [cls._np_from_shared(d, shapes, dtype) for d in data] def get_free_slot(): while not stop_event.is_set(): i = buffer_tokens.get() if stop_event.is_set() or i is None or i == "EOF": break yield WorkerBuffer(i, np_data[i], stop_event, result_tokens) args = tuple((get_free_slot(), )) + tuple(args) try: target(*args, **kwargs) except Exception as ex: logger.exception(ex) stop_event.set() result_tokens.put(None) logger.debug("FixedProducerDispatcher worker for %s shutdown", str(target))
def _runner(cls, data=None, stop_event=None, target=None, buffer_tokens=None, result_tokens=None, dtype=None, shapes=None, log_queue=None, log_level=None, args=None, kwargs=None): """ Shared Memory Object runner """ # Fork inherits the queue handler, so skip registration with "fork" set_root_logger(log_level, queue=log_queue) logger.debug("FixedProducerDispatcher worker for %s started", str(target)) np_data = [cls._np_from_shared(d, shapes, dtype) for d in data] def get_free_slot(): while not stop_event.is_set(): i = buffer_tokens.get() if stop_event.is_set() or i is None or i == "EOF": break yield WorkerBuffer(i, np_data[i], stop_event, result_tokens) args = tuple((get_free_slot(),)) + tuple(args) try: target(*args, **kwargs) except Exception as ex: logger.exception(ex) stop_event.set() result_tokens.put(None) logger.debug("FixedProducerDispatcher worker for %s shutdown", str(target))