def _start_batch_workers(self, queue: mp.JoinableQueue, ctx) -> List[BaseProcess]: Tqdm.set_lock(mp.RLock()) workers: List[BaseProcess] = [] for worker_id in range(self.num_workers): worker: BaseProcess = ctx.Process( target=self._batch_worker, args=(worker_id, queue, Tqdm.get_lock()), daemon=True ) worker.start() workers.append(worker) return workers
def _start_batch_workers( self, queue: mp.JoinableQueue, ctx) -> Tuple[List[BaseProcess], List[Connection]]: Tqdm.set_lock(mp.RLock()) workers: List[BaseProcess] = [] txs: List[Connection] = [] for worker_id in range(self.num_workers): rx, tx = ctx.Pipe(duplex=False) worker: BaseProcess = ctx.Process(target=self._batch_worker, args=(worker_id, queue, Tqdm.get_lock(), rx), daemon=True) worker.start() workers.append(worker) txs.append(tx) return workers, txs