def finalize(self):
if len(self._workers) == 0:
return
for rank in range(self._nr_workers):
self._task_queue.put(None)
map_exec_method('join', self._workers)
def main():
pull, pushs = make_gather_pair('jaincle-test', nr_workers=4, mode='ipc')
push_procs = [
mp.Process(target=mainloop_push, args=(i, p))
for i, p in enumerate(pushs)
]
map_exec_method('start', push_procs)
mainloop_pull(pull)
def main():
push, pulls = make_broadcast_pair('jaincle-test', nr_workers=1, mode='ipc')
pull_procs = [
mp.Process(target=mainloop_pull, args=(i, p))
for i, p in enumerate(pulls)
]
map_exec_method('start', pull_procs)
mainloop_push(push)
def start(self):
assert not self.__started
self._task_queue = self.Queue(maxsize=self._nr_workers * 8)
self._result_queue = self.Queue(maxsize=self._nr_workers * 8)
self._worker_pool = [
self.Process(target=self._worker, args=(i, ), daemon=True)
for i in range(self._nr_workers)
]
map_exec_method('start', self._worker_pool)
self._task_dispatcher_thread = threading.Thread(
target=self._task_dispatcher, daemon=True)
self._task_dispatcher_queue = queue.Queue(maxsize=1)
self._task_dispatcher_result = queue.Queue(maxsize=1)
self._task_dispatcher_thread.start()
self.__started = True
def terminate(self):
self._task_dispatcher_queue.put(None)
self._task_dispatcher_thread.join()
map_exec_method('join', self._worker_pool)