def __init__(self):
self._reader, self._writer = Pipe(duplex=False)
self._rlock = Lock()
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = Lock()
self._make_methods()
def submit(self, func, *args, **kwargs):
# noinspection PyUnresolvedReferences
from multiprocess import Process, Pipe
from concurrent.futures import Future
fut, (c0, c1) = Future(), Pipe(False)
task = Process(target=self._target, args=(c1.send, func, args, kwargs))
self.tasks[fut] = task
task.start()
return self._set_future(fut, c0.recv())
def __init__(self, num_cores):
self.num_cores = num_cores
self.names = set()
self.global_queue = Queue()
self.pool = []
self.channels = []
self.__terminated = False
atexit.register(self.__del__)
queues = [Queue() for _ in range(num_cores)]
for i in range(num_cores):
pipe_master, pipe_slave = Pipe()
state = Value('i', 0)
process = MRServer(i, queues, pipe_slave, state, self.global_queue)
self.channels.append(Channel(queues[i], pipe_master, state))
self.pool.append(process)
process.start()
def __init__(self, maxsize=0):
if maxsize <= 0:
maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
self._maxsize = maxsize
self._reader, self._writer = Pipe(duplex=False)
self._rlock = Lock()
self._opid = os.getpid()
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = Lock()
self._sem = BoundedSemaphore(maxsize)
self._after_fork()
if sys.platform != 'win32':
register_after_fork(self, Queue._after_fork)
def __init__(self,
difficulty=1,
pool_size=4,
queue_size=16
): # NOTE: `queue_size` must be larger than 2x `pool_size`!!!
self._args_queue = Queue(queue_size)
# pipes for data dispatching
self._args_pipes = [Pipe() for _ in range(pool_size)]
self._return_queue = Queue(queue_size)
self._envs = [
self._factory_env(difficulty=difficulty) for _ in range(pool_size)
]
self._service_process = Process(target=self._service_job,
args=(self._envs, self._args_pipes,
self._args_queue,
self._return_queue))
self._service_process.start()
print('[LOG] environment service started!')
