def listen(self) -> None:
"""
Listen for messages from the broadcaster.
"""
while self.connection:
for ready_connection in connection.wait([self.connection]):
message = ready_connection.recv()
def map(self, func, iterable, chunksize=None):
args = iter(iterable)
existing_processes = {}
arg = get_arg(args, chunksize)
while len(existing_processes) < self.processes and arg:
add_process(existing_processes, loop_function, (func, arg))
arg = get_arg(args, chunksize)
while existing_processes:
failed = set()
for s in wait(existing_processes):
if existing_processes[s].exitcode is not None:
if s in failed:
log.error('%s: Process sentinel is ready and exitcode is now available as %s' %
(s, existing_processes[s].exitcode))
if existing_processes[s].exitcode != 0:
raise ChildProcessError('Exit code %s' % existing_processes[s].exitcode)
del existing_processes[s]
if arg:
add_process(existing_processes, loop_function, (func, arg))
arg = get_arg(args, chunksize)
else:
if s not in failed:
log.error('%s: Process sentinel is ready but exitcode is %s' %
(s, existing_processes[s].exitcode))
failed.add(s)
def wait(self, timeout=None):
if sys.version_info < (3, 3):
import time
if timeout is None:
return self.poll(0)
deadline = time.time() + timeout
delay = 0.0005
while 1:
res = self.poll()
if res is not None:
break
remaining = deadline - time.time()
if remaining <= 0:
break
delay = min(delay * 2, remaining, 0.05)
time.sleep(delay)
return res
if self.returncode is None:
if timeout is not None:
from multiprocessing.connection import wait
if not wait([self.sentinel], timeout):
return None
# This shouldn't block if wait() returned successfully.
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
return self.returncode
def parent(funcs: list, collection) -> tuple:
"""create workers and colect result via pipes
:return list of results from chields
"""
# pipe is a tuple (parent , chield)
update_time = current_datetime_tz()
pipes = [multiprocessing.Pipe(duplex=False) for _ in funcs]
processes_news = [multiprocessing.Process(target=workerANDconnector, args=(params[0],),
kwargs={'connector': params[1][1], 'collection': collection,
'update_time': update_time})
for params in zip(funcs, pipes)]
for process, pipe in zip(processes_news, pipes):
process.start()
pipe[1].close()
readers = [r for r, w in pipes]
result = []
while readers:
for r in wait(readers):
try:
result_r = r.recv()
except EOFError:
readers.remove(r)
else:
result.append(result_r)
for process in processes_news:
process.join()
return reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]), result)
def get_their_work(self, pipe_package):
pipe_package.setChunkedsObjects(chunk(pipe_package.get_objects(), self.nb_process))
if not self.processs:
self._start(pipe_package)
else:
for i in range(self.nb_process):
#print('send things')
pipe_package.setCurrentProcessId(i)
self.writers_pipes[i].send(pipe_package)
things_done_collection = []
reader_useds = []
while self.readers_pipes:
for r in wait(self.readers_pipes):
try:
things_dones = r.recv()
except EOFError:
reader_useds.append(r)
self.readers_pipes.remove(r)
else:
reader_useds.append(r)
self.readers_pipes.remove(r)
for thing_done in things_dones:
things_done_collection.append(thing_done)
self.readers_pipes = reader_useds
return things_done_collection
def run(self): try: while self.keep_looping: for conn in wait(self.connections): try: msg = conn.recv() except EOFError: self.connections.remove(conn) except ConnectionResetError: pass else: if conn is self.gui_pipe: c="GUI" elif conn is self.server_conn: c="Server" else: c=conn self.handle_rpc(c,msg) if not self.keep_looping: break #if not all([p.is_alive() for p in self.procs]): # self.keep_looping=False finally: self.close()
def wait_childs(self):
""""""
for r in wait(self.readers):
try:
msg = r.recv()
except EOFError:
self.readers.remove(r)
else:
if len(msg)==3 and msg[self.K_TYPE] in self.TYPE_MSG :
cpid = self.getcpid(msg[self.K_ID])
if msg[self.K_TYPE] == self.MSG_INIT :
if hasattr(self.onstart_bind, '__call__'):
self.onstart_bind(msg[self.K_ID], cpid, msg[self.K_DATA])
elif msg[self.K_TYPE] == self.MSG_PRINT :
if Sys.g.DEBUG :
if not Sys.g.GUI :
for item in msg[self.K_DATA] :
Sys.echo(item[0], Sys.clzdic[item[1]], False, True)
Sys.dprint('')
#~ else :
Sys.wlog(msg[self.K_DATA])
elif msg[self.K_TYPE] == self.MSG_DATA :
if hasattr(self.onrun_bind, '__call__'):
self.onrun_bind(msg[self.K_ID], cpid, msg[self.K_DATA])
elif msg[self.K_TYPE] == self.MSG_END :
if hasattr(self.onend_bind, '__call__'):
self.onend_bind(msg[self.K_ID], cpid, msg[self.K_DATA])
def process_poll(self,right_conn):
clients_connected = self.clients_connected = []
clients_logged = self.clients_logged = []
while True:
wait_lst = clients_connected + clients_logged + [right_conn]
for c in wait(wait_lst):
if c is right_conn:
conn = c.recv()
client = Client(conn,self.create_id())
clients_connected.append(client)
print("new client:",clients_connected[-1])
#broadcast change
elif c in clients_connected:
try:
msg = c.conn.recv()
except (EOFError,ConnectionResetError):
clients_connected.remove(c)
print("[disconnected]",c)
else:
self.handle_rpc(c,msg)
else:
try:
msg = c.conn.recv()
except (EOFError,ConnectionResetError):
self.logout(c)
else:
self.handle_rpc(c,msg)
def collect(self):
while True:
inbound = list(self._publishers.values())
for p in connection.wait(inbound):
try:
msg = p.recv_bytes()
self._holding.put(msg)
except EOFError:
inbound.remove(p)
def wait(self, timeout=None):
if self.returncode is None:
if timeout is not None:
from multiprocessing.connection import wait
if not wait([self.sentinel], timeout):
return None
# This shouldn't block if wait() returned successfully.
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
return self.returncode
def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, result_queue):
executor = None
def shutting_down():
return _shutdown or (executor is None or executor._shutdown_thread)
def shutdown_worker():
nb_children_alive = sum(p.is_alive() for p in processes.values())
for i in range(0, nb_children_alive):
call_queue.put_nowait(None)
call_queue.close()
for p in processes.values():
p.join()
reader = result_queue._reader
while True:
_add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue)
sentinels = [p.sentinel for p in processes.values()]
ready = wait([reader] + sentinels)
if reader in ready:
result_item = reader.recv()
else:
executor = executor_reference()
if executor is not None:
executor._broken = True
executor._shutdown_thread = True
executor = None
for (work_id, work_item) in pending_work_items.items():
work_item.future.set_exception(BrokenProcessPool('A process in the process pool was terminated abruptly while the future was running or pending.'))
pending_work_items.clear()
for p in processes.values():
p.terminate()
shutdown_worker()
return
if isinstance(result_item, int):
p = processes.pop(result_item)
p.join()
if not processes:
shutdown_worker()
return
elif result_item is not None:
work_item = pending_work_items.pop(result_item.work_id, None)
if work_item is not None:
if result_item.exception:
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
executor = executor_reference()
if shutting_down():
try:
if not pending_work_items:
shutdown_worker()
return
except Full:
pass
executor = None
def poll(self, flag=os.WNOHANG):
if self.returncode is None:
from multiprocessing.connection import wait
timeout = 0 if flag == os.WNOHANG else None
if not wait([self.sentinel], timeout):
return None
try:
self.returncode = forkserver.read_unsigned(self.sentinel)
except (OSError, EOFError):
# The process ended abnormally perhaps because of a signal
self.returncode = 255
return self.returncode
def run(self):
with connection.Listener(address=SessionServer._address,
family='AF_UNIX',
authkey=self.__key.bytes) as listener:
with listener.accept() as client:
print("connection accepted")
while True:
for ready_client in connection.wait([client]):
try:
cmd_line = ready_client.recv()
print(cmd_line)
except EOFError:
print("end of file?")
def run_keeped_process(target, main_write_pipe, process_read_pipe, new_things):
while new_things != 'stop':
things_dones = target(new_things)
main_write_pipe.send(things_dones)
readers = [process_read_pipe]
while readers:
for r in wait(readers):
try:
new_things = r.recv()
except EOFError:
pass
finally:
readers.remove(r)
def get_their_work(self, things_to_do):
chunked_things = chunk(things_to_do, self.nb_process)
self._start(chunked_things)
things_done_collection = []
while self.readers_pipes:
for r in wait(self.readers_pipes):
try:
things_dones = r.recv()
except EOFError:
self.readers_pipes.remove(r)
else:
self.readers_pipes.remove(r)
things_done_collection.append(things_dones)
return things_done_collection
def poll(self, flag=os.WNOHANG):
if self.returncode is None:
from multiprocessing.connection import wait
timeout = 0 if flag == os.WNOHANG else None
if not wait([self.sentinel], timeout):
return None
try:
self.returncode = forkserver.read_signed(self.sentinel)
except (OSError, EOFError):
# This should not happen usually, but perhaps the forkserver
# process itself got killed
self.returncode = 255
return self.returncode
def run_scan_stream(self):
while True:
comment_process = mp.Process(
target=self.scan_content_stream,
args=("comment",),
)
comment_process.start()
submission_process = mp.Process(
target=self.scan_content_stream,
args=("submission",),
)
submission_process.start()
mpc.wait([
comment_process.sentinel,
submission_process.sentinel
])
print("Something went wrong - restarting processes.")
submission_process.terminate()
comment_process.terminate()
print("Sleeping for 15 seconds.")
time.sleep(15)
def event_loop(incoming_channels):
while(True):
num_processes = len(incoming_channels)
if num_processes == 0:
break
channels_with_data = wait(incoming_channels)
if channels_with_data is None or len(channels_with_data) == 0:
continue
for incoming in channels_with_data:
try:
msg = incoming.recv()
except EOFError:
incoming_messages.remove(incoming)
else:
process(msg)
def run_keepedalive_process(main_write_pipe, process_read_pipe, obj):
"""
Procees who don't finish while job to do
"""
while obj != 'stop':
oneshot_in_process(obj)
# Send to main process "I've done my job"
main_write_pipe.send('job is done')
# Wait for new job to do (this part can be simplified no ?)
readers = [process_read_pipe]
while readers:
for r in wait(readers):
try:
obj = r.recv()
except EOFError:
pass
finally:
readers.remove(r)
def _predict_batch_worker(self):
"""
Thread worker which listens on each pipe in self.pipes for an observation, and then outputs
the predictions for the policy and value networks when the observations come in. Repeats.
"""
while True:
ready = connection.wait(self.pipes,timeout=0.001)
if not ready:
continue
data, result_pipes = [], []
for pipe in ready:
while pipe.poll():
data.append(pipe.recv())
result_pipes.append(pipe)
data = np.asarray(data, dtype=np.float32)
policy_ary, value_ary = self.agent_model.model.predict_on_batch(data)
for pipe, p, v in zip(result_pipes, policy_ary, value_ary):
pipe.send((p, float(v)))
def computer(bugs, nb_process=4):
bugs_iterables = chunk(bugs, nb_process)
readers = []
for i in range(nb_process):
r, w = Pipe(duplex=False)
readers.append(r)
p = Process(target=run_worker, args=(w, bugs_iterables[i]))
p.start()
w.close()
while readers:
for r in wait(readers):
try:
bugs_sendeds = r.recv()
except EOFError:
readers.remove(r)
else:
for bug in bugs_sendeds:
pass#print(bug.id, bug.willmove)
def run_keeped_process(target, main_write_pipe, process_read_pipe, things):
keep_alive = True
while keep_alive:
things_dones = target(things)
main_write_pipe.send(things_dones)
readers = [process_read_pipe]
readers_used = []
while readers:
for r in wait(readers):
try:
things = r.recv()
#print('p: things received')
except EOFError:
pass
finally:
readers.remove(r)
#print('p: continue')
if things == 'stop':
keep_alive = False
def nqueens(n, nb_process=2):
columns = range(n)
boards = list(permutations(columns))
readers = []
for i in range(nb_process):
r, w = Pipe(duplex=False)
readers.append(r)
p = Process(target=check_nqueens, args=(w, n, columns, boards[i::nb_process]))
p.start()
w.close()
while readers:
for r in wait(readers):
try:
msg = r.recv()
except EOFError:
readers.remove(r)
else:
print_board(n, msg)
def wait( # type: ignore
workers: List["SubprocEnvWorker"],
wait_num: int,
timeout: Optional[float] = None,
) -> List["SubprocEnvWorker"]:
remain_conns = conns = [x.parent_remote for x in workers]
ready_conns: List[connection.Connection] = []
remain_time, t1 = timeout, time.time()
while len(remain_conns) > 0 and len(ready_conns) < wait_num:
if timeout:
remain_time = timeout - (time.time() - t1)
if remain_time <= 0:
break
# connection.wait hangs if the list is empty
new_ready_conns = connection.wait(remain_conns,
timeout=remain_time)
ready_conns.extend(new_ready_conns) # type: ignore
remain_conns = [
conn for conn in remain_conns if conn not in ready_conns
]
return [workers[conns.index(con)] for con in ready_conns]
def __listen(self):
print("[IPCServer] [INFO] Now listening to client messages")
# connection verification:
# so normally the client would send "ping" to the server, but in this case we want to verify if the clients
# have two way connectivity (since we boss them around), so we will send the ping request
# send the message multiple times to make sure it gets through
for i in range(6):
self.transmit({"message": "ping", "my_id": self.agent_id})
print(f"[IPCServer] [DEBUG] Transmitting ping #{i}")
while self.clients:
for conn in wait(self.clients):
try:
msg = conn.recv()
print(f"[IPCServer] [DEBUG] Message from {conn}: {msg}")
self.__handle_verification(msg)
self.event_handler(msg)
except EOFError:
print(f"[IPCServer] [WARN] A client caused an EOFError! Disconnecting it", file=sys.stderr)
self.clients.remove(conn)
def __work(self):
""" This method does the actual work of taking batches of requests and
send the responses back to the clients.
"""
while self.do_run:
try:
ready = wait(self.connections, timeout=0.0001)
except OSError:
# If there is any connection closed (our side), we delete it.
with self.conn_lock:
self.__delete_closed_conns()
if not ready:
continue
data, result_conns = [], []
for i, conn in enumerate(ready):
if conn in self.to_ignore:
continue
while conn.poll():
try:
g = conn.recv()
except EOFError:
# In case of the cliend closed the other end of the
# connection. We close our end and ignore it.
conn.close()
self.to_ignore.add(conn)
break
data.append(netencoder.get_game_state(g))
result_conns.append(conn)
if len(data) > 0:
data = np.asarray(data, dtype=np.float16)
policies, values = self.model.predict(data)
for conn, p, v in zip(result_conns, policies, values):
conn.send((p, float(v)))
def predict_batch_worker(self):
if self.config.internet.distributed and self.need_reload:
self.try_reload_model_from_internet()
last_model_check_time = time()
while not self.done:
if last_model_check_time + 600 < time() and self.need_reload:
self.try_reload_model()
last_model_check_time = time()
ready = connection.wait(self.pipes, timeout=0.001)
if not ready:
continue
data, result_pipes, data_len = [], [], []
for pipe in ready:
while pipe.poll():
try:
tmp = pipe.recv()
except EOFError as e:
logger.error(f"EOF error: {e}")
pipe.close()
else:
data.extend(tmp)
data_len.append(len(tmp))
result_pipes.append(pipe)
if not data:
continue
data = np.asarray(data, dtype=np.float32)
with self.agent_model.graph.as_default():
policy_ary, value_ary = self.agent_model.model.predict_on_batch(
data)
buf = []
k, i = 0, 0
for p, v in zip(policy_ary, value_ary):
buf.append((p, float(v)))
k += 1
if k >= data_len[i]:
result_pipes[i].send(buf)
buf = []
k = 0
i += 1
def __call__(self):
def models():
for i, model in enumerate(self.models_):
for parameters in model():
yield i, parameters
self.generator_ = models()
self.left_ = 0
self.results_ = []
while self.generator_ or self.left_ > 0:
for source in wait(
[self.listener_._listener._socket, *self.clients_]):
if source == self.listener_._listener._socket:
source = self.listener_.accept()
self.clients_.append(source)
else:
self.results_.append(source.recv())
self.left_ -= 1
if self.generator_:
try:
data = next(self.generator_)
self.left_ += 1
model, parameters = data
print(f'Running: ' +
', '.join(f'{key}={repr(value)}'
for key, value in self.models_[model].
values(parameters).items()))
source.send(data)
except StopIteration:
self.generator_ = None
if not self.generator_:
source.send(None)
source.close()
self.clients_.remove(source)
self.results_ = sorted(self.results_,
key=lambda v: v[-1],
reverse=True)
def start_in_keepedalive_processes(obj, nb_process):
"""
Start nb_process and keep them alive. Send job to them multiple times, then close thems.
"""
processes = []
readers_pipes = []
writers_pipes = []
for i in range(nb_process):
# Start process with Pipes for communicate
local_read_pipe, local_write_pipe = Pipe(duplex=False)
process_read_pipe, process_write_pipe = Pipe(duplex=False)
readers_pipes.append(local_read_pipe)
writers_pipes.append(process_write_pipe)
p = Process(target=run_keepedalive_process, args=(local_write_pipe, process_read_pipe, obj))
p.start()
processes.append(p)
# Send to process some job to do
for job in range(3):
print('send new job to processes:')
for process_number in range(nb_process):
# Send data to process
writers_pipes[process_number].send(obj)
reader_useds = []
# Wait response from processes
while readers_pipes:
for r in wait(readers_pipes):
try:
r.recv()
except EOFError:
pass
finally:
reader_useds.append(r)
readers_pipes.remove(r)
readers_pipes = reader_useds
# Kill processes
for writer_pipe in writers_pipes:
writer_pipe.send('stop')
def predict_batch_worker(self):
"""
把各个有用管道收集来的数据集中eval，再在原来的管道发送出去
:return:
"""
while not self.done:
ready = connection.wait(self.pipes,
timeout=0.001) # 等待有通信管道可用，返回可用的管道
if not ready:
continue
data, result_pipes, data_len = [], [], []
for pipe in ready:
while pipe.poll(): # 不停地返回false，直到连接到有用数据，就返回true
try:
tmp = pipe.recv(
) # 如果没有消息可接收，recv方法会一直阻塞。如果连接的另外一端已经关闭，那么recv方法会抛出EOFError。
except EOFError as e:
logger.error(f"EOF error: {e}")
pipe.close() # 另一端关闭，这端就关闭了
else:
data.extend(tmp)
data_len.append(len(tmp))
result_pipes.append(pipe)
if not data:
continue
data = np.asarray(data, dtype=np.float32)
with self.agent_model.graph.as_default():
policy, value = self.agent_model.eval(data)
buf = []
k, i = 0, 0
for p, v in zip(policy, value):
buf.append((p, float(v)))
k += 1
if k >= data_len[i]:
result_pipes[i].send(buf)
buf = []
k = 0
i += 1
def get_their_work(self, things_to_do):
chunked_things = chunk(things_to_do, self.nb_process)
if not self.processs:
self._start(chunked_things)
else:
for i in range(self.nb_process):
#print('send things')
self.writers_pipes[i].send(chunked_things[i])
things_done_collection = []
reader_useds = []
while self.readers_pipes:
for r in wait(self.readers_pipes):
try:
things_dones = r.recv()
except EOFError:
reader_useds.append(r)
self.readers_pipes.remove(r)
else:
reader_useds.append(r)
self.readers_pipes.remove(r)
things_done_collection.append(things_dones)
self.readers_pipes = reader_useds
return things_done_collection
def predict_batch_worker(self):
last_model_check_time = time()
while True:
if last_model_check_time + 600 < time():
self.try_reload_model()
last_model_check_time = time()
ready = connection.wait(self.pipes, timeout=0.001)
if not ready:
continue
data, result_pipes = [], []
for pipe in ready:
while pipe.poll():
try:
data.append(pipe.recv())
result_pipes.append(pipe)
except EOFError as e:
pipe.close()
data = np.asarray(data, dtype=np.float32)
with self.agent_model.graph.as_default():
policy_ary, value_ary = self.agent_model.model.predict_on_batch(
data)
for pipe, p, v in zip(result_pipes, policy_ary, value_ary):
pipe.send((p, float(v)))
def loop(self): # NOQA: C901
self.exit_msg = ''
check_for_events = True
while check_for_events:
event_handlers = list(self.event_handlers)
if self.receiver:
event_handlers.append(self.receiver)
if len(event_handlers) == 0:
break
input_conn_list = wait(event_handlers)
for conn in input_conn_list:
if conn == self.receiver:
item = conn.recv()
if item is None:
self.receiver = None
else:
self.on_input_func(item)
continue
event = conn.recv()
event_label = self.event_manager_to_plugin(conn)
if __debug__:
self.logger.debug("GOT %s FROM %s", event, event_label)
if event == "started":
self.on_started(event_label)
if event is None: # Got "None" event - plugin was terminated
exit_msg = conn.recv()
self.on_exit(event_label, exit_msg)
self.remove_event_source(conn)
if event == "error":
return 1, "Error"
print(self.exit_msg)
return 0, "Finished"
def send_data(conversion):
"""
Sends the data to the loader processes.
:param conversion: Conversion
:return: None
"""
if len(conversion.data_pool) == 0:
return
params_list = [[conversion.config, meta] for meta in conversion.data_pool]
reader_connections = []
number_of_workers = min(
conversion.max_each_db_connection_pool_size,
len(conversion.data_pool),
cpu_count()
)
with ProcessPoolExecutor(max_workers=number_of_workers) as executor:
while len(params_list) != 0:
reader_connection, writer_connection = Pipe(duplex=False)
reader_connections.append(reader_connection)
params = params_list.pop()
params.append(writer_connection)
executor.submit(DataLoader._load, *params)
while reader_connections:
for reader_connection in connection.wait(object_list=reader_connections):
just_populated_table_name = ''
try:
just_populated_table_name = reader_connection.recv()
reader_connection.close()
finally:
reader_connections.remove(reader_connection)
ConstraintsProcessor.process_constraints_per_table(conversion, just_populated_table_name)
MigrationStateManager.set(conversion, 'per_table_constraints_loaded')
def process(self, pipe_to_bus, pipe_to_ui):
print("process<{}> run".format(self.__class__.__name__))
#self.bus_pipe = bus_pipe
self.ui_pipe = pipe_to_ui
close_handle = False
while not close_handle: #FIXME: there is somd end command
all_pipes = self.get_bus_devices_pipe_line()
all_pipes.append(pipe_to_bus)
all_pipes.append(pipe_to_ui)
for r in wait(all_pipes[:]):
#try:
msg = r.recv()
#print("Process<{}> get: {}".format(self.__class__.__name__, msg))
location = msg.loc()
if location == Message.BUS or location == Message.DEVICE:
self.handle_bus_message(msg)
elif location == Message.UI:
self.handle_ui_command(msg)
else:
pass
"""
except EOFError:
#readers.remove(r)
ServerError("Pipe {} report EOFError", r) #FIXME: should process the ERROR?
if r == server_to_bus_pipe:
close_handle = True
break
else:
print("Unexpected error: {}".format(self.__class__.__name__))
"""
self.process_bus_command()
self.report_ui_message(pipe_to_ui)
pipe_to_bus.close()
pipe_to_ui.close()
def parent(funcs: list, collection) -> tuple:
"""create workers and colect result via pipes
:return list of results from chields
"""
# pipe is a tuple (parent , chield)
update_time = datetime.utcnow()
pipes = [multiprocessing.Pipe(duplex=False) for _ in funcs]
processes_news = [
multiprocessing.Process(target=workerANDconnector,
args=(params[0], ),
kwargs={
'connector': params[1][1],
'collection': collection,
'update_time': update_time
}) for params in zip(funcs, pipes)
]
logger.debug('created process num= {}'.format(len(processes_news)))
for process, pipe in zip(processes_news, pipes):
process.start()
pipe[1].close()
readers = [r for r, w in pipes]
result = []
while readers:
for r in wait(readers):
try:
result_r = r.recv()
except EOFError:
readers.remove(r)
else:
result.append(result_r)
for process in processes_news:
process.join()
return reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]), result)
def process(self):
super(Hid_Device, self).process()
# try:
self.open_dev()
all_pipes = [self.logic_pipe()]
close_handle = False
while not close_handle:
#try:
for r in wait(all_pipes, timeout=self.poll_interval()):
if r:
try:
msg = r.recv()
except EOFError:
print("Process EOF: {}".format(
self.__class__.__name__))
close_handle = True
self.close_dev()
# self.pipe_hid_event.close()
# self.pipe_hid_recv.close()
return
#print("Process<{}> get: {}".format(self.__class__.__name__, msg))
self.handle_bus_command(msg)
# location = msg.loc()
# if location == PhyMessage.HID_DEVICE:
# #self.handle_phy_message(msg)
# elif location == PhyMessage.SERVER:
# self.handle_bus_command(msg)
# else:
# pass
self.send_command()
self.handle_timeout_command()
def loadEditions(editions):
processes = []
outPipes = []
cores = 4
logger.info('Processing {} editions'.format(len(editions)))
chunkSize = math.ceil(len(editions) / cores)
for i in range(cores):
start = i * chunkSize
end = start + chunkSize
logger.info('Processing chunk {} to {}'.format(start, end))
pConn, cConn = Pipe(duplex=False)
proc = Process(target=parseChunk, args=(editions[start:end], cConn))
processes.append(proc)
outPipes.append(pConn)
proc.start()
cConn.close()
outEds = []
while outPipes:
for p in wait(outPipes):
try:
ed = p.recv()
if ed == 'DONE':
outPipes.remove(p)
else:
outEds.append(ed)
except EOFError:
outPipes.remove(p)
for proc in processes:
proc.join()
return outEds
def _predict_batch_worker(self):
"""
Thread worker which listens on each pipe in self.pipes for an observation, and then outputs
the predictions for the policy and value networks when the observations come in. Repeats.
## CITE
"""
while True:
ready = connection.wait(self.return_policy_value,timeout=0.001)
if not ready:
continue
data, result_pipes = [], []
for pipe in ready:
while pipe.poll():
data.append(pipe.recv())
result_pipes.append(pipe)
data = np.asarray(data, dtype=np.float32)
# print (data.shape)
policy_array, value_array = self.model.predict_on_batch(data)
# print (policy_array, value_array)
for pipe, policy, value in zip(result_pipes, policy_array, value_array):
pipe.send((policy, float(value)))
readers = []
for i in range(4):
r, w = Pipe(duplex=False)
readers.append(r)
p = Process(target=foo, args=(w,))
p.start()
# We close the writable end of the pipe now to be sure that
# p is the only process which owns a handle for it. This
# ensures that when p closes its handle for the writable end,
# wait() will promptly report the readable end as being ready.
w.close()
while readers:
for r in wait(readers):
try:
msg = r.recv()
except EOFError:
readers.remove(r)
else:
print(msg)
# from multiprocessing import Process, Pipe
# """
# 返回的两个连接对象 Pipe() 表示管道的两端。每个连接对象都有 send() 和 recv() 方法（相互之间的）。
# 如果两个进程（或线程）同时尝试读取或写入管道的 同一 端，则管道中的数据可能会损坏。当然，同时使用管道的不同端的进程不存在损坏的风险。
# """
#
# def f(conn):
# conn.send([42, None, 'hello'])
def _queue_management_worker(executor_reference, processes, pending_work_items,
work_ids_queue, call_queue, result_queue,
thread_wakeup):
"""Manages the communication between this process and the worker processes.
This function is run in a local thread.
Args:
executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
this thread. Used to determine if the ProcessPoolExecutor has been
garbage collected and that this function can exit.
process: A list of the ctx.Process instances used as
workers.
pending_work_items: A dict mapping work ids to _WorkItems e.g.
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
call_queue: A ctx.Queue that will be filled with _CallItems
derived from _WorkItems for processing by the process workers.
result_queue: A ctx.SimpleQueue of _ResultItems generated by the
process workers.
thread_wakeup: A _ThreadWakeup to allow waking up the
queue_manager_thread from the main Thread and avoid deadlocks
caused by permanently locked queues.
"""
executor = None
def shutting_down():
return (_global_shutdown or executor is None
or executor._shutdown_thread)
def shutdown_worker():
# This is an upper bound on the number of children alive.
n_children_alive = sum(p.is_alive() for p in processes.values())
n_children_to_stop = n_children_alive
n_sentinels_sent = 0
# Send the right number of sentinels, to make sure all children are
# properly terminated.
while n_sentinels_sent < n_children_to_stop and n_children_alive > 0:
for i in range(n_children_to_stop - n_sentinels_sent):
try:
call_queue.put_nowait(None)
n_sentinels_sent += 1
except Full:
break
n_children_alive = sum(p.is_alive() for p in processes.values())
# Release the queue's resources as soon as possible.
call_queue.close()
# If .join() is not called on the created processes then
# some ctx.Queue methods may deadlock on Mac OS X.
for p in processes.values():
p.join()
result_reader = result_queue._reader
wakeup_reader = thread_wakeup._reader
readers = [result_reader, wakeup_reader]
while True:
_add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue)
# Wait for a result to be ready in the result_queue while checking
# that all worker processes are still running, or for a wake up
# signal send. The wake up signals come either from new tasks being
# submitted, from the executor being shutdown/gc-ed, or from the
# shutdown of the python interpreter.
worker_sentinels = [p.sentinel for p in processes.values()]
ready = wait(readers + worker_sentinels)
cause = None
is_broken = True
if result_reader in ready:
try:
result_item = result_reader.recv()
is_broken = False
except BaseException as e:
cause = traceback.format_exception(type(e), e, e.__traceback__)
elif wakeup_reader in ready:
is_broken = False
result_item = None
thread_wakeup.clear()
if is_broken:
# Mark the process pool broken so that submits fail right now.
executor = executor_reference()
if executor is not None:
executor._broken = ('A child process terminated '
'abruptly, the process pool is not '
'usable anymore')
executor._shutdown_thread = True
executor = None
bpe = BrokenProcessPool("A process in the process pool was "
"terminated abruptly while the future was "
"running or pending.")
if cause is not None:
bpe.__cause__ = _RemoteTraceback(f"\n'''\n{''.join(cause)}'''")
# All futures in flight must be marked failed
for work_id, work_item in pending_work_items.items():
work_item.future.set_exception(bpe)
# Delete references to object. See issue16284
del work_item
pending_work_items.clear()
# Terminate remaining workers forcibly: the queues or their
# locks may be in a dirty state and block forever.
for p in processes.values():
p.terminate()
shutdown_worker()
return
if isinstance(result_item, int):
# Clean shutdown of a worker using its PID
# (avoids marking the executor broken)
assert shutting_down()
p = processes.pop(result_item)
p.join()
if not processes:
shutdown_worker()
return
elif result_item is not None:
work_item = pending_work_items.pop(result_item.work_id, None)
# work_item can be None if another process terminated (see above)
if work_item is not None:
if result_item.exception:
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
# Delete references to object. See issue16284
del work_item
# Delete reference to result_item
del result_item
# Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
# - The interpreter is shutting down OR
# - The executor that owns this worker has been collected OR
# - The executor that owns this worker has been shutdown.
if shutting_down():
try:
# Flag the executor as shutting down as early as possible if it
# is not gc-ed yet.
if executor is not None:
executor._shutdown_thread = True
# Since no new work items can be added, it is safe to shutdown
# this thread if there are no pending work items.
if not pending_work_items:
shutdown_worker()
return
except Full:
# This is not a problem: we will eventually be woken up (in
# result_queue.get()) and be able to send a sentinel again.
pass
executor = None
def is_alive(self):
from multiprocessing.connection import wait
return not wait([self._sentinel], timeout=0)
def join(self, timeout=None):
'''
Wait until parent process terminates
'''
from multiprocessing.connection import wait
wait([self._sentinel], timeout=timeout)
def read(self):
try:
readers = wait([w.outputReader for w in self.writers])
return [r.recv() for r in readers]
except (EOFError, OSError):
return None
def _queue_management_worker(executor_reference,
processes,
pending_work_items,
work_ids_queue,
call_queue,
result_queue,
thread_wakeup):
"""Manages the communication between this process and the worker processes.
This function is run in a local thread.
Args:
executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
this thread. Used to determine if the ProcessPoolExecutor has been
garbage collected and that this function can exit.
process: A list of the ctx.Process instances used as
workers.
pending_work_items: A dict mapping work ids to _WorkItems e.g.
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
call_queue: A ctx.Queue that will be filled with _CallItems
derived from _WorkItems for processing by the process workers.
result_queue: A ctx.SimpleQueue of _ResultItems generated by the
process workers.
thread_wakeup: A _ThreadWakeup to allow waking up the
queue_manager_thread from the main Thread and avoid deadlocks
caused by permanently locked queues.
"""
executor = None
def shutting_down():
return (_global_shutdown or executor is None
or executor._shutdown_thread)
def shutdown_worker():
# This is an upper bound on the number of children alive.
n_children_alive = sum(p.is_alive() for p in processes.values())
n_children_to_stop = n_children_alive
n_sentinels_sent = 0
# Send the right number of sentinels, to make sure all children are
# properly terminated.
while n_sentinels_sent < n_children_to_stop and n_children_alive > 0:
for i in range(n_children_to_stop - n_sentinels_sent):
try:
call_queue.put_nowait(None)
n_sentinels_sent += 1
except Full:
break
n_children_alive = sum(p.is_alive() for p in processes.values())
# Release the queue's resources as soon as possible.
call_queue.close()
# If .join() is not called on the created processes then
# some ctx.Queue methods may deadlock on Mac OS X.
for p in processes.values():
p.join()
result_reader = result_queue._reader
wakeup_reader = thread_wakeup._reader
readers = [result_reader, wakeup_reader]
while True:
_add_call_item_to_queue(pending_work_items,
work_ids_queue,
call_queue)
# Wait for a result to be ready in the result_queue while checking
# that all worker processes are still running, or for a wake up
# signal send. The wake up signals come either from new tasks being
# submitted, from the executor being shutdown/gc-ed, or from the
# shutdown of the python interpreter.
worker_sentinels = [p.sentinel for p in processes.values()]
ready = wait(readers + worker_sentinels)
cause = None
is_broken = True
if result_reader in ready:
try:
result_item = result_reader.recv()
is_broken = False
except BaseException as e:
cause = traceback.format_exception(type(e), e, e.__traceback__)
elif wakeup_reader in ready:
is_broken = False
result_item = None
thread_wakeup.clear()
if is_broken:
# Mark the process pool broken so that submits fail right now.
executor = executor_reference()
if executor is not None:
executor._broken = ('A child process terminated '
'abruptly, the process pool is not '
'usable anymore')
executor._shutdown_thread = True
executor = None
bpe = BrokenProcessPool("A process in the process pool was "
"terminated abruptly while the future was "
"running or pending.")
if cause is not None:
bpe.__cause__ = _RemoteTraceback(
f"\n'''\n{''.join(cause)}'''")
# All futures in flight must be marked failed
for work_id, work_item in pending_work_items.items():
work_item.future.set_exception(bpe)
# Delete references to object. See issue16284
del work_item
pending_work_items.clear()
# Terminate remaining workers forcibly: the queues or their
# locks may be in a dirty state and block forever.
for p in processes.values():
p.terminate()
shutdown_worker()
return
if isinstance(result_item, int):
# Clean shutdown of a worker using its PID
# (avoids marking the executor broken)
assert shutting_down()
p = processes.pop(result_item)
p.join()
if not processes:
shutdown_worker()
return
elif result_item is not None:
work_item = pending_work_items.pop(result_item.work_id, None)
# work_item can be None if another process terminated (see above)
if work_item is not None:
if result_item.exception:
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
# Delete references to object. See issue16284
del work_item
# Delete reference to result_item
del result_item
# Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
# - The interpreter is shutting down OR
# - The executor that owns this worker has been collected OR
# - The executor that owns this worker has been shutdown.
if shutting_down():
try:
# Flag the executor as shutting down as early as possible if it
# is not gc-ed yet.
if executor is not None:
executor._shutdown_thread = True
# Since no new work items can be added, it is safe to shutdown
# this thread if there are no pending work items.
if not pending_work_items:
shutdown_worker()
return
except Full:
# This is not a problem: we will eventually be woken up (in
# result_queue.get()) and be able to send a sentinel again.
pass
executor = None
def run(self) -> None:
"""
Loop over the legs of the continuous-time evolution of the event-chain Monte Carlo algorithm.
This method is called by run.py and resume.py. The loop should only be interrupted when a
base.exceptions.EndOfRun exception is raised, which is caught in the scripts.
"""
while True:
# Extract active global state
active_global_state = self._state_handler.extract_active_global_state(
)
# Fetch event handlers to activate
event_handlers_in_state_dictionary = self._activator.get_event_handlers_to_run(
active_global_state,
self._event_handler_with_shortest_event_time)
if self._logger_enabled_for_debug:
self._logger.debug(
"Event handlers that will be run with their in-state identifiers: {0}"
.format({
event_handler.__class__.__name__: in_state_identifier
for event_handler, in_state_identifier in
event_handlers_in_state_dictionary.items()
}))
# Fetch in-states
for event_handler, in_state_identifiers in event_handlers_in_state_dictionary.items(
):
if in_state_identifiers is not None:
event_handlers_in_state_dictionary[event_handler] = [
self._state_handler.extract_from_global_state(
identifier) for identifier in in_state_identifiers
]
# Send in-states
pipes = []
for event_handler, in_state in event_handlers_in_state_dictionary.items(
):
pipe = self._pipes[event_handler]
if self._event_handlers_state[pipe] == EventHandlerState.idle:
self._start_events[pipe].set()
else:
raise MediatorError("Event Process not ready!")
if event_handler.number_send_event_time_arguments:
assert in_state is not None
pipe.send(in_state)
self._event_handlers_state[
pipe] = EventHandlerState.event_time_started
pipes.append(pipe)
# Receive event times, and let them continue on out-state if there are idle os-processes.
pipes_time_received = collections.deque()
event_times_received = 0
while event_times_received < len(
event_handlers_in_state_dictionary):
for pipe in connection.wait(pipes):
if self._event_handlers_state[
pipe] == EventHandlerState.event_time_started:
self._event_handlers_state[
pipe] = EventHandlerState.suspended
returned = pipe.recv()
try:
event_time, self._out_state_arguments[
pipe] = returned
except TypeError:
event_time, self._out_state_arguments[
pipe] = returned, []
# If no arguments for out-state, one can calculate it in advance if there are resources
if not self._event_handlers[
pipe].number_send_out_state_arguments:
pipes_time_received.append(pipe)
event_times_received += 1
if (0 < len(event_handlers_in_state_dictionary) -
event_times_received < self._number_cores - 1
and len(pipes_time_received)):
next_pipe = pipes_time_received.popleft()
self._send_out_state_events[next_pipe].set()
if self._event_handlers[
next_pipe].number_send_out_state_arguments:
next_pipe.send(
self._out_state_arguments_methods[
self._event_handlers[next_pipe]]
(*self._out_state_arguments[next_pipe]))
self._event_handlers_state[
next_pipe] = EventHandlerState.out_state_started
identifier = self._event_storage.push_to_storage(
self._event_handlers[pipe])
self._scheduler.push_event(event_time, identifier)
if self._logger_enabled_for_debug:
self._logger.debug(
"Pushed candidate event time to the scheduler: {0} ({1})"
.format(
event_time, self._event_handlers[pipe].
__class__.__name__))
elif self._event_handlers_state[
pipe] == EventHandlerState.out_state_started:
self._event_handlers_state[
pipe] = EventHandlerState.idle
if len(pipes_time_received):
next_pipe = pipes_time_received.popleft()
self._send_out_state_events[next_pipe].set()
if self._event_handlers[
next_pipe].number_send_out_state_arguments:
next_pipe.send(
self._out_state_arguments_methods[
self._event_handlers[next_pipe]]
(*self._out_state_arguments[next_pipe]))
self._event_handlers_state[
next_pipe] = EventHandlerState.out_state_started
self._event_storage.append_to_stored_value(
self._event_handlers[pipe], pipe.recv())
else:
raise MediatorError(
"Event process with pipe {0} to mediator is already finished"
" and shouldn't receive anything anymore!".format(
pipe))
# Pop the earliest event handler from scheduler, and let it calculate out-state
identifier = self._scheduler.get_succeeding_event()
event = self._event_storage.get_event_handler(identifier)
self._event_handler_with_shortest_event_time = event[0]
if self._logger_enabled_for_debug:
self._logger.debug(
"Event handler which created the event with the shortest event time: {0}"
.format(self._event_handler_with_shortest_event_time.
__class__.__name__))
pipe_with_shortest_event_time = self._pipes[
self._event_handler_with_shortest_event_time]
if self._event_handlers_state[
pipe_with_shortest_event_time] == EventHandlerState.suspended:
self._send_out_state_events[pipe_with_shortest_event_time].set(
)
self._event_handlers_state[
pipe_with_shortest_event_time] = EventHandlerState.out_state_started
if self._event_handler_with_shortest_event_time.number_send_out_state_arguments:
pipe_with_shortest_event_time.send(
self._out_state_arguments_methods[
self._event_handler_with_shortest_event_time]
(*self.
_out_state_arguments[pipe_with_shortest_event_time]))
if self._event_handlers_state[
pipe_with_shortest_event_time] == EventHandlerState.out_state_started:
out_state = pipe_with_shortest_event_time.recv()
self._event_handlers_state[
pipe_with_shortest_event_time] = EventHandlerState.idle
self._event_storage.append_to_stored_value(
self._event_handler_with_shortest_event_time, out_state)
event = [
self._event_handler_with_shortest_event_time, out_state
]
# Request shortest time
self._event_handler_with_shortest_event_time, out_state = event
pipe_with_shortest_event_time = self._pipes[
self._event_handler_with_shortest_event_time]
assert self._event_handlers_state[
pipe_with_shortest_event_time] == EventHandlerState.idle
# Commit out-state
self._state_handler.insert_into_global_state(out_state)
# Trash
for event_handler in self._activator.get_trashable_events(
self._event_handler_with_shortest_event_time):
if self._logger_enabled_for_debug:
self._logger.debug(
"Event handler trashed in the scheduler: {0}".format(
event_handler.__class__.__name__))
event_identifier = self._event_storage.get_event_identifier(
event_handler)
self._scheduler.trash_event(event_identifier)
self._event_storage.delete_identifier(event_identifier,
event_handler)
pipe = self._pipes[event_handler]
if self._event_handlers_state[
pipe] == EventHandlerState.suspended:
self._event_handlers_state[pipe] = EventHandlerState.idle
elif self._event_handlers_state[
pipe] == EventHandlerState.out_state_started:
pipe.recv()
self._event_handlers_state[pipe] = EventHandlerState.idle
# Other optional operations, mainly output
self._mediating_methods.get(
self._event_handler_with_shortest_event_time, lambda: None)()
def train(config):
started_at = monotonic()
def log(message):
return print("%06.1f %s" % (monotonic() - started_at, message))
model = config.model
position = config.initial_state.position()
label_shape = config.model(np.expand_dims(position, 0)).shape[1:]
buffer = ReplayBuffer(max_size=config.buffer_size,
features_shape=position.shape,
label_shape=label_shape)
optimizer = tf.keras.optimizers.Adam(learning_rate=0.0,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
log(f"spawning {config.workers} worker(s)")
pipes, process = zip(*(_spawn_worker(config)
for i in range(config.workers)))
step = 0
games_played = 0
while step < config.steps:
log(f"waiting up to 1s for worker commands")
states_for_evaluation = []
wins = 0
losses = 0
draws = 0
for pipe in wait(pipes, 1):
buffered_pipe = _BufferedPipe(pipe)
for command, *args in pipe.recv():
if command == _EVALUATE:
game_state, = args
states_for_evaluation.append((game_state, buffered_pipe))
elif command == _RESULT:
games_played += 1
score, history = args
if abs(score) < 1e-5:
score = 0
draws += 1
else:
assert abs(score) > 0.99
if score > 0:
score = 1
wins += 1
else:
score = -1
losses += 1
for i, (game_state,
search_probabilities) in enumerate(history):
position = game_state.position()
label = np.zeros(label_shape)
label[0] = score
if len(search_probabilities) == 0:
for i in range(1, label.shape[0]):
label[i] = 1.0 / (label.shape[0] - 1)
else:
for move, probability in search_probabilities:
label[1 + move] = probability
assert abs(np.sum(label[1:]) - 1) < 1e-5
buffer.insert(position, label)
score = -score
else:
assert False, f"invalid command {command}"
log(f"received {len(states_for_evaluation)} state(s) for evaluation")
log(f"ingested {wins + losses + draws} game result(s), w/l/d {wins}/{losses}/{draws}"
)
log(f"played {games_played} game(s) total thus far")
if len(states_for_evaluation) > 0:
evaluation_features = np.stack(
[state.position() for state, _ in states_for_evaluation])
log(f"evaluating {evaluation_features.shape[0]} position(s)")
evaluations = model(evaluation_features)
log(f"evaluation complete; emitting responses")
for i, (state, pipe) in enumerate(states_for_evaluation):
av = float(evaluations[i, 0])
priors = evaluations[i, 1:]
denom = sum(priors[move] for move in state.moves())
expansion = [(move, state.play(move), priors[move] / denom)
for move in state.moves()]
pipe.send((_EVALUATION, av, expansion))
for _, pipe in states_for_evaluation:
pipe.flush()
log(f"done")
if len(buffer) >= 4 * config.batch_size:
step += 1
learning_rate = None
for threshold, lr in config.lr_schedule:
if step >= threshold:
learning_rate = lr
assert learning_rate is not None
optimizer.learning_rate = learning_rate
features, labels = buffer.sample(config.batch_size)
log(f"training against {features.shape[0]} of {len(buffer)} example(s) (step {step})"
)
with tf.GradientTape() as tape:
predictions = model(features)
loss = tf.reduce_sum((predictions[:, 0] - labels[:, 0])**2)
loss += tf.reduce_sum(
tf.losses.categorical_crossentropy(predictions[:, 1:],
labels[:, 1:]))
loss /= features.shape[0]
for variable in model.trainable_variables:
loss = loss + config.weight_decay * tf.reduce_sum(
tf.nn.l2_loss(variable))
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients,
model.trainable_variables))
log(f"done")
predicted_outcomes = predictions[:, 0].numpy()
min_po = np.amin(predicted_outcomes)
avg_po = np.sum(predicted_outcomes) / features.shape[0]
max_po = np.amax(predicted_outcomes)
log(f"min., avg., max. predicted outcome: {min_po}, {avg_po}, {max_po}"
)
log(f"loss: {float(loss)}")
if step % config.checkpoint_every == 0:
path = f"{config.model_name}_step{step}.h5"
log(f"saving model after {step} steps to {repr(path)}")
model.save_weights(path)
else:
log(f"collected {len(buffer)} example(s); training starts at {4 * config.batch_size}"
)
log(f"trained for {config.steps} step(s)")
for pipe in pipes:
pipe.send((_TERMINATE, ))
pipe.flush()
log("waiting up to 10s for workers to exit")
waiting_at = monotonic()
for process in processes:
process.join(max(10 - (monotonic - waiting_at), 0.01))
def foo(w):
for i in range(10):
w.send((i, current_process().name))
w.close()
if __name__ == '__main__':
readers = []
for i in range(4):
r, w = Pipe(duplex=False)
readers.append(r)
p = Process(target=foo, args=(w,))
p.start()
# We close the writable end of the pipe now to be sure that
# p is the only process which owns a handle for it. This
# ensures that when p closes its handle for the writable end,
# wait() will promptly report the readable end as being ready.
w.close()
while readers:
for r in wait(readers):
try:
msg = r.recv()
except EOFError:
readers.remove(r)
else:
print(msg)
def start_shards(self):
for number in range(self.shard_count):
r, w = Pipe(duplex=False)
self.r_pipes.append(r)
journal.send("INFO:AngelBot Manager: Starting Shard {}".format(number))
temp = Process(target=self._run, args=(number, self.shard_count, w), daemon=False)
temp.start()
self.shards[number] = {'Pipe': r, 'Process': temp}
journal.send("STATUS:Shard {}: Shard Started.".format(number))
while self.shards.keys():
for shard in self.shards.keys():
if not self.shards[shard]['Process'].is_alive():
del self.shards[shard]
journal.send("QUIT:Shard {0}: Shard {0} Exited.".format(shard))
journal.send("INFO:AngelBot Manage: Attempting to restart Shard {}".format(shard))
r, w = Pipe(duplex=False)
self.r_pipes.append(r)
journal.send("INFO:AngelBot Manager: Starting Shard {}".format(shard))
temp = Process(target=self._run, args=(shard, self.shard_count, w), daemon=False)
temp.start()
self.shards[shard] = {'Pipe': r, 'Process': temp}
journal.send("STATUS:Shard {}: Shard restarted.".format(shard))
for reader in wait(self.r_pipes):
try:
msg = reader.recv()
except EOFError:
self.r_pipes.remove(reader)
else:
if 'QUIT' in msg:
shard = int(msg.split(":")[1])
if shard in self.shards:
del self.shards[shard]
journal.send("QUIT:Shard {0}: Shard {0} Exited.".format(shard))
journal.send("INFO:AngelBot Manager: Attempting to restart Shard {}".format(shard))
r, w = Pipe(duplex=False)
self.r_pipes.append(r)
journal.send("INFO:AngelBot Manager: Starting Shard {}".format(shard))
temp = Process(target=self._run, args=(shard, self.shard_count, w), daemon=False)
temp.start()
self.shards[shard] = {'Pipe': r, 'Process': temp}
journal.send("STATUS:Shard {}: Shard restarted.".format(shard))
elif 'STATUS' in msg:
_, sid, servs, members = msg.split(":")
sid = int(sid)
journal.send("UPDATE:Shard {0}: Shard {0} reporting stats. {1} servers. {2} members.".format(sid,
servs,
members))
self.shards[sid]['stats'] = {'servers': int(servs), 'users': int(members)}
if all('stats' in self.shards[x] for x in range(self.shard_count)):
total_stats = {'Servers': sum(self.shards[x]['stats']['servers'] for x in range(self.shard_count)),
'Users': sum(self.shards[x]['stats']['users'] for x in range(self.shard_count))}
rdb = redis.StrictRedis(host='localhost', port=6379, db=1, decode_responses=True)
ckey = rdb.get("CarbonKey")
lkey = rdb.get("ListBoat")
r = requests.post("https://www.carbonitex.net/discord/data/botdata.php",
data=json.dumps({'key': ckey, 'servercount': total_stats['Servers']}),
headers={'Content-Type': 'application/json'})
if r.status_code != 200:
journal.send("ERROR:AngelBot Manager: Attempted to update carbonitex but got {}".format(r.status_code))
journal.send("ERROR DETAILS: {}".format(r.text))
r = requests.post("https://bots.discord.pw/api/bots/168925517079248896/stats",
data=json.dumps({'server_count': total_stats['Servers']}),
headers={"Authorization": lkey, "Content-Type": "application/json"})
if r.status_code != 200:
journal.send("ERROR:AngelBot Manager: Attempted to update bots.discord.pw but got {}".format(r.status_code))
journal.send("ERROR DETAILS: {}".format(r.text))
rdb.hset("stats", "users", total_stats['Users'])
rdb.hset("stats", "servers", total_stats['Servers'])
journal.send("INFO:AngelBot Manager: Finished updating stats.")
for shard in range(self.shard_count):
del self.shards[shard]['stats']
journal.send("INFO:AngelBot Manager: Shards Exhausted. Shutting down.")
def _execute(
self,
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
task_finished: Callable,
):
if use_threads and max_workers == 1:
with self.pbar_class(**tqdm_kwargs) as pbar:
for args in task_arguments:
result = task(args)
task_finished(result, pbar)
return
task_arguments = list(task_arguments)
grouped_args = list(
zip_longest(*list(split_every(max_workers, task_arguments)))
)
if not grouped_args:
return
processes: List[Process] = []
connections: List[Connection] = []
for chunk in grouped_args:
parent_conn, child_conn = Pipe()
worker_args = [args for args in chunk if args is not None]
process = Process(
target=process_loop,
args=(
child_conn,
worker_initializer,
logging.getLogger("").level,
task,
worker_args,
),
)
process.daemon = True
processes.append(process)
connections.append(parent_conn)
for process in processes:
process.start()
with self.pbar_class(**tqdm_kwargs) as pbar:
while connections:
for r in wait(connections):
if not isinstance(r, Connection):
raise NotImplementedError("We only support Connection()")
try:
msg_type, msg = r.recv()
except EOFError:
connections.remove(r)
continue
if msg_type == MessageType.result:
if task_finished:
task_finished(msg, pbar)
elif msg_type == 'log':
record = msg
logger = logging.getLogger(record.name)
logger.handle(record)
elif msg_type == MessageType.complete:
connections.remove(r)
elif msg_type == MessageType.exception:
for process in processes:
process.terminate()
raise msg
for process in processes:
process.join()
def _provide_inference(self, pipe_endings):
print('provide inference...')
#use_random = True
#cdef double[:, :, :, ::1] batch_state_planes_view = self.batch_state_planes
#cdef double[::1] batch_value_results_view = self.batch_value_results
#cdef double[:, ::1] batch_policy_results = self.batch_policy_results
send_batches = False #True
while self.running is True:
filled_pipes = connection.wait(pipe_endings)
if filled_pipes:
if True or len(filled_pipes) >= self.batch_size: # 1
if send_batches is True:
planes_batch = []
pipes_pred_output = []
for pipe in filled_pipes: #[:self.batch_size]:
while pipe.poll():
planes_batch.append(pipe.recv())
pipes_pred_output.append(pipe)
# logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes)))
state_planes_mxnet = mx.nd.array(
planes_batch, ctx=self.net.get_ctx())
else:
planes_ids = []
pipes_pred_output = []
for pipe in filled_pipes: #[:self.batch_size]:
while pipe.poll():
planes_ids.append(pipe.recv())
pipes_pred_output.append(pipe)
#logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes)))
state_planes_mxnet = mx.nd.array(
self.batch_state_planes[planes_ids],
ctx=self.net.get_ctx())
#pred = self.net.get_executor().forward(is_train=False, data=state_planes_mxnet)
pred = self.net.get_net()(state_planes_mxnet)
#print('pred: %.3f' % (time()-t_s)*1000)
#t_s = time()
value_preds = pred[0].asnumpy()
# for the policy prediction we still have to apply the softmax activation
# because it's not done by the neural net
#policy_preds = pred[1].softmax().asnumpy()
policy_preds = pred[1].softmax().asnumpy()
#if use_random is True:
# value_preds = np.random.random(len(filled_pipes))
# policy_preds = np.random.random((len(filled_pipes), NB_LABELS))
# send the predictions back to the according workers
for i, pipe in enumerate(pipes_pred_output):
if send_batches is True:
pipe.send([value_preds[i], policy_preds[i]])
else:
# get the according channel index for setting the result
channel_idx = planes_ids[i]
# set the value result
self.batch_value_results[
channel_idx] = value_preds[i]
self.batch_policy_results[
channel_idx] = policy_preds[i]
# give the thread the signal that the result has been set by sending back his channel_idx
pipe.send(channel_idx)
def upload_results_to_s3(self):
"""
Run a separate process for each file that is to be uploaded to s3
:return:
"""
try:
file_lists = upload_file_utils.get_files_for_multiprocessing(
path=self.path)
num_files = sum(len(s) for s in file_lists)
logging.debug("Num files found {}".format(num_files))
kwargs = {
'bucket': self.bucket,
'key': self.key,
'compression': self.compression,
'max_uncompressed': self.max_uncompressed,
'max_cpu_usage': self.max_cpu_usage,
'storage': self.storage,
'temp_dir': self.temp_path
}
i = 0
# While there is anything to process keep going
while any(file_lists) or any(self.gzip_file_list) or any(
self.running_processes):
"""
Check the gzip list first because gzipping is the longest running process so we want
to run as many gzip processes as we can at a time
Otherwise we are keeping track of the order of processes
"""
for gzip_file in self.gzip_file_list:
f_size = gzip_file['file_size']
if self.can_add_gzip_process(f_size=f_size):
self.gzip_file_list.remove(gzip_file)
process, parent_conn = self.add_gzip_upload_process(
args=gzip_file, kwargs=kwargs)
self.running_processes[parent_conn] = {
'process': process,
'size': f_size
}
"""
Check all the processes to see if any have reported back that they are finished
If gzip is finished we want to release the memory we are tracking in order to allow
more gzip processes to run
"""
if any(self.running_processes):
keys = list(self.running_processes.keys())
while len(keys) > 0:
# there is a maximum of 63 elements in a list for the wait call
wait_keys = keys[0:min(len(keys), 62)]
keys = keys[min(len(keys), 62):]
for listener in wait(wait_keys, timeout=0.01):
try:
while listener.poll(0.01):
response = dict(listener.recv())
logging.info(response)
if response[
'action'] == const.CREATED_PART:
self.gzip_file_list.append(response)
elif response[
'action'] == const.GZIPPED_PART:
self.gzip_running_size -= response[
'file_size']
elif response[
'action'] == const.FINISHED_CREATING_PARTS:
self.file_running_size -= min(
response['file_size'],
self.max_uncompressed)
elif response[
'action'] == const.UPLOADED_PART:
pass
else:
logging.error(
"UNKNOWN RESPONSE {}".format(
response))
except (EOFError, BrokenPipeError):
pass
proc_dict = self.running_processes.get(listener)
"""
Kill dead processes
"""
if not proc_dict['process'].is_alive():
current_proc = self.running_processes.pop(
listener)['process']
current_proc.join()
current_proc.terminate()
if any(file_lists):
f_size = file_lists[i][-1].size
if self.can_add_file_process(f_size=f_size):
file_to_upload = file_lists[i].pop()
process, parent_conn = self.add_divide_file_process(
file_to_upload=file_to_upload, kwargs=kwargs)
self.running_processes[parent_conn] = {
'process': process,
'size': f_size
}
if not any(file_lists[i]):
file_lists.remove(file_lists[i])
i = i + 1 if i + 1 < len(file_lists) else 0
if not any(self.running_processes) and not any(
file_lists) and not any(self.gzip_file_list):
logging.debug("All processes finished")
except (FileExistsError, FloatingPointError) as e:
raise e
except:
raise Exception(sys.exc_info())
def mws_proc(self, **kwargs):
"""
Continuously runs its MWS function until the previous MWS function reports that it's all done.
When the previous MWS function (or Walmart's Routine, in the case of gmpfId) is done, this function
will update its queue with the rest of the items, run through all those until they're complete, and
then report itself as all done.
"""
for key, value in kwargs.items():
if key == 'op':
op = value
if key == 'func':
func = value
if key == 'ammo':
ammo = value
if key == 'triggs':
triggs = value
# if op == 'gmpfId':
# print ("Asd")
# else:
# time.sleep(999999)
if type(self.qDefs[op]['qry']) is str:
isQry = True
q = deque()
# A dict with keys equal to the keys in triggs['recv'], and all values False
finishUps = {recvEnd: False for recvEnd in triggs['recv']}
elif type(self.qDefs[op]['qry']) in (list, tuple):
isQry = False
q = deque(self.qDefs[op]['qry'])
finishUps = {'bluhhuuuuurrrpp': True}
else:
print("Invalid value for self.qDefs[op]['qry'] in routine.Routine.mws_proc")
return
funcName = func.__name__
throtGap = 1.1
qGap = 8
while True:
prev = time.time()
finished = all(q for q in finishUps.values()) # Will be T if all values in finishUps are T, else F
# If we don't have enough items for a full call, check to see if more items are ready
if len(q) < self.throt[op]['maxpercall']:
if isQry:
q = self.fill_q(op, q)
if finished and len(q) == 0:
print("{} signing off!".format(funcName))
for sendEnd in triggs['send']:
triggs['send'][sendEnd].send('')
triggs['send'][sendEnd].close()
break
# Enough items in q to warrant an MWS call (which is anything > 0 if <finished> is True)
if len(q) >= (1 - finished) * self.throt[op]['minpercall']:
num = min(len(q), self.throt[op]['maxpercall'])
if ammo['val'].value >= num: # Not gonna get throttled
margs = [q.pop() for _ in range(num)]
if margs:
if len(margs[0]) == 1: # Convert a list of single-length tuples/lists to just a list
margs = [elem[0] for elem in margs]
# Run the mwsutils function
print("{} - starting".format(funcName))
if op == 'gmpfId':
func('Walmart', 'upc', margs)
elif op == 'gcpfAsin':
func(margs)
elif op == 'glolfAsin':
func(margs)
elif op == 'gmfe':
func(margs)
elif op == 'lis':
func(margs)
print("{} - leaving".format(funcName))
with ammo['lock']:
ammo['val'].value -= len(margs)
else:
time.sleep(max(prev + throtGap - time.time(), 0))
else:
# The only way I could find to trip a condition after a message is sent down a pipe in another process.
# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.connection.wait
# Checks each receive Pipe end for a message. <finishedUps> records which receive Pipe ends have gotten
# their message.
for recvEnd in triggs['recv']:
if wait([triggs['recv'][recvEnd]], 0.1) and not finishUps[recvEnd]:
try:
_ = triggs['recv'][recvEnd].recv()
finishUps[recvEnd] = True
except EOFError:
print("{} giving EOFError".format(op))
finishUps[recvEnd] = True
time.sleep(max(prev + qGap - time.time(), 0))
def solve_subproblems(network_name,
subproblems,
property_path,
runtimes,
tree_states,
init_to=0,
num_workers=16,
online_split=1,
to_factor=1.5,
strategy=3,
input_name="input:0",
seed=4,
log_file=None):
"""
Solve the given set of subqueries on the network using divide-and-conquer algorithm
Args:
network_name (str) Name of the network
subproblems (list of pairs) A list of {query-timeout} pairs
property_path (str) Path to the property file
runtimes (dict) Dictionary that maps sub-query's id to its runtime
tree_states (dict) Dictionary that maps sub-query's id to its tree states
init_to (int) Initial timeout
num_workers (int) Number of worker threads to create
online_split (int) Number of times to bisect a subquery when timeout occurs
to_factor (float) The timeout factor
strategy (int) Heuristic used to decide the influence of an input interval
input_name (str) Name of the input variable in the .pb representation of the network
seed (int) Random seed
log_file (str) If not None, log the stats of each subproblem while solving
Return:
Empty list if UNSAT
A list containing a satisfying assignment if SAT
"""
inputs = []
for p in subproblems:
inputs.append((p, init_to))
random.Random(seed).shuffle(inputs)
# Create LIFO Queue for each worker
L = []
for i in range(num_workers):
L.append(mp.Manager().list())
for i, input_ in enumerate(inputs):
L[i % num_workers].append(input_)
# Create and initiate workers
num_tasks = Value("i", len(subproblems))
processes = []
readers = []
return_lst = []
lock = Lock()
for i in range(num_workers):
r, w = Pipe(duplex=True)
readers.append(r)
p = mp.Process(target=worker,
args=(network_name, property_path, L, num_tasks,
online_split, to_factor, log_file, w, lock, i,
strategy, input_name))
processes.append(p)
p.daemon = True
p.start()
w.close()
# Wait for results
while readers:
for r in wait(readers):
try:
val_, results = r.recv()
return_lst.append(results)
if len(val_) > 0:
for results in return_lst:
for result in results:
val, t_id, runtime, has_TO, num_tree_states = result
if has_TO:
runtimes[t_id] = -runtime
else:
runtimes[t_id] = runtime
tree_states[t_id] = num_tree_states
with num_tasks.get_lock():
num_tasks.value = 0
for p in processes:
while p.is_alive():
subprocess.run(["kill", "-9", str(p.pid)])
return val_
except EOFError:
readers.remove(r)
for p in processes:
p.join()
for results in return_lst:
for result in results:
val, t_id, runtime, has_TO, num_tree_states = result
if has_TO:
runtimes[t_id] = -runtime
else:
runtimes[t_id] = runtime
tree_states[t_id] = num_tree_states
return []
def _predict(self):
"""
Method to receive game histories from several processes in order to prepare, predict and eventually
dispatch policy vectors to the process associated with the data.
Note: Also checks the cache if the FEN has been infered before. If yes, respective policy vector will get
returned immediately.
"""
logging.basicConfig(filename='ModelApiLog.log', level=logging.DEBUG)
breaking = False
while not self.end:
# listen and wait for incoming data
ready = connection.wait(self.pipes,
timeout=self.config.timeout_listen)
if not ready:
continue
recv_data, result_pipes, recv_fen = [], [], []
# collect data from pipes
for pipe in ready:
if breaking:
break
while pipe.poll():
try:
recv_data.append(pipe.recv())
result_pipes.append(pipe)
except Exception as exc:
logging.exception("pipe recv error, exception: " +
str(exc))
breaking = True
break
# make fens from game histories
recv_fen = self.make_fen(recv_data)
# check the cache to which incoming state has already been predicted
# if all prediction requests got answered, continue
if len(recv_data) == 0:
continue
# image preparation
data = self.make_image(recv_data)
data_images = np.asarray(data, dtype=np.float32)
# predict either with TensorRT model or tf.keras.model
if self.config.use_tensorRT:
value, policy = self._infer_trt(data_images)
else:
value, policy = self.model.predict_on_batch(data_images)
for pipe, val, pol, fen in zip(result_pipes, value, policy,
recv_fen):
# send back value and policy vectors
try:
pipe.send((val, pol))
except Exception as exc:
logging.exception("pipe send error, exception: " +
str(exc))
break
# write value and policy vectors of associated game state to the cache
for pipe_to_close in self.pipes:
pipe_to_close.close()
def _discard_rollout_promises(self, train_type):
ready_promises = wait(self.rollout_promise_list[train_type], timeout=0)
for rollout_promise in ready_promises:
self._free_rollout_promise(rollout_promise)
def _queue_management_worker(executor_reference,
processes,
pending_work_items,
work_ids_queue,
call_queue,
result_queue):
"""Manages the communication between this process and the worker processes.
This function is run in a local thread.
Args:
executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
this thread. Used to determine if the ProcessPoolExecutor has been
garbage collected and that this function can exit.
process: A list of the multiprocessing.Process instances used as
workers.
pending_work_items: A dict mapping work ids to _WorkItems e.g.
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
call_queue: A multiprocessing.Queue that will be filled with _CallItems
derived from _WorkItems for processing by the process workers.
result_queue: A multiprocessing.Queue of _ResultItems generated by the
process workers.
"""
executor = None
def shutting_down():
return _shutdown or executor is None or executor._shutdown_thread
def shutdown_worker():
# This is an upper bound
nb_children_alive = sum(p.is_alive() for p in processes.values())
for i in range(0, nb_children_alive):
call_queue.put_nowait(None)
# Release the queue's resources as soon as possible.
call_queue.close()
# If .join() is not called on the created processes then
# some multiprocessing.Queue methods may deadlock on Mac OS X.
for p in processes.values():
p.join()
reader = result_queue._reader
while True:
_add_call_item_to_queue(pending_work_items,
work_ids_queue,
call_queue)
sentinels = [p.sentinel for p in processes.values()]
assert sentinels
ready = wait([reader] + sentinels)
if reader in ready:
result_item = reader.recv()
else:
# Mark the process pool broken so that submits fail right now.
executor = executor_reference()
if executor is not None:
executor._broken = True
executor._shutdown_thread = True
executor = None
# All futures in flight must be marked failed
for work_id, work_item in pending_work_items.items():
work_item.future.set_exception(
BrokenProcessPool(
"A process in the process pool was "
"terminated abruptly while the future was "
"running or pending."
))
# Delete references to object. See issue16284
del work_item
pending_work_items.clear()
# Terminate remaining workers forcibly: the queues or their
# locks may be in a dirty state and block forever.
for p in processes.values():
p.terminate()
shutdown_worker()
return
if isinstance(result_item, int):
# Clean shutdown of a worker using its PID
# (avoids marking the executor broken)
assert shutting_down()
p = processes.pop(result_item)
p.join()
if not processes:
shutdown_worker()
return
elif result_item is not None:
work_item = pending_work_items.pop(result_item.work_id, None)
# work_item can be None if another process terminated (see above)
if work_item is not None:
if result_item.exception:
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
# Delete references to object. See issue16284
del work_item
# Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
# - The interpreter is shutting down OR
# - The executor that owns this worker has been collected OR
# - The executor that owns this worker has been shutdown.
if shutting_down():
try:
# Since no new work items can be added, it is safe to shutdown
# this thread if there are no pending work items.
if not pending_work_items:
shutdown_worker()
return
except Full:
# This is not a problem: we will eventually be woken up (in
# result_queue.get()) and be able to send a sentinel again.
pass
executor = None
def _queue_management_worker(executor_reference, _manager,
pending_work_items, work_ids_queue,
_call_queue, _result_queue):
"""Manages the communication between this process and the worker processes."""
executor = None
def shutting_down():
return process._shutdown or executor is None or executor._shutdown_thread
def shutdown_worker():
# This is an upper bound
if _manager.is_alive():
_call_queue.put_nowait(None)
# Release the queue's resources as soon as possible.
_call_queue.close()
# If .join() is not called on the created processes then
# some multiprocessing.Queue methods may deadlock on Mac OS X.
_manager.join()
reader = _result_queue._reader
while True:
_add_call_item_to_queue(pending_work_items, work_ids_queue,
_call_queue)
sentinel = _manager.sentinel
assert sentinel
ready = wait([reader, sentinel])
if reader in ready:
result_item = reader.recv()
else:
# Mark the process pool broken so that submits fail right now.
executor = executor_reference()
if executor is not None:
executor._broken = True
executor._shutdown_thread = True
executor = None
# All futures in flight must be marked failed
for work_id, work_item in pending_work_items.items():
work_item.future.set_exception(
BrokenProcessPool(
"A process in the process pool was "
"terminated abruptly while the future was "
"running or pending."))
# Delete references to object. See issue16284
del work_item
# pending_work_items.clear()
del pending_work_items[:]
# Terminate remaining workers forcibly: the queues or their
# locks may be in a dirty state and block forever.
_manager.terminate()
shutdown_worker()
return
if isinstance(result_item, int):
# Clean shutdown of a worker using its PID
# (avoids marking the executor broken)
assert shutting_down()
_manager.join()
if _manager is None:
shutdown_worker()
return
elif result_item is not None:
work_item = pending_work_items.pop(result_item.work_id, None)
# work_item can be None if another process terminated (see above)
if work_item is not None:
if result_item.exception:
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
# Delete references to object. See issue16284
del work_item
# Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
# - The interpreter is shutting down OR
# - The executor that owns this worker has been collected OR
# - The executor that owns this worker has been shutdown.
if shutting_down():
try:
# Since no new work items can be added, it is safe to shutdown
# this thread if there are no pending work items.
if not pending_work_items:
shutdown_worker()
return
except queue.Full:
# This is not a problem: we will eventually be woken up (in
# _result_queue.get()) and be able to send a sentinel again.
pass
executor = None
def _queue_management_worker(executor_reference, processes, pending_work_items,
work_ids_queue, call_queue, result_queue):
"""Manages the communication between this process and the worker processes.
This function is run in a local thread.
Args:
executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
this thread. Used to determine if the ProcessPoolExecutor has been
garbage collected and that this function can exit.
process: A list of the multiprocessing.Process instances used as
workers.
pending_work_items: A dict mapping work ids to _WorkItems e.g.
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
call_queue: A multiprocessing.Queue that will be filled with _CallItems
derived from _WorkItems for processing by the process workers.
result_queue: A multiprocessing.Queue of _ResultItems generated by the
process workers.
"""
executor = None
def shutting_down():
return _shutdown or executor is None or executor._shutdown_thread
def shutdown_worker():
# This is an upper bound
nb_children_alive = sum(p.is_alive() for p in processes.values())
for i in range(0, nb_children_alive):
call_queue.put_nowait(None)
# Release the queue's resources as soon as possible.
call_queue.close()
# If .join() is not called on the created processes then
# some multiprocessing.Queue methods may deadlock on Mac OS X.
for p in processes.values():
p.join()
reader = result_queue._reader
while True:
_add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue)
sentinels = [p.sentinel for p in processes.values()]
assert sentinels
ready = wait([reader] + sentinels)
if reader in ready:
result_item = reader.recv()
else:
# Mark the process pool broken so that submits fail right now.
executor = executor_reference()
if executor is not None:
executor._broken = True
executor._shutdown_thread = True
executor = None
# All futures in flight must be marked failed
for work_id, work_item in pending_work_items.items():
work_item.future.set_exception(
BrokenProcessPool(
"A process in the process pool was "
"terminated abruptly while the future was "
"running or pending."))
# Delete references to object. See issue16284
del work_item
pending_work_items.clear()
# Terminate remaining workers forcibly: the queues or their
# locks may be in a dirty state and block forever.
for p in processes.values():
p.terminate()
shutdown_worker()
return
if isinstance(result_item, int):
# Clean shutdown of a worker using its PID
# (avoids marking the executor broken)
assert shutting_down()
p = processes.pop(result_item)
p.join()
if not processes:
shutdown_worker()
return
elif result_item is not None:
work_item = pending_work_items.pop(result_item.work_id, None)
# work_item can be None if another process terminated (see above)
if work_item is not None:
if result_item.exception:
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
# Delete references to object. See issue16284
del work_item
# Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
# - The interpreter is shutting down OR
# - The executor that owns this worker has been collected OR
# - The executor that owns this worker has been shutdown.
if shutting_down():
try:
# Since no new work items can be added, it is safe to shutdown
# this thread if there are no pending work items.
if not pending_work_items:
shutdown_worker()
return
except Full:
# This is not a problem: we will eventually be woken up (in
# result_queue.get()) and be able to send a sentinel again.
pass
executor = None
for i in range(10):
w.send((i,mp.current_process().name))
print("send:",i,mp.current_process().name)
w.close()
if __name__ == "__main__":
readers = []
for i in range(4):
r,w = mp.Pipe(duplex=False)
readers.append(r)
p = mp.Process(target=foo,args=(w,))
p.start()
w.close()
while readers:
for r in connection.wait(readers):
print('recv:',r)
try:
msg = r.recv()
except EOFError:
print(r," is done")
readers.remove(r)
else:
print(msg)
def test2():
address = ('localhost',6000)
with connection.Client(address,authkey=b'123') as client:
print(client.recv())
