def listen(self) -> None: """ Listen for messages from the broadcaster. """ while self.connection: for ready_connection in connection.wait([self.connection]): message = ready_connection.recv()
def map(self, func, iterable, chunksize=None): args = iter(iterable) existing_processes = {} arg = get_arg(args, chunksize) while len(existing_processes) < self.processes and arg: add_process(existing_processes, loop_function, (func, arg)) arg = get_arg(args, chunksize) while existing_processes: failed = set() for s in wait(existing_processes): if existing_processes[s].exitcode is not None: if s in failed: log.error('%s: Process sentinel is ready and exitcode is now available as %s' % (s, existing_processes[s].exitcode)) if existing_processes[s].exitcode != 0: raise ChildProcessError('Exit code %s' % existing_processes[s].exitcode) del existing_processes[s] if arg: add_process(existing_processes, loop_function, (func, arg)) arg = get_arg(args, chunksize) else: if s not in failed: log.error('%s: Process sentinel is ready but exitcode is %s' % (s, existing_processes[s].exitcode)) failed.add(s)
def wait(self, timeout=None): if sys.version_info < (3, 3): import time if timeout is None: return self.poll(0) deadline = time.time() + timeout delay = 0.0005 while 1: res = self.poll() if res is not None: break remaining = deadline - time.time() if remaining <= 0: break delay = min(delay * 2, remaining, 0.05) time.sleep(delay) return res if self.returncode is None: if timeout is not None: from multiprocessing.connection import wait if not wait([self.sentinel], timeout): return None # This shouldn't block if wait() returned successfully. return self.poll(os.WNOHANG if timeout == 0.0 else 0) return self.returncode
def parent(funcs: list, collection) -> tuple: """create workers and colect result via pipes :return list of results from chields """ # pipe is a tuple (parent , chield) update_time = current_datetime_tz() pipes = [multiprocessing.Pipe(duplex=False) for _ in funcs] processes_news = [multiprocessing.Process(target=workerANDconnector, args=(params[0],), kwargs={'connector': params[1][1], 'collection': collection, 'update_time': update_time}) for params in zip(funcs, pipes)] for process, pipe in zip(processes_news, pipes): process.start() pipe[1].close() readers = [r for r, w in pipes] result = [] while readers: for r in wait(readers): try: result_r = r.recv() except EOFError: readers.remove(r) else: result.append(result_r) for process in processes_news: process.join() return reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]), result)
def get_their_work(self, pipe_package): pipe_package.setChunkedsObjects(chunk(pipe_package.get_objects(), self.nb_process)) if not self.processs: self._start(pipe_package) else: for i in range(self.nb_process): #print('send things') pipe_package.setCurrentProcessId(i) self.writers_pipes[i].send(pipe_package) things_done_collection = [] reader_useds = [] while self.readers_pipes: for r in wait(self.readers_pipes): try: things_dones = r.recv() except EOFError: reader_useds.append(r) self.readers_pipes.remove(r) else: reader_useds.append(r) self.readers_pipes.remove(r) for thing_done in things_dones: things_done_collection.append(thing_done) self.readers_pipes = reader_useds return things_done_collection
def run(self): try: while self.keep_looping: for conn in wait(self.connections): try: msg = conn.recv() except EOFError: self.connections.remove(conn) except ConnectionResetError: pass else: if conn is self.gui_pipe: c="GUI" elif conn is self.server_conn: c="Server" else: c=conn self.handle_rpc(c,msg) if not self.keep_looping: break #if not all([p.is_alive() for p in self.procs]): # self.keep_looping=False finally: self.close()
def wait_childs(self): """""" for r in wait(self.readers): try: msg = r.recv() except EOFError: self.readers.remove(r) else: if len(msg)==3 and msg[self.K_TYPE] in self.TYPE_MSG : cpid = self.getcpid(msg[self.K_ID]) if msg[self.K_TYPE] == self.MSG_INIT : if hasattr(self.onstart_bind, '__call__'): self.onstart_bind(msg[self.K_ID], cpid, msg[self.K_DATA]) elif msg[self.K_TYPE] == self.MSG_PRINT : if Sys.g.DEBUG : if not Sys.g.GUI : for item in msg[self.K_DATA] : Sys.echo(item[0], Sys.clzdic[item[1]], False, True) Sys.dprint('') #~ else : Sys.wlog(msg[self.K_DATA]) elif msg[self.K_TYPE] == self.MSG_DATA : if hasattr(self.onrun_bind, '__call__'): self.onrun_bind(msg[self.K_ID], cpid, msg[self.K_DATA]) elif msg[self.K_TYPE] == self.MSG_END : if hasattr(self.onend_bind, '__call__'): self.onend_bind(msg[self.K_ID], cpid, msg[self.K_DATA])
def process_poll(self,right_conn): clients_connected = self.clients_connected = [] clients_logged = self.clients_logged = [] while True: wait_lst = clients_connected + clients_logged + [right_conn] for c in wait(wait_lst): if c is right_conn: conn = c.recv() client = Client(conn,self.create_id()) clients_connected.append(client) print("new client:",clients_connected[-1]) #broadcast change elif c in clients_connected: try: msg = c.conn.recv() except (EOFError,ConnectionResetError): clients_connected.remove(c) print("[disconnected]",c) else: self.handle_rpc(c,msg) else: try: msg = c.conn.recv() except (EOFError,ConnectionResetError): self.logout(c) else: self.handle_rpc(c,msg)
def collect(self): while True: inbound = list(self._publishers.values()) for p in connection.wait(inbound): try: msg = p.recv_bytes() self._holding.put(msg) except EOFError: inbound.remove(p)
def wait(self, timeout=None): if self.returncode is None: if timeout is not None: from multiprocessing.connection import wait if not wait([self.sentinel], timeout): return None # This shouldn't block if wait() returned successfully. return self.poll(os.WNOHANG if timeout == 0.0 else 0) return self.returncode
def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, result_queue): executor = None def shutting_down(): return _shutdown or (executor is None or executor._shutdown_thread) def shutdown_worker(): nb_children_alive = sum(p.is_alive() for p in processes.values()) for i in range(0, nb_children_alive): call_queue.put_nowait(None) call_queue.close() for p in processes.values(): p.join() reader = result_queue._reader while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) sentinels = [p.sentinel for p in processes.values()] ready = wait([reader] + sentinels) if reader in ready: result_item = reader.recv() else: executor = executor_reference() if executor is not None: executor._broken = True executor._shutdown_thread = True executor = None for (work_id, work_item) in pending_work_items.items(): work_item.future.set_exception(BrokenProcessPool('A process in the process pool was terminated abruptly while the future was running or pending.')) pending_work_items.clear() for p in processes.values(): p.terminate() shutdown_worker() return if isinstance(result_item, int): p = processes.pop(result_item) p.join() if not processes: shutdown_worker() return elif result_item is not None: work_item = pending_work_items.pop(result_item.work_id, None) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) executor = executor_reference() if shutting_down(): try: if not pending_work_items: shutdown_worker() return except Full: pass executor = None
def poll(self, flag=os.WNOHANG): if self.returncode is None: from multiprocessing.connection import wait timeout = 0 if flag == os.WNOHANG else None if not wait([self.sentinel], timeout): return None try: self.returncode = forkserver.read_unsigned(self.sentinel) except (OSError, EOFError): # The process ended abnormally perhaps because of a signal self.returncode = 255 return self.returncode
def run(self): with connection.Listener(address=SessionServer._address, family='AF_UNIX', authkey=self.__key.bytes) as listener: with listener.accept() as client: print("connection accepted") while True: for ready_client in connection.wait([client]): try: cmd_line = ready_client.recv() print(cmd_line) except EOFError: print("end of file?")
def run_keeped_process(target, main_write_pipe, process_read_pipe, new_things): while new_things != 'stop': things_dones = target(new_things) main_write_pipe.send(things_dones) readers = [process_read_pipe] while readers: for r in wait(readers): try: new_things = r.recv() except EOFError: pass finally: readers.remove(r)
def get_their_work(self, things_to_do): chunked_things = chunk(things_to_do, self.nb_process) self._start(chunked_things) things_done_collection = [] while self.readers_pipes: for r in wait(self.readers_pipes): try: things_dones = r.recv() except EOFError: self.readers_pipes.remove(r) else: self.readers_pipes.remove(r) things_done_collection.append(things_dones) return things_done_collection
def poll(self, flag=os.WNOHANG): if self.returncode is None: from multiprocessing.connection import wait timeout = 0 if flag == os.WNOHANG else None if not wait([self.sentinel], timeout): return None try: self.returncode = forkserver.read_signed(self.sentinel) except (OSError, EOFError): # This should not happen usually, but perhaps the forkserver # process itself got killed self.returncode = 255 return self.returncode
def run_scan_stream(self): while True: comment_process = mp.Process( target=self.scan_content_stream, args=("comment",), ) comment_process.start() submission_process = mp.Process( target=self.scan_content_stream, args=("submission",), ) submission_process.start() mpc.wait([ comment_process.sentinel, submission_process.sentinel ]) print("Something went wrong - restarting processes.") submission_process.terminate() comment_process.terminate() print("Sleeping for 15 seconds.") time.sleep(15)
def event_loop(incoming_channels): while(True): num_processes = len(incoming_channels) if num_processes == 0: break channels_with_data = wait(incoming_channels) if channels_with_data is None or len(channels_with_data) == 0: continue for incoming in channels_with_data: try: msg = incoming.recv() except EOFError: incoming_messages.remove(incoming) else: process(msg)
def run_keepedalive_process(main_write_pipe, process_read_pipe, obj): """ Procees who don't finish while job to do """ while obj != 'stop': oneshot_in_process(obj) # Send to main process "I've done my job" main_write_pipe.send('job is done') # Wait for new job to do (this part can be simplified no ?) readers = [process_read_pipe] while readers: for r in wait(readers): try: obj = r.recv() except EOFError: pass finally: readers.remove(r)
def _predict_batch_worker(self): """ Thread worker which listens on each pipe in self.pipes for an observation, and then outputs the predictions for the policy and value networks when the observations come in. Repeats. """ while True: ready = connection.wait(self.pipes,timeout=0.001) if not ready: continue data, result_pipes = [], [] for pipe in ready: while pipe.poll(): data.append(pipe.recv()) result_pipes.append(pipe) data = np.asarray(data, dtype=np.float32) policy_ary, value_ary = self.agent_model.model.predict_on_batch(data) for pipe, p, v in zip(result_pipes, policy_ary, value_ary): pipe.send((p, float(v)))
def computer(bugs, nb_process=4): bugs_iterables = chunk(bugs, nb_process) readers = [] for i in range(nb_process): r, w = Pipe(duplex=False) readers.append(r) p = Process(target=run_worker, args=(w, bugs_iterables[i])) p.start() w.close() while readers: for r in wait(readers): try: bugs_sendeds = r.recv() except EOFError: readers.remove(r) else: for bug in bugs_sendeds: pass#print(bug.id, bug.willmove)
def run_keeped_process(target, main_write_pipe, process_read_pipe, things): keep_alive = True while keep_alive: things_dones = target(things) main_write_pipe.send(things_dones) readers = [process_read_pipe] readers_used = [] while readers: for r in wait(readers): try: things = r.recv() #print('p: things received') except EOFError: pass finally: readers.remove(r) #print('p: continue') if things == 'stop': keep_alive = False
def nqueens(n, nb_process=2): columns = range(n) boards = list(permutations(columns)) readers = [] for i in range(nb_process): r, w = Pipe(duplex=False) readers.append(r) p = Process(target=check_nqueens, args=(w, n, columns, boards[i::nb_process])) p.start() w.close() while readers: for r in wait(readers): try: msg = r.recv() except EOFError: readers.remove(r) else: print_board(n, msg)
def wait( # type: ignore workers: List["SubprocEnvWorker"], wait_num: int, timeout: Optional[float] = None, ) -> List["SubprocEnvWorker"]: remain_conns = conns = [x.parent_remote for x in workers] ready_conns: List[connection.Connection] = [] remain_time, t1 = timeout, time.time() while len(remain_conns) > 0 and len(ready_conns) < wait_num: if timeout: remain_time = timeout - (time.time() - t1) if remain_time <= 0: break # connection.wait hangs if the list is empty new_ready_conns = connection.wait(remain_conns, timeout=remain_time) ready_conns.extend(new_ready_conns) # type: ignore remain_conns = [ conn for conn in remain_conns if conn not in ready_conns ] return [workers[conns.index(con)] for con in ready_conns]
def __listen(self): print("[IPCServer] [INFO] Now listening to client messages") # connection verification: # so normally the client would send "ping" to the server, but in this case we want to verify if the clients # have two way connectivity (since we boss them around), so we will send the ping request # send the message multiple times to make sure it gets through for i in range(6): self.transmit({"message": "ping", "my_id": self.agent_id}) print(f"[IPCServer] [DEBUG] Transmitting ping #{i}") while self.clients: for conn in wait(self.clients): try: msg = conn.recv() print(f"[IPCServer] [DEBUG] Message from {conn}: {msg}") self.__handle_verification(msg) self.event_handler(msg) except EOFError: print(f"[IPCServer] [WARN] A client caused an EOFError! Disconnecting it", file=sys.stderr) self.clients.remove(conn)
def __work(self): """ This method does the actual work of taking batches of requests and send the responses back to the clients. """ while self.do_run: try: ready = wait(self.connections, timeout=0.0001) except OSError: # If there is any connection closed (our side), we delete it. with self.conn_lock: self.__delete_closed_conns() if not ready: continue data, result_conns = [], [] for i, conn in enumerate(ready): if conn in self.to_ignore: continue while conn.poll(): try: g = conn.recv() except EOFError: # In case of the cliend closed the other end of the # connection. We close our end and ignore it. conn.close() self.to_ignore.add(conn) break data.append(netencoder.get_game_state(g)) result_conns.append(conn) if len(data) > 0: data = np.asarray(data, dtype=np.float16) policies, values = self.model.predict(data) for conn, p, v in zip(result_conns, policies, values): conn.send((p, float(v)))
def predict_batch_worker(self): if self.config.internet.distributed and self.need_reload: self.try_reload_model_from_internet() last_model_check_time = time() while not self.done: if last_model_check_time + 600 < time() and self.need_reload: self.try_reload_model() last_model_check_time = time() ready = connection.wait(self.pipes, timeout=0.001) if not ready: continue data, result_pipes, data_len = [], [], [] for pipe in ready: while pipe.poll(): try: tmp = pipe.recv() except EOFError as e: logger.error(f"EOF error: {e}") pipe.close() else: data.extend(tmp) data_len.append(len(tmp)) result_pipes.append(pipe) if not data: continue data = np.asarray(data, dtype=np.float32) with self.agent_model.graph.as_default(): policy_ary, value_ary = self.agent_model.model.predict_on_batch( data) buf = [] k, i = 0, 0 for p, v in zip(policy_ary, value_ary): buf.append((p, float(v))) k += 1 if k >= data_len[i]: result_pipes[i].send(buf) buf = [] k = 0 i += 1
def __call__(self): def models(): for i, model in enumerate(self.models_): for parameters in model(): yield i, parameters self.generator_ = models() self.left_ = 0 self.results_ = [] while self.generator_ or self.left_ > 0: for source in wait( [self.listener_._listener._socket, *self.clients_]): if source == self.listener_._listener._socket: source = self.listener_.accept() self.clients_.append(source) else: self.results_.append(source.recv()) self.left_ -= 1 if self.generator_: try: data = next(self.generator_) self.left_ += 1 model, parameters = data print(f'Running: ' + ', '.join(f'{key}={repr(value)}' for key, value in self.models_[model]. values(parameters).items())) source.send(data) except StopIteration: self.generator_ = None if not self.generator_: source.send(None) source.close() self.clients_.remove(source) self.results_ = sorted(self.results_, key=lambda v: v[-1], reverse=True)
def start_in_keepedalive_processes(obj, nb_process): """ Start nb_process and keep them alive. Send job to them multiple times, then close thems. """ processes = [] readers_pipes = [] writers_pipes = [] for i in range(nb_process): # Start process with Pipes for communicate local_read_pipe, local_write_pipe = Pipe(duplex=False) process_read_pipe, process_write_pipe = Pipe(duplex=False) readers_pipes.append(local_read_pipe) writers_pipes.append(process_write_pipe) p = Process(target=run_keepedalive_process, args=(local_write_pipe, process_read_pipe, obj)) p.start() processes.append(p) # Send to process some job to do for job in range(3): print('send new job to processes:') for process_number in range(nb_process): # Send data to process writers_pipes[process_number].send(obj) reader_useds = [] # Wait response from processes while readers_pipes: for r in wait(readers_pipes): try: r.recv() except EOFError: pass finally: reader_useds.append(r) readers_pipes.remove(r) readers_pipes = reader_useds # Kill processes for writer_pipe in writers_pipes: writer_pipe.send('stop')
def predict_batch_worker(self): """ 把各个有用管道收集来的数据集中eval,再在原来的管道发送出去 :return: """ while not self.done: ready = connection.wait(self.pipes, timeout=0.001) # 等待有通信管道可用,返回可用的管道 if not ready: continue data, result_pipes, data_len = [], [], [] for pipe in ready: while pipe.poll(): # 不停地返回false,直到连接到有用数据,就返回true try: tmp = pipe.recv( ) # 如果没有消息可接收,recv方法会一直阻塞。如果连接的另外一端已经关闭,那么recv方法会抛出EOFError。 except EOFError as e: logger.error(f"EOF error: {e}") pipe.close() # 另一端关闭,这端就关闭了 else: data.extend(tmp) data_len.append(len(tmp)) result_pipes.append(pipe) if not data: continue data = np.asarray(data, dtype=np.float32) with self.agent_model.graph.as_default(): policy, value = self.agent_model.eval(data) buf = [] k, i = 0, 0 for p, v in zip(policy, value): buf.append((p, float(v))) k += 1 if k >= data_len[i]: result_pipes[i].send(buf) buf = [] k = 0 i += 1
def get_their_work(self, things_to_do): chunked_things = chunk(things_to_do, self.nb_process) if not self.processs: self._start(chunked_things) else: for i in range(self.nb_process): #print('send things') self.writers_pipes[i].send(chunked_things[i]) things_done_collection = [] reader_useds = [] while self.readers_pipes: for r in wait(self.readers_pipes): try: things_dones = r.recv() except EOFError: reader_useds.append(r) self.readers_pipes.remove(r) else: reader_useds.append(r) self.readers_pipes.remove(r) things_done_collection.append(things_dones) self.readers_pipes = reader_useds return things_done_collection
def predict_batch_worker(self): last_model_check_time = time() while True: if last_model_check_time + 600 < time(): self.try_reload_model() last_model_check_time = time() ready = connection.wait(self.pipes, timeout=0.001) if not ready: continue data, result_pipes = [], [] for pipe in ready: while pipe.poll(): try: data.append(pipe.recv()) result_pipes.append(pipe) except EOFError as e: pipe.close() data = np.asarray(data, dtype=np.float32) with self.agent_model.graph.as_default(): policy_ary, value_ary = self.agent_model.model.predict_on_batch( data) for pipe, p, v in zip(result_pipes, policy_ary, value_ary): pipe.send((p, float(v)))
def loop(self): # NOQA: C901 self.exit_msg = '' check_for_events = True while check_for_events: event_handlers = list(self.event_handlers) if self.receiver: event_handlers.append(self.receiver) if len(event_handlers) == 0: break input_conn_list = wait(event_handlers) for conn in input_conn_list: if conn == self.receiver: item = conn.recv() if item is None: self.receiver = None else: self.on_input_func(item) continue event = conn.recv() event_label = self.event_manager_to_plugin(conn) if __debug__: self.logger.debug("GOT %s FROM %s", event, event_label) if event == "started": self.on_started(event_label) if event is None: # Got "None" event - plugin was terminated exit_msg = conn.recv() self.on_exit(event_label, exit_msg) self.remove_event_source(conn) if event == "error": return 1, "Error" print(self.exit_msg) return 0, "Finished"
def send_data(conversion): """ Sends the data to the loader processes. :param conversion: Conversion :return: None """ if len(conversion.data_pool) == 0: return params_list = [[conversion.config, meta] for meta in conversion.data_pool] reader_connections = [] number_of_workers = min( conversion.max_each_db_connection_pool_size, len(conversion.data_pool), cpu_count() ) with ProcessPoolExecutor(max_workers=number_of_workers) as executor: while len(params_list) != 0: reader_connection, writer_connection = Pipe(duplex=False) reader_connections.append(reader_connection) params = params_list.pop() params.append(writer_connection) executor.submit(DataLoader._load, *params) while reader_connections: for reader_connection in connection.wait(object_list=reader_connections): just_populated_table_name = '' try: just_populated_table_name = reader_connection.recv() reader_connection.close() finally: reader_connections.remove(reader_connection) ConstraintsProcessor.process_constraints_per_table(conversion, just_populated_table_name) MigrationStateManager.set(conversion, 'per_table_constraints_loaded')
def process(self, pipe_to_bus, pipe_to_ui): print("process<{}> run".format(self.__class__.__name__)) #self.bus_pipe = bus_pipe self.ui_pipe = pipe_to_ui close_handle = False while not close_handle: #FIXME: there is somd end command all_pipes = self.get_bus_devices_pipe_line() all_pipes.append(pipe_to_bus) all_pipes.append(pipe_to_ui) for r in wait(all_pipes[:]): #try: msg = r.recv() #print("Process<{}> get: {}".format(self.__class__.__name__, msg)) location = msg.loc() if location == Message.BUS or location == Message.DEVICE: self.handle_bus_message(msg) elif location == Message.UI: self.handle_ui_command(msg) else: pass """ except EOFError: #readers.remove(r) ServerError("Pipe {} report EOFError", r) #FIXME: should process the ERROR? if r == server_to_bus_pipe: close_handle = True break else: print("Unexpected error: {}".format(self.__class__.__name__)) """ self.process_bus_command() self.report_ui_message(pipe_to_ui) pipe_to_bus.close() pipe_to_ui.close()
def parent(funcs: list, collection) -> tuple: """create workers and colect result via pipes :return list of results from chields """ # pipe is a tuple (parent , chield) update_time = datetime.utcnow() pipes = [multiprocessing.Pipe(duplex=False) for _ in funcs] processes_news = [ multiprocessing.Process(target=workerANDconnector, args=(params[0], ), kwargs={ 'connector': params[1][1], 'collection': collection, 'update_time': update_time }) for params in zip(funcs, pipes) ] logger.debug('created process num= {}'.format(len(processes_news))) for process, pipe in zip(processes_news, pipes): process.start() pipe[1].close() readers = [r for r, w in pipes] result = [] while readers: for r in wait(readers): try: result_r = r.recv() except EOFError: readers.remove(r) else: result.append(result_r) for process in processes_news: process.join() return reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]), result)
def process(self): super(Hid_Device, self).process() # try: self.open_dev() all_pipes = [self.logic_pipe()] close_handle = False while not close_handle: #try: for r in wait(all_pipes, timeout=self.poll_interval()): if r: try: msg = r.recv() except EOFError: print("Process EOF: {}".format( self.__class__.__name__)) close_handle = True self.close_dev() # self.pipe_hid_event.close() # self.pipe_hid_recv.close() return #print("Process<{}> get: {}".format(self.__class__.__name__, msg)) self.handle_bus_command(msg) # location = msg.loc() # if location == PhyMessage.HID_DEVICE: # #self.handle_phy_message(msg) # elif location == PhyMessage.SERVER: # self.handle_bus_command(msg) # else: # pass self.send_command() self.handle_timeout_command()
def loadEditions(editions): processes = [] outPipes = [] cores = 4 logger.info('Processing {} editions'.format(len(editions))) chunkSize = math.ceil(len(editions) / cores) for i in range(cores): start = i * chunkSize end = start + chunkSize logger.info('Processing chunk {} to {}'.format(start, end)) pConn, cConn = Pipe(duplex=False) proc = Process(target=parseChunk, args=(editions[start:end], cConn)) processes.append(proc) outPipes.append(pConn) proc.start() cConn.close() outEds = [] while outPipes: for p in wait(outPipes): try: ed = p.recv() if ed == 'DONE': outPipes.remove(p) else: outEds.append(ed) except EOFError: outPipes.remove(p) for proc in processes: proc.join() return outEds
def _predict_batch_worker(self): """ Thread worker which listens on each pipe in self.pipes for an observation, and then outputs the predictions for the policy and value networks when the observations come in. Repeats. ## CITE """ while True: ready = connection.wait(self.return_policy_value,timeout=0.001) if not ready: continue data, result_pipes = [], [] for pipe in ready: while pipe.poll(): data.append(pipe.recv()) result_pipes.append(pipe) data = np.asarray(data, dtype=np.float32) # print (data.shape) policy_array, value_array = self.model.predict_on_batch(data) # print (policy_array, value_array) for pipe, policy, value in zip(result_pipes, policy_array, value_array): pipe.send((policy, float(value)))
readers = [] for i in range(4): r, w = Pipe(duplex=False) readers.append(r) p = Process(target=foo, args=(w,)) p.start() # We close the writable end of the pipe now to be sure that # p is the only process which owns a handle for it. This # ensures that when p closes its handle for the writable end, # wait() will promptly report the readable end as being ready. w.close() while readers: for r in wait(readers): try: msg = r.recv() except EOFError: readers.remove(r) else: print(msg) # from multiprocessing import Process, Pipe # """ # 返回的两个连接对象 Pipe() 表示管道的两端。每个连接对象都有 send() 和 recv() 方法(相互之间的)。 # 如果两个进程(或线程)同时尝试读取或写入管道的 同一 端,则管道中的数据可能会损坏。当然,同时使用管道的不同端的进程不存在损坏的风险。 # """ # # def f(conn): # conn.send([42, None, 'hello'])
def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, result_queue, thread_wakeup): """Manages the communication between this process and the worker processes. This function is run in a local thread. Args: executor_reference: A weakref.ref to the ProcessPoolExecutor that owns this thread. Used to determine if the ProcessPoolExecutor has been garbage collected and that this function can exit. process: A list of the ctx.Process instances used as workers. pending_work_items: A dict mapping work ids to _WorkItems e.g. {5: <_WorkItem...>, 6: <_WorkItem...>, ...} work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]). call_queue: A ctx.Queue that will be filled with _CallItems derived from _WorkItems for processing by the process workers. result_queue: A ctx.SimpleQueue of _ResultItems generated by the process workers. thread_wakeup: A _ThreadWakeup to allow waking up the queue_manager_thread from the main Thread and avoid deadlocks caused by permanently locked queues. """ executor = None def shutting_down(): return (_global_shutdown or executor is None or executor._shutdown_thread) def shutdown_worker(): # This is an upper bound on the number of children alive. n_children_alive = sum(p.is_alive() for p in processes.values()) n_children_to_stop = n_children_alive n_sentinels_sent = 0 # Send the right number of sentinels, to make sure all children are # properly terminated. while n_sentinels_sent < n_children_to_stop and n_children_alive > 0: for i in range(n_children_to_stop - n_sentinels_sent): try: call_queue.put_nowait(None) n_sentinels_sent += 1 except Full: break n_children_alive = sum(p.is_alive() for p in processes.values()) # Release the queue's resources as soon as possible. call_queue.close() # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on Mac OS X. for p in processes.values(): p.join() result_reader = result_queue._reader wakeup_reader = thread_wakeup._reader readers = [result_reader, wakeup_reader] while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) # Wait for a result to be ready in the result_queue while checking # that all worker processes are still running, or for a wake up # signal send. The wake up signals come either from new tasks being # submitted, from the executor being shutdown/gc-ed, or from the # shutdown of the python interpreter. worker_sentinels = [p.sentinel for p in processes.values()] ready = wait(readers + worker_sentinels) cause = None is_broken = True if result_reader in ready: try: result_item = result_reader.recv() is_broken = False except BaseException as e: cause = traceback.format_exception(type(e), e, e.__traceback__) elif wakeup_reader in ready: is_broken = False result_item = None thread_wakeup.clear() if is_broken: # Mark the process pool broken so that submits fail right now. executor = executor_reference() if executor is not None: executor._broken = ('A child process terminated ' 'abruptly, the process pool is not ' 'usable anymore') executor._shutdown_thread = True executor = None bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") if cause is not None: bpe.__cause__ = _RemoteTraceback(f"\n'''\n{''.join(cause)}'''") # All futures in flight must be marked failed for work_id, work_item in pending_work_items.items(): work_item.future.set_exception(bpe) # Delete references to object. See issue16284 del work_item pending_work_items.clear() # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. for p in processes.values(): p.terminate() shutdown_worker() return if isinstance(result_item, int): # Clean shutdown of a worker using its PID # (avoids marking the executor broken) assert shutting_down() p = processes.pop(result_item) p.join() if not processes: shutdown_worker() return elif result_item is not None: work_item = pending_work_items.pop(result_item.work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) # Delete references to object. See issue16284 del work_item # Delete reference to result_item del result_item # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this worker has been collected OR # - The executor that owns this worker has been shutdown. if shutting_down(): try: # Flag the executor as shutting down as early as possible if it # is not gc-ed yet. if executor is not None: executor._shutdown_thread = True # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not pending_work_items: shutdown_worker() return except Full: # This is not a problem: we will eventually be woken up (in # result_queue.get()) and be able to send a sentinel again. pass executor = None
def is_alive(self): from multiprocessing.connection import wait return not wait([self._sentinel], timeout=0)
def join(self, timeout=None): ''' Wait until parent process terminates ''' from multiprocessing.connection import wait wait([self._sentinel], timeout=timeout)
def read(self): try: readers = wait([w.outputReader for w in self.writers]) return [r.recv() for r in readers] except (EOFError, OSError): return None
def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, result_queue, thread_wakeup): """Manages the communication between this process and the worker processes. This function is run in a local thread. Args: executor_reference: A weakref.ref to the ProcessPoolExecutor that owns this thread. Used to determine if the ProcessPoolExecutor has been garbage collected and that this function can exit. process: A list of the ctx.Process instances used as workers. pending_work_items: A dict mapping work ids to _WorkItems e.g. {5: <_WorkItem...>, 6: <_WorkItem...>, ...} work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]). call_queue: A ctx.Queue that will be filled with _CallItems derived from _WorkItems for processing by the process workers. result_queue: A ctx.SimpleQueue of _ResultItems generated by the process workers. thread_wakeup: A _ThreadWakeup to allow waking up the queue_manager_thread from the main Thread and avoid deadlocks caused by permanently locked queues. """ executor = None def shutting_down(): return (_global_shutdown or executor is None or executor._shutdown_thread) def shutdown_worker(): # This is an upper bound on the number of children alive. n_children_alive = sum(p.is_alive() for p in processes.values()) n_children_to_stop = n_children_alive n_sentinels_sent = 0 # Send the right number of sentinels, to make sure all children are # properly terminated. while n_sentinels_sent < n_children_to_stop and n_children_alive > 0: for i in range(n_children_to_stop - n_sentinels_sent): try: call_queue.put_nowait(None) n_sentinels_sent += 1 except Full: break n_children_alive = sum(p.is_alive() for p in processes.values()) # Release the queue's resources as soon as possible. call_queue.close() # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on Mac OS X. for p in processes.values(): p.join() result_reader = result_queue._reader wakeup_reader = thread_wakeup._reader readers = [result_reader, wakeup_reader] while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) # Wait for a result to be ready in the result_queue while checking # that all worker processes are still running, or for a wake up # signal send. The wake up signals come either from new tasks being # submitted, from the executor being shutdown/gc-ed, or from the # shutdown of the python interpreter. worker_sentinels = [p.sentinel for p in processes.values()] ready = wait(readers + worker_sentinels) cause = None is_broken = True if result_reader in ready: try: result_item = result_reader.recv() is_broken = False except BaseException as e: cause = traceback.format_exception(type(e), e, e.__traceback__) elif wakeup_reader in ready: is_broken = False result_item = None thread_wakeup.clear() if is_broken: # Mark the process pool broken so that submits fail right now. executor = executor_reference() if executor is not None: executor._broken = ('A child process terminated ' 'abruptly, the process pool is not ' 'usable anymore') executor._shutdown_thread = True executor = None bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") if cause is not None: bpe.__cause__ = _RemoteTraceback( f"\n'''\n{''.join(cause)}'''") # All futures in flight must be marked failed for work_id, work_item in pending_work_items.items(): work_item.future.set_exception(bpe) # Delete references to object. See issue16284 del work_item pending_work_items.clear() # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. for p in processes.values(): p.terminate() shutdown_worker() return if isinstance(result_item, int): # Clean shutdown of a worker using its PID # (avoids marking the executor broken) assert shutting_down() p = processes.pop(result_item) p.join() if not processes: shutdown_worker() return elif result_item is not None: work_item = pending_work_items.pop(result_item.work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) # Delete references to object. See issue16284 del work_item # Delete reference to result_item del result_item # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this worker has been collected OR # - The executor that owns this worker has been shutdown. if shutting_down(): try: # Flag the executor as shutting down as early as possible if it # is not gc-ed yet. if executor is not None: executor._shutdown_thread = True # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not pending_work_items: shutdown_worker() return except Full: # This is not a problem: we will eventually be woken up (in # result_queue.get()) and be able to send a sentinel again. pass executor = None
def run(self) -> None: """ Loop over the legs of the continuous-time evolution of the event-chain Monte Carlo algorithm. This method is called by run.py and resume.py. The loop should only be interrupted when a base.exceptions.EndOfRun exception is raised, which is caught in the scripts. """ while True: # Extract active global state active_global_state = self._state_handler.extract_active_global_state( ) # Fetch event handlers to activate event_handlers_in_state_dictionary = self._activator.get_event_handlers_to_run( active_global_state, self._event_handler_with_shortest_event_time) if self._logger_enabled_for_debug: self._logger.debug( "Event handlers that will be run with their in-state identifiers: {0}" .format({ event_handler.__class__.__name__: in_state_identifier for event_handler, in_state_identifier in event_handlers_in_state_dictionary.items() })) # Fetch in-states for event_handler, in_state_identifiers in event_handlers_in_state_dictionary.items( ): if in_state_identifiers is not None: event_handlers_in_state_dictionary[event_handler] = [ self._state_handler.extract_from_global_state( identifier) for identifier in in_state_identifiers ] # Send in-states pipes = [] for event_handler, in_state in event_handlers_in_state_dictionary.items( ): pipe = self._pipes[event_handler] if self._event_handlers_state[pipe] == EventHandlerState.idle: self._start_events[pipe].set() else: raise MediatorError("Event Process not ready!") if event_handler.number_send_event_time_arguments: assert in_state is not None pipe.send(in_state) self._event_handlers_state[ pipe] = EventHandlerState.event_time_started pipes.append(pipe) # Receive event times, and let them continue on out-state if there are idle os-processes. pipes_time_received = collections.deque() event_times_received = 0 while event_times_received < len( event_handlers_in_state_dictionary): for pipe in connection.wait(pipes): if self._event_handlers_state[ pipe] == EventHandlerState.event_time_started: self._event_handlers_state[ pipe] = EventHandlerState.suspended returned = pipe.recv() try: event_time, self._out_state_arguments[ pipe] = returned except TypeError: event_time, self._out_state_arguments[ pipe] = returned, [] # If no arguments for out-state, one can calculate it in advance if there are resources if not self._event_handlers[ pipe].number_send_out_state_arguments: pipes_time_received.append(pipe) event_times_received += 1 if (0 < len(event_handlers_in_state_dictionary) - event_times_received < self._number_cores - 1 and len(pipes_time_received)): next_pipe = pipes_time_received.popleft() self._send_out_state_events[next_pipe].set() if self._event_handlers[ next_pipe].number_send_out_state_arguments: next_pipe.send( self._out_state_arguments_methods[ self._event_handlers[next_pipe]] (*self._out_state_arguments[next_pipe])) self._event_handlers_state[ next_pipe] = EventHandlerState.out_state_started identifier = self._event_storage.push_to_storage( self._event_handlers[pipe]) self._scheduler.push_event(event_time, identifier) if self._logger_enabled_for_debug: self._logger.debug( "Pushed candidate event time to the scheduler: {0} ({1})" .format( event_time, self._event_handlers[pipe]. __class__.__name__)) elif self._event_handlers_state[ pipe] == EventHandlerState.out_state_started: self._event_handlers_state[ pipe] = EventHandlerState.idle if len(pipes_time_received): next_pipe = pipes_time_received.popleft() self._send_out_state_events[next_pipe].set() if self._event_handlers[ next_pipe].number_send_out_state_arguments: next_pipe.send( self._out_state_arguments_methods[ self._event_handlers[next_pipe]] (*self._out_state_arguments[next_pipe])) self._event_handlers_state[ next_pipe] = EventHandlerState.out_state_started self._event_storage.append_to_stored_value( self._event_handlers[pipe], pipe.recv()) else: raise MediatorError( "Event process with pipe {0} to mediator is already finished" " and shouldn't receive anything anymore!".format( pipe)) # Pop the earliest event handler from scheduler, and let it calculate out-state identifier = self._scheduler.get_succeeding_event() event = self._event_storage.get_event_handler(identifier) self._event_handler_with_shortest_event_time = event[0] if self._logger_enabled_for_debug: self._logger.debug( "Event handler which created the event with the shortest event time: {0}" .format(self._event_handler_with_shortest_event_time. __class__.__name__)) pipe_with_shortest_event_time = self._pipes[ self._event_handler_with_shortest_event_time] if self._event_handlers_state[ pipe_with_shortest_event_time] == EventHandlerState.suspended: self._send_out_state_events[pipe_with_shortest_event_time].set( ) self._event_handlers_state[ pipe_with_shortest_event_time] = EventHandlerState.out_state_started if self._event_handler_with_shortest_event_time.number_send_out_state_arguments: pipe_with_shortest_event_time.send( self._out_state_arguments_methods[ self._event_handler_with_shortest_event_time] (*self. _out_state_arguments[pipe_with_shortest_event_time])) if self._event_handlers_state[ pipe_with_shortest_event_time] == EventHandlerState.out_state_started: out_state = pipe_with_shortest_event_time.recv() self._event_handlers_state[ pipe_with_shortest_event_time] = EventHandlerState.idle self._event_storage.append_to_stored_value( self._event_handler_with_shortest_event_time, out_state) event = [ self._event_handler_with_shortest_event_time, out_state ] # Request shortest time self._event_handler_with_shortest_event_time, out_state = event pipe_with_shortest_event_time = self._pipes[ self._event_handler_with_shortest_event_time] assert self._event_handlers_state[ pipe_with_shortest_event_time] == EventHandlerState.idle # Commit out-state self._state_handler.insert_into_global_state(out_state) # Trash for event_handler in self._activator.get_trashable_events( self._event_handler_with_shortest_event_time): if self._logger_enabled_for_debug: self._logger.debug( "Event handler trashed in the scheduler: {0}".format( event_handler.__class__.__name__)) event_identifier = self._event_storage.get_event_identifier( event_handler) self._scheduler.trash_event(event_identifier) self._event_storage.delete_identifier(event_identifier, event_handler) pipe = self._pipes[event_handler] if self._event_handlers_state[ pipe] == EventHandlerState.suspended: self._event_handlers_state[pipe] = EventHandlerState.idle elif self._event_handlers_state[ pipe] == EventHandlerState.out_state_started: pipe.recv() self._event_handlers_state[pipe] = EventHandlerState.idle # Other optional operations, mainly output self._mediating_methods.get( self._event_handler_with_shortest_event_time, lambda: None)()
def train(config): started_at = monotonic() def log(message): return print("%06.1f %s" % (monotonic() - started_at, message)) model = config.model position = config.initial_state.position() label_shape = config.model(np.expand_dims(position, 0)).shape[1:] buffer = ReplayBuffer(max_size=config.buffer_size, features_shape=position.shape, label_shape=label_shape) optimizer = tf.keras.optimizers.Adam(learning_rate=0.0, beta_1=0.9, beta_2=0.999, amsgrad=False) log(f"spawning {config.workers} worker(s)") pipes, process = zip(*(_spawn_worker(config) for i in range(config.workers))) step = 0 games_played = 0 while step < config.steps: log(f"waiting up to 1s for worker commands") states_for_evaluation = [] wins = 0 losses = 0 draws = 0 for pipe in wait(pipes, 1): buffered_pipe = _BufferedPipe(pipe) for command, *args in pipe.recv(): if command == _EVALUATE: game_state, = args states_for_evaluation.append((game_state, buffered_pipe)) elif command == _RESULT: games_played += 1 score, history = args if abs(score) < 1e-5: score = 0 draws += 1 else: assert abs(score) > 0.99 if score > 0: score = 1 wins += 1 else: score = -1 losses += 1 for i, (game_state, search_probabilities) in enumerate(history): position = game_state.position() label = np.zeros(label_shape) label[0] = score if len(search_probabilities) == 0: for i in range(1, label.shape[0]): label[i] = 1.0 / (label.shape[0] - 1) else: for move, probability in search_probabilities: label[1 + move] = probability assert abs(np.sum(label[1:]) - 1) < 1e-5 buffer.insert(position, label) score = -score else: assert False, f"invalid command {command}" log(f"received {len(states_for_evaluation)} state(s) for evaluation") log(f"ingested {wins + losses + draws} game result(s), w/l/d {wins}/{losses}/{draws}" ) log(f"played {games_played} game(s) total thus far") if len(states_for_evaluation) > 0: evaluation_features = np.stack( [state.position() for state, _ in states_for_evaluation]) log(f"evaluating {evaluation_features.shape[0]} position(s)") evaluations = model(evaluation_features) log(f"evaluation complete; emitting responses") for i, (state, pipe) in enumerate(states_for_evaluation): av = float(evaluations[i, 0]) priors = evaluations[i, 1:] denom = sum(priors[move] for move in state.moves()) expansion = [(move, state.play(move), priors[move] / denom) for move in state.moves()] pipe.send((_EVALUATION, av, expansion)) for _, pipe in states_for_evaluation: pipe.flush() log(f"done") if len(buffer) >= 4 * config.batch_size: step += 1 learning_rate = None for threshold, lr in config.lr_schedule: if step >= threshold: learning_rate = lr assert learning_rate is not None optimizer.learning_rate = learning_rate features, labels = buffer.sample(config.batch_size) log(f"training against {features.shape[0]} of {len(buffer)} example(s) (step {step})" ) with tf.GradientTape() as tape: predictions = model(features) loss = tf.reduce_sum((predictions[:, 0] - labels[:, 0])**2) loss += tf.reduce_sum( tf.losses.categorical_crossentropy(predictions[:, 1:], labels[:, 1:])) loss /= features.shape[0] for variable in model.trainable_variables: loss = loss + config.weight_decay * tf.reduce_sum( tf.nn.l2_loss(variable)) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) log(f"done") predicted_outcomes = predictions[:, 0].numpy() min_po = np.amin(predicted_outcomes) avg_po = np.sum(predicted_outcomes) / features.shape[0] max_po = np.amax(predicted_outcomes) log(f"min., avg., max. predicted outcome: {min_po}, {avg_po}, {max_po}" ) log(f"loss: {float(loss)}") if step % config.checkpoint_every == 0: path = f"{config.model_name}_step{step}.h5" log(f"saving model after {step} steps to {repr(path)}") model.save_weights(path) else: log(f"collected {len(buffer)} example(s); training starts at {4 * config.batch_size}" ) log(f"trained for {config.steps} step(s)") for pipe in pipes: pipe.send((_TERMINATE, )) pipe.flush() log("waiting up to 10s for workers to exit") waiting_at = monotonic() for process in processes: process.join(max(10 - (monotonic - waiting_at), 0.01))
def foo(w): for i in range(10): w.send((i, current_process().name)) w.close() if __name__ == '__main__': readers = [] for i in range(4): r, w = Pipe(duplex=False) readers.append(r) p = Process(target=foo, args=(w,)) p.start() # We close the writable end of the pipe now to be sure that # p is the only process which owns a handle for it. This # ensures that when p closes its handle for the writable end, # wait() will promptly report the readable end as being ready. w.close() while readers: for r in wait(readers): try: msg = r.recv() except EOFError: readers.remove(r) else: print(msg)
def start_shards(self): for number in range(self.shard_count): r, w = Pipe(duplex=False) self.r_pipes.append(r) journal.send("INFO:AngelBot Manager: Starting Shard {}".format(number)) temp = Process(target=self._run, args=(number, self.shard_count, w), daemon=False) temp.start() self.shards[number] = {'Pipe': r, 'Process': temp} journal.send("STATUS:Shard {}: Shard Started.".format(number)) while self.shards.keys(): for shard in self.shards.keys(): if not self.shards[shard]['Process'].is_alive(): del self.shards[shard] journal.send("QUIT:Shard {0}: Shard {0} Exited.".format(shard)) journal.send("INFO:AngelBot Manage: Attempting to restart Shard {}".format(shard)) r, w = Pipe(duplex=False) self.r_pipes.append(r) journal.send("INFO:AngelBot Manager: Starting Shard {}".format(shard)) temp = Process(target=self._run, args=(shard, self.shard_count, w), daemon=False) temp.start() self.shards[shard] = {'Pipe': r, 'Process': temp} journal.send("STATUS:Shard {}: Shard restarted.".format(shard)) for reader in wait(self.r_pipes): try: msg = reader.recv() except EOFError: self.r_pipes.remove(reader) else: if 'QUIT' in msg: shard = int(msg.split(":")[1]) if shard in self.shards: del self.shards[shard] journal.send("QUIT:Shard {0}: Shard {0} Exited.".format(shard)) journal.send("INFO:AngelBot Manager: Attempting to restart Shard {}".format(shard)) r, w = Pipe(duplex=False) self.r_pipes.append(r) journal.send("INFO:AngelBot Manager: Starting Shard {}".format(shard)) temp = Process(target=self._run, args=(shard, self.shard_count, w), daemon=False) temp.start() self.shards[shard] = {'Pipe': r, 'Process': temp} journal.send("STATUS:Shard {}: Shard restarted.".format(shard)) elif 'STATUS' in msg: _, sid, servs, members = msg.split(":") sid = int(sid) journal.send("UPDATE:Shard {0}: Shard {0} reporting stats. {1} servers. {2} members.".format(sid, servs, members)) self.shards[sid]['stats'] = {'servers': int(servs), 'users': int(members)} if all('stats' in self.shards[x] for x in range(self.shard_count)): total_stats = {'Servers': sum(self.shards[x]['stats']['servers'] for x in range(self.shard_count)), 'Users': sum(self.shards[x]['stats']['users'] for x in range(self.shard_count))} rdb = redis.StrictRedis(host='localhost', port=6379, db=1, decode_responses=True) ckey = rdb.get("CarbonKey") lkey = rdb.get("ListBoat") r = requests.post("https://www.carbonitex.net/discord/data/botdata.php", data=json.dumps({'key': ckey, 'servercount': total_stats['Servers']}), headers={'Content-Type': 'application/json'}) if r.status_code != 200: journal.send("ERROR:AngelBot Manager: Attempted to update carbonitex but got {}".format(r.status_code)) journal.send("ERROR DETAILS: {}".format(r.text)) r = requests.post("https://bots.discord.pw/api/bots/168925517079248896/stats", data=json.dumps({'server_count': total_stats['Servers']}), headers={"Authorization": lkey, "Content-Type": "application/json"}) if r.status_code != 200: journal.send("ERROR:AngelBot Manager: Attempted to update bots.discord.pw but got {}".format(r.status_code)) journal.send("ERROR DETAILS: {}".format(r.text)) rdb.hset("stats", "users", total_stats['Users']) rdb.hset("stats", "servers", total_stats['Servers']) journal.send("INFO:AngelBot Manager: Finished updating stats.") for shard in range(self.shard_count): del self.shards[shard]['stats'] journal.send("INFO:AngelBot Manager: Shards Exhausted. Shutting down.")
def _execute( self, *, use_threads: bool, max_workers: int, tqdm_kwargs: dict, worker_initializer: Callable, task: Callable, task_arguments: Iterable, task_finished: Callable, ): if use_threads and max_workers == 1: with self.pbar_class(**tqdm_kwargs) as pbar: for args in task_arguments: result = task(args) task_finished(result, pbar) return task_arguments = list(task_arguments) grouped_args = list( zip_longest(*list(split_every(max_workers, task_arguments))) ) if not grouped_args: return processes: List[Process] = [] connections: List[Connection] = [] for chunk in grouped_args: parent_conn, child_conn = Pipe() worker_args = [args for args in chunk if args is not None] process = Process( target=process_loop, args=( child_conn, worker_initializer, logging.getLogger("").level, task, worker_args, ), ) process.daemon = True processes.append(process) connections.append(parent_conn) for process in processes: process.start() with self.pbar_class(**tqdm_kwargs) as pbar: while connections: for r in wait(connections): if not isinstance(r, Connection): raise NotImplementedError("We only support Connection()") try: msg_type, msg = r.recv() except EOFError: connections.remove(r) continue if msg_type == MessageType.result: if task_finished: task_finished(msg, pbar) elif msg_type == 'log': record = msg logger = logging.getLogger(record.name) logger.handle(record) elif msg_type == MessageType.complete: connections.remove(r) elif msg_type == MessageType.exception: for process in processes: process.terminate() raise msg for process in processes: process.join()
def _provide_inference(self, pipe_endings): print('provide inference...') #use_random = True #cdef double[:, :, :, ::1] batch_state_planes_view = self.batch_state_planes #cdef double[::1] batch_value_results_view = self.batch_value_results #cdef double[:, ::1] batch_policy_results = self.batch_policy_results send_batches = False #True while self.running is True: filled_pipes = connection.wait(pipe_endings) if filled_pipes: if True or len(filled_pipes) >= self.batch_size: # 1 if send_batches is True: planes_batch = [] pipes_pred_output = [] for pipe in filled_pipes: #[:self.batch_size]: while pipe.poll(): planes_batch.append(pipe.recv()) pipes_pred_output.append(pipe) # logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes))) state_planes_mxnet = mx.nd.array( planes_batch, ctx=self.net.get_ctx()) else: planes_ids = [] pipes_pred_output = [] for pipe in filled_pipes: #[:self.batch_size]: while pipe.poll(): planes_ids.append(pipe.recv()) pipes_pred_output.append(pipe) #logging.debug('planes_batch length: %d %d' % (len(planes_batch), len(filled_pipes))) state_planes_mxnet = mx.nd.array( self.batch_state_planes[planes_ids], ctx=self.net.get_ctx()) #pred = self.net.get_executor().forward(is_train=False, data=state_planes_mxnet) pred = self.net.get_net()(state_planes_mxnet) #print('pred: %.3f' % (time()-t_s)*1000) #t_s = time() value_preds = pred[0].asnumpy() # for the policy prediction we still have to apply the softmax activation # because it's not done by the neural net #policy_preds = pred[1].softmax().asnumpy() policy_preds = pred[1].softmax().asnumpy() #if use_random is True: # value_preds = np.random.random(len(filled_pipes)) # policy_preds = np.random.random((len(filled_pipes), NB_LABELS)) # send the predictions back to the according workers for i, pipe in enumerate(pipes_pred_output): if send_batches is True: pipe.send([value_preds[i], policy_preds[i]]) else: # get the according channel index for setting the result channel_idx = planes_ids[i] # set the value result self.batch_value_results[ channel_idx] = value_preds[i] self.batch_policy_results[ channel_idx] = policy_preds[i] # give the thread the signal that the result has been set by sending back his channel_idx pipe.send(channel_idx)
def upload_results_to_s3(self): """ Run a separate process for each file that is to be uploaded to s3 :return: """ try: file_lists = upload_file_utils.get_files_for_multiprocessing( path=self.path) num_files = sum(len(s) for s in file_lists) logging.debug("Num files found {}".format(num_files)) kwargs = { 'bucket': self.bucket, 'key': self.key, 'compression': self.compression, 'max_uncompressed': self.max_uncompressed, 'max_cpu_usage': self.max_cpu_usage, 'storage': self.storage, 'temp_dir': self.temp_path } i = 0 # While there is anything to process keep going while any(file_lists) or any(self.gzip_file_list) or any( self.running_processes): """ Check the gzip list first because gzipping is the longest running process so we want to run as many gzip processes as we can at a time Otherwise we are keeping track of the order of processes """ for gzip_file in self.gzip_file_list: f_size = gzip_file['file_size'] if self.can_add_gzip_process(f_size=f_size): self.gzip_file_list.remove(gzip_file) process, parent_conn = self.add_gzip_upload_process( args=gzip_file, kwargs=kwargs) self.running_processes[parent_conn] = { 'process': process, 'size': f_size } """ Check all the processes to see if any have reported back that they are finished If gzip is finished we want to release the memory we are tracking in order to allow more gzip processes to run """ if any(self.running_processes): keys = list(self.running_processes.keys()) while len(keys) > 0: # there is a maximum of 63 elements in a list for the wait call wait_keys = keys[0:min(len(keys), 62)] keys = keys[min(len(keys), 62):] for listener in wait(wait_keys, timeout=0.01): try: while listener.poll(0.01): response = dict(listener.recv()) logging.info(response) if response[ 'action'] == const.CREATED_PART: self.gzip_file_list.append(response) elif response[ 'action'] == const.GZIPPED_PART: self.gzip_running_size -= response[ 'file_size'] elif response[ 'action'] == const.FINISHED_CREATING_PARTS: self.file_running_size -= min( response['file_size'], self.max_uncompressed) elif response[ 'action'] == const.UPLOADED_PART: pass else: logging.error( "UNKNOWN RESPONSE {}".format( response)) except (EOFError, BrokenPipeError): pass proc_dict = self.running_processes.get(listener) """ Kill dead processes """ if not proc_dict['process'].is_alive(): current_proc = self.running_processes.pop( listener)['process'] current_proc.join() current_proc.terminate() if any(file_lists): f_size = file_lists[i][-1].size if self.can_add_file_process(f_size=f_size): file_to_upload = file_lists[i].pop() process, parent_conn = self.add_divide_file_process( file_to_upload=file_to_upload, kwargs=kwargs) self.running_processes[parent_conn] = { 'process': process, 'size': f_size } if not any(file_lists[i]): file_lists.remove(file_lists[i]) i = i + 1 if i + 1 < len(file_lists) else 0 if not any(self.running_processes) and not any( file_lists) and not any(self.gzip_file_list): logging.debug("All processes finished") except (FileExistsError, FloatingPointError) as e: raise e except: raise Exception(sys.exc_info())
def mws_proc(self, **kwargs): """ Continuously runs its MWS function until the previous MWS function reports that it's all done. When the previous MWS function (or Walmart's Routine, in the case of gmpfId) is done, this function will update its queue with the rest of the items, run through all those until they're complete, and then report itself as all done. """ for key, value in kwargs.items(): if key == 'op': op = value if key == 'func': func = value if key == 'ammo': ammo = value if key == 'triggs': triggs = value # if op == 'gmpfId': # print ("Asd") # else: # time.sleep(999999) if type(self.qDefs[op]['qry']) is str: isQry = True q = deque() # A dict with keys equal to the keys in triggs['recv'], and all values False finishUps = {recvEnd: False for recvEnd in triggs['recv']} elif type(self.qDefs[op]['qry']) in (list, tuple): isQry = False q = deque(self.qDefs[op]['qry']) finishUps = {'bluhhuuuuurrrpp': True} else: print("Invalid value for self.qDefs[op]['qry'] in routine.Routine.mws_proc") return funcName = func.__name__ throtGap = 1.1 qGap = 8 while True: prev = time.time() finished = all(q for q in finishUps.values()) # Will be T if all values in finishUps are T, else F # If we don't have enough items for a full call, check to see if more items are ready if len(q) < self.throt[op]['maxpercall']: if isQry: q = self.fill_q(op, q) if finished and len(q) == 0: print("{} signing off!".format(funcName)) for sendEnd in triggs['send']: triggs['send'][sendEnd].send('') triggs['send'][sendEnd].close() break # Enough items in q to warrant an MWS call (which is anything > 0 if <finished> is True) if len(q) >= (1 - finished) * self.throt[op]['minpercall']: num = min(len(q), self.throt[op]['maxpercall']) if ammo['val'].value >= num: # Not gonna get throttled margs = [q.pop() for _ in range(num)] if margs: if len(margs[0]) == 1: # Convert a list of single-length tuples/lists to just a list margs = [elem[0] for elem in margs] # Run the mwsutils function print("{} - starting".format(funcName)) if op == 'gmpfId': func('Walmart', 'upc', margs) elif op == 'gcpfAsin': func(margs) elif op == 'glolfAsin': func(margs) elif op == 'gmfe': func(margs) elif op == 'lis': func(margs) print("{} - leaving".format(funcName)) with ammo['lock']: ammo['val'].value -= len(margs) else: time.sleep(max(prev + throtGap - time.time(), 0)) else: # The only way I could find to trip a condition after a message is sent down a pipe in another process. # https://docs.python.org/3/library/multiprocessing.html#multiprocessing.connection.wait # Checks each receive Pipe end for a message. <finishedUps> records which receive Pipe ends have gotten # their message. for recvEnd in triggs['recv']: if wait([triggs['recv'][recvEnd]], 0.1) and not finishUps[recvEnd]: try: _ = triggs['recv'][recvEnd].recv() finishUps[recvEnd] = True except EOFError: print("{} giving EOFError".format(op)) finishUps[recvEnd] = True time.sleep(max(prev + qGap - time.time(), 0))
def solve_subproblems(network_name, subproblems, property_path, runtimes, tree_states, init_to=0, num_workers=16, online_split=1, to_factor=1.5, strategy=3, input_name="input:0", seed=4, log_file=None): """ Solve the given set of subqueries on the network using divide-and-conquer algorithm Args: network_name (str) Name of the network subproblems (list of pairs) A list of {query-timeout} pairs property_path (str) Path to the property file runtimes (dict) Dictionary that maps sub-query's id to its runtime tree_states (dict) Dictionary that maps sub-query's id to its tree states init_to (int) Initial timeout num_workers (int) Number of worker threads to create online_split (int) Number of times to bisect a subquery when timeout occurs to_factor (float) The timeout factor strategy (int) Heuristic used to decide the influence of an input interval input_name (str) Name of the input variable in the .pb representation of the network seed (int) Random seed log_file (str) If not None, log the stats of each subproblem while solving Return: Empty list if UNSAT A list containing a satisfying assignment if SAT """ inputs = [] for p in subproblems: inputs.append((p, init_to)) random.Random(seed).shuffle(inputs) # Create LIFO Queue for each worker L = [] for i in range(num_workers): L.append(mp.Manager().list()) for i, input_ in enumerate(inputs): L[i % num_workers].append(input_) # Create and initiate workers num_tasks = Value("i", len(subproblems)) processes = [] readers = [] return_lst = [] lock = Lock() for i in range(num_workers): r, w = Pipe(duplex=True) readers.append(r) p = mp.Process(target=worker, args=(network_name, property_path, L, num_tasks, online_split, to_factor, log_file, w, lock, i, strategy, input_name)) processes.append(p) p.daemon = True p.start() w.close() # Wait for results while readers: for r in wait(readers): try: val_, results = r.recv() return_lst.append(results) if len(val_) > 0: for results in return_lst: for result in results: val, t_id, runtime, has_TO, num_tree_states = result if has_TO: runtimes[t_id] = -runtime else: runtimes[t_id] = runtime tree_states[t_id] = num_tree_states with num_tasks.get_lock(): num_tasks.value = 0 for p in processes: while p.is_alive(): subprocess.run(["kill", "-9", str(p.pid)]) return val_ except EOFError: readers.remove(r) for p in processes: p.join() for results in return_lst: for result in results: val, t_id, runtime, has_TO, num_tree_states = result if has_TO: runtimes[t_id] = -runtime else: runtimes[t_id] = runtime tree_states[t_id] = num_tree_states return []
def _predict(self): """ Method to receive game histories from several processes in order to prepare, predict and eventually dispatch policy vectors to the process associated with the data. Note: Also checks the cache if the FEN has been infered before. If yes, respective policy vector will get returned immediately. """ logging.basicConfig(filename='ModelApiLog.log', level=logging.DEBUG) breaking = False while not self.end: # listen and wait for incoming data ready = connection.wait(self.pipes, timeout=self.config.timeout_listen) if not ready: continue recv_data, result_pipes, recv_fen = [], [], [] # collect data from pipes for pipe in ready: if breaking: break while pipe.poll(): try: recv_data.append(pipe.recv()) result_pipes.append(pipe) except Exception as exc: logging.exception("pipe recv error, exception: " + str(exc)) breaking = True break # make fens from game histories recv_fen = self.make_fen(recv_data) # check the cache to which incoming state has already been predicted # if all prediction requests got answered, continue if len(recv_data) == 0: continue # image preparation data = self.make_image(recv_data) data_images = np.asarray(data, dtype=np.float32) # predict either with TensorRT model or tf.keras.model if self.config.use_tensorRT: value, policy = self._infer_trt(data_images) else: value, policy = self.model.predict_on_batch(data_images) for pipe, val, pol, fen in zip(result_pipes, value, policy, recv_fen): # send back value and policy vectors try: pipe.send((val, pol)) except Exception as exc: logging.exception("pipe send error, exception: " + str(exc)) break # write value and policy vectors of associated game state to the cache for pipe_to_close in self.pipes: pipe_to_close.close()
def _discard_rollout_promises(self, train_type): ready_promises = wait(self.rollout_promise_list[train_type], timeout=0) for rollout_promise in ready_promises: self._free_rollout_promise(rollout_promise)
def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, result_queue): """Manages the communication between this process and the worker processes. This function is run in a local thread. Args: executor_reference: A weakref.ref to the ProcessPoolExecutor that owns this thread. Used to determine if the ProcessPoolExecutor has been garbage collected and that this function can exit. process: A list of the multiprocessing.Process instances used as workers. pending_work_items: A dict mapping work ids to _WorkItems e.g. {5: <_WorkItem...>, 6: <_WorkItem...>, ...} work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]). call_queue: A multiprocessing.Queue that will be filled with _CallItems derived from _WorkItems for processing by the process workers. result_queue: A multiprocessing.Queue of _ResultItems generated by the process workers. """ executor = None def shutting_down(): return _shutdown or executor is None or executor._shutdown_thread def shutdown_worker(): # This is an upper bound nb_children_alive = sum(p.is_alive() for p in processes.values()) for i in range(0, nb_children_alive): call_queue.put_nowait(None) # Release the queue's resources as soon as possible. call_queue.close() # If .join() is not called on the created processes then # some multiprocessing.Queue methods may deadlock on Mac OS X. for p in processes.values(): p.join() reader = result_queue._reader while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) sentinels = [p.sentinel for p in processes.values()] assert sentinels ready = wait([reader] + sentinels) if reader in ready: result_item = reader.recv() else: # Mark the process pool broken so that submits fail right now. executor = executor_reference() if executor is not None: executor._broken = True executor._shutdown_thread = True executor = None # All futures in flight must be marked failed for work_id, work_item in pending_work_items.items(): work_item.future.set_exception( BrokenProcessPool( "A process in the process pool was " "terminated abruptly while the future was " "running or pending." )) # Delete references to object. See issue16284 del work_item pending_work_items.clear() # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. for p in processes.values(): p.terminate() shutdown_worker() return if isinstance(result_item, int): # Clean shutdown of a worker using its PID # (avoids marking the executor broken) assert shutting_down() p = processes.pop(result_item) p.join() if not processes: shutdown_worker() return elif result_item is not None: work_item = pending_work_items.pop(result_item.work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) # Delete references to object. See issue16284 del work_item # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this worker has been collected OR # - The executor that owns this worker has been shutdown. if shutting_down(): try: # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not pending_work_items: shutdown_worker() return except Full: # This is not a problem: we will eventually be woken up (in # result_queue.get()) and be able to send a sentinel again. pass executor = None
def _queue_management_worker(executor_reference, _manager, pending_work_items, work_ids_queue, _call_queue, _result_queue): """Manages the communication between this process and the worker processes.""" executor = None def shutting_down(): return process._shutdown or executor is None or executor._shutdown_thread def shutdown_worker(): # This is an upper bound if _manager.is_alive(): _call_queue.put_nowait(None) # Release the queue's resources as soon as possible. _call_queue.close() # If .join() is not called on the created processes then # some multiprocessing.Queue methods may deadlock on Mac OS X. _manager.join() reader = _result_queue._reader while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, _call_queue) sentinel = _manager.sentinel assert sentinel ready = wait([reader, sentinel]) if reader in ready: result_item = reader.recv() else: # Mark the process pool broken so that submits fail right now. executor = executor_reference() if executor is not None: executor._broken = True executor._shutdown_thread = True executor = None # All futures in flight must be marked failed for work_id, work_item in pending_work_items.items(): work_item.future.set_exception( BrokenProcessPool( "A process in the process pool was " "terminated abruptly while the future was " "running or pending.")) # Delete references to object. See issue16284 del work_item # pending_work_items.clear() del pending_work_items[:] # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. _manager.terminate() shutdown_worker() return if isinstance(result_item, int): # Clean shutdown of a worker using its PID # (avoids marking the executor broken) assert shutting_down() _manager.join() if _manager is None: shutdown_worker() return elif result_item is not None: work_item = pending_work_items.pop(result_item.work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) # Delete references to object. See issue16284 del work_item # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this worker has been collected OR # - The executor that owns this worker has been shutdown. if shutting_down(): try: # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not pending_work_items: shutdown_worker() return except queue.Full: # This is not a problem: we will eventually be woken up (in # _result_queue.get()) and be able to send a sentinel again. pass executor = None
def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, result_queue): """Manages the communication between this process and the worker processes. This function is run in a local thread. Args: executor_reference: A weakref.ref to the ProcessPoolExecutor that owns this thread. Used to determine if the ProcessPoolExecutor has been garbage collected and that this function can exit. process: A list of the multiprocessing.Process instances used as workers. pending_work_items: A dict mapping work ids to _WorkItems e.g. {5: <_WorkItem...>, 6: <_WorkItem...>, ...} work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]). call_queue: A multiprocessing.Queue that will be filled with _CallItems derived from _WorkItems for processing by the process workers. result_queue: A multiprocessing.Queue of _ResultItems generated by the process workers. """ executor = None def shutting_down(): return _shutdown or executor is None or executor._shutdown_thread def shutdown_worker(): # This is an upper bound nb_children_alive = sum(p.is_alive() for p in processes.values()) for i in range(0, nb_children_alive): call_queue.put_nowait(None) # Release the queue's resources as soon as possible. call_queue.close() # If .join() is not called on the created processes then # some multiprocessing.Queue methods may deadlock on Mac OS X. for p in processes.values(): p.join() reader = result_queue._reader while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) sentinels = [p.sentinel for p in processes.values()] assert sentinels ready = wait([reader] + sentinels) if reader in ready: result_item = reader.recv() else: # Mark the process pool broken so that submits fail right now. executor = executor_reference() if executor is not None: executor._broken = True executor._shutdown_thread = True executor = None # All futures in flight must be marked failed for work_id, work_item in pending_work_items.items(): work_item.future.set_exception( BrokenProcessPool( "A process in the process pool was " "terminated abruptly while the future was " "running or pending.")) # Delete references to object. See issue16284 del work_item pending_work_items.clear() # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. for p in processes.values(): p.terminate() shutdown_worker() return if isinstance(result_item, int): # Clean shutdown of a worker using its PID # (avoids marking the executor broken) assert shutting_down() p = processes.pop(result_item) p.join() if not processes: shutdown_worker() return elif result_item is not None: work_item = pending_work_items.pop(result_item.work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) # Delete references to object. See issue16284 del work_item # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this worker has been collected OR # - The executor that owns this worker has been shutdown. if shutting_down(): try: # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not pending_work_items: shutdown_worker() return except Full: # This is not a problem: we will eventually be woken up (in # result_queue.get()) and be able to send a sentinel again. pass executor = None
for i in range(10): w.send((i,mp.current_process().name)) print("send:",i,mp.current_process().name) w.close() if __name__ == "__main__": readers = [] for i in range(4): r,w = mp.Pipe(duplex=False) readers.append(r) p = mp.Process(target=foo,args=(w,)) p.start() w.close() while readers: for r in connection.wait(readers): print('recv:',r) try: msg = r.recv() except EOFError: print(r," is done") readers.remove(r) else: print(msg) def test2(): address = ('localhost',6000) with connection.Client(address,authkey=b'123') as client: print(client.recv())