async def iomain(window: MyGame, loop): ctx = Context() sub_sock: Socket = ctx.socket(zmq.SUB) sub_sock.connect('tcp://localhost:25000') sub_sock.subscribe('') # Required for PUB+SUB push_sock: Socket = ctx.socket(zmq.PUSH) push_sock.connect('tcp://localhost:25001') async def send_player_input(): """ Task A """ while True: d = asdict(window.player_input) msg = dict(event=d) await push_sock.send_json(msg) await asyncio.sleep(1 / UPDATE_TICK) async def receive_game_state(): """ Task B """ while True: gs_string = await sub_sock.recv_string() window.game_state.from_json(gs_string) ps = window.game_state.player_states[0] window.position_buffer.append( (Vec2d(ps.x, ps.y), time.time()) ) window.time_since_state_update = 0 window.player_position_snapshot = copy.copy(window.player.position) try: await asyncio.gather(send_player_input(), receive_game_state()) finally: sub_sock.close(1) push_sock.close(1) ctx.destroy(linger=1)
async def main(): fut = asyncio.Future() # IGNORE! app = App(signal=fut) # IGNORE! gs = GameState(player_states=[PlayerState(speed=150)]) ctx = Context() # "Task A" (ZeroMQ) sock_B: Socket = ctx.socket(zmq.PULL) sock_B.bind('tcp://*:25001') task_B = create_task(update_from_client(gs, sock_B)) sock_C: Socket = ctx.socket(zmq.PUB) sock_C.bind('tcp://*:25000') task_C = create_task(push_game_state(gs, sock_C)) try: await asyncio.wait([task_B, task_C, fut], return_when=asyncio.FIRST_COMPLETED) except CancelledError: print('Cancelled') finally: sock_B.close(1) sock_C.close(1) ctx.destroy(linger=1)
async def main(): fut = asyncio.Future() app = App(signal=fut) ctx = Context() sock_push_gamestate: Socket = ctx.socket(zmq.PUB) sock_push_gamestate.bind('tcp://*:25000') sock_recv_player_evts: Socket = ctx.socket(zmq.PULL) sock_recv_player_evts.bind('tcp://*:25001') ticker_task = asyncio.create_task( ticker(sock_push_gamestate, sock_recv_player_evts), ) try: await asyncio.wait( [ticker_task, fut], return_when=asyncio.FIRST_COMPLETED ) except CancelledError: print('Cancelled') finally: ticker_task.cancel() await ticker_task sock_push_gamestate.close(1) sock_recv_player_evts.close(1) ctx.destroy(linger=1000)
async def thread_main(window: MyGame, loop): ctx = Context() sub_sock: Socket = ctx.socket(zmq.SUB) sub_sock.connect('tcp://localhost:25000') sub_sock.subscribe('') push_sock: Socket = ctx.socket(zmq.PUSH) push_sock.connect('tcp://localhost:25001') async def pusher(): """Push the player's INPUT state 60 times per second""" while True: d = window.player_event.asdict() msg = dict(counter=1, event=d) await push_sock.send_json(msg) await asyncio.sleep(1 / UPDATE_TICK) async def receive_game_state(): while True: gs_string = await sub_sock.recv_string() # logger.debug('.', end='', flush=True) window.game_state.from_json(gs_string) ps = window.game_state.player_states[0] t = time.time() window.position_buffer.append((Vec2d(ps.x, ps.y), t)) window.t = 0 window.player_position_snapshot = copy.copy(window.player.position) try: await asyncio.gather(pusher(), receive_game_state()) finally: sub_sock.close(1) push_sock.close(1) ctx.destroy(linger=1)
async def main(): ctx = Context() sock = ctx.socket(zmq.PUB) sock.bind('tcp://127.0.0.1:12345') try: await sender(sock) finally: sock.close(1) ctx.destroy()
async def main(): ctx = Context() sock = ctx.socket(zmq.SUB) sock.connect('tcp://127.0.0.1:12345') sock.subscribe(b'') try: await receiver(sock) finally: sock.close(1) ctx.destroy()
def main(): import sys name = sys.argv[1] store = StorageServerStore() context = Context() try: asyncio.run(storage_server(store, context, "storage+" + name)) except KeyboardInterrupt: context.destroy() print('') except BaseException as e: context.destroy() raise e
def main(): import sys command = sys.argv[1] arg = sys.argv[2] store = StorageServerStore() context = Context() try: asyncio.run(app(store, context, "app", command, arg)) except KeyboardInterrupt: context.destroy() print('') except BaseException as e: context.destroy() raise e
async def zmain(): logger.info(123) ctx = Context() logger.info(456) sock: Socket = ctx.socket(zmq.DEALER) logger.info(789) idd = settings.IDENTITY logger.info(idd) sock.identity = idd.encode() logger.info(345) connection_string = f'tcp://{settings.TARGET_SERVER_URL()}:{settings.TARGET_SERVER_PORT():d}' logger.info(f'Connecting to {connection_string}') sock.connect(connection_string) try: await process_messages(sock) except asyncio.CancelledError: pass finally: sock.close(1) ctx.destroy()
async def app(store: StorageServerStore, context: Context, name: str, command: str, arg: str): print("Starting...") dirserv_commands = context.socket(zmq.REQ) dirserv_commands.connect("tcp://127.0.0.1:5350") print("App is started") if command == "declare": self_addr = await asyncio.wait_for(ping(dirserv_commands, name), 5) print("Directory server report this client is run on {}".format( self_addr)) command_port: Socket = context.socket(zmq.ROUTER) port = command_port.bind_to_random_port("tcp://127.0.0.1") self_entrypoint_addr = "tcp://{}:{}".format(self_addr, port) await asyncio.wait_for( cast_address(dirserv_commands, name, self_entrypoint_addr), 5) print("Address {} casted on directory server".format( self_entrypoint_addr)) with open(arg, mode='rb') as f: store.files[arg] = VirtualFile(arg, f.read(), []) await declare_file(dirserv_commands, arg, name) print("File {} is declared, serving file...".format(arg)) poller = Poller() poller.register(command_port, zmq.POLLIN) while True: events: List[Tuple[Socket, int]] = await poller.poll() for socket, mark in events: frames: List[Frame] = await socket.recv_multipart(copy=False) id_frame = frames.pop(0) frames.pop(0) command_frame = frames.pop(0) command = str(command_frame.bytes, 'utf8') if socket == command_port: if command == 'fs.read_file': await read_file_handler(store, frames, socket, id_frame) elif command == "disown": await disown_file(dirserv_commands, arg, name) context.destroy() return elif command == "show": content = await download_file(context, dirserv_commands, arg) print("==== Content of '{}' ====".format(arg)) print(str(content, 'utf8')) context.destroy() return else: print("Unknown command {}".format(command)) context.destroy() return
class NodeCommunicator(object): LAST_MSG = None def __init__(self, peers_config, my_id, linger_timeout): self.peers_config = peers_config self.my_id = my_id self.bytes_sent = 0 self.benchmark_logger = logging.LoggerAdapter( logging.getLogger("benchmark_logger"), {"node_id": my_id} ) self._dealer_tasks = [] self._router_task = None self.linger_timeout = linger_timeout self.zmq_context = Context(io_threads=cpu_count()) n = len(peers_config) self._receiver_queue = asyncio.Queue() self._sender_queues = [None] * n for i in range(n): if i == self.my_id: self._sender_queues[i] = self._receiver_queue else: self._sender_queues[i] = asyncio.Queue() def send(self, node_id, msg): msg = (self.my_id, msg) if node_id == self.my_id else msg self._sender_queues[node_id].put_nowait(msg) async def recv(self): return await self._receiver_queue.get() async def __aenter__(self): await self._setup() return self async def __aexit__(self, exc_type, exc, tb): # Add None to the sender queues and drain out all the messages. for i in range(len(self._sender_queues)): if i != self.my_id: self._sender_queues[i].put_nowait(NodeCommunicator.LAST_MSG) await asyncio.gather(*self._dealer_tasks) logging.debug("Dealer tasks finished.") self._router_task.cancel() logging.debug("Router task cancelled.") self.zmq_context.destroy(linger=self.linger_timeout * 1000) self.benchmark_logger.info("Total bytes sent out: %d", self.bytes_sent) async def _setup(self): # Setup one router for a party, this acts as a # server for receiving messages from other parties. router = self.zmq_context.socket(ROUTER) router.bind(f"tcp://*:{self.peers_config[self.my_id].port}") # Start a task to receive messages on this node. self._router_task = asyncio.create_task(self._recv_loop(router)) self._router_task.add_done_callback(print_exception_callback) # Setup one dealer per receving party. This is used # as a client to send messages to other parties. for i in range(len(self.peers_config)): if i != self.my_id: dealer = self.zmq_context.socket(DEALER) # This identity is sent with each message. Setting it to my_id, this is # used to appropriately route the message. This is not a good idea since # a node can pretend to send messages on behalf of other nodes. dealer.setsockopt(IDENTITY, str(self.my_id).encode()) dealer.connect( f"tcp://{self.peers_config[i].ip}:{self.peers_config[i].port}" ) # Setup a task which reads messages intended for this # party from a queue and then sends them to this node. task = asyncio.create_task( self._process_node_messages( i, self._sender_queues[i], dealer.send_multipart ) ) self._dealer_tasks.append(task) async def _recv_loop(self, router): while True: sender_id, raw_msg = await router.recv_multipart() msg = loads(raw_msg) # logging.debug("[RECV] FROM: %s, MSG: %s,", sender_id, msg) self._receiver_queue.put_nowait((int(sender_id), msg)) async def _process_node_messages(self, node_id, node_msg_queue, send_to_node): while True: msg = await node_msg_queue.get() if msg is NodeCommunicator.LAST_MSG: logging.debug("No more messages to Node: %d can be sent.", node_id) break raw_msg = dumps(msg) self.bytes_sent += len(raw_msg) # logging.debug("[SEND] TO: %d, MSG: %s", node_id, msg) await send_to_node([raw_msg])
for task in asyncio.Task.all_tasks(loop): task.cancel() def signal_cancel_tasks(*args): loop = asyncio.get_event_loop() for task in asyncio.Task.all_tasks(loop): task.cancel() if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') loop = asyncio.get_event_loop() # not implemented in Windows: # loop.add_signal_handler(signal.SIGINT, cancel_tasks, loop) # loop.add_signal_handler(signal.SIGTERM, cancel_tasks, loop) # alternative signal.signal(signal.SIGINT, signal_cancel_tasks) signal.signal(signal.SIGTERM, signal_cancel_tasks) context = Context() #zmq.asyncio.Context() try: loop.run_until_complete(main(context)) except Exception as ex: logging.error("Base interruption: {}".format(ex)) traceback.print_tb(ex.__traceback__) finally: loop.close() context.destroy()
def test_query(): responses = [] dbname = '34feba8fb61144dfb5cb9d4ecdd08683' sql = 'SELECT * from frame_posting WHERE frame_id = 1' url = f'http://127.0.0.1:8080/{dbname}/{parse.quote_plus(sql)}' printl(url) payload = json.dumps(dict(dbname=dbname, query=sql)) print(1) ctx = Context() print(2) s: Socket = ctx.socket(zmq.ROUTER) s.setsockopt(zmq.ROUTER_MANDATORY, 1) print(3) port = s.bind_to_random_port(addr='tcp://*') print(f'Bound to port: {port}') async def test(): printl(time.perf_counter()) msg = payload async def receiver(): while True: try: print(f'Waiting for reply...') m = await asyncio.wait_for(s.recv_multipart(), .01) break except asyncio.TimeoutError: print('timed out waiting') continue print('got message back') responses.append(m[1].decode()) print(m) return async with server(): t = asyncio.create_task(receiver()) print(f'Sending string: {msg}') while True: try: print('trying to send...') await s.send_multipart([b'NO_IDENTITY', msg.encode()]) break except Exception as e: print(e) await asyncio.sleep(0.1) continue await t with biodome.env_change('TARGET_SERVER_PORT', port), \ biodome.env_change('TARGET_SERVER_URL', '127.0.0.1'): asyncio.run(test()) s.close(1) ctx.destroy() printl(responses) print('leaving test!') assert responses[0] == ( '[(1, 0, 194), (1, 1, 1101), (1, 2, 12), (1, 3, 23), (1, 4, 247), (1, 5, 2), ' '(1, 6, 94), (1, 7, 196), (1, 8, 56), (1, 9, 9), (1, 10, 127), (1, 11, 13), ' '(1, 12, 11), (1, 13, 4), (1, 14, 508), (1, 15, 181), (1, 16, 314), (1, 17, ' '19), (1, 18, 144), (1, 19, 35), (1, 20, 128), (1, 21, 75), (1, 22, 15), (1, ' '23, 1), (1, 24, 426), (1, 25, 2), (1, 26, 160), (1, 27, 102), (1, 28, 9), ' '(1, 29, 245), (1, 30, 3817), (1, 31, 1), (1, 32, 276), (1, 33, 4), (1, 34, ' '23), (1, 35, 98), (1, 36, 940), (1, 37, 2), (1, 38, 1), (1, 39, 57), (1, 40, ' '4), (1, 41, 1734), (1, 42, 1344), (1, 43, 4), (1, 44, 18), (1, 45, 75), (1, ' '46, 3), (1, 47, 17), (1, 48, 350), (1, 49, 177), (1, 50, 313), (1, 51, 1), ' '(1, 52, 2117), (1, 53, 20), (1, 54, 4), (1, 55, 198), (1, 56, 890), (1, 57, ' '3), (1, 58, 290), (1, 59, 4), (1, 60, 92), (1, 61, 2), (1, 62, 774), (1, 63, ' '13), (1, 64, 36), (1, 65, 56), (1, 66, 80), (1, 67, 39), (1, 68, 92), (1, ' '69, 521), (1, 70, 2), (1, 71, 6), (1, 72, 1276), (1, 73, 5099), (1, 74, ' '389), (1, 75, 31), (1, 76, 199), (1, 77, 23), (1, 78, 30), (1, 79, 593), (1, ' '80, 8), (1, 81, 5740)]' )