def test_profiling_info_endpoint(shutdown_only): ray.init(num_cpus=1) redis_client = ray.worker.global_worker.redis_client node_ip = ray.nodes()[0]["NodeManagerAddress"] while True: reporter_port = redis_client.get("REPORTER_PORT:{}".format(node_ip)) if reporter_port: break reporter_channel = grpc.insecure_channel("{}:{}".format( node_ip, int(reporter_port))) reporter_stub = reporter_pb2_grpc.ReporterServiceStub(reporter_channel) @ray.remote(num_cpus=1) class ActorA: def __init__(self): pass def getpid(self): return os.getpid() a = ActorA.remote() actor_pid = ray.get(a.getpid.remote()) reply = reporter_stub.GetProfilingStats( reporter_pb2.GetProfilingStatsRequest(pid=actor_pid, duration=10)) profiling_stats = json.loads(reply.profiling_stats) assert profiling_stats is not None
async def _update_stubs(self, change): if change.new: ip, port = next(iter(change.new.items())) channel = aiogrpc.insecure_channel("{}:{}".format(ip, int(port))) stub = reporter_pb2_grpc.ReporterServiceStub(channel) self._stubs[ip] = stub if change.old: ip, port = next(iter(change.old.items())) self._stubs.pop(ip)
async def _update_stubs(self, change): if change.old: ip, port = change.old self._stubs.pop(ip) if change.new: ip, ports = change.new channel = aiogrpc.insecure_channel(f"{ip}:{ports[1]}") stub = reporter_pb2_grpc.ReporterServiceStub(channel) self._stubs[ip] = stub
async def _update_stubs(self, change): if change.old: node_id, port = change.old ip = DataSource.node_id_to_ip[node_id] self._stubs.pop(ip) if change.new: node_id, ports = change.new ip = DataSource.node_id_to_ip[node_id] channel = aiogrpc.insecure_channel(f"{ip}:{ports[1]}") stub = reporter_pb2_grpc.ReporterServiceStub(channel) self._stubs[ip] = stub
async def _update_stubs(self, change): if change.old: node_id, port = change.old ip = DataSource.node_id_to_ip[node_id] self._stubs.pop(ip) if change.new: node_id, ports = change.new ip = DataSource.node_id_to_ip[node_id] options = (("grpc.enable_http_proxy", 0), ) channel = aiogrpc.insecure_channel( f"{ip}:{ports[1]}", options=options) stub = reporter_pb2_grpc.ReporterServiceStub(channel) self._stubs[ip] = stub
async def _update_stubs(self, change): if change.old: node_id, port = change.old ip = DataSource.node_id_to_ip[node_id] self._stubs.pop(ip) if change.new: node_id, ports = change.new ip = DataSource.node_id_to_ip[node_id] options = (("grpc.enable_http_proxy", 0), ) channel = ray._private.utils.init_grpc_channel( f"{ip}:{ports[1]}", options=options, asynchronous=True) stub = reporter_pb2_grpc.ReporterServiceStub(channel) self._stubs[ip] = stub
def _update_nodes(self): with self.nodes_lock: self.nodes = ray.nodes() node_ids = [node["NodeID"] for node in self.nodes] # First remove node connections of disconnected nodes. for node_id in self.stubs.keys(): if node_id not in node_ids: stub = self.stubs.pop(node_id) stub.close() reporter_stub = self.reporter_stubs.pop(node_id) reporter_stub.close() # Now add node connections of new nodes. for node in self.nodes: node_id = node["NodeID"] if node_id not in self.stubs: node_ip = node["NodeManagerAddress"] channel = grpc.insecure_channel("{}:{}".format( node_ip, node["NodeManagerPort"])) stub = node_manager_pb2_grpc.NodeManagerServiceStub( channel) self.stubs[node_id] = stub # Block wait until the reporter for the node starts. while True: reporter_port = self.redis_client.get( "REPORTER_PORT:{}".format(node_ip)) if reporter_port: break reporter_channel = grpc.insecure_channel("{}:{}".format( node_ip, int(reporter_port))) reporter_stub = reporter_pb2_grpc.ReporterServiceStub( reporter_channel) self.reporter_stubs[node_id] = reporter_stub assert len(self.stubs) == len( self.reporter_stubs), (self.stubs.keys(), self.reporter_stubs.keys())