def test_profiling_info_endpoint(shutdown_only):
ray.init(num_cpus=1)
redis_client = ray.worker.global_worker.redis_client
node_ip = ray.nodes()[0]["NodeManagerAddress"]
while True:
reporter_port = redis_client.get("REPORTER_PORT:{}".format(node_ip))
if reporter_port:
break
reporter_channel = grpc.insecure_channel("{}:{}".format(
node_ip, int(reporter_port)))
reporter_stub = reporter_pb2_grpc.ReporterServiceStub(reporter_channel)
@ray.remote(num_cpus=1)
class ActorA:
def __init__(self):
pass
def getpid(self):
return os.getpid()
a = ActorA.remote()
actor_pid = ray.get(a.getpid.remote())
reply = reporter_stub.GetProfilingStats(
reporter_pb2.GetProfilingStatsRequest(pid=actor_pid, duration=10))
profiling_stats = json.loads(reply.profiling_stats)
assert profiling_stats is not None
async def _update_stubs(self, change):
if change.new:
ip, port = next(iter(change.new.items()))
channel = aiogrpc.insecure_channel("{}:{}".format(ip, int(port)))
stub = reporter_pb2_grpc.ReporterServiceStub(channel)
self._stubs[ip] = stub
if change.old:
ip, port = next(iter(change.old.items()))
self._stubs.pop(ip)
async def _update_stubs(self, change):
if change.old:
ip, port = change.old
self._stubs.pop(ip)
if change.new:
ip, ports = change.new
channel = aiogrpc.insecure_channel(f"{ip}:{ports[1]}")
stub = reporter_pb2_grpc.ReporterServiceStub(channel)
self._stubs[ip] = stub
async def _update_stubs(self, change):
if change.old:
node_id, port = change.old
ip = DataSource.node_id_to_ip[node_id]
self._stubs.pop(ip)
if change.new:
node_id, ports = change.new
ip = DataSource.node_id_to_ip[node_id]
channel = aiogrpc.insecure_channel(f"{ip}:{ports[1]}")
stub = reporter_pb2_grpc.ReporterServiceStub(channel)
self._stubs[ip] = stub
async def _update_stubs(self, change):
if change.old:
node_id, port = change.old
ip = DataSource.node_id_to_ip[node_id]
self._stubs.pop(ip)
if change.new:
node_id, ports = change.new
ip = DataSource.node_id_to_ip[node_id]
options = (("grpc.enable_http_proxy", 0), )
channel = aiogrpc.insecure_channel(
f"{ip}:{ports[1]}", options=options)
stub = reporter_pb2_grpc.ReporterServiceStub(channel)
self._stubs[ip] = stub
async def _update_stubs(self, change):
if change.old:
node_id, port = change.old
ip = DataSource.node_id_to_ip[node_id]
self._stubs.pop(ip)
if change.new:
node_id, ports = change.new
ip = DataSource.node_id_to_ip[node_id]
options = (("grpc.enable_http_proxy", 0), )
channel = ray._private.utils.init_grpc_channel(
f"{ip}:{ports[1]}", options=options, asynchronous=True)
stub = reporter_pb2_grpc.ReporterServiceStub(channel)
self._stubs[ip] = stub
def _update_nodes(self):
with self.nodes_lock:
self.nodes = ray.nodes()
node_ids = [node["NodeID"] for node in self.nodes]
# First remove node connections of disconnected nodes.
for node_id in self.stubs.keys():
if node_id not in node_ids:
stub = self.stubs.pop(node_id)
stub.close()
reporter_stub = self.reporter_stubs.pop(node_id)
reporter_stub.close()
# Now add node connections of new nodes.
for node in self.nodes:
node_id = node["NodeID"]
if node_id not in self.stubs:
node_ip = node["NodeManagerAddress"]
channel = grpc.insecure_channel("{}:{}".format(
node_ip, node["NodeManagerPort"]))
stub = node_manager_pb2_grpc.NodeManagerServiceStub(
channel)
self.stubs[node_id] = stub
# Block wait until the reporter for the node starts.
while True:
reporter_port = self.redis_client.get(
"REPORTER_PORT:{}".format(node_ip))
if reporter_port:
break
reporter_channel = grpc.insecure_channel("{}:{}".format(
node_ip, int(reporter_port)))
reporter_stub = reporter_pb2_grpc.ReporterServiceStub(
reporter_channel)
self.reporter_stubs[node_id] = reporter_stub
assert len(self.stubs) == len(
self.reporter_stubs), (self.stubs.keys(),
self.reporter_stubs.keys())