def test_global_state_actor_table(ray_start_regular):
@ray.remote
class Actor:
def ready(self):
pass
# actor table should be empty at first
assert len(ray.actors()) == 0
# actor table should contain only one entry
a = Actor.remote()
ray.get(a.ready.remote())
assert len(ray.actors()) == 1
# actor table should contain only this entry
# even when the actor goes out of scope
del a
def get_state():
return list(ray.actors().values())[0]["State"]
dead_state = ray.gcs_utils.ActorTableData.DEAD
for _ in range(10):
if get_state() == dead_state:
break
else:
time.sleep(0.5)
assert get_state() == dead_state
def test_global_state_api(shutdown_only):
error_message = ("The ray global state API cannot be used "
"before ray.init has been called.")
with pytest.raises(Exception, match=error_message):
ray.objects()
with pytest.raises(Exception, match=error_message):
ray.actors()
with pytest.raises(Exception, match=error_message):
ray.nodes()
with pytest.raises(Exception, match=error_message):
ray.jobs()
ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1})
assert ray.cluster_resources()["CPU"] == 5
assert ray.cluster_resources()["GPU"] == 3
assert ray.cluster_resources()["CustomResource"] == 1
assert ray.objects() == {}
job_id = ray.utils.compute_job_id_from_driver(
ray.WorkerID(ray.worker.global_worker.worker_id))
client_table = ray.nodes()
node_ip_address = ray.worker.global_worker.node_ip_address
assert len(client_table) == 1
assert client_table[0]["NodeManagerAddress"] == node_ip_address
@ray.remote
class Actor:
def __init__(self):
pass
_ = Actor.remote() # noqa: F841
# Wait for actor to be created
wait_for_num_actors(1)
actor_table = ray.actors()
assert len(actor_table) == 1
actor_info, = actor_table.values()
assert actor_info["JobID"] == job_id.hex()
assert "IPAddress" in actor_info["Address"]
assert "IPAddress" in actor_info["OwnerAddress"]
assert actor_info["Address"]["Port"] != actor_info["OwnerAddress"]["Port"]
job_table = ray.jobs()
assert len(job_table) == 1
assert job_table[0]["JobID"] == job_id.hex()
assert job_table[0]["NodeManagerAddress"] == node_ip_address
def wait_until_actor_dead(handle):
actor_status = ray.actors(actor_id=handle._actor_id.hex())
max_wait_time = 10
wait_time = 0
while actor_status["State"] != ray.gcs_utils.ActorTableData.DEAD:
actor_status = ray.actors(actor_id=handle._actor_id.hex())
time.sleep(1.0)
wait_time += 1
if wait_time >= max_wait_time:
assert None, (
"It took too much time to kill an actor: {}".format(
handle._actor_id))
def kill_actor_and_wait_for_failure(actor, timeout=10, retry_interval_ms=100):
actor_id = actor._actor_id.hex()
current_num_restarts = ray.actors(actor_id)["NumRestarts"]
ray.kill(actor)
start = time.time()
while time.time() - start <= timeout:
actor_status = ray.actors(actor_id)
if actor_status["State"] == ray.gcs_utils.ActorTableData.DEAD \
or actor_status["NumRestarts"] > current_num_restarts:
return
time.sleep(retry_interval_ms / 1000.0)
raise RuntimeError(
"It took too much time to kill an actor: {}".format(actor_id))
def test_http_head_only():
cluster = Cluster()
head_node = cluster.add_node(num_cpus=4)
cluster.add_node(num_cpus=4)
ray.init(head_node.address)
node_ids = ray.state.node_ids()
assert len(node_ids) == 2
client = serve.start(http_options={
"port": new_port(),
"location": "HeadOnly"
})
# Only the controller and head node actor should be started
assert len(ray.actors()) == 2
# They should all be placed on the head node
cpu_per_nodes = {
r["CPU"]
for r in ray.state.state._available_resources_per_node().values()
}
assert cpu_per_nodes == {2, 4}
client.shutdown()
ray.shutdown()
cluster.shutdown()
def wait_for_num_actors(num_actors, timeout=10):
start_time = time.time()
while time.time() - start_time < timeout:
if len(ray.actors()) >= num_actors:
return
time.sleep(0.1)
raise RayTestTimeoutException("Timed out while waiting for global state.")
def create_and_kill_actor(actor_name):
# Make sure same name is creatable after killing it.
detached_actor = DetachedActor.options(name=actor_name).remote()
# Wait for detached actor creation.
assert ray.get(detached_actor.ping.remote()) == "pong"
ray.kill(detached_actor)
# Wait until actor dies.
actor_status = ray.actors(actor_id=detached_actor._actor_id.hex())
max_wait_time = 10
wait_time = 0
while actor_status["State"] != 3:
actor_status = ray.actors(actor_id=detached_actor._actor_id.hex())
time.sleep(1.0)
wait_time += 1
if wait_time >= max_wait_time:
assert None, (
"It took too much time to kill an actor: {}".format(
detached_actor._actor_id))
def test_placement_group_strict_spread(ray_start_cluster):
@ray.remote(num_cpus=2)
class Actor(object):
def __init__(self):
self.n = 0
def value(self):
return self.n
cluster = ray_start_cluster
num_nodes = 3
for _ in range(num_nodes):
cluster.add_node(num_cpus=4)
ray.init(address=cluster.address)
placement_group = ray.util.placement_group(
name="name",
strategy="STRICT_SPREAD",
bundles=[{
"CPU": 2
}, {
"CPU": 2
}, {
"CPU": 2
}])
ray.get(placement_group.ready())
actor_1 = Actor.options(
placement_group=placement_group,
placement_group_bundle_index=0).remote()
actor_2 = Actor.options(
placement_group=placement_group,
placement_group_bundle_index=1).remote()
actor_3 = Actor.options(
placement_group=placement_group,
placement_group_bundle_index=2).remote()
ray.get(actor_1.value.remote())
ray.get(actor_2.value.remote())
ray.get(actor_3.value.remote())
# Get all actors.
actor_infos = ray.actors()
# Make sure all actors in counter_list are located in separate nodes.
actor_info_1 = actor_infos.get(actor_1._actor_id.hex())
actor_info_2 = actor_infos.get(actor_2._actor_id.hex())
actor_info_3 = actor_infos.get(actor_3._actor_id.hex())
assert actor_info_1 and actor_info_2 and actor_info_3
node_of_actor_1 = actor_info_1["Address"]["NodeID"]
node_of_actor_2 = actor_info_2["Address"]["NodeID"]
node_of_actor_3 = actor_info_3["Address"]["NodeID"]
assert node_of_actor_1 != node_of_actor_2
assert node_of_actor_1 != node_of_actor_3
assert node_of_actor_2 != node_of_actor_3
def wait_for_num_actors(num_actors, state=None, timeout=10):
start_time = time.time()
while time.time() - start_time < timeout:
if len([
_ for _ in ray.actors().values()
if state is None or _["State"] == state
]) >= num_actors:
return
time.sleep(0.1)
raise RayTestTimeoutException("Timed out while waiting for global state.")
def test_capture_child_tasks(ray_start_cluster):
cluster = ray_start_cluster
total_num_actors = 4
for _ in range(2):
cluster.add_node(num_cpus=total_num_actors)
ray.init(address=cluster.address)
pg = ray.util.placement_group([{
"CPU": 2
}, {
"CPU": 2
}],
strategy="STRICT_PACK")
ray.get(pg.ready(), timeout=5)
# If get_current_placement_group is used when the current worker/driver
# doesn't belong to any of placement group, it should return None.
assert get_current_placement_group() is None
@ray.remote(num_cpus=1)
class NestedActor:
def ready(self):
return True
@ray.remote(num_cpus=1)
class Actor:
def __init__(self):
self.actors = []
def ready(self):
return True
def schedule_nested_actor(self):
actor = NestedActor.options(
placement_group=get_current_placement_group()).remote()
ray.get(actor.ready.remote())
self.actors.append(actor)
a = Actor.options(placement_group=pg).remote()
ray.get(a.ready.remote())
# 1 top level actor + 3 children.
for _ in range(total_num_actors - 1):
ray.get(a.schedule_nested_actor.remote())
# Make sure all the actors are scheduled on the same node.
# (why? The placement group has STRICT_PACK strategy).
node_id_set = set()
for actor_info in ray.actors().values():
node_id = actor_info["Address"]["NodeID"]
node_id_set.add(node_id)
# Since all node id should be identical, set should be equal to 1.
assert len(node_id_set) == 1
def test_actor_creation(tmpdir, ray_start_2_cpus, num_workers):
"""Tests whether the appropriate number of training actors are created."""
model = BoringModel()
def check_num_actor():
assert len(ray.actors()) == num_workers
model.on_epoch_end = check_num_actor
plugin = RayPlugin(num_workers=num_workers)
trainer = get_trainer(tmpdir, plugins=[plugin])
trainer.fit(model)
assert all(actor["State"] == ray.gcs_utils.ActorTableData.DEAD
for actor in list(ray.actors().values()))
def test_global_state_api(shutdown_only):
ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1})
assert ray.cluster_resources()["CPU"] == 5
assert ray.cluster_resources()["GPU"] == 3
assert ray.cluster_resources()["CustomResource"] == 1
# A driver/worker creates a temporary object during startup. Although the
# temporary object is freed immediately, in a rare case, we can still find
# the object ref in GCS because Raylet removes the object ref from GCS
# asynchronously.
# Because we can't control when workers create the temporary objects, so
# We can't assert that `ray.objects()` returns an empty dict. Here we just
# make sure `ray.objects()` succeeds.
assert len(ray.objects()) >= 0
job_id = ray.utils.compute_job_id_from_driver(
ray.WorkerID(ray.worker.global_worker.worker_id))
client_table = ray.nodes()
node_ip_address = ray.worker.global_worker.node_ip_address
assert len(client_table) == 1
assert client_table[0]["NodeManagerAddress"] == node_ip_address
@ray.remote
class Actor:
def __init__(self):
pass
_ = Actor.options(name="test_actor").remote() # noqa: F841
# Wait for actor to be created
wait_for_num_actors(1)
actor_table = ray.actors()
assert len(actor_table) == 1
actor_info, = actor_table.values()
assert actor_info["JobID"] == job_id.hex()
assert actor_info["Name"] == "test_actor"
assert "IPAddress" in actor_info["Address"]
assert "IPAddress" in actor_info["OwnerAddress"]
assert actor_info["Address"]["Port"] != actor_info["OwnerAddress"]["Port"]
job_table = ray.jobs()
assert len(job_table) == 1
assert job_table[0]["JobID"] == job_id.hex()
assert job_table[0]["DriverIPAddress"] == node_ip_address
def test_placement_group_pack(ray_start_cluster):
@ray.remote(num_cpus=2)
class Actor(object):
def __init__(self):
self.n = 0
def value(self):
return self.n
cluster = ray_start_cluster
num_nodes = 2
for _ in range(num_nodes):
cluster.add_node(num_cpus=4)
ray.init(address=cluster.address)
placement_group = ray.util.placement_group(
name="name",
strategy="PACK",
bundles=[
{
"CPU": 2,
"GPU": 0 # Test 0 resource spec doesn't break tests.
},
{
"CPU": 2
}
])
ray.get(placement_group.ready())
actor_1 = Actor.options(
placement_group=placement_group,
placement_group_bundle_index=0).remote()
actor_2 = Actor.options(
placement_group=placement_group,
placement_group_bundle_index=1).remote()
ray.get(actor_1.value.remote())
ray.get(actor_2.value.remote())
# Get all actors.
actor_infos = ray.actors()
# Make sure all actors in counter_list are collocated in one node.
actor_info_1 = actor_infos.get(actor_1._actor_id.hex())
actor_info_2 = actor_infos.get(actor_2._actor_id.hex())
assert actor_info_1 and actor_info_2
node_of_actor_1 = actor_info_1["Address"]["NodeID"]
node_of_actor_2 = actor_info_2["Address"]["NodeID"]
assert node_of_actor_1 == node_of_actor_2
def _mock_train(*args, _training_state, **kwargs):
try:
results = _train(
*args, _training_state=_training_state, **kwargs)
return results
except Exception:
raise
finally:
assert len(_training_state.actors) == num_actors
if not any(a is None for a in _training_state.actors):
actor_infos = ray.actors()
actor_nodes = []
for a in _training_state.actors:
actor_info = actor_infos.get(a._actor_id.hex())
actor_node = actor_info["Address"]["NodeID"]
actor_nodes.append(actor_node)
assert actor_nodes[0] == actor_nodes[1]
def test_no_http(ray_shutdown):
# The following should have the same effect.
options = [
{
"http_host": None
},
{
"http_options": {
"host": None
}
},
{
"http_options": {
"location": None
}
},
{
"http_options": {
"location": "NoServer"
}
},
]
ray.init(num_cpus=16)
for i, option in enumerate(options):
print(f"[{i+1}/{len(options)}] Running with {option}")
serve.start(**option)
# Only controller actor should exist
live_actors = [
actor for actor in ray.actors().values()
if actor["State"] == ray.gcs_utils.ActorTableData.ALIVE
]
assert len(live_actors) == 1
controller = serve.api._global_client._controller
assert len(ray.get(controller.get_http_proxies.remote())) == 0
# Test that the handle still works.
def hello(*args):
return "hello"
serve.create_backend("backend", hello)
serve.create_endpoint("endpoint", backend="backend")
assert ray.get(serve.get_handle("endpoint").remote()) == "hello"
serve.shutdown()
def test_placement_group_strict_pack(ray_start_cluster):
@ray.remote(num_cpus=2)
class Actor(object):
def __init__(self):
self.n = 0
def value(self):
return self.n
cluster = ray_start_cluster
num_nodes = 2
for _ in range(num_nodes):
cluster.add_node(num_cpus=4)
ray.init(address=cluster.address)
placement_group_id = ray.experimental.placement_group(
name="name", strategy="STRICT_PACK", bundles=[{
"CPU": 2
}, {
"CPU": 2
}])
actor_1 = Actor.options(
placement_group_id=placement_group_id,
placement_group_bundle_index=0).remote()
actor_2 = Actor.options(
placement_group_id=placement_group_id,
placement_group_bundle_index=1).remote()
print(ray.get(actor_1.value.remote()))
print(ray.get(actor_2.value.remote()))
# Get all actors.
actor_infos = ray.actors()
# Make sure all actors in counter_list are collocated in one node.
actor_info_1 = actor_infos.get(actor_1._actor_id.hex())
actor_info_2 = actor_infos.get(actor_2._actor_id.hex())
assert actor_info_1 and actor_info_2
node_of_actor_1 = actor_info_1["Address"]["NodeID"]
node_of_actor_2 = actor_info_2["Address"]["NodeID"]
assert node_of_actor_1 == node_of_actor_2
def _get_actor(name):
if ray._raylet.gcs_actor_service_enabled():
worker = ray.worker.global_worker
handle = worker.core_worker.get_named_actor_handle(name)
else:
actor_name = _calculate_key(name)
pickled_state = _internal_kv_get(actor_name)
if pickled_state is None:
raise ValueError(
"The actor with name={} doesn't exist".format(name))
handle = pickle.loads(pickled_state)
# If the actor state is dead, that means that this name is reusable.
# We don't delete the name entry from key value store when
# the actor is killed because ray.kill is asynchronous,
# and it can cause worker leaks.
actor_info = ray.actors(actor_id=handle._actor_id.hex())
actor_state = actor_info.get("State", None)
if actor_state and actor_state == ActorTableData.DEAD:
raise ValueError("The actor with name={} is dead.".format(name))
return handle
def test_job_gc(call_ray_start):
address = call_ray_start
ray.init(address=address)
driver = """
import ray
ray.init(address="{}")
@ray.remote
class Actor:
def __init__(self):
pass
_ = Actor.remote()
""".format(address)
p = run_string_as_driver_nonblocking(driver)
# Wait for actor to be created
wait_for_num_actors(1)
actor_table = ray.actors()
assert len(actor_table) == 1
job_table = ray.jobs()
assert len(job_table) == 2 # dash
# Kill the driver process.
p.kill()
p.wait()
def actor_finish():
actor_table = ray.actors()
if (len(actor_table) == 0):
return True
else:
return False
wait_for_condition(actor_finish)
def test_job_gc_with_detached_actor(call_ray_start):
address = call_ray_start
ray.init(address=address)
driver = """
import ray
ray.init(address="{}")
@ray.remote
class Actor:
def __init__(self):
pass
def value(self):
return 1
_ = Actor.options(lifetime="detached", name="DetachedActor").remote()
# Make sure the actor is created before the driver exits.
ray.get(_.value.remote())
""".format(address)
p = run_string_as_driver_nonblocking(driver)
# Wait for actor to be created
wait_for_num_actors(1, ray.gcs_utils.ActorTableData.ALIVE)
actor_table = ray.actors()
assert len(actor_table) == 1
job_table = ray.jobs()
assert len(job_table) == 2 # dash
# Kill the driver process.
p.kill()
p.wait()
detached_actor = ray.get_actor("DetachedActor")
assert ray.get(detached_actor.value.remote()) == 1
def test_job_gc_with_detached_actor(call_ray_start):
address = call_ray_start
ray.init(address=address)
driver = """
import ray
ray.init(address="{}")
@ray.remote
class Actor:
def __init__(self):
pass
def value(self):
return 1
_ = Actor.options(name="DetachedActor").remote()
""".format(address)
p = run_string_as_driver_nonblocking(driver)
# Wait for actor to be created
wait_for_num_actors(1)
actor_table = ray.actors()
assert len(actor_table) == 1
job_table = ray.jobs()
assert len(job_table) == 2
# Kill the driver process.
p.kill()
p.wait()
detached_actor = ray.get_actor("DetachedActor")
assert ray.get(detached_actor.value.remote()) == 1
def test_no_http():
# The following should have the same effect.
options = [
{
"http_host": None
},
{
"http_options": {
"host": None
}
},
{
"http_options": {
"location": None
}
},
{
"http_options": {
"location": "NoServer"
}
},
]
ray.init()
for option in options:
client = serve.start(**option)
# Only controller actor should exist
live_actors = [
actor for actor in ray.actors().values()
if actor["State"] == ray.gcs_utils.ActorTableData.ALIVE
]
assert len(live_actors) == 1
client.shutdown()
ray.shutdown()
def test_global_state_actor_entry(ray_start_regular):
@ray.remote
class Actor:
def ready(self):
pass
# actor table should be empty at first
assert len(ray.actors()) == 0
a = Actor.remote()
b = Actor.remote()
ray.get(a.ready.remote())
ray.get(b.ready.remote())
assert len(ray.actors()) == 2
a_actor_id = a._actor_id.hex()
b_actor_id = b._actor_id.hex()
assert ray.actors(actor_id=a_actor_id)["ActorID"] == a_actor_id
assert ray.actors(actor_id=a_actor_id)["State"] == 1
assert ray.actors(actor_id=b_actor_id)["ActorID"] == b_actor_id
assert ray.actors(actor_id=b_actor_id)["State"] == 1
def _all_actors_dead():
return all(actor["State"] == ray.gcs_utils.ActorTableData.DEAD
for actor in list(ray.actors().values()))
def get_node_id_for_actor(actor_handle):
"""Given an actor handle, return the node id it's placed on."""
return ray.actors()[actor_handle._actor_id.hex()]["Address"]["NodeID"]
def test_capture_child_actors(ray_start_cluster):
cluster = ray_start_cluster
total_num_actors = 4
for _ in range(2):
cluster.add_node(num_cpus=total_num_actors)
ray.init(address=cluster.address)
pg = ray.util.placement_group(
[{
"CPU": 2
}, {
"CPU": 2
}], strategy="STRICT_PACK")
ray.get(pg.ready())
# If get_current_placement_group is used when the current worker/driver
# doesn't belong to any of placement group, it should return None.
assert get_current_placement_group() is None
# Test actors first.
@ray.remote(num_cpus=1)
class NestedActor:
def ready(self):
return True
@ray.remote(num_cpus=1)
class Actor:
def __init__(self):
self.actors = []
def ready(self):
return True
def schedule_nested_actor(self):
# Make sure we can capture the current placement group.
assert get_current_placement_group() is not None
# Actors should be implicitly captured.
actor = NestedActor.remote()
ray.get(actor.ready.remote())
self.actors.append(actor)
def schedule_nested_actor_outside_pg(self):
# Don't use placement group.
actor = NestedActor.options(placement_group=None).remote()
ray.get(actor.ready.remote())
self.actors.append(actor)
a = Actor.options(placement_group=pg).remote()
ray.get(a.ready.remote())
# 1 top level actor + 3 children.
for _ in range(total_num_actors - 1):
ray.get(a.schedule_nested_actor.remote())
# Make sure all the actors are scheduled on the same node.
# (why? The placement group has STRICT_PACK strategy).
node_id_set = set()
for actor_info in ray.actors().values():
node_id = actor_info["Address"]["NodeID"]
node_id_set.add(node_id)
# Since all node id should be identical, set should be equal to 1.
assert len(node_id_set) == 1
# Kill an actor and wait until it is killed.
ray.kill(a)
with pytest.raises(ray.exceptions.RayActorError):
ray.get(a.ready.remote())
# Now create an actor, but do not capture the current tasks
a = Actor.options(
placement_group=pg,
placement_group_capture_child_tasks=False).remote()
ray.get(a.ready.remote())
# 1 top level actor + 3 children.
for _ in range(total_num_actors - 1):
ray.get(a.schedule_nested_actor.remote())
# Make sure all the actors are not scheduled on the same node.
# It is because the child tasks are not scheduled on the same
# placement group.
node_id_set = set()
for actor_info in ray.actors().values():
node_id = actor_info["Address"]["NodeID"]
node_id_set.add(node_id)
assert len(node_id_set) == 2
# Kill an actor and wait until it is killed.
ray.kill(a)
with pytest.raises(ray.exceptions.RayActorError):
ray.get(a.ready.remote())
# Lastly, make sure when None is specified, actors are not scheduled
# on the same placement group.
a = Actor.options(placement_group=pg).remote()
ray.get(a.ready.remote())
# 1 top level actor + 3 children.
for _ in range(total_num_actors - 1):
ray.get(a.schedule_nested_actor_outside_pg.remote())
# Make sure all the actors are not scheduled on the same node.
# It is because the child tasks are not scheduled on the same
# placement group.
node_id_set = set()
for actor_info in ray.actors().values():
node_id = actor_info["Address"]["NodeID"]
node_id_set.add(node_id)
assert len(node_id_set) == 2
def check_num_actor():
assert len(ray.actors()) == num_workers
def assert_alive_num_actor(expected_num_actor):
alive_num_actor = 0
for actor_info in ray.actors().values():
if actor_info["State"] == ray.gcs_utils.ActorTableData.ALIVE:
alive_num_actor += 1
return alive_num_actor == expected_num_actor
def run(self):
p = self.redis_client.pubsub(ignore_subscribe_messages=True)
p.psubscribe(self.redis_key)
logger.info("NodeStats: subscribed to {}".format(self.redis_key))
log_channel = ray.gcs_utils.LOG_FILE_CHANNEL
p.subscribe(log_channel)
logger.info("NodeStats: subscribed to {}".format(log_channel))
error_channel = ray.gcs_utils.TablePubsub.Value("ERROR_INFO_PUBSUB")
p.subscribe(error_channel)
logger.info("NodeStats: subscribed to {}".format(error_channel))
actor_channel = ray.gcs_utils.TablePubsub.Value("ACTOR_PUBSUB")
p.subscribe(actor_channel)
logger.info("NodeStats: subscribed to {}".format(actor_channel))
current_actor_table = ray.actors()
with self._node_stats_lock:
for actor_data in current_actor_table.values():
addr = (actor_data["Address"]["IPAddress"],
str(actor_data["Address"]["Port"]))
owner_addr = (actor_data["OwnerAddress"]["IPAddress"],
str(actor_data["OwnerAddress"]["Port"]))
self._addr_to_owner_addr[addr] = owner_addr
self._addr_to_actor_id[addr] = actor_data["ActorID"]
self._addr_to_extra_info_dict[addr] = {
"jobId": actor_data["JobID"],
"state": actor_data["State"],
"isDirectCall": actor_data["IsDirectCall"],
"timestamp": actor_data["Timestamp"]
}
for x in p.listen():
try:
with self._node_stats_lock:
channel = ray.utils.decode(x["channel"])
data = x["data"]
if channel == log_channel:
data = json.loads(ray.utils.decode(data))
ip = data["ip"]
pid = str(data["pid"])
self._logs[ip][pid].extend(data["lines"])
elif channel == str(error_channel):
gcs_entry = ray.gcs_utils.GcsEntry.FromString(data)
error_data = ray.gcs_utils.ErrorTableData.FromString(
gcs_entry.entries[0])
message = error_data.error_message
message = re.sub(r"\x1b\[\d+m", "", message)
match = re.search(r"\(pid=(\d+), ip=(.*?)\)", message)
if match:
pid = match.group(1)
ip = match.group(2)
self._errors[ip][pid].append({
"message":
message,
"timestamp":
error_data.timestamp,
"type":
error_data.type
})
elif channel == str(actor_channel):
gcs_entry = ray.gcs_utils.GcsEntry.FromString(data)
actor_data = ray.gcs_utils.ActorTableData.FromString(
gcs_entry.entries[0])
addr = (actor_data.address.ip_address,
str(actor_data.address.port))
owner_addr = (actor_data.owner_address.ip_address,
str(actor_data.owner_address.port))
self._addr_to_owner_addr[addr] = owner_addr
self._addr_to_actor_id[addr] = ray.utils.binary_to_hex(
actor_data.actor_id)
self._addr_to_extra_info_dict[addr] = {
"jobId":
ray.utils.binary_to_hex(actor_data.job_id),
"state": actor_data.state,
"isDirectCall": actor_data.is_direct_call,
"timestamp": actor_data.timestamp
}
else:
data = json.loads(ray.utils.decode(data))
self._node_stats[data["hostname"]] = data
except Exception:
logger.exception(traceback.format_exc())
continue
def get_state():
return list(ray.actors().values())[0]["State"]
def test_cond():
alive_actors = [
v for v in real_ray.actors().values()
if v["State"] != ActorTableData.DEAD
]
return len(alive_actors) == 0
