def test_two_subscribers(ray_start_regular):
"""Tests concurrently subscribing to two channels work."""
address_info = ray_start_regular
gcs_server_addr = address_info["gcs_address"]
num_messages = 100
errors = []
error_subscriber = GcsErrorSubscriber(address=gcs_server_addr)
# Make sure subscription is registered before publishing starts.
error_subscriber.subscribe()
def receive_errors():
while len(errors) < num_messages:
_, msg = error_subscriber.poll()
errors.append(msg)
t1 = threading.Thread(target=receive_errors)
t1.start()
logs = []
log_subscriber = GcsLogSubscriber(address=gcs_server_addr)
# Make sure subscription is registered before publishing starts.
log_subscriber.subscribe()
def receive_logs():
while len(logs) < num_messages:
log_batch = log_subscriber.poll()
logs.append(log_batch)
t2 = threading.Thread(target=receive_logs)
t2.start()
publisher = GcsPublisher(address=gcs_server_addr)
for i in range(0, num_messages):
publisher.publish_error(b"msg_id", ErrorTableData(error_message=f"error {i}"))
publisher.publish_logs(
{
"ip": "127.0.0.1",
"pid": "gcs",
"job": "0001",
"is_err": False,
"lines": [f"log {i}"],
"actor_name": "test actor",
"task_name": "test task",
}
)
t1.join(timeout=10)
assert len(errors) == num_messages, str(errors)
assert not t1.is_alive(), str(errors)
t2.join(timeout=10)
assert len(logs) == num_messages, str(logs)
assert not t2.is_alive(), str(logs)
for i in range(0, num_messages):
assert errors[i].error_message == f"error {i}", str(errors)
assert logs[i]["lines"][0] == f"log {i}", str(logs)
def init_error_pubsub():
"""Initialize redis error info pub/sub"""
if gcs_pubsub_enabled():
s = GcsErrorSubscriber(address=ray.worker.global_worker.gcs_client.address)
s.subscribe()
else:
s = ray.worker.global_worker.redis_client.pubsub(ignore_subscribe_messages=True)
s.psubscribe(gcs_utils.RAY_ERROR_PUBSUB_PATTERN)
return s
def test_publish_and_subscribe_error_info(ray_start_regular):
address_info = ray_start_regular
gcs_server_addr = address_info["gcs_address"]
subscriber = GcsErrorSubscriber(address=gcs_server_addr)
subscriber.subscribe()
publisher = GcsPublisher(address=gcs_server_addr)
err1 = ErrorTableData(error_message="test error message 1")
err2 = ErrorTableData(error_message="test error message 2")
publisher.publish_error(b"aaa_id", err1)
publisher.publish_error(b"bbb_id", err2)
assert subscriber.poll() == (b"aaa_id", err1)
assert subscriber.poll() == (b"bbb_id", err2)
subscriber.close()
def test_publish_and_subscribe_error_info(ray_start_regular):
address_info = ray_start_regular
redis = ray._private.services.create_redis_client(
address_info["redis_address"],
password=ray.ray_constants.REDIS_DEFAULT_PASSWORD)
gcs_server_addr = gcs_utils.get_gcs_address_from_redis(redis)
subscriber = GcsErrorSubscriber(address=gcs_server_addr)
subscriber.subscribe()
publisher = GcsPublisher(address=gcs_server_addr)
err1 = ErrorTableData(error_message="test error message 1")
err2 = ErrorTableData(error_message="test error message 2")
publisher.publish_error(b"aaa_id", err1)
publisher.publish_error(b"bbb_id", err2)
assert subscriber.poll() == (b"aaa_id", err1)
assert subscriber.poll() == (b"bbb_id", err2)
subscriber.close()
def init_error_pubsub():
"""Initialize error info pub/sub"""
s = GcsErrorSubscriber(
address=ray._private.worker.global_worker.gcs_client.address)
s.subscribe()
return s
def receive_errors():
subscriber = GcsErrorSubscriber(address=gcs_server_addr)
subscriber.subscribe()
while len(errors) < num_messages:
_, msg = subscriber.poll()
errors.append(msg)
