def verify():
tasks = list_tasks()
# Actor.__init__: 1 finished
# Actor.call: 1 running, 9 waiting for execution (queued).
correct_num_tasks = len(tasks) == 11
waiting_for_execution = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_EXECUTION",
tasks,
)))
scheduled = len(
list(
filter(lambda task: task["scheduling_state"] == "SCHEDULED",
tasks)))
waiting_for_dep = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_DEPENDENCIES",
tasks,
)))
running = len(
list(
filter(
lambda task: task["scheduling_state"] == "RUNNING",
tasks,
)))
return (correct_num_tasks and running == 1 and waiting_for_dep == 0
and waiting_for_execution == 9 and scheduled == 0)
def verify():
tasks = list_tasks()
correct_num_tasks = len(tasks) == 5
waiting_for_execution = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_EXECUTION",
tasks,
)))
scheduled = len(
list(
filter(lambda task: task["scheduling_state"] == "SCHEDULED",
tasks)))
waiting_for_dep = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_DEPENDENCIES",
tasks,
)))
running = len(
list(
filter(
lambda task: task["scheduling_state"] == "RUNNING",
tasks,
)))
return (correct_num_tasks and running == 2 and waiting_for_dep == 1
and waiting_for_execution == 0 and scheduled == 2)
def test_network_partial_failures(ray_start_cluster):
"""When the request fails due to network failure,
verifies it prints proper warning."""
cluster = ray_start_cluster
cluster.add_node(num_cpus=2)
ray.init(address=cluster.address)
n = cluster.add_node(num_cpus=2)
@ray.remote
def f():
import time
time.sleep(30)
a = [f.remote() for _ in range(4)] # noqa
wait_for_condition(lambda: len(list_tasks()) == 4)
# Make sure when there's 0 node failure, it doesn't print the error.
with pytest.warns(None) as record:
list_tasks(_explain=True)
assert len(record) == 0
# Kill raylet so that list_tasks will have network error on querying raylets.
cluster.remove_node(n, allow_graceful=False)
with pytest.warns(RuntimeWarning):
list_tasks(_explain=True)
# Make sure when _explain == False, warning is not printed.
with pytest.warns(None) as record:
list_tasks(_explain=False)
assert len(record) == 0
def tasks(ctx, format: str):
url = ctx.obj["api_server_url"]
format = AvailableFormat(format)
print(
get_state_api_output_to_print(
list_tasks(api_server_url=url, _explain=_should_explain(format)),
format=format,
)
)
def test_list_tasks(shutdown_only):
ray.init(num_cpus=2)
@ray.remote
def f():
import time
time.sleep(30)
@ray.remote
def g(dep):
import time
time.sleep(30)
@ray.remote(num_gpus=1)
def impossible():
pass
out = [f.remote() for _ in range(2)] # noqa
g_out = g.remote(f.remote()) # noqa
im = impossible.remote() # noqa
def verify():
tasks = list_tasks()
correct_num_tasks = len(tasks) == 5
waiting_for_execution = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_EXECUTION",
tasks,
)))
scheduled = len(
list(
filter(lambda task: task["scheduling_state"] == "SCHEDULED",
tasks)))
waiting_for_dep = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_DEPENDENCIES",
tasks,
)))
running = len(
list(
filter(
lambda task: task["scheduling_state"] == "RUNNING",
tasks,
)))
return (correct_num_tasks and running == 2 and waiting_for_dep == 1
and waiting_for_execution == 0 and scheduled == 2)
wait_for_condition(verify)
print(list_tasks())
def test_network_failure(shutdown_only):
"""When the request fails due to network failure,
verifies it raises an exception."""
ray.init()
@ray.remote
def f():
import time
time.sleep(30)
a = [f.remote() for _ in range(4)] # noqa
wait_for_condition(lambda: len(list_tasks()) == 4)
# Kill raylet so that list_tasks will have network error on querying raylets.
ray._private.worker._global_node.kill_raylet()
with pytest.raises(RayStateApiException):
list_tasks(_explain=True)
def tasks(ctx, format: str, filter: List[Tuple[str, str]]):
url = ctx.obj["api_server_url"]
format = AvailableFormat(format)
print(
get_state_api_output_to_print(
list_tasks(
api_server_url=url,
filters=filter,
_explain=_should_explain(format),
),
format=format,
)
)
def test_list_actor_tasks(shutdown_only):
ray.init(num_cpus=2)
@ray.remote
class Actor:
def call(self):
import time
time.sleep(30)
a = Actor.remote()
calls = [a.call.remote() for _ in range(10)] # noqa
def verify():
tasks = list_tasks()
# Actor.__init__: 1 finished
# Actor.call: 1 running, 9 waiting for execution (queued).
correct_num_tasks = len(tasks) == 11
waiting_for_execution = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_EXECUTION",
tasks,
)))
scheduled = len(
list(
filter(lambda task: task["scheduling_state"] == "SCHEDULED",
tasks)))
waiting_for_dep = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_DEPENDENCIES",
tasks,
)))
running = len(
list(
filter(
lambda task: task["scheduling_state"] == "RUNNING",
tasks,
)))
return (correct_num_tasks and running == 1 and waiting_for_dep == 0
and waiting_for_execution == 9 and scheduled == 0)
wait_for_condition(verify)
print(list_tasks())
def verify():
tasks = list(list_tasks().values())
correct_num_tasks = len(tasks) == 4
scheduled = len(
list(
filter(lambda task: task["scheduling_state"] == "SCHEDULED",
tasks)))
waiting_for_dep = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_DEPENDENCIES",
tasks,
)))
return correct_num_tasks and scheduled == 3 and waiting_for_dep == 1
def test_list_tasks(shutdown_only):
ray.init(num_cpus=2)
@ray.remote
def f():
import time
time.sleep(30)
@ray.remote
def g(dep):
import time
time.sleep(30)
out = [f.remote() for _ in range(2)] # noqa
g_out = g.remote(f.remote()) # noqa
def verify():
tasks = list(list_tasks().values())
correct_num_tasks = len(tasks) == 4
scheduled = len(
list(
filter(lambda task: task["scheduling_state"] == "SCHEDULED",
tasks)))
waiting_for_dep = len(
list(
filter(
lambda task: task["scheduling_state"] ==
"WAITING_FOR_DEPENDENCIES",
tasks,
)))
return correct_num_tasks and scheduled == 3 and waiting_for_dep == 1
wait_for_condition(verify)
print(list_tasks())
def tasks(ctx):
url = ctx.obj["api_server_url"]
pprint(list_tasks(url))
def verify():
with pytest.warns(None) as record:
list_tasks(_explain=True, timeout=5)
return len(record) == 1
def tasks(ctx, format: str):
url = ctx.obj["api_server_url"]
print(
get_state_api_output_to_print(list_tasks(api_server_url=url),
format=AvailableFormat(format)))
