g.remove_node(node2) g.remove_node(node) assert not any(n.any_processes_alive() for n in [node, node2]) def test_shutdown(): g = Cluster(initialize_head=False) node = g.add_node() node2 = g.add_node() g.shutdown() assert not any(n.any_processes_alive() for n in [node, node2]) @pytest.mark.parametrize( "ray_start_cluster_head", [generate_internal_config_map(num_heartbeats_timeout=20)], indirect=True) def test_internal_config(ray_start_cluster_head): """Checks that the internal configuration setting works. We set the cluster to timeout nodes after 2 seconds of no timeouts. We then remove a node, wait for 1 second to check that the cluster is out of sync, then wait another 2 seconds (giving 1 second of leeway) to check that the client has timed out. """ cluster = ray_start_cluster_head worker = cluster.add_node() cluster.wait_for_nodes() cluster.remove_node(worker, allow_graceful=False) time.sleep(1)
def increase(self): self.value += 1 return self.value remote_actor = Actor.remote() assert ray.get(RetryableTask.remote(remote_actor)) == 3 # NOTE(hchen): we set initial_reconstruction_timeout_milliseconds to 1s for # this test. Because if this value is too small, suprious task reconstruction # may happen and cause the test fauilure. If the value is too large, this test # could be very slow. We can remove this once we support dynamic timeout. @pytest.mark.parametrize( "ray_start_cluster_head", [ generate_internal_config_map( initial_reconstruction_timeout_milliseconds=1000) ], indirect=True) def test_multiple_actor_restart(ray_start_cluster_head): cluster = ray_start_cluster_head # This test can be made more stressful by increasing the numbers below. # The total number of actors created will be # num_actors_at_a_time * num_nodes. num_nodes = 5 num_actors_at_a_time = 3 num_function_calls_at_a_time = 10 worker_nodes = [ cluster.add_node( num_cpus=3, _internal_config=json.dumps({
g.remove_node(node2) g.remove_node(node) assert not any(n.any_processes_alive() for n in [node, node2]) def test_shutdown(): g = Cluster(initialize_head=False) node = g.add_node() node2 = g.add_node() g.shutdown() assert not any(n.any_processes_alive() for n in [node, node2]) @pytest.mark.parametrize("ray_start_cluster_head", [ generate_internal_config_map( num_heartbeats_timeout=20, initial_reconstruction_timeout_milliseconds=12345) ], indirect=True) def test_internal_config(ray_start_cluster_head): """Checks that the internal configuration setting works. We set the cluster to timeout nodes after 2 seconds of no timeouts. We then remove a node, wait for 1 second to check that the cluster is out of sync, then wait another 2 seconds (giving 1 second of leeway) to check that the client has timed out. We also check to see if the config is set. """ cluster = ray_start_cluster_head worker = cluster.add_node() cluster.wait_for_nodes()