def test_global_state_task_object_api(shutdown_only): ray.init() job_id = ray.utils.compute_job_id_from_driver( ray.WorkerID(ray.worker.global_worker.worker_id)) driver_task_id = ray.worker.global_worker.current_task_id.hex() nil_actor_id_hex = ray.ActorID.nil().hex() @ray.remote def f(*xs): return 1 x_id = ray.put(1) result_id = f.remote(1, "hi", x_id) # Wait for one additional task to complete. wait_for_num_tasks(1 + 1) task_table = ray.tasks() assert len(task_table) == 1 + 1 task_id_set = set(task_table.keys()) task_id_set.remove(driver_task_id) task_id = list(task_id_set)[0] task_spec = task_table[task_id]["TaskSpec"] assert task_spec["ActorID"] == nil_actor_id_hex assert task_spec["Args"] == [ signature.DUMMY_TYPE, 1, signature.DUMMY_TYPE, "hi", signature.DUMMY_TYPE, x_id ] assert task_spec["JobID"] == job_id.hex() assert task_spec["ReturnObjectIDs"] == [result_id] assert task_table[task_id] == ray.tasks(task_id) # Wait for two objects, one for the x_id and one for result_id. wait_for_num_objects(2) def wait_for_object_table(): timeout = 10 start_time = time.time() while time.time() - start_time < timeout: object_table = ray.objects() tables_ready = (object_table[x_id]["ManagerIDs"] is not None and object_table[result_id]["ManagerIDs"] is not None) if tables_ready: return time.sleep(0.1) raise RayTestTimeoutException( "Timed out while waiting for object table to " "update.") object_table = ray.objects() assert len(object_table) == 2 assert object_table[x_id] == ray.objects(x_id) object_table_entry = ray.objects(result_id) assert object_table[result_id] == object_table_entry
def wait_for_num_tasks(num_tasks, timeout=10): start_time = time.time() while time.time() - start_time < timeout: if len(ray.tasks()) >= num_tasks: return time.sleep(0.1) raise RayTestTimeoutException("Timed out while waiting for global state.")
def test_free_objects_multi_node(ray_start_cluster): # This test will do following: # 1. Create 3 raylets that each hold an actor. # 2. Each actor creates an object which is the deletion target. # 3. Wait 0.1 second for the objects to be deleted. # 4. Check that the deletion targets have been deleted. # Caution: if remote functions are used instead of actor methods, # one raylet may create more than one worker to execute the # tasks, so the flushing operations may be executed in different # workers and the plasma client holding the deletion target # may not be flushed. cluster = ray_start_cluster config = json.dumps({"object_manager_repeated_push_delay_ms": 1000}) for i in range(3): cluster.add_node( num_cpus=1, resources={"Custom{}".format(i): 1}, _internal_config=config) ray.init(address=cluster.address) class RawActor(object): def get(self): return ray.worker.global_worker.node.unique_id ActorOnNode0 = ray.remote(resources={"Custom0": 1})(RawActor) ActorOnNode1 = ray.remote(resources={"Custom1": 1})(RawActor) ActorOnNode2 = ray.remote(resources={"Custom2": 1})(RawActor) def create(actors): a = actors[0].get.remote() b = actors[1].get.remote() c = actors[2].get.remote() (l1, l2) = ray.wait([a, b, c], num_returns=3) assert len(l1) == 3 assert len(l2) == 0 return (a, b, c) def run_one_test(actors, local_only, delete_creating_tasks): (a, b, c) = create(actors) # The three objects should be generated on different object stores. assert ray.get(a) != ray.get(b) assert ray.get(a) != ray.get(c) assert ray.get(c) != ray.get(b) ray.internal.free( [a, b, c], local_only=local_only, delete_creating_tasks=delete_creating_tasks) # Wait for the objects to be deleted. time.sleep(0.1) return (a, b, c) actors = [ ActorOnNode0.remote(), ActorOnNode1.remote(), ActorOnNode2.remote() ] # Case 1: run this local_only=False. All 3 objects will be deleted. (a, b, c) = run_one_test(actors, False, False) (l1, l2) = ray.wait([a, b, c], timeout=0.01, num_returns=1) # All the objects are deleted. assert len(l1) == 0 assert len(l2) == 3 # Case 2: run this local_only=True. Only 1 object will be deleted. (a, b, c) = run_one_test(actors, True, False) (l1, l2) = ray.wait([a, b, c], timeout=0.01, num_returns=3) # One object is deleted and 2 objects are not. assert len(l1) == 2 assert len(l2) == 1 # The deleted object will have the same store with the driver. local_return = ray.worker.global_worker.node.unique_id for object_id in l1: assert ray.get(object_id) != local_return # Case3: These cases test the deleting creating tasks for the object. (a, b, c) = run_one_test(actors, False, False) task_table = ray.tasks() for obj in [a, b, c]: assert ray._raylet.compute_task_id(obj).hex() in task_table (a, b, c) = run_one_test(actors, False, True) task_table = ray.tasks() for obj in [a, b, c]: assert ray._raylet.compute_task_id(obj).hex() not in task_table
def _xray_clean_up_entries_for_job(self, job_id): """Remove this job's object/task entries from redis. Removes control-state entries of all tasks and task return objects belonging to the driver. Args: job_id: The job id. """ xray_task_table_prefix = ( ray.gcs_utils.TablePrefix_RAYLET_TASK_string.encode("ascii")) xray_object_table_prefix = ( ray.gcs_utils.TablePrefix_OBJECT_string.encode("ascii")) task_table_objects = ray.tasks() job_id_hex = binary_to_hex(job_id) job_task_id_bins = set() for task_id_hex, task_info in task_table_objects.items(): task_table_object = task_info["TaskSpec"] task_job_id_hex = task_table_object["JobID"] if job_id_hex != task_job_id_hex: # Ignore tasks that aren't from this driver. continue job_task_id_bins.add(hex_to_binary(task_id_hex)) # Get objects associated with the driver. object_table_objects = ray.objects() job_object_id_bins = set() for object_id, _ in object_table_objects.items(): task_id_bin = ray._raylet.compute_task_id(object_id).binary() if task_id_bin in job_task_id_bins: job_object_id_bins.add(object_id.binary()) def to_shard_index(id_bin): if len(id_bin) == ray.TaskID.size(): return binary_to_task_id(id_bin).redis_shard_hash() % len( ray.state.state.redis_clients) else: return binary_to_object_id(id_bin).redis_shard_hash() % len( ray.state.state.redis_clients) # Form the redis keys to delete. sharded_keys = [[] for _ in range(len(ray.state.state.redis_clients))] for task_id_bin in job_task_id_bins: sharded_keys[to_shard_index(task_id_bin)].append( xray_task_table_prefix + task_id_bin) for object_id_bin in job_object_id_bins: sharded_keys[to_shard_index(object_id_bin)].append( xray_object_table_prefix + object_id_bin) # Remove with best effort. for shard_index in range(len(sharded_keys)): keys = sharded_keys[shard_index] if len(keys) == 0: continue redis = ray.state.state.redis_clients[shard_index] num_deleted = redis.delete(*keys) logger.info("Monitor: " "Removed {} dead redis entries of the " "driver from redis shard {}.".format( num_deleted, shard_index)) if num_deleted != len(keys): logger.warning("Monitor: " "Failed to remove {} relevant redis " "entries from redis shard {}.".format( len(keys) - num_deleted, shard_index))
def task_table(self, task_id=None): logger.warning( "ray.global_state.task_table() is deprecated and will be " "removed in a subsequent release. Use ray.tasks() instead.") return ray.tasks(task_id=task_id)
def StateSummary(): obj_tbl_len = len(ray.objects()) task_tbl_len = len(ray.tasks()) return obj_tbl_len, task_tbl_len
def test_global_state_api(shutdown_only): error_message = ("The ray global state API cannot be used " "before ray.init has been called.") with pytest.raises(Exception, match=error_message): ray.objects() with pytest.raises(Exception, match=error_message): ray.actors() with pytest.raises(Exception, match=error_message): ray.tasks() with pytest.raises(Exception, match=error_message): ray.nodes() with pytest.raises(Exception, match=error_message): ray.jobs() ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1}) assert ray.cluster_resources()["CPU"] == 5 assert ray.cluster_resources()["GPU"] == 3 assert ray.cluster_resources()["CustomResource"] == 1 assert ray.objects() == {} job_id = ray.utils.compute_job_id_from_driver( ray.WorkerID(ray.worker.global_worker.worker_id)) driver_task_id = ray.worker.global_worker.current_task_id.hex() # One task is put in the task table which corresponds to this driver. wait_for_num_tasks(1) task_table = ray.tasks() assert len(task_table) == 1 assert driver_task_id == list(task_table.keys())[0] task_spec = task_table[driver_task_id]["TaskSpec"] nil_unique_id_hex = ray.UniqueID.nil().hex() nil_actor_id_hex = ray.ActorID.nil().hex() assert task_spec["TaskID"] == driver_task_id assert task_spec["ActorID"] == nil_actor_id_hex assert task_spec["Args"] == [] assert task_spec["JobID"] == job_id.hex() assert task_spec["FunctionID"] == nil_unique_id_hex assert task_spec["ReturnObjectIDs"] == [] client_table = ray.nodes() node_ip_address = ray.worker.global_worker.node_ip_address assert len(client_table) == 1 assert client_table[0]["NodeManagerAddress"] == node_ip_address @ray.remote class Actor: def __init__(self): pass _ = Actor.remote() # Wait for actor to be created wait_for_num_actors(1) actor_table = ray.actors() assert len(actor_table) == 1 actor_info, = actor_table.values() assert actor_info["JobID"] == job_id.hex() assert "IPAddress" in actor_info["Address"] assert "IPAddress" in actor_info["OwnerAddress"] assert actor_info["Address"]["Port"] != actor_info["OwnerAddress"]["Port"] job_table = ray.jobs() assert len(job_table) == 1 assert job_table[0]["JobID"] == job_id.hex() assert job_table[0]["NodeManagerAddress"] == node_ip_address
def test_global_state_api(shutdown_only): error_message = ("The ray global state API cannot be used " "before ray.init has been called.") with pytest.raises(Exception, match=error_message): ray.objects() with pytest.raises(Exception, match=error_message): ray.tasks() with pytest.raises(Exception, match=error_message): ray.nodes() with pytest.raises(Exception, match=error_message): ray.jobs() ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1}) assert ray.cluster_resources()["CPU"] == 5 assert ray.cluster_resources()["GPU"] == 3 assert ray.cluster_resources()["CustomResource"] == 1 assert ray.objects() == {} job_id = ray.utils.compute_job_id_from_driver( ray.WorkerID(ray.worker.global_worker.worker_id)) driver_task_id = ray.worker.global_worker.current_task_id.hex() # One task is put in the task table which corresponds to this driver. wait_for_num_tasks(1) task_table = ray.tasks() assert len(task_table) == 1 assert driver_task_id == list(task_table.keys())[0] task_spec = task_table[driver_task_id]["TaskSpec"] nil_unique_id_hex = ray.UniqueID.nil().hex() nil_actor_id_hex = ray.ActorID.nil().hex() assert task_spec["TaskID"] == driver_task_id assert task_spec["ActorID"] == nil_actor_id_hex assert task_spec["Args"] == [] assert task_spec["JobID"] == job_id.hex() assert task_spec["FunctionID"] == nil_unique_id_hex assert task_spec["ReturnObjectIDs"] == [] client_table = ray.nodes() node_ip_address = ray.worker.global_worker.node_ip_address assert len(client_table) == 1 assert client_table[0]["NodeManagerAddress"] == node_ip_address @ray.remote def f(*xs): return 1 x_id = ray.put(1) result_id = f.remote(1, "hi", x_id) # Wait for one additional task to complete. wait_for_num_tasks(1 + 1) task_table = ray.tasks() assert len(task_table) == 1 + 1 task_id_set = set(task_table.keys()) task_id_set.remove(driver_task_id) task_id = list(task_id_set)[0] task_spec = task_table[task_id]["TaskSpec"] assert task_spec["ActorID"] == nil_actor_id_hex assert task_spec["Args"] == [ signature.DUMMY_TYPE, 1, signature.DUMMY_TYPE, "hi", signature.DUMMY_TYPE, x_id ] assert task_spec["JobID"] == job_id.hex() assert task_spec["ReturnObjectIDs"] == [result_id] assert task_table[task_id] == ray.tasks(task_id) # Wait for two objects, one for the x_id and one for result_id. wait_for_num_objects(2) def wait_for_object_table(): timeout = 10 start_time = time.time() while time.time() - start_time < timeout: object_table = ray.objects() tables_ready = (object_table[x_id]["ManagerIDs"] is not None and object_table[result_id]["ManagerIDs"] is not None) if tables_ready: return time.sleep(0.1) raise RayTestTimeoutException( "Timed out while waiting for object table to " "update.") object_table = ray.objects() assert len(object_table) == 2 assert object_table[x_id] == ray.objects(x_id) object_table_entry = ray.objects(result_id) assert object_table[result_id] == object_table_entry job_table = ray.jobs() assert len(job_table) == 1 assert job_table[0]["JobID"] == job_id.hex() assert job_table[0]["NodeManagerAddress"] == node_ip_address