def recursive(num_tasks_left):
if num_tasks_left == 0:
return put_object(
np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put), os.getpid()
return ray.get(recursive.remote(num_tasks_left - 1))
def test_worker_holding_serialized_reference(one_worker_100MiB, use_ray_put,
failure):
@ray.remote(max_retries=1)
def child(dep1, dep2):
if failure:
os._exit(0)
return
@ray.remote
def launch_pending_task(ref, signal):
return child.remote(ref[0], signal.wait.remote())
signal = SignalActor.remote()
# Test that the reference held by the actor isn't evicted.
array_oid = put_object(np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
child_return_id = ray.get(launch_pending_task.remote([array_oid], signal))
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Test that the reference prevents the object from being evicted.
_fill_object_store_and_get(array_oid_bytes)
ray.get(signal.send.remote())
try:
ray.get(child_return_id)
assert not failure
except (ray.exceptions.WorkerCrashedError, ray.exceptions.ObjectLostError):
assert failure
del child_return_id
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_basic_serialized_reference(one_worker_100MiB, use_ray_put, failure):
@ray.remote(max_retries=1)
def pending(ref, dep):
ray.get(ref[0])
if failure:
os._exit(0)
array_oid = put_object(np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
signal = SignalActor.remote()
obj_ref = pending.remote([array_oid], signal.wait.remote())
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the task to finish.
ray.get(signal.send.remote())
try:
ray.get(obj_ref)
assert not failure
except ray.exceptions.WorkerCrashedError:
assert failure
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_actor_holding_serialized_reference(one_worker_100MiB, use_ray_put,
failure):
@ray.remote
class GreedyActor(object):
def __init__(self):
pass
def set_ref1(self, ref):
self.ref1 = ref
def add_ref2(self, new_ref):
self.ref2 = new_ref
def delete_ref1(self):
self.ref1 = None
def delete_ref2(self):
self.ref2 = None
# Test that the reference held by the actor isn't evicted.
array_oid = put_object(np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
actor = GreedyActor.remote()
actor.set_ref1.remote([array_oid])
# Test that giving the same actor a duplicate reference works.
ray.get(actor.add_ref2.remote([array_oid]))
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Test that the remote references still pin the object.
_fill_object_store_and_get(array_oid_bytes)
# Test that removing only the first reference doesn't unpin the object.
ray.get(actor.delete_ref1.remote())
_fill_object_store_and_get(array_oid_bytes)
if failure:
# Test that the actor exiting stops the reference from being pinned.
# Kill the actor and wait for the actor to exit.
kill_actor_and_wait_for_failure(actor)
with pytest.raises(ray.exceptions.RayActorError):
ray.get(actor.delete_ref1.remote())
else:
# Test that deleting the second reference stops it from being pinned.
ray.get(actor.delete_ref2.remote())
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_recursively_nest_ids(one_worker_100MiB, use_ray_put, failure):
@ray.remote(max_retries=1)
def recursive(ref, signal, max_depth, depth=0):
unwrapped = ray.get(ref[0])
if depth == max_depth:
ray.get(signal.wait.remote())
if failure:
os._exit(0)
return
else:
return recursive.remote(unwrapped, signal, max_depth, depth + 1)
signal = SignalActor.remote()
max_depth = 5
array_oid = put_object(
np.zeros(20 * 1024 * 1024, dtype=np.uint8), use_ray_put)
nested_oid = array_oid
for _ in range(max_depth):
nested_oid = ray.put([nested_oid])
head_oid = recursive.remote([nested_oid], signal, max_depth)
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid, nested_oid
tail_oid = head_oid
for _ in range(max_depth):
tail_oid = ray.get(tail_oid)
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the tail task to finish.
ray.get(signal.send.remote())
if not failure:
ray.get(tail_oid)
else:
# There is only 1 core, so the same worker will execute all `recursive`
# tasks. Therefore, if we kill the worker during the last task, its
# owner (the worker that executed the second-to-last task) will also
# have died.
with pytest.raises(ray.exceptions.OwnerDiedError):
ray.get(tail_oid)
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_recursively_nest_ids(one_worker_100MiB, use_ray_put, failure):
@ray.remote(max_retries=1)
def recursive(ref, signal, max_depth, depth=0):
unwrapped = ray.get(ref[0])
if depth == max_depth:
ray.get(signal.wait.remote())
if failure:
os._exit(0)
return
else:
return recursive.remote(unwrapped, signal, max_depth, depth + 1)
signal = SignalActor.remote()
max_depth = 5
array_oid = put_object(np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
nested_oid = array_oid
for _ in range(max_depth):
nested_oid = ray.put([nested_oid])
head_oid = recursive.remote([nested_oid], signal, max_depth)
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid, nested_oid
tail_oid = head_oid
for _ in range(max_depth):
tail_oid = ray.get(tail_oid)
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the tail task to finish.
ray.get(signal.send.remote())
try:
ray.get(tail_oid)
assert not failure
# TODO(edoakes): this should raise WorkerError.
except ray.exceptions.ObjectLostError:
assert failure
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def return_an_id():
return put_object(
np.zeros(20 * 1024 * 1024, dtype=np.uint8), use_ray_put)