def test_worker_holding_serialized_reference(one_worker_100MiB, use_ray_put,
failure):
@ray.remote(max_retries=1)
def child(dep1, dep2):
if failure:
os._exit(0)
return
@ray.remote
def launch_pending_task(ref, signal):
return child.remote(ref[0], signal.wait.remote())
signal = SignalActor.remote()
# Test that the reference held by the actor isn't evicted.
array_oid = put_object(np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
child_return_id = ray.get(launch_pending_task.remote([array_oid], signal))
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Test that the reference prevents the object from being evicted.
_fill_object_store_and_get(array_oid_bytes)
ray.get(signal.send.remote())
try:
ray.get(child_return_id)
assert not failure
except (ray.exceptions.WorkerCrashedError, ray.exceptions.ObjectLostError):
assert failure
del child_return_id
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def recursive(num_tasks_left):
if num_tasks_left == 0:
return put_object(
np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put), os.getpid()
return ray.get(recursive.remote(num_tasks_left - 1))
def test_basic_serialized_reference(one_worker_100MiB, use_ray_put, failure):
@ray.remote(max_retries=1)
def pending(ref, dep):
ray.get(ref[0])
if failure:
os._exit(0)
array_oid = put_object(np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
signal = SignalActor.remote()
obj_ref = pending.remote([array_oid], signal.wait.remote())
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the task to finish.
ray.get(signal.send.remote())
try:
ray.get(obj_ref)
assert not failure
except ray.exceptions.WorkerCrashedError:
assert failure
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def recursive(num_tasks_left):
if num_tasks_left == 0:
return put_object(np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
final_id = ray.get(recursive.remote(num_tasks_left - 1))
ray.get(final_id)
return final_id
def test_actor_holding_serialized_reference(one_worker_100MiB, use_ray_put,
failure):
@ray.remote
class GreedyActor(object):
def __init__(self):
pass
def set_ref1(self, ref):
self.ref1 = ref
def add_ref2(self, new_ref):
self.ref2 = new_ref
def delete_ref1(self):
self.ref1 = None
def delete_ref2(self):
self.ref2 = None
# Test that the reference held by the actor isn't evicted.
array_oid = put_object(
np.zeros(40 * 1024 * 1024, dtype=np.uint8), use_ray_put)
actor = GreedyActor.remote()
actor.set_ref1.remote([array_oid])
# Test that giving the same actor a duplicate reference works.
ray.get(actor.add_ref2.remote([array_oid]))
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Test that the remote references still pin the object.
_fill_object_store_and_get(array_oid_bytes)
# Test that removing only the first reference doesn't unpin the object.
ray.get(actor.delete_ref1.remote())
_fill_object_store_and_get(array_oid_bytes)
if failure:
# Test that the actor exiting stops the reference from being pinned.
ray.kill(actor)
# Wait for the actor to exit.
try:
ray.get(actor.delete_ref1.remote())
except ray.exceptions.RayActorError:
pass
else:
# Test that deleting the second reference stops it from being pinned.
ray.get(actor.delete_ref2.remote())
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_recursively_nest_ids(one_worker_100MiB, use_ray_put):
@ray.remote(num_cpus=0)
class Signal:
def __init__(self):
self.ready_event = asyncio.Event()
def send(self):
self.ready_event.set()
async def wait(self):
await self.ready_event.wait()
@ray.remote
def recursive(ref, signal, max_depth, depth=0):
unwrapped = ray.get(ref[0])
if depth == max_depth:
return ray.get(signal.wait.remote())
else:
return recursive.remote(unwrapped, signal, max_depth, depth + 1)
signal = SignalActor.remote()
max_depth = 5
array_oid = put_object(np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
nested_oid = array_oid
for _ in range(max_depth):
nested_oid = ray.put([nested_oid])
head_oid = recursive.remote([nested_oid], signal, max_depth)
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid, nested_oid
tail_oid = head_oid
for _ in range(max_depth):
tail_oid = ray.get(tail_oid)
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the tail task to finish.
ray.get(signal.send.remote())
ray.get(tail_oid)
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_recursively_nest_ids(one_worker_100MiB, use_ray_put, failure):
@ray.remote(max_retries=1)
def recursive(ref, signal, max_depth, depth=0):
unwrapped = ray.get(ref[0])
if depth == max_depth:
ray.get(signal.wait.remote())
if failure:
os._exit(0)
return
else:
return recursive.remote(unwrapped, signal, max_depth, depth + 1)
signal = SignalActor.remote()
max_depth = 5
array_oid = put_object(np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
nested_oid = array_oid
for _ in range(max_depth):
nested_oid = ray.put([nested_oid])
head_oid = recursive.remote([nested_oid], signal, max_depth)
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid, nested_oid
tail_oid = head_oid
for _ in range(max_depth):
tail_oid = ray.get(tail_oid)
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the tail task to finish.
ray.get(signal.send.remote())
try:
ray.get(tail_oid)
assert not failure
# TODO(edoakes): this should raise WorkerError.
except ray.exceptions.ObjectLostError:
assert failure
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_worker_holding_serialized_reference(one_worker_100MiB, use_ray_put):
@ray.remote(num_cpus=0)
class Signal:
def __init__(self):
self.ready_event = asyncio.Event()
def send(self):
self.ready_event.set()
async def wait(self):
await self.ready_event.wait()
@ray.remote
def child(dep1, dep2):
return
@ray.remote
def launch_pending_task(ref, signal):
return child.remote(ref[0], signal.wait.remote())
signal = SignalActor.remote()
# Test that the reference held by the actor isn't evicted.
array_oid = put_object(np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
child_return_id = ray.get(launch_pending_task.remote([array_oid], signal))
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Test that the reference prevents the object from being evicted.
_fill_object_store_and_get(array_oid_bytes)
ray.get(signal.send.remote())
ray.get(child_return_id)
del child_return_id
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def test_basic_serialized_reference(one_worker_100MiB, use_ray_put):
@ray.remote
def pending(ref, dep):
ray.get(ref[0])
array_oid = put_object(np.zeros(40 * 1024 * 1024, dtype=np.uint8),
use_ray_put)
signal = SignalActor.remote()
oid = pending.remote([array_oid], signal.wait.remote())
# Remove the local reference.
array_oid_bytes = array_oid.binary()
del array_oid
# Check that the remote reference pins the object.
_fill_object_store_and_get(array_oid_bytes)
# Fulfill the dependency, causing the task to finish.
ray.get(signal.send.remote())
ray.get(oid)
# Reference should be gone, check that array gets evicted.
_fill_object_store_and_get(array_oid_bytes, succeed=False)
def return_an_id():
return put_object(np.zeros(20 * 1024 * 1024, dtype=np.uint8),
use_ray_put)