def test_cleanup_on_driver_exit(call_ray_start):
# This test will create a driver that creates a bunch of objects and then
# exits. The entries in the object table should be cleaned up.
address = call_ray_start
ray.init(address=address)
# Define a driver that creates a bunch of objects and exits.
driver_script = """
import time
import ray
import numpy as np
from ray._private.test_utils import object_memory_usage
import os
ray.init(address="{}")
object_refs = [ray.put(np.zeros(200 * 1024, dtype=np.uint8))
for i in range(1000)]
start_time = time.time()
while time.time() - start_time < 30:
if object_memory_usage() > 0:
break
else:
raise Exception("Objects did not appear in object table.")
@ray.remote
def f():
time.sleep(1)
print("success")
# Submit some tasks without waiting for them to finish. Their workers should
# still get cleaned up eventually, even if they get started after the driver
# exits.
[f.remote() for _ in range(10)]
""".format(address)
out = run_string_as_driver(driver_script)
assert "success" in out
# Make sure the objects are removed from the object table.
start_time = time.time()
while time.time() - start_time < 30:
if object_memory_usage() == 0:
break
else:
raise Exception("Objects were not all removed from object table.")
def all_workers_exited():
result = True
print("list of idle workers:")
for proc in psutil.process_iter():
if ray_constants.WORKER_PROCESS_TYPE_IDLE_WORKER in proc.name():
print(f"{proc}")
result = False
return result
# Check that workers are eventually cleaned up.
wait_for_condition(all_workers_exited, timeout=15, retry_interval_ms=1000)
def on_trial_result(self, *args, **kwargs):
self.iter_ += 1
all_files = self.process.open_files()
if self.verbose:
print("Iteration", self.iter_)
print("=" * 10)
print("Object memory use: ", object_memory_usage())
print("Virtual Mem:", self.get_virt_mem() >> 30, "gb")
print("File Descriptors:", len(all_files))
assert len(all_files) < 20
def test_lease_request_leak(shutdown_only):
ray.init(num_cpus=1, _system_config={"object_timeout_milliseconds": 200})
@ray.remote
def f(x):
time.sleep(0.1)
return
# Submit pairs of tasks. Tasks in a pair can reuse the same worker leased
# from the raylet.
tasks = []
for _ in range(10):
obj_ref = ray.put(1)
for _ in range(2):
tasks.append(f.remote(obj_ref))
del obj_ref
ray.get(tasks)
wait_for_condition(lambda: object_memory_usage() == 0)
def save(self, *args, **kwargs):
checkpoint = super(CustomExecutor, self).save(*args, **kwargs)
assert object_memory_usage() <= (12 * 80e6)
return checkpoint
def test_cond():
return object_memory_usage() == 0
