def debug(address): """Show all active breakpoints and exceptions in the Ray debugger.""" from telnetlib import Telnet if not address: address = services.get_ray_address_to_use_or_die() logger.info(f"Connecting to Ray instance at {address}.") ray.init(address=address) while True: active_sessions = ray.experimental.internal_kv._internal_kv_list( "RAY_PDB_") print("Active breakpoints:") for i, active_session in enumerate(active_sessions): data = json.loads( ray.experimental.internal_kv._internal_kv_get(active_session)) print( str(i) + ": " + data["proctitle"] + " | " + data["filename"] + ":" + str(data["lineno"])) print(data["traceback"]) inp = input("Enter breakpoint index or press enter to refresh: ") if inp == "": print() continue else: index = int(inp) session = json.loads( ray.experimental.internal_kv._internal_kv_get( active_sessions[index])) host, port = session["pdb_address"].split(":") with Telnet(host, int(port)) as tn: tn.interact()
def status(address): """Print cluster status, including autoscaling info.""" if not address: address = services.get_ray_address_to_use_or_die() logger.info(f"Connecting to Ray instance at {address}.") ray.init(address=address) print(debug_status())
def memory(address, redis_password): """Print object references held in a Ray cluster.""" if not address: address = services.get_ray_address_to_use_or_die() logger.info(f"Connecting to Ray instance at {address}.") ray.init(address=address, _redis_password=redis_password) print(ray.internal.internal_api.memory_summary())
def global_gc(address): """Trigger Python garbage collection on all cluster workers.""" if not address: address = services.get_ray_address_to_use_or_die() logger.info(f"Connecting to Ray instance at {address}.") ray.init(address=address) ray.internal.internal_api.global_gc() print("Triggered gc.collect() on all workers.")
def timeline(address): """Take a Chrome tracing timeline for a Ray cluster.""" if not address: address = services.get_ray_address_to_use_or_die() logger.info(f"Connecting to Ray instance at {address}.") ray.init(address=address) time = datetime.today().strftime("%Y-%m-%d_%H-%M-%S") filename = os.path.join(ray.utils.get_user_temp_dir(), f"ray-timeline-{time}.json") ray.timeline(filename=filename) size = os.path.getsize(filename) logger.info(f"Trace file written to {filename} ({size} bytes).") logger.info( "You can open this with chrome://tracing in the Chrome browser.")
def memory_summary(address=None, redis_password=ray_constants.REDIS_DEFAULT_PASSWORD, group_by="NODE_ADDRESS", sort_by="OBJECT_SIZE", line_wrap=True, stats_only=False): from ray.new_dashboard.memory_utils import memory_summary if not address: address = services.get_ray_address_to_use_or_die() state = GlobalState() state._initialize_global_state(address, redis_password) if stats_only: return get_store_stats(state) return (memory_summary(state, group_by, sort_by, line_wrap) + get_store_stats(state))