def get_store_stats_summary(redis_address, redis_password) -> str: state = GlobalState() state._initialize_global_state(redis_address, redis_password) raylet = state.node_table()[0] stats = node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"]) store_summary = store_stats_summary(stats) return store_summary
def memory_summary(state, group_by="NODE_ADDRESS", sort_by="OBJECT_SIZE", line_wrap=True, unit="B", num_entries=None) -> str: from ray.dashboard.modules.node.node_head\ import node_stats_to_dict # Get terminal size import shutil size = shutil.get_terminal_size((80, 20)).columns line_wrap_threshold = 137 # Unit conversions units = {"B": 10**0, "KB": 10**3, "MB": 10**6, "GB": 10**9} # Fetch core memory worker stats, store as a dictionary core_worker_stats = [] for raylet in state.node_table(): stats = node_stats_to_dict( node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"])) core_worker_stats.extend(stats["coreWorkersStats"]) assert type(stats) is dict and "coreWorkersStats" in stats # Build memory table with "group_by" and "sort_by" parameters group_by, sort_by = get_group_by_type(group_by), get_sorting_type(sort_by) memory_table = construct_memory_table(core_worker_stats, group_by, sort_by).as_dict() assert "summary" in memory_table and "group" in memory_table # Build memory summary mem = "" group_by, sort_by = group_by.name.lower().replace( "_", " "), sort_by.name.lower().replace("_", " ") summary_labels = [ "Mem Used by Objects", "Local References", "Pinned", "Pending Tasks", "Captured in Objects", "Actor Handles" ] summary_string = "{:<19} {:<16} {:<12} {:<13} {:<19} {:<13}\n" object_ref_labels = [ "IP Address", "PID", "Type", "Call Site", "Size", "Reference Type", "Object Ref" ] object_ref_string = "{:<13} | {:<8} | {:<7} | {:<9} \ | {:<8} | {:<14} | {:<10}\n" if size > line_wrap_threshold and line_wrap: object_ref_string = "{:<15} {:<5} {:<6} {:<22} {:<6} {:<18} \ {:<56}\n" mem += f"Grouping by {group_by}...\ Sorting by {sort_by}...\ Display {num_entries if num_entries is not None else 'all'}\ entries per group...\n\n\n" for key, group in memory_table["group"].items(): # Group summary summary = group["summary"] ref_size = track_reference_size(group) for k, v in summary.items(): if k == "total_object_size": summary[k] = str(v / units[unit]) + f" {unit}" else: summary[k] = str(v) + f", ({ref_size[k] / units[unit]} {unit})" mem += f"--- Summary for {group_by}: {key} ---\n" mem += summary_string\ .format(*summary_labels) mem += summary_string\ .format(*summary.values()) + "\n" # Memory table per group mem += f"--- Object references for {group_by}: {key} ---\n" mem += object_ref_string\ .format(*object_ref_labels) n = 1 # Counter for num entries per group for entry in group["entries"]: if num_entries is not None and n > num_entries: break entry["object_size"] = str( entry["object_size"] / units[unit]) + f" {unit}" if entry["object_size"] > -1 else "?" num_lines = 1 if size > line_wrap_threshold and line_wrap: call_site_length = 22 if len(entry["call_site"]) == 0: entry["call_site"] = ["disabled"] else: entry["call_site"] = [ entry["call_site"][i:i + call_site_length] for i in range(0, len(entry["call_site"]), call_site_length) ] num_lines = len(entry["call_site"]) else: mem += "\n" object_ref_values = [ entry["node_ip_address"], entry["pid"], entry["type"], entry["call_site"], entry["object_size"], entry["reference_type"], entry["object_ref"] ] for i in range(len(object_ref_values)): if not isinstance(object_ref_values[i], list): object_ref_values[i] = [object_ref_values[i]] object_ref_values[i].extend( ["" for x in range(num_lines - len(object_ref_values[i]))]) for i in range(num_lines): row = [elem[i] for elem in object_ref_values] mem += object_ref_string\ .format(*row) mem += "\n" n += 1 mem += "To record callsite information for each ObjectRef created, set " \ "env variable RAY_record_ref_creation_sites=1\n\n" return mem
def get_memory_summary(group_by, sort_by, line_wrap) -> str: from ray.new_dashboard.modules.stats_collector.stats_collector_head\ import node_stats_to_dict # Get terminal size import shutil size = shutil.get_terminal_size((80, 20)).columns line_wrap_threshold = 137 # Fetch core memory worker stats, store as a dictionary core_worker_stats = [] for raylet in ray.nodes(): stats = node_stats_to_dict( node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"])) core_worker_stats.extend(stats["coreWorkersStats"]) assert type(stats) is dict and "coreWorkersStats" in stats # Build memory table with "group_by" and "sort_by" parameters group_by, sort_by = get_group_by_type(group_by), get_sorting_type(sort_by) memory_table = construct_memory_table(core_worker_stats, group_by, sort_by).as_dict() assert "summary" in memory_table and "group" in memory_table # Build memory summary mem = "" group_by, sort_by = group_by.name.lower().replace( "_", " "), sort_by.name.lower().replace("_", " ") summary_labels = [ "Mem Used by Objects", "Local References", "Pinned Count", "Pending Tasks", "Captured in Objects", "Actor Handles" ] summary_string = "{:<19} {:<16} {:<12} {:<13} {:<19} {:<13}\n" object_ref_labels = [ "IP Address", "PID", "Type", "Call Site", "Size", "Reference Type", "Object Ref" ] object_ref_string = "{:<8} {:<3} {:<4} {:<9} {:<4} {:<14} {:<10}\n" if size > line_wrap_threshold and line_wrap: object_ref_string = "{:<12} {:<5} {:<6} {:<22} {:<6} {:<18} \ {:<56}\n" mem += f"Grouping by {group_by}...\ Sorting by {sort_by}...\n\n\n\n" for key, group in memory_table["group"].items(): # Group summary summary = group["summary"] summary["total_object_size"] = str(summary["total_object_size"]) + " B" mem += f"--- Summary for {group_by}: {key} ---\n" mem += summary_string\ .format(*summary_labels) mem += summary_string\ .format(*summary.values()) + "\n" # Memory table per group mem += f"--- Object references for {group_by}: {key} ---\n" mem += object_ref_string\ .format(*object_ref_labels) for entry in group["entries"]: entry["object_size"] = str(entry["object_size"]) + " B" if entry[ "object_size"] > -1 else "?" num_lines = 1 if size > line_wrap_threshold and line_wrap: call_site_length = 22 entry["call_site"] = [ entry["call_site"][i:i + call_site_length] for i in range( 0, len(entry["call_site"]), call_site_length) ] num_lines = len(entry["call_site"]) object_ref_values = [ entry["node_ip_address"], entry["pid"], entry["type"], entry["call_site"], entry["object_size"], entry["reference_type"], entry["object_ref"] ] for i in range(len(object_ref_values)): if not isinstance(object_ref_values[i], list): object_ref_values[i] = [object_ref_values[i]] object_ref_values[i].extend( ["" for x in range(num_lines - len(object_ref_values[i]))]) for i in range(num_lines): row = [elem[i] for elem in object_ref_values] mem += object_ref_string\ .format(*row) mem += "\n" mem += "\n\n\n" return mem