def get_store_stats_summary(redis_address, redis_password) -> str:
state = GlobalState()
state._initialize_global_state(redis_address, redis_password)
raylet = state.node_table()[0]
stats = node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"])
store_summary = store_stats_summary(stats)
return store_summary
def memory_summary(address=None,
redis_password=ray_constants.REDIS_DEFAULT_PASSWORD,
group_by="NODE_ADDRESS",
sort_by="OBJECT_SIZE",
line_wrap=True,
stats_only=False):
from ray.new_dashboard.memory_utils import memory_summary
if not address:
address = services.get_ray_address_to_use_or_die()
state = GlobalState()
state._initialize_global_state(address, redis_password)
if stats_only:
return get_store_stats(state)
return (memory_summary(state, group_by, sort_by, line_wrap) +
get_store_stats(state))
def memory_summary(address=None,
redis_password=ray_constants.REDIS_DEFAULT_PASSWORD,
group_by="NODE_ADDRESS",
sort_by="OBJECT_SIZE",
units="B",
line_wrap=True,
stats_only=False,
num_entries=None):
from ray.dashboard.memory_utils import memory_summary
address = services.canonicalize_bootstrap_address(address)
state = GlobalState()
state._initialize_global_state(
GcsClientOptions.from_redis_address(address, redis_password))
if stats_only:
return get_store_stats(state)
return (memory_summary(state, group_by, sort_by, line_wrap, units,
num_entries) + get_store_stats(state))
def get_memory_summary(redis_address, redis_password, group_by, sort_by,
line_wrap, stats_only) -> str:
from ray.new_dashboard.modules.stats_collector.stats_collector_head\
import node_stats_to_dict
# Get terminal size
import shutil
size = shutil.get_terminal_size((80, 20)).columns
line_wrap_threshold = 137
# Fetch core memory worker stats, store as a dictionary
state = GlobalState()
state._initialize_global_state(redis_address, redis_password)
core_worker_stats = []
for raylet in state.node_table():
stats = node_stats_to_dict(
node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"],
(not stats_only)))
core_worker_stats.extend(stats["coreWorkersStats"])
assert type(stats) is dict and "coreWorkersStats" in stats
# Build memory table with "group_by" and "sort_by" parameters
group_by, sort_by = get_group_by_type(group_by), get_sorting_type(sort_by)
memory_table = construct_memory_table(core_worker_stats, group_by,
sort_by).as_dict()
assert "summary" in memory_table and "group" in memory_table
# Build memory summary
mem = ""
group_by, sort_by = group_by.name.lower().replace(
"_", " "), sort_by.name.lower().replace("_", " ")
summary_labels = [
"Mem Used by Objects", "Local References", "Pinned Count",
"Pending Tasks", "Captured in Objects", "Actor Handles"
]
summary_string = "{:<19} {:<16} {:<12} {:<13} {:<19} {:<13}\n"
object_ref_labels = [
"IP Address", "PID", "Type", "Call Site", "Size", "Reference Type",
"Object Ref"
]
object_ref_string = "{:<8} {:<3} {:<4} {:<9} {:<4} {:<14} {:<10}\n"
if size > line_wrap_threshold and line_wrap:
object_ref_string = "{:<12} {:<5} {:<6} {:<22} {:<6} {:<18} \
{:<56}\n"
mem += f"Grouping by {group_by}...\
Sorting by {sort_by}...\n\n\n\n"
for key, group in memory_table["group"].items():
# Group summary
summary = group["summary"]
summary["total_object_size"] = str(summary["total_object_size"]) + " B"
mem += f"--- Summary for {group_by}: {key} ---\n"
mem += summary_string\
.format(*summary_labels)
mem += summary_string\
.format(*summary.values()) + "\n"
# Memory table per group
mem += f"--- Object references for {group_by}: {key} ---\n"
mem += object_ref_string\
.format(*object_ref_labels)
for entry in group["entries"]:
entry["object_size"] = str(entry["object_size"]) + " B" if entry[
"object_size"] > -1 else "?"
num_lines = 1
if size > line_wrap_threshold and line_wrap:
call_site_length = 22
entry["call_site"] = [
entry["call_site"][i:i + call_site_length] for i in range(
0, len(entry["call_site"]), call_site_length)
]
num_lines = len(entry["call_site"])
object_ref_values = [
entry["node_ip_address"], entry["pid"], entry["type"],
entry["call_site"], entry["object_size"],
entry["reference_type"], entry["object_ref"]
]
for i in range(len(object_ref_values)):
if not isinstance(object_ref_values[i], list):
object_ref_values[i] = [object_ref_values[i]]
object_ref_values[i].extend(
["" for x in range(num_lines - len(object_ref_values[i]))])
for i in range(num_lines):
row = [elem[i] for elem in object_ref_values]
mem += object_ref_string\
.format(*row)
mem += "\n"
mem += "\n\n\n"
return mem