def get_store_stats_summary(redis_address, redis_password) -> str:
state = GlobalState()
state._initialize_global_state(redis_address, redis_password)
raylet = state.node_table()[0]
stats = node_stats(raylet["NodeManagerAddress"], raylet["NodeManagerPort"])
store_summary = store_stats_summary(stats)
return store_summary
def memory_summary(state,
group_by="NODE_ADDRESS",
sort_by="OBJECT_SIZE",
line_wrap=True,
unit="B",
num_entries=None) -> str:
from ray.dashboard.modules.node.node_head\
import node_stats_to_dict
# Get terminal size
import shutil
size = shutil.get_terminal_size((80, 20)).columns
line_wrap_threshold = 137
# Unit conversions
units = {"B": 10**0, "KB": 10**3, "MB": 10**6, "GB": 10**9}
# Fetch core memory worker stats, store as a dictionary
core_worker_stats = []
for raylet in state.node_table():
stats = node_stats_to_dict(
node_stats(raylet["NodeManagerAddress"],
raylet["NodeManagerPort"]))
core_worker_stats.extend(stats["coreWorkersStats"])
assert type(stats) is dict and "coreWorkersStats" in stats
# Build memory table with "group_by" and "sort_by" parameters
group_by, sort_by = get_group_by_type(group_by), get_sorting_type(sort_by)
memory_table = construct_memory_table(core_worker_stats, group_by,
sort_by).as_dict()
assert "summary" in memory_table and "group" in memory_table
# Build memory summary
mem = ""
group_by, sort_by = group_by.name.lower().replace(
"_", " "), sort_by.name.lower().replace("_", " ")
summary_labels = [
"Mem Used by Objects", "Local References", "Pinned", "Pending Tasks",
"Captured in Objects", "Actor Handles"
]
summary_string = "{:<19} {:<16} {:<12} {:<13} {:<19} {:<13}\n"
object_ref_labels = [
"IP Address", "PID", "Type", "Call Site", "Size", "Reference Type",
"Object Ref"
]
object_ref_string = "{:<13} | {:<8} | {:<7} | {:<9} \
| {:<8} | {:<14} | {:<10}\n"
if size > line_wrap_threshold and line_wrap:
object_ref_string = "{:<15} {:<5} {:<6} {:<22} {:<6} {:<18} \
{:<56}\n"
mem += f"Grouping by {group_by}...\
Sorting by {sort_by}...\
Display {num_entries if num_entries is not None else 'all'}\
entries per group...\n\n\n"
for key, group in memory_table["group"].items():
# Group summary
summary = group["summary"]
ref_size = track_reference_size(group)
for k, v in summary.items():
if k == "total_object_size":
summary[k] = str(v / units[unit]) + f" {unit}"
else:
summary[k] = str(v) + f", ({ref_size[k] / units[unit]} {unit})"
mem += f"--- Summary for {group_by}: {key} ---\n"
mem += summary_string\
.format(*summary_labels)
mem += summary_string\
.format(*summary.values()) + "\n"
# Memory table per group
mem += f"--- Object references for {group_by}: {key} ---\n"
mem += object_ref_string\
.format(*object_ref_labels)
n = 1 # Counter for num entries per group
for entry in group["entries"]:
if num_entries is not None and n > num_entries:
break
entry["object_size"] = str(
entry["object_size"] /
units[unit]) + f" {unit}" if entry["object_size"] > -1 else "?"
num_lines = 1
if size > line_wrap_threshold and line_wrap:
call_site_length = 22
if len(entry["call_site"]) == 0:
entry["call_site"] = ["disabled"]
else:
entry["call_site"] = [
entry["call_site"][i:i + call_site_length] for i in
range(0, len(entry["call_site"]), call_site_length)
]
num_lines = len(entry["call_site"])
else:
mem += "\n"
object_ref_values = [
entry["node_ip_address"], entry["pid"], entry["type"],
entry["call_site"], entry["object_size"],
entry["reference_type"], entry["object_ref"]
]
for i in range(len(object_ref_values)):
if not isinstance(object_ref_values[i], list):
object_ref_values[i] = [object_ref_values[i]]
object_ref_values[i].extend(
["" for x in range(num_lines - len(object_ref_values[i]))])
for i in range(num_lines):
row = [elem[i] for elem in object_ref_values]
mem += object_ref_string\
.format(*row)
mem += "\n"
n += 1
mem += "To record callsite information for each ObjectRef created, set " \
"env variable RAY_record_ref_creation_sites=1\n\n"
return mem
def get_memory_summary(group_by, sort_by, line_wrap) -> str:
from ray.new_dashboard.modules.stats_collector.stats_collector_head\
import node_stats_to_dict
# Get terminal size
import shutil
size = shutil.get_terminal_size((80, 20)).columns
line_wrap_threshold = 137
# Fetch core memory worker stats, store as a dictionary
core_worker_stats = []
for raylet in ray.nodes():
stats = node_stats_to_dict(
node_stats(raylet["NodeManagerAddress"],
raylet["NodeManagerPort"]))
core_worker_stats.extend(stats["coreWorkersStats"])
assert type(stats) is dict and "coreWorkersStats" in stats
# Build memory table with "group_by" and "sort_by" parameters
group_by, sort_by = get_group_by_type(group_by), get_sorting_type(sort_by)
memory_table = construct_memory_table(core_worker_stats, group_by,
sort_by).as_dict()
assert "summary" in memory_table and "group" in memory_table
# Build memory summary
mem = ""
group_by, sort_by = group_by.name.lower().replace(
"_", " "), sort_by.name.lower().replace("_", " ")
summary_labels = [
"Mem Used by Objects", "Local References", "Pinned Count",
"Pending Tasks", "Captured in Objects", "Actor Handles"
]
summary_string = "{:<19} {:<16} {:<12} {:<13} {:<19} {:<13}\n"
object_ref_labels = [
"IP Address", "PID", "Type", "Call Site", "Size", "Reference Type",
"Object Ref"
]
object_ref_string = "{:<8} {:<3} {:<4} {:<9} {:<4} {:<14} {:<10}\n"
if size > line_wrap_threshold and line_wrap:
object_ref_string = "{:<12} {:<5} {:<6} {:<22} {:<6} {:<18} \
{:<56}\n"
mem += f"Grouping by {group_by}...\
Sorting by {sort_by}...\n\n\n\n"
for key, group in memory_table["group"].items():
# Group summary
summary = group["summary"]
summary["total_object_size"] = str(summary["total_object_size"]) + " B"
mem += f"--- Summary for {group_by}: {key} ---\n"
mem += summary_string\
.format(*summary_labels)
mem += summary_string\
.format(*summary.values()) + "\n"
# Memory table per group
mem += f"--- Object references for {group_by}: {key} ---\n"
mem += object_ref_string\
.format(*object_ref_labels)
for entry in group["entries"]:
entry["object_size"] = str(entry["object_size"]) + " B" if entry[
"object_size"] > -1 else "?"
num_lines = 1
if size > line_wrap_threshold and line_wrap:
call_site_length = 22
entry["call_site"] = [
entry["call_site"][i:i + call_site_length] for i in range(
0, len(entry["call_site"]), call_site_length)
]
num_lines = len(entry["call_site"])
object_ref_values = [
entry["node_ip_address"], entry["pid"], entry["type"],
entry["call_site"], entry["object_size"],
entry["reference_type"], entry["object_ref"]
]
for i in range(len(object_ref_values)):
if not isinstance(object_ref_values[i], list):
object_ref_values[i] = [object_ref_values[i]]
object_ref_values[i].extend(
["" for x in range(num_lines - len(object_ref_values[i]))])
for i in range(num_lines):
row = [elem[i] for elem in object_ref_values]
mem += object_ref_string\
.format(*row)
mem += "\n"
mem += "\n\n\n"
return mem