def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
(ip, port) = redis_address.split(":")
self.gcs_client = connect_to_gcs(ip, int(port), redis_password)
# Initialize the gcs stub for getting all node resource usage.
gcs_address = self.redis.get("GcsServerAddress").decode("utf-8")
options = (("grpc.enable_http_proxy", 0), )
gcs_channel = grpc.insecure_channel(gcs_address, options=options)
self.gcs_node_resources_stub = \
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.gcs_client = self.gcs_client
worker.mode = 0
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
self.prefix_cluster_info = prefix_cluster_info
self.autoscaling_config = autoscaling_config
self.autoscaler = None
logger.info("Monitor: Started")
def __init__(self, redis_address, autoscaling_config, redis_password=None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
self.global_state_accessor = GlobalStateAccessor(
redis_address, redis_password, False)
self.global_state_accessor.connect()
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Setup subscriptions to the primary Redis server and the Redis shards.
self.primary_subscribe_client = self.redis.pubsub(
ignore_subscribe_messages=True)
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
if autoscaling_config:
self.autoscaler = StandardAutoscaler(autoscaling_config,
self.load_metrics)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def testScaleUpLoadMetrics(self):
config = MULTI_WORKER_CLUSTER.copy()
config["min_workers"] = 0
config["max_workers"] = 50
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
lm = LoadMetrics()
autoscaler = StandardAutoscaler(config_path,
lm,
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(0)
autoscaler.update()
lm.update("1.2.3.4", {},
True, {},
True, {},
waiting_bundles=[{
"GPU": 1
}],
infeasible_bundles=[{
"CPU": 16
}])
autoscaler.update()
self.waitForNodes(2)
nodes = {
self.provider.mock_nodes[0].node_type,
self.provider.mock_nodes[1].node_type
}
assert nodes == {"p2.xlarge", "m4.4xlarge"}
def _setup_autoscaler(self):
self.runner = MockProcessRunner()
self.config = yaml.safe_load(open(self.config_path).read())
self.provider.create_node(
{},
{
TAG_RAY_NODE_KIND: NODE_KIND_HEAD,
TAG_RAY_USER_NODE_TYPE: self.config["head_node_type"],
},
1,
)
self.head_ip = self.provider.non_terminated_node_ips({})[0]
self.load_metrics = LoadMetrics(local_ip=self.head_ip)
self.autoscaler = StandardAutoscaler(
self.config_path,
self.load_metrics,
# Don't let the autoscaler start any node launchers. Instead, we
# will launch nodes ourself after every update call.
max_concurrent_launches=0,
max_failures=0,
process_runner=self.runner,
update_interval_s=0,
)
# Manually create a node launcher. Note that we won't start it as a
# separate thread.
self.node_launcher = NodeLauncher(
provider=self.autoscaler.provider,
queue=self.autoscaler.launch_queue,
index=0,
pending=self.autoscaler.pending_launches,
node_types=self.autoscaler.available_node_types,
)
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
self.global_state_accessor = GlobalStateAccessor(
redis_address, redis_password, False)
self.global_state_accessor.connect()
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
if autoscaling_config:
self.autoscaler = StandardAutoscaler(
autoscaling_config,
self.load_metrics,
prefix_cluster_info=prefix_cluster_info)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False,
monitor_ip=None,
stop_event: Optional[Event] = None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
if monitor_ip:
self.redis.set("AutoscalerMetricsAddress",
f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}")
(ip, port) = redis_address.split(":")
self.gcs_client = connect_to_gcs(ip, int(port), redis_password)
# Initialize the gcs stub for getting all node resource usage.
gcs_address = self.redis.get("GcsServerAddress").decode("utf-8")
options = (("grpc.enable_http_proxy", 0), )
gcs_channel = grpc.insecure_channel(gcs_address, options=options)
self.gcs_node_resources_stub = \
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.gcs_client = self.gcs_client
worker.mode = 0
head_node_ip = redis_address.split(":")[0]
self.redis_address = redis_address
self.redis_password = redis_password
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
self.prefix_cluster_info = prefix_cluster_info
# Can be used to signal graceful exit from monitor loop.
self.stop_event = stop_event # type: Optional[Event]
self.autoscaling_config = autoscaling_config
self.autoscaler = None
self.prom_metrics = AutoscalerPrometheusMetrics()
if monitor_ip:
# If monitor_ip wasn't passed in, then don't attempt to start the
# metric server to keep behavior identical to before metrics were
# introduced
try:
logger.info(
"Starting autoscaler metrics server on port {}".format(
AUTOSCALER_METRIC_PORT))
prometheus_client.start_http_server(
AUTOSCALER_METRIC_PORT,
registry=self.prom_metrics.registry)
except Exception:
logger.exception(
"An exception occurred while starting the metrics server.")
logger.info("Monitor: Started")
def testResourceDemandVector(self):
lm = LoadMetrics()
lm.update("1.1.1.1", {"CPU": 2}, {"CPU": 1}, {},
waiting_bundles=[{
"GPU": 1
}],
infeasible_bundles=[{
"CPU": 16
}])
assert same_elements(lm.get_resource_demand_vector(), [{
"CPU": 16
}, {
"GPU": 1
}])
def testPlacementGroupLoad(self):
lm = LoadMetrics()
pending_placement_groups = [
PlacementGroupTableData(
state=PlacementGroupTableData.RESCHEDULING,
strategy=PlacementStrategy.PACK,
bundles=([Bundle(unit_resources={"GPU": 2})] * 2)),
PlacementGroupTableData(
state=PlacementGroupTableData.RESCHEDULING,
strategy=PlacementStrategy.SPREAD,
bundles=([Bundle(unit_resources={"GPU": 2})] * 2)),
]
lm.update("1.1.1.1", {}, {}, {},
pending_placement_groups=pending_placement_groups)
assert lm.get_pending_placement_groups() == pending_placement_groups
def testRequestBundlesAccountsForHeadNode(self):
config = MULTI_WORKER_CLUSTER.copy()
config["head_node_type"] = "p2.8xlarge"
config["min_workers"] = 0
config["max_workers"] = 50
config_path = self.write_config(config)
self.provider = MockProvider()
self.provider.create_node({}, {
TAG_RAY_USER_NODE_TYPE: "p2.8xlarge",
TAG_RAY_NODE_KIND: "head"
}, 1)
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 1
# These requests fit on the head node.
autoscaler.update()
self.waitForNodes(1)
autoscaler.request_resources([{"CPU": 1}])
autoscaler.update()
self.waitForNodes(1)
assert len(self.provider.mock_nodes) == 1
autoscaler.request_resources([{"GPU": 8}])
autoscaler.update()
self.waitForNodes(1)
# This request requires an additional worker node.
autoscaler.request_resources([{"GPU": 8}] * 2)
autoscaler.update()
self.waitForNodes(2)
assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
def testRequestBundles(self):
config = MULTI_WORKER_CLUSTER.copy()
config["min_workers"] = 0
config["max_workers"] = 50
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(0)
autoscaler.request_resources([{"CPU": 1}])
autoscaler.update()
self.waitForNodes(1)
assert self.provider.mock_nodes[0].node_type == "m4.large"
autoscaler.request_resources([{"GPU": 8}])
autoscaler.update()
self.waitForNodes(2)
assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
autoscaler.request_resources([{"CPU": 32}] * 4)
autoscaler.update()
self.waitForNodes(4)
assert self.provider.mock_nodes[2].node_type == "m4.16xlarge"
assert self.provider.mock_nodes[3].node_type == "m4.16xlarge"
def testScaleUpIgnoreUsed(self):
config = MULTI_WORKER_CLUSTER.copy()
# Commenting out this line causes the test case to fail?!?!
config["min_workers"] = 0
config["target_utilization_fraction"] = 1.0
config_path = self.write_config(config)
self.provider = MockProvider()
self.provider.create_node({}, {
TAG_RAY_NODE_KIND: "head",
TAG_RAY_USER_NODE_TYPE: "p2.xlarge"
}, 1)
head_ip = self.provider.non_terminated_node_ips({})[0]
self.provider.finish_starting_nodes()
runner = MockProcessRunner()
lm = LoadMetrics(local_ip=head_ip)
autoscaler = StandardAutoscaler(config_path,
lm,
max_failures=0,
process_runner=runner,
update_interval_s=0)
autoscaler.update()
self.waitForNodes(1)
lm.update(head_ip, {"CPU": 4, "GPU": 1}, {}, {})
self.waitForNodes(1)
lm.update(head_ip, {
"CPU": 4,
"GPU": 1
}, {"GPU": 0}, {},
waiting_bundles=[{
"GPU": 1
}])
autoscaler.update()
self.waitForNodes(2)
assert self.provider.mock_nodes[1].node_type == "p2.xlarge"
def testScaleUpMinWorkers(self):
config = copy.deepcopy(MULTI_WORKER_CLUSTER)
config["min_workers"] = 2
config["max_workers"] = 50
config["idle_timeout_minutes"] = 1
# Since config["min_workers"] > 1, the remaining worker is started
# with the default worker node type.
config["available_node_types"]["p2.8xlarge"]["min_workers"] = 1
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
lm = LoadMetrics()
autoscaler = StandardAutoscaler(
config_path,
lm,
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(2)
assert len(self.provider.mock_nodes) == 2
assert {
self.provider.mock_nodes[0].node_type,
self.provider.mock_nodes[1].node_type
} == {"p2.8xlarge", "m4.large"}
self.provider.create_node({}, {
TAG_RAY_USER_NODE_TYPE: "p2.8xlarge",
TAG_RAY_NODE_KIND: NODE_KIND_WORKER
}, 2)
self.provider.create_node({}, {
TAG_RAY_USER_NODE_TYPE: "m4.16xlarge",
TAG_RAY_NODE_KIND: NODE_KIND_WORKER
}, 2)
assert len(self.provider.non_terminated_nodes({})) == 6
# Make sure that after idle_timeout_minutes we don't kill idle
# min workers.
for node_id in self.provider.non_terminated_nodes({}):
lm.last_used_time_by_ip[self.provider.internal_ip(node_id)] = -60
autoscaler.update()
self.waitForNodes(2)
cnt = 0
for id in self.provider.mock_nodes:
if self.provider.mock_nodes[id].state == "running" or \
self.provider.mock_nodes[id].state == "pending":
assert self.provider.mock_nodes[id].node_type in {
"p2.8xlarge", "m4.large"
}
cnt += 1
assert cnt == 2
def testScaleUpMinSanity(self):
config_path = self.write_config(MULTI_WORKER_CLUSTER)
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(2)
autoscaler.update()
self.waitForNodes(2)
def testCommandPassing(self):
t = "custom"
config = MULTI_WORKER_CLUSTER.copy()
config["available_node_types"]["p2.8xlarge"][
"worker_setup_commands"] = ["new_worker_setup_command"]
config["available_node_types"]["p2.xlarge"][
"initialization_commands"] = ["new_worker_initialization_cmd"]
config["available_node_types"]["p2.xlarge"]["resources"][t] = 1
# Commenting out this line causes the test case to fail?!?!
config["min_workers"] = 0
config["max_workers"] = 10
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(
config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(0)
autoscaler.request_resources([{"CPU": 1}])
autoscaler.update()
self.waitForNodes(1)
assert self.provider.mock_nodes[0].node_type == "m4.large"
autoscaler.request_resources([{"GPU": 8}])
autoscaler.update()
self.waitForNodes(2)
assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
autoscaler.request_resources([{"GPU": 1}] * 9)
autoscaler.update()
self.waitForNodes(3)
assert self.provider.mock_nodes[2].node_type == "p2.xlarge"
autoscaler.update()
sleep(0.1)
runner.assert_has_call(self.provider.mock_nodes[1].internal_ip,
"new_worker_setup_command")
runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip,
"setup_cmd")
runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip,
"worker_setup_cmd")
runner.assert_has_call(self.provider.mock_nodes[2].internal_ip,
"new_worker_initialization_cmd")
runner.assert_not_has_call(self.provider.mock_nodes[2].internal_ip,
"init_cmd")
def testUpdateConfig(self):
config = MULTI_WORKER_CLUSTER.copy()
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(2)
config["min_workers"] = 0
config["available_node_types"]["m4.large"]["node_config"][
"field_changed"] = 1
config_path = self.write_config(config)
autoscaler.update()
self.waitForNodes(0)
def testResourcePassing(self):
config = MULTI_WORKER_CLUSTER.copy()
config["min_workers"] = 0
config["max_workers"] = 50
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(
config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(0)
autoscaler.request_resources([{"CPU": 1}])
autoscaler.update()
self.waitForNodes(1)
assert self.provider.mock_nodes[0].node_type == "m4.large"
autoscaler.request_resources([{"GPU": 8}])
autoscaler.update()
self.waitForNodes(2)
assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
# TODO (Alex): Autoscaler creates the node during one update then
# starts the updater in the enxt update. The sleep is largely
# unavoidable because the updater runs in its own thread and we have no
# good way of ensuring that the commands are sent in time.
autoscaler.update()
sleep(0.1)
# These checks are done separately because we have no guarantees on the
# order the dict is serialized in.
runner.assert_has_call("172.0.0.0", "RAY_OVERRIDE_RESOURCES=")
runner.assert_has_call("172.0.0.0", "\"CPU\":2")
runner.assert_has_call("172.0.0.1", "RAY_OVERRIDE_RESOURCES=")
runner.assert_has_call("172.0.0.1", "\"CPU\":32")
runner.assert_has_call("172.0.0.1", "\"GPU\":8")
class Monitor:
"""Autoscaling monitor.
This process periodically collects stats from the GCS and triggers
autoscaler updates.
Attributes:
redis: A connection to the Redis server.
"""
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False,
monitor_ip=None,
stop_event: Optional[Event] = None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(
redis_address, redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
if monitor_ip:
self.redis.set("AutoscalerMetricsAddress",
f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}")
(ip, port) = redis_address.split(":")
# Initialize the gcs stub for getting all node resource usage.
gcs_address = self.redis.get("GcsServerAddress").decode("utf-8")
options = (("grpc.enable_http_proxy", 0), )
gcs_channel = ray._private.utils.init_grpc_channel(
gcs_address, options)
# TODO: Use gcs client for this
self.gcs_node_resources_stub = \
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)
self.gcs_node_info_stub = \
gcs_service_pb2_grpc.NodeInfoGcsServiceStub(gcs_channel)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
gcs_client = GcsClient.create_from_redis(self.redis)
_initialize_internal_kv(gcs_client)
worker.mode = 0
head_node_ip = redis_address.split(":")[0]
self.redis_address = redis_address
self.redis_password = redis_password
if os.environ.get("RAY_FAKE_CLUSTER"):
self.load_metrics = LoadMetrics(local_ip=FAKE_HEAD_NODE_ID)
else:
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
self.prefix_cluster_info = prefix_cluster_info
# Can be used to signal graceful exit from monitor loop.
self.stop_event = stop_event # type: Optional[Event]
self.autoscaling_config = autoscaling_config
self.autoscaler = None
# If set, we are in a manually created cluster (non-autoscaling) and
# simply mirroring what the GCS tells us the cluster node types are.
self.readonly_config = None
self.prom_metrics = AutoscalerPrometheusMetrics()
if monitor_ip and prometheus_client:
# If monitor_ip wasn't passed in, then don't attempt to start the
# metric server to keep behavior identical to before metrics were
# introduced
try:
logger.info(
"Starting autoscaler metrics server on port {}".format(
AUTOSCALER_METRIC_PORT))
prometheus_client.start_http_server(
port=AUTOSCALER_METRIC_PORT,
addr="127.0.0.1" if head_node_ip == "127.0.0.1" else "",
registry=self.prom_metrics.registry)
except Exception:
logger.exception(
"An exception occurred while starting the metrics server.")
elif not prometheus_client:
logger.warning("`prometheus_client` not found, so metrics will "
"not be exported.")
logger.info("Monitor: Started")
def _initialize_autoscaler(self):
if self.autoscaling_config:
autoscaling_config = self.autoscaling_config
else:
# This config mirrors the current setup of the manually created
# cluster. Each node gets its own unique node type.
self.readonly_config = BASE_READONLY_CONFIG
# Note that the "available_node_types" of the config can change.
def get_latest_readonly_config():
return self.readonly_config
autoscaling_config = get_latest_readonly_config
self.autoscaler = StandardAutoscaler(
autoscaling_config,
self.load_metrics,
self.gcs_node_info_stub,
prefix_cluster_info=self.prefix_cluster_info,
event_summarizer=self.event_summarizer,
prom_metrics=self.prom_metrics)
def update_load_metrics(self):
"""Fetches resource usage data from GCS and updates load metrics."""
request = gcs_service_pb2.GetAllResourceUsageRequest()
response = self.gcs_node_resources_stub.GetAllResourceUsage(
request, timeout=60)
resources_batch_data = response.resource_usage_data
# Tell the readonly node provider what nodes to report.
if self.readonly_config:
new_nodes = []
for msg in list(resources_batch_data.batch):
node_id = msg.node_id.hex()
new_nodes.append((node_id, msg.node_manager_address))
self.autoscaler.provider._set_nodes(new_nodes)
mirror_node_types = {}
cluster_full = False
for resource_message in resources_batch_data.batch:
node_id = resource_message.node_id
# Generate node type config based on GCS reported node list.
if self.readonly_config:
# Keep prefix in sync with ReadonlyNodeProvider.
node_type = format_readonly_node_type(node_id.hex())
resources = {}
for k, v in resource_message.resources_total.items():
resources[k] = v
mirror_node_types[node_type] = {
"resources": resources,
"node_config": {},
"max_workers": 1,
}
if (hasattr(resource_message, "cluster_full_of_actors_detected")
and resource_message.cluster_full_of_actors_detected):
# Aggregate this flag across all batches.
cluster_full = True
resource_load = dict(resource_message.resource_load)
total_resources = dict(resource_message.resources_total)
available_resources = dict(resource_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
resources_batch_data.resource_load_by_shape)
pending_placement_groups = list(
resources_batch_data.placement_group_load.placement_group_data)
use_node_id_as_ip = (self.autoscaler is not None
and self.autoscaler.config["provider"].get(
"use_node_id_as_ip", False))
# "use_node_id_as_ip" is a hack meant to address situations in
# which there's more than one Ray node residing at a given ip.
# TODO (Dmitri): Stop using ips as node identifiers.
# https://github.com/ray-project/ray/issues/19086
if use_node_id_as_ip:
peloton_id = total_resources.get("NODE_ID_AS_RESOURCE")
# Legacy support https://github.com/ray-project/ray/pull/17312
if peloton_id is not None:
ip = str(int(peloton_id))
else:
ip = node_id.hex()
else:
ip = resource_message.node_manager_address
self.load_metrics.update(ip, node_id, total_resources,
available_resources, resource_load,
waiting_bundles, infeasible_bundles,
pending_placement_groups, cluster_full)
if self.readonly_config:
self.readonly_config["available_node_types"].update(
mirror_node_types)
def update_resource_requests(self):
"""Fetches resource requests from the internal KV and updates load."""
if not _internal_kv_initialized():
return
data = _internal_kv_get(
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
if data:
try:
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
except Exception:
logger.exception("Error parsing resource requests")
def _run(self):
"""Run the monitor loop."""
while True:
if self.stop_event and self.stop_event.is_set():
break
self.update_load_metrics()
self.update_resource_requests()
self.update_event_summary()
status = {
"load_metrics_report": self.load_metrics.summary()._asdict(),
"time": time.time(),
"monitor_pid": os.getpid()
}
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.autoscaler.update()
status[
"autoscaler_report"] = self.autoscaler.summary()._asdict()
for msg in self.event_summarizer.summary():
logger.info("{}{}".format(
ray_constants.LOG_PREFIX_EVENT_SUMMARY, msg))
self.event_summarizer.clear()
as_json = json.dumps(status)
if _internal_kv_initialized():
_internal_kv_put(
DEBUG_AUTOSCALING_STATUS, as_json, overwrite=True)
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def update_event_summary(self):
"""Report the current size of the cluster.
To avoid log spam, only cluster size changes (CPU or GPU count change)
are reported to the event summarizer. The event summarizer will report
only the latest cluster size per batch.
"""
avail_resources = self.load_metrics.resources_avail_summary()
if (not self.readonly_config
and avail_resources != self.last_avail_resources):
self.event_summarizer.add(
"Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs.
quantity=avail_resources,
aggregate=lambda old, new: new)
self.last_avail_resources = avail_resources
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def _handle_failure(self, error):
logger.exception("Error in monitor loop")
if self.autoscaler is not None and \
os.environ.get("RAY_AUTOSCALER_FATESHARE_WORKERS", "") == "1":
self.autoscaler.kill_workers()
# Take down autoscaler workers if necessary.
self.destroy_autoscaler_workers()
# Something went wrong, so push an error to all current and future
# drivers.
message = f"The autoscaler failed with the following error:\n{error}"
if _internal_kv_initialized():
_internal_kv_put(DEBUG_AUTOSCALING_ERROR, message, overwrite=True)
redis_client = ray._private.services.create_redis_client(
self.redis_address, password=self.redis_password)
gcs_publisher = None
if args.gcs_address:
gcs_publisher = GcsPublisher(address=args.gcs_address)
elif gcs_pubsub_enabled():
gcs_publisher = GcsPublisher(
address=get_gcs_address_from_redis(redis_client))
from ray._private.utils import publish_error_to_driver
publish_error_to_driver(
ray_constants.MONITOR_DIED_ERROR,
message,
redis_client=redis_client,
gcs_publisher=gcs_publisher)
def _signal_handler(self, sig, frame):
self._handle_failure(f"Terminated with signal {sig}\n" +
"".join(traceback.format_stack(frame)))
sys.exit(sig + 128)
def run(self):
# Register signal handlers for autoscaler termination.
signal.signal(signal.SIGINT, self._signal_handler)
signal.signal(signal.SIGTERM, self._signal_handler)
try:
if _internal_kv_initialized():
# Delete any previous autoscaling errors.
_internal_kv_del(DEBUG_AUTOSCALING_ERROR)
self._initialize_autoscaler()
self._run()
except Exception:
self._handle_failure(traceback.format_exc())
raise
class Monitor:
"""Autoscaling monitor.
This process periodically collects stats from the GCS and triggers
autoscaler updates.
Attributes:
redis: A connection to the Redis server.
"""
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False,
monitor_ip=None,
stop_event: Optional[Event] = None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
if monitor_ip:
self.redis.set("AutoscalerMetricsAddress",
f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}")
(ip, port) = redis_address.split(":")
self.gcs_client = connect_to_gcs(ip, int(port), redis_password)
# Initialize the gcs stub for getting all node resource usage.
gcs_address = self.redis.get("GcsServerAddress").decode("utf-8")
options = (("grpc.enable_http_proxy", 0), )
gcs_channel = grpc.insecure_channel(gcs_address, options=options)
self.gcs_node_resources_stub = \
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.gcs_client = self.gcs_client
worker.mode = 0
head_node_ip = redis_address.split(":")[0]
self.redis_address = redis_address
self.redis_password = redis_password
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
self.prefix_cluster_info = prefix_cluster_info
# Can be used to signal graceful exit from monitor loop.
self.stop_event = stop_event # type: Optional[Event]
self.autoscaling_config = autoscaling_config
self.autoscaler = None
self.prom_metrics = AutoscalerPrometheusMetrics()
if monitor_ip:
# If monitor_ip wasn't passed in, then don't attempt to start the
# metric server to keep behavior identical to before metrics were
# introduced
try:
logger.info(
"Starting autoscaler metrics server on port {}".format(
AUTOSCALER_METRIC_PORT))
prometheus_client.start_http_server(
AUTOSCALER_METRIC_PORT,
registry=self.prom_metrics.registry)
except Exception:
logger.exception(
"An exception occurred while starting the metrics server.")
logger.info("Monitor: Started")
def __del__(self):
disconnect_from_gcs(self.gcs_client)
def _initialize_autoscaler(self):
if self.autoscaling_config:
self.autoscaler = StandardAutoscaler(
self.autoscaling_config,
self.load_metrics,
prefix_cluster_info=self.prefix_cluster_info,
event_summarizer=self.event_summarizer,
prom_metrics=self.prom_metrics)
def update_load_metrics(self):
"""Fetches resource usage data from GCS and updates load metrics."""
request = gcs_service_pb2.GetAllResourceUsageRequest()
response = self.gcs_node_resources_stub.GetAllResourceUsage(request,
timeout=4)
resources_batch_data = response.resource_usage_data
for resource_message in resources_batch_data.batch:
resource_load = dict(resource_message.resource_load)
total_resources = dict(resource_message.resources_total)
available_resources = dict(resource_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
resources_batch_data.resource_load_by_shape)
pending_placement_groups = list(
resources_batch_data.placement_group_load.placement_group_data)
ip = resource_message.node_manager_address
self.load_metrics.update(ip, total_resources, available_resources,
resource_load, waiting_bundles,
infeasible_bundles,
pending_placement_groups)
def update_resource_requests(self):
"""Fetches resource requests from the internal KV and updates load."""
if not _internal_kv_initialized():
return
data = _internal_kv_get(
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
if data:
try:
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
except Exception:
logger.exception("Error parsing resource requests")
def _run(self):
"""Run the monitor loop."""
while True:
if self.stop_event and self.stop_event.is_set():
break
self.update_load_metrics()
self.update_resource_requests()
self.update_event_summary()
status = {
"load_metrics_report": self.load_metrics.summary()._asdict(),
"time": time.time(),
"monitor_pid": os.getpid()
}
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.autoscaler.update()
status["autoscaler_report"] = self.autoscaler.summary(
)._asdict()
for msg in self.event_summarizer.summary():
logger.info(":event_summary:{}".format(msg))
self.event_summarizer.clear()
as_json = json.dumps(status)
if _internal_kv_initialized():
_internal_kv_put(DEBUG_AUTOSCALING_STATUS,
as_json,
overwrite=True)
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def update_event_summary(self):
"""Report the current size of the cluster.
To avoid log spam, only cluster size changes (CPU or GPU count change)
are reported to the event summarizer. The event summarizer will report
only the latest cluster size per batch.
"""
avail_resources = self.load_metrics.resources_avail_summary()
if avail_resources != self.last_avail_resources:
self.event_summarizer.add(
"Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs.
quantity=avail_resources,
aggregate=lambda old, new: new)
self.last_avail_resources = avail_resources
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def _handle_failure(self, error):
logger.exception("Error in monitor loop")
if self.autoscaler is not None and \
os.environ.get("RAY_AUTOSCALER_FATESHARE_WORKERS", "") == "1":
self.autoscaler.kill_workers()
# Take down autoscaler workers if necessary.
self.destroy_autoscaler_workers()
# Something went wrong, so push an error to all current and future
# drivers.
message = f"The autoscaler failed with the following error:\n{error}"
if _internal_kv_initialized():
_internal_kv_put(DEBUG_AUTOSCALING_ERROR, message, overwrite=True)
redis_client = ray._private.services.create_redis_client(
self.redis_address, password=self.redis_password)
from ray._private.utils import push_error_to_driver_through_redis
push_error_to_driver_through_redis(redis_client,
ray_constants.MONITOR_DIED_ERROR,
message)
def _signal_handler(self, sig, frame):
self._handle_failure(f"Terminated with signal {sig}\n" +
"".join(traceback.format_stack(frame)))
sys.exit(sig + 128)
def run(self):
# Register signal handlers for autoscaler termination.
signal.signal(signal.SIGINT, self._signal_handler)
signal.signal(signal.SIGTERM, self._signal_handler)
try:
if _internal_kv_initialized():
# Delete any previous autoscaling errors.
_internal_kv_del(DEBUG_AUTOSCALING_ERROR)
self._initialize_autoscaler()
self._run()
except Exception:
self._handle_failure(traceback.format_exc())
raise
def __init__(
self,
address: str,
autoscaling_config: Union[str, Callable[[], Dict[str, Any]]],
redis_password: Optional[str] = None,
prefix_cluster_info: bool = False,
monitor_ip: Optional[str] = None,
stop_event: Optional[Event] = None,
retry_on_failure: bool = True,
):
gcs_address = address
options = (("grpc.enable_http_proxy", 0), )
gcs_channel = ray._private.utils.init_grpc_channel(
gcs_address, options)
# TODO: Use gcs client for this
self.gcs_node_resources_stub = (
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel))
self.gcs_node_info_stub = gcs_service_pb2_grpc.NodeInfoGcsServiceStub(
gcs_channel)
if redis_password is not None:
logger.warning("redis_password has been deprecated.")
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
gcs_client = GcsClient(address=gcs_address)
if monitor_ip:
monitor_addr = f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}"
gcs_client.internal_kv_put(b"AutoscalerMetricsAddress",
monitor_addr.encode(), True, None)
_initialize_internal_kv(gcs_client)
if monitor_ip:
monitor_addr = f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}"
gcs_client.internal_kv_put(b"AutoscalerMetricsAddress",
monitor_addr.encode(), True, None)
worker.mode = 0
head_node_ip = gcs_address.split(":")[0]
self.load_metrics = LoadMetrics()
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
self.prefix_cluster_info = prefix_cluster_info
# Can be used to signal graceful exit from monitor loop.
self.stop_event = stop_event # type: Optional[Event]
self.retry_on_failure = retry_on_failure
self.autoscaling_config = autoscaling_config
self.autoscaler = None
# If set, we are in a manually created cluster (non-autoscaling) and
# simply mirroring what the GCS tells us the cluster node types are.
self.readonly_config = None
self.prom_metrics = AutoscalerPrometheusMetrics()
if monitor_ip and prometheus_client:
# If monitor_ip wasn't passed in, then don't attempt to start the
# metric server to keep behavior identical to before metrics were
# introduced
try:
logger.info(
"Starting autoscaler metrics server on port {}".format(
AUTOSCALER_METRIC_PORT))
prometheus_client.start_http_server(
port=AUTOSCALER_METRIC_PORT,
addr="127.0.0.1" if head_node_ip == "127.0.0.1" else "",
registry=self.prom_metrics.registry,
)
except Exception:
logger.exception(
"An exception occurred while starting the metrics server.")
elif not prometheus_client:
logger.warning(
"`prometheus_client` not found, so metrics will not be exported."
)
logger.info("Monitor: Started")
class Monitor:
"""A monitor for Ray processes.
The monitor is in charge of cleaning up the tables in the global state
after processes have died. The monitor is currently not responsible for
detecting component failures.
Attributes:
redis: A connection to the Redis server.
primary_subscribe_client: A pubsub client for the Redis server.
This is used to receive notifications about failed components.
"""
def __init__(self, redis_address, autoscaling_config, redis_password=None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Setup subscriptions to the primary Redis server and the Redis shards.
self.primary_subscribe_client = self.redis.pubsub(
ignore_subscribe_messages=True)
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
self.light_heartbeat_enabled = ray._config.light_heartbeat_enabled()
self.load_metrics = LoadMetrics()
if autoscaling_config:
self.autoscaler = StandardAutoscaler(autoscaling_config,
self.load_metrics)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def __del__(self):
"""Destruct the monitor object."""
# We close the pubsub client to avoid leaking file descriptors.
try:
primary_subscribe_client = self.primary_subscribe_client
except AttributeError:
primary_subscribe_client = None
if primary_subscribe_client is not None:
primary_subscribe_client.close()
def subscribe(self, channel):
"""Subscribe to the given channel on the primary Redis shard.
Args:
channel (str): The channel to subscribe to.
Raises:
Exception: An exception is raised if the subscription fails.
"""
self.primary_subscribe_client.subscribe(channel)
def psubscribe(self, pattern):
"""Subscribe to the given pattern on the primary Redis shard.
Args:
pattern (str): The pattern to subscribe to.
Raises:
Exception: An exception is raised if the subscription fails.
"""
self.primary_subscribe_client.psubscribe(pattern)
def parse_resource_demands(self, resource_load_by_shape):
"""Handle the message.resource_load_by_shape protobuf for the demand
based autoscaling. Catch and log all exceptions so this doesn't
interfere with the utilization based autoscaler until we're confident
this is stable.
Args:
resource_load_by_shape (pb2.gcs.ResourceLoad): The resource demands
in protobuf form or None.
"""
waiting_bundles, infeasible_bundles = [], []
try:
if self.autoscaler:
for resource_demand_pb in list(
resource_load_by_shape.resource_demands):
request_shape = dict(resource_demand_pb.shape)
for _ in range(
resource_demand_pb.num_ready_requests_queued):
waiting_bundles.append(request_shape)
for _ in range(
resource_demand_pb.num_infeasible_requests_queued):
infeasible_bundles.append(request_shape)
except Exception as e:
logger.exception(e)
return waiting_bundles, infeasible_bundles
def xray_heartbeat_batch_handler(self, unused_channel, data):
"""Handle an xray heartbeat batch message from Redis."""
pub_message = ray.gcs_utils.PubSubMessage.FromString(data)
heartbeat_data = pub_message.data
message = ray.gcs_utils.HeartbeatBatchTableData.FromString(
heartbeat_data)
for heartbeat_message in message.batch:
resource_load = dict(heartbeat_message.resource_load)
total_resources = dict(heartbeat_message.resources_total)
available_resources = dict(heartbeat_message.resources_available)
waiting_bundles, infeasible_bundles = \
self.parse_resource_demands(message.resource_load_by_shape)
# Update the load metrics for this raylet.
client_id = ray.utils.binary_to_hex(heartbeat_message.client_id)
ip = self.raylet_id_to_ip_map.get(client_id)
if ip:
update_available_resources = not self.light_heartbeat_enabled \
or heartbeat_message.resources_available_changed()
update_resource_load = not self.light_heartbeat_enabled \
or heartbeat_message.resource_load_changed()
self.load_metrics.update(ip, total_resources,
update_available_resources,
available_resources,
update_resource_load, resource_load,
waiting_bundles, infeasible_bundles)
else:
logger.warning(
f"Monitor: could not find ip for client {client_id}")
def xray_job_notification_handler(self, unused_channel, data):
"""Handle a notification that a job has been added or removed.
Args:
unused_channel: The message channel.
data: The message data.
"""
pub_message = ray.gcs_utils.PubSubMessage.FromString(data)
job_data = pub_message.data
message = ray.gcs_utils.JobTableData.FromString(job_data)
job_id = message.job_id
if message.is_dead:
logger.info("Monitor: "
"XRay Driver {} has been removed.".format(
binary_to_hex(job_id)))
def autoscaler_resource_request_handler(self, _, data):
"""Handle a notification of a resource request for the autoscaler.
This channel and method are only used by the manual
`ray.autoscaler.sdk.request_resources` api.
Args:
channel: unused
data: a resource request as JSON, e.g. {"CPU": 1}
"""
if not self.autoscaler:
return
try:
self.autoscaler.request_resources(json.loads(data))
except Exception:
# We don't want this to kill the monitor.
traceback.print_exc()
def process_messages(self, max_messages=10000):
"""Process all messages ready in the subscription channels.
This reads messages from the subscription channels and calls the
appropriate handlers until there are no messages left.
Args:
max_messages: The maximum number of messages to process before
returning.
"""
subscribe_clients = [self.primary_subscribe_client]
for subscribe_client in subscribe_clients:
for _ in range(max_messages):
message = None
try:
message = subscribe_client.get_message()
except redis.exceptions.ConnectionError:
pass
if message is None:
# Continue on to the next subscribe client.
break
# Parse the message.
pattern = message["pattern"]
channel = message["channel"]
data = message["data"]
# Determine the appropriate message handler.
if pattern == ray.gcs_utils.XRAY_HEARTBEAT_BATCH_PATTERN:
# Similar functionality as raylet info channel
message_handler = self.xray_heartbeat_batch_handler
elif pattern == ray.gcs_utils.XRAY_JOB_PATTERN:
# Handles driver death.
message_handler = self.xray_job_notification_handler
elif (channel ==
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL):
message_handler = self.autoscaler_resource_request_handler
else:
assert False, "This code should be unreachable."
# Call the handler.
message_handler(channel, data)
def update_raylet_map(self, _append_port=False):
"""Updates internal raylet map.
Args:
_append_port (bool): Defaults to False. Appending the port is
useful in testing, as mock clusters have many nodes with
the same IP and cannot be uniquely identified.
"""
all_raylet_nodes = ray.nodes()
self.raylet_id_to_ip_map = {}
for raylet_info in all_raylet_nodes:
node_id = (raylet_info.get("DBClientID") or raylet_info["NodeID"])
ip_address = (raylet_info.get("AuxAddress")
or raylet_info["NodeManagerAddress"]).split(":")[0]
if _append_port:
ip_address += ":" + str(raylet_info["NodeManagerPort"])
self.raylet_id_to_ip_map[node_id] = ip_address
def _run(self):
"""Run the monitor.
This function loops forever, checking for messages about dead database
clients and cleaning up state accordingly.
"""
# Initialize the mapping from raylet client ID to IP address.
self.update_raylet_map()
# Initialize the subscription channel.
self.psubscribe(ray.gcs_utils.XRAY_HEARTBEAT_BATCH_PATTERN)
self.psubscribe(ray.gcs_utils.XRAY_JOB_PATTERN)
if self.autoscaler:
self.subscribe(
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
# TODO(rkn): If there were any dead clients at startup, we should clean
# up the associated state in the state tables.
# Handle messages from the subscription channels.
while True:
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.update_raylet_map()
self.autoscaler.update()
# Process a round of messages.
self.process_messages()
# Wait for a heartbeat interval before processing the next round of
# messages.
time.sleep(ray._config.raylet_heartbeat_timeout_milliseconds() *
1e-3)
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def run(self):
try:
self._run()
except Exception:
logger.exception("Error in monitor loop")
if self.autoscaler:
self.autoscaler.kill_workers()
raise
class Simulator:
"""This autoscaler simulator consists of a few components.
State is stored in 3 main data structures:
* Resource management state is stored in self.ip_to_nodes
* The scheduler's work queue is stored in self.work_queue
* An event queue which acts as the simulation's "timeline" in
self.event_queue
The logic is organized into 3 functions (and their helpers):
* self.run_autoscaler plays the role of `monitor.py` and translates
resource management state for load_metrics to consume.
* self.schedule is the only consumer of the work queue. It dispatches
work to the appropriate schedulers, which mutate cluster state and
produce events for the event queue.
* self.process_event is the sole consumer of the event queue. It
dispatches work to the appropriate event handlers.
There are 3 main ways of interacting with the simulator:
* simulator.submit: To submit tasks
* simulator.step: To go to the next "event"
* task/actor/placement group start/done callbacks
"""
def __init__(
self,
config_path,
provider,
autoscaler_update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S,
node_startup_delay_s=120,
):
self.config_path = config_path
self.provider = provider
self.autoscaler_update_interval_s = autoscaler_update_interval_s
self.node_startup_delay_s = node_startup_delay_s
self._setup_autoscaler()
self._setup_simulator()
def _setup_autoscaler(self):
self.runner = MockProcessRunner()
self.config = yaml.safe_load(open(self.config_path).read())
self.provider.create_node(
{},
{
TAG_RAY_NODE_KIND: NODE_KIND_HEAD,
TAG_RAY_USER_NODE_TYPE: self.config["head_node_type"],
},
1,
)
self.head_ip = self.provider.non_terminated_node_ips({})[0]
self.load_metrics = LoadMetrics(local_ip=self.head_ip)
self.autoscaler = StandardAutoscaler(
self.config_path,
self.load_metrics,
# Don't let the autoscaler start any node launchers. Instead, we
# will launch nodes ourself after every update call.
max_concurrent_launches=0,
max_failures=0,
process_runner=self.runner,
update_interval_s=0,
)
# Manually create a node launcher. Note that we won't start it as a
# separate thread.
self.node_launcher = NodeLauncher(
provider=self.autoscaler.provider,
queue=self.autoscaler.launch_queue,
index=0,
pending=self.autoscaler.pending_launches,
node_types=self.autoscaler.available_node_types,
)
def _setup_simulator(self):
self.virtual_time = 0
self.ip_to_nodes = {}
self._update_cluster_state(join_immediately=True)
self.work_queue = []
self.event_queue = PriorityQueue()
self.event_queue.put(Event(0, SIMULATOR_EVENT_AUTOSCALER_UPDATE))
def _update_cluster_state(self, join_immediately=False):
nodes = self.provider.non_terminated_nodes(tag_filters={})
for node_id in nodes:
ip = self.provider.internal_ip(node_id)
if ip in self.ip_to_nodes:
continue
node_tags = self.provider.node_tags(node_id)
if TAG_RAY_USER_NODE_TYPE in node_tags:
node_type = node_tags[TAG_RAY_USER_NODE_TYPE]
resources = self.config["available_node_types"][node_type].get(
"resources", {})
node = Node(resources, join_immediately, node_type,
self.virtual_time)
self.ip_to_nodes[ip] = node
if not join_immediately:
join_time = self.virtual_time + self.node_startup_delay_s
self.event_queue.put(
Event(join_time, SIMULATOR_EVENT_NODE_JOINED, node))
def submit(self, work):
if isinstance(work, list):
self.work_queue.extend(work)
else:
self.work_queue.append(work)
def _get_node_to_run(self, bundle, nodes):
for ip, node in nodes.items():
if node.bundle_fits(bundle):
return ip, node
return None, None
def _schedule_placement_group(self, pg, nodes):
# This scheduling algorithm is bad, but it is approximately as bad as
# the real placement group scheduler.
to_allocate = []
if (pg.strategy == PlacementStrategy.STRICT_PACK
or pg.strategy == PlacementStrategy.PACK):
combined = collections.defaultdict(float)
for bundle in pg.bundles:
for k, v in bundle.items():
combined[k] += v
ip, node_to_run = self._get_node_to_run(combined, nodes)
if node_to_run is None:
return False
to_allocate.append((combined, ip))
elif (pg.strategy == PlacementStrategy.STRICT_SPREAD
or pg.strategy == PlacementStrategy.SPREAD):
# TODO (Alex): More accurate handling of non-STRICT_PACK groups.
remaining_nodes = nodes.copy()
for bundle in pg.bundles:
ip, node_to_run = self._get_node_to_run(
bundle, remaining_nodes)
if node_to_run is None:
return False
del remaining_nodes[ip]
to_allocate.append((bundle, ip))
for bundle, ip in to_allocate:
node = self.ip_to_nodes[ip]
node.allocate(bundle)
pg.start_time = self.virtual_time
end_time = self.virtual_time + pg.duration
self.event_queue.put(
Event(end_time, SIMULATOR_EVENT_PG_DONE, (pg, to_allocate)))
if pg.start_callback:
pg.start_callback()
return True
def _schedule_task(self, task, nodes):
ip, node = self._get_node_to_run(task.resources, nodes)
if node is None:
return False
node.allocate(task.resources)
task.node = node
task.start_time = self.virtual_time
end_time = self.virtual_time + task.duration
self.event_queue.put(Event(end_time, SIMULATOR_EVENT_TASK_DONE, task))
if task.start_callback:
task.start_callback()
return True
def schedule(self):
# TODO (Alex): Implement a more realistic scheduling algorithm.
new_work_queue = []
for work in self.work_queue:
if isinstance(work, Task):
scheduled = self._schedule_task(work, self.ip_to_nodes)
elif isinstance(work, PlacementGroup):
scheduled = self._schedule_placement_group(
work, self.ip_to_nodes)
else:
assert False, "Unknown work object!"
if scheduled is False:
new_work_queue.append(work)
self.work_queue = new_work_queue
def _launch_nodes(self):
"""Launch all queued nodes. Since this will be run serially after
`autoscaler.update` there are no race conditions in checking if the
queue is empty.
"""
while not self.node_launcher.queue.empty():
config, count, node_type = self.node_launcher.queue.get()
try:
self.node_launcher._launch_node(config, count, node_type)
except Exception:
pass
finally:
self.node_launcher.pending.dec(node_type, count)
def _infeasible(self, bundle):
for node in self.ip_to_nodes.values():
if node.feasible(bundle):
return False
return True
def run_autoscaler(self):
waiting_bundles = []
infeasible_bundles = []
placement_groups = []
for work in self.work_queue:
if isinstance(work, Task):
shape = work.resources
if self._infeasible(shape):
infeasible_bundles.append(shape)
else:
waiting_bundles.append(shape)
if isinstance(work, PlacementGroup):
placement_groups.append(
PlacementGroupTableData(
state=PlacementGroupTableData.PENDING,
strategy=work.strategy,
bundles=[
Bundle(unit_resources=bundle)
for bundle in work.bundles
],
))
for ip, node in self.ip_to_nodes.items():
if not node.in_cluster:
continue
self.load_metrics.update(
ip=ip,
static_resources=node.total_resources,
dynamic_resources=node.available_resources,
resource_load={},
waiting_bundles=waiting_bundles,
infeasible_bundles=infeasible_bundles,
pending_placement_groups=placement_groups,
)
self.autoscaler.update()
self._launch_nodes()
self._update_cluster_state()
def process_event(self, event):
if event.event_type == SIMULATOR_EVENT_AUTOSCALER_UPDATE:
self.run_autoscaler()
next_update = self.virtual_time + self.autoscaler_update_interval_s
self.event_queue.put(
Event(next_update, SIMULATOR_EVENT_AUTOSCALER_UPDATE))
elif event.event_type == SIMULATOR_EVENT_TASK_DONE:
task = event.data
task.node.free(task.resources)
if task.done_callback:
task.done_callback()
elif event.event_type == SIMULATOR_EVENT_NODE_JOINED:
node = event.data
node.in_cluster = True
elif event.event_type == SIMULATOR_EVENT_PG_DONE:
pg, allocated = event.data
for bundle, ip in allocated:
self.ip_to_nodes[ip].free(bundle)
if pg.done_callback:
pg.done_callback()
else:
assert False, "Unknown event!"
def step(self):
self.virtual_time = self.event_queue.queue[0].time
while self.event_queue.queue[0].time == self.virtual_time:
event = self.event_queue.get()
self.process_event(event)
self.schedule()
print(self.info_string())
return self.virtual_time
def node_costs(self):
"""Returns the cost of nodes. Cost is measured in terms of cumulative hours of
runtime per node type.
"""
costs = collections.defaultdict(float)
for node in self.ip_to_nodes.values():
if not node.in_cluster:
continue
runtime = self.virtual_time - node.start_time
costs[node.node_type] += runtime
return costs
def info_string(self):
num_connected_nodes = len(
[node for node in self.ip_to_nodes.values() if node.in_cluster])
num_pending_nodes = len(self.ip_to_nodes) - num_connected_nodes
return (f"[t={self.virtual_time}] "
f"Connected: {num_connected_nodes}, "
f"Pending: {num_pending_nodes}, "
f"Remaining: {len(self.work_queue)}")
def testPlacementGroup(self):
# Note this is mostly an integration test. See
# testPlacementGroupScaling for more comprehensive tests.
config = copy.deepcopy(MULTI_WORKER_CLUSTER)
config["min_workers"] = 0
config["max_workers"] = 999
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
lm = LoadMetrics()
autoscaler = StandardAutoscaler(config_path,
lm,
max_failures=0,
process_runner=runner,
update_interval_s=0)
self.provider.create_node({}, {
TAG_RAY_NODE_KIND: "head",
TAG_RAY_USER_NODE_TYPE: "m4.4xlarge"
}, 1)
head_ip = self.provider.non_terminated_node_ips({})[0]
assert len(self.provider.non_terminated_nodes({})) == 1
autoscaler.update()
self.waitForNodes(1)
pending_placement_groups = [
PlacementGroupTableData(
state=PlacementGroupTableData.RESCHEDULING,
strategy=PlacementStrategy.STRICT_SPREAD,
bundles=[Bundle(unit_resources={"GPU": 2})] * 3),
PlacementGroupTableData(
state=PlacementGroupTableData.RESCHEDULING,
strategy=PlacementStrategy.PACK,
bundles=([Bundle(unit_resources={"GPU": 2})] * 5)),
]
# Since placement groups are implemented with custom resources, this is
# an example of the accompanying resource demands. Note the resource
# demand autoscaler will be unable to fulfill these demands, but we
# should still handle the other infeasible/waiting bundles.
placement_group_resource_demands = [{
"GPU_group_0_6c2506ac733bc37496295b02c4fad446":
0.0101,
"GPU_group_6c2506ac733bc37496295b02c4fad446":
0.0101
}]
lm.update(head_ip, {"CPU": 16},
True, {"CPU": 16},
False, {},
infeasible_bundles=placement_group_resource_demands,
waiting_bundles=[{
"GPU": 8
}],
pending_placement_groups=pending_placement_groups)
autoscaler.update()
self.waitForNodes(5)
for i in range(1, 5):
assert self.provider.mock_nodes[i].node_type == "p2.8xlarge"
pending_placement_groups = [
PlacementGroupTableData(
state=PlacementGroupTableData.RESCHEDULING,
strategy=PlacementStrategy.STRICT_PACK,
bundles=([Bundle(unit_resources={"GPU": 2})] * 4)),
PlacementGroupTableData(
state=PlacementGroupTableData.RESCHEDULING,
strategy=PlacementStrategy.SPREAD,
bundles=([Bundle(unit_resources={"GPU": 2})] * 2)),
]
class Monitor:
"""A monitor for Ray processes.
The monitor is in charge of cleaning up the tables in the global state
after processes have died. The monitor is currently not responsible for
detecting component failures.
Attributes:
redis: A connection to the Redis server.
primary_subscribe_client: A pubsub client for the Redis server.
This is used to receive notifications about failed components.
"""
def __init__(self, redis_address, autoscaling_config, redis_password=None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
self.global_state_accessor = GlobalStateAccessor(
redis_address, redis_password, False)
self.global_state_accessor.connect()
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Setup subscriptions to the primary Redis server and the Redis shards.
self.primary_subscribe_client = self.redis.pubsub(
ignore_subscribe_messages=True)
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
if autoscaling_config:
self.autoscaler = StandardAutoscaler(autoscaling_config,
self.load_metrics)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def __del__(self):
"""Destruct the monitor object."""
# We close the pubsub client to avoid leaking file descriptors.
try:
primary_subscribe_client = self.primary_subscribe_client
except AttributeError:
primary_subscribe_client = None
if primary_subscribe_client is not None:
primary_subscribe_client.close()
if self.global_state_accessor is not None:
self.global_state_accessor.disconnect()
self.global_state_accessor = None
def subscribe(self, channel):
"""Subscribe to the given channel on the primary Redis shard.
Args:
channel (str): The channel to subscribe to.
Raises:
Exception: An exception is raised if the subscription fails.
"""
self.primary_subscribe_client.subscribe(channel)
def update_load_metrics(self):
"""Fetches heartbeat data from GCS and updates load metrics."""
all_heartbeat = self.global_state_accessor.get_all_heartbeat()
heartbeat_batch_data = \
ray.gcs_utils.HeartbeatBatchTableData.FromString(all_heartbeat)
for heartbeat_message in heartbeat_batch_data.batch:
resource_load = dict(heartbeat_message.resource_load)
total_resources = dict(heartbeat_message.resources_total)
available_resources = dict(heartbeat_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
heartbeat_batch_data.resource_load_by_shape)
pending_placement_groups = list(
heartbeat_batch_data.placement_group_load.placement_group_data)
# Update the load metrics for this raylet.
node_id = ray.utils.binary_to_hex(heartbeat_message.node_id)
ip = self.raylet_id_to_ip_map.get(node_id)
if ip:
self.load_metrics.update(ip, total_resources,
available_resources, resource_load,
waiting_bundles, infeasible_bundles,
pending_placement_groups)
else:
logger.warning(
f"Monitor: could not find ip for node {node_id}")
def autoscaler_resource_request_handler(self, _, data):
"""Handle a notification of a resource request for the autoscaler.
This channel and method are only used by the manual
`ray.autoscaler.sdk.request_resources` api.
Args:
channel: unused
data: a resource request as JSON, e.g. {"CPU": 1}
"""
if not self.autoscaler:
return
try:
self.autoscaler.request_resources(json.loads(data))
except Exception:
# We don't want this to kill the monitor.
traceback.print_exc()
def process_messages(self, max_messages=10000):
"""Process all messages ready in the subscription channels.
This reads messages from the subscription channels and calls the
appropriate handlers until there are no messages left.
Args:
max_messages: The maximum number of messages to process before
returning.
"""
subscribe_clients = [self.primary_subscribe_client]
for subscribe_client in subscribe_clients:
for _ in range(max_messages):
message = None
try:
message = subscribe_client.get_message()
except redis.exceptions.ConnectionError:
pass
if message is None:
# Continue on to the next subscribe client.
break
# Parse the message.
channel = message["channel"]
data = message["data"]
if (channel ==
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL):
message_handler = self.autoscaler_resource_request_handler
else:
assert False, "This code should be unreachable."
# Call the handler.
message_handler(channel, data)
def update_raylet_map(self, _append_port=False):
"""Updates internal raylet map.
Args:
_append_port (bool): Defaults to False. Appending the port is
useful in testing, as mock clusters have many nodes with
the same IP and cannot be uniquely identified.
"""
all_raylet_nodes = ray.nodes()
self.raylet_id_to_ip_map = {}
for raylet_info in all_raylet_nodes:
node_id = (raylet_info.get("DBClientID") or raylet_info["NodeID"])
ip_address = (raylet_info.get("AuxAddress")
or raylet_info["NodeManagerAddress"]).split(":")[0]
if _append_port:
ip_address += ":" + str(raylet_info["NodeManagerPort"])
self.raylet_id_to_ip_map[node_id] = ip_address
def _run(self):
"""Run the monitor.
This function loops forever, checking for messages about dead database
clients and cleaning up state accordingly.
"""
self.subscribe(ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
# Handle messages from the subscription channels.
while True:
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.update_raylet_map()
self.update_load_metrics()
self.autoscaler.update()
# Process a round of messages.
self.process_messages()
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def run(self):
try:
self._run()
except Exception:
logger.exception("Error in monitor loop")
if self.autoscaler:
self.autoscaler.kill_workers()
raise
class Monitor:
"""Autoscaling monitor.
This process periodically collects stats from the GCS and triggers
autoscaler updates.
Attributes:
redis: A connection to the Redis server.
"""
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
self.global_state_accessor = GlobalStateAccessor(
redis_address, redis_password, False)
self.global_state_accessor.connect()
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
if autoscaling_config:
self.autoscaler = StandardAutoscaler(
autoscaling_config,
self.load_metrics,
prefix_cluster_info=prefix_cluster_info)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def __del__(self):
"""Destruct the monitor object."""
# We close the pubsub client to avoid leaking file descriptors.
if self.global_state_accessor is not None:
self.global_state_accessor.disconnect()
self.global_state_accessor = None
def update_load_metrics(self):
"""Fetches resource usage data from GCS and updates load metrics."""
all_resources = self.global_state_accessor.get_all_resource_usage()
resources_batch_data = \
ray.gcs_utils.ResourceUsageBatchData.FromString(all_resources)
for resource_message in resources_batch_data.batch:
resource_load = dict(resource_message.resource_load)
total_resources = dict(resource_message.resources_total)
available_resources = dict(resource_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
resources_batch_data.resource_load_by_shape)
pending_placement_groups = list(
resources_batch_data.placement_group_load.placement_group_data)
# Update the load metrics for this raylet.
node_id = ray.utils.binary_to_hex(resource_message.node_id)
ip = self.raylet_id_to_ip_map.get(node_id)
if ip:
self.load_metrics.update(ip, total_resources,
available_resources, resource_load,
waiting_bundles, infeasible_bundles,
pending_placement_groups)
else:
logger.warning(
f"Monitor: could not find ip for node {node_id}")
def update_resource_requests(self):
"""Fetches resource requests from the internal KV and updates load."""
if not _internal_kv_initialized():
return
data = _internal_kv_get(
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
if data:
try:
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
except Exception:
logger.exception("Error parsing resource requests")
def autoscaler_resource_request_handler(self, _, data):
"""Handle a notification of a resource request for the autoscaler.
This channel and method are only used by the manual
`ray.autoscaler.sdk.request_resources` api.
Args:
channel: unused
data: a resource request as JSON, e.g. {"CPU": 1}
"""
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
def update_raylet_map(self, _append_port=False):
"""Updates internal raylet map.
Args:
_append_port (bool): Defaults to False. Appending the port is
useful in testing, as mock clusters have many nodes with
the same IP and cannot be uniquely identified.
"""
all_raylet_nodes = ray.nodes()
self.raylet_id_to_ip_map = {}
for raylet_info in all_raylet_nodes:
node_id = (raylet_info.get("DBClientID") or raylet_info["NodeID"])
ip_address = (raylet_info.get("AuxAddress")
or raylet_info["NodeManagerAddress"]).split(":")[0]
if _append_port:
ip_address += ":" + str(raylet_info["NodeManagerPort"])
self.raylet_id_to_ip_map[node_id] = ip_address
def _run(self):
"""Run the monitor loop."""
while True:
self.update_raylet_map()
self.update_load_metrics()
self.update_resource_requests()
status = {
"load_metrics_report": self.load_metrics.summary()._asdict()
}
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.autoscaler.update()
status["autoscaler_report"] = self.autoscaler.summary(
)._asdict()
as_json = json.dumps(status)
if _internal_kv_initialized():
_internal_kv_put(DEBUG_AUTOSCALING_STATUS,
as_json,
overwrite=True)
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def run(self):
try:
self._run()
except Exception:
logger.exception("Error in monitor loop")
if self.autoscaler:
self.autoscaler.kill_workers()
raise
def testDockerWorkers(self):
config = MULTI_WORKER_CLUSTER.copy()
config["available_node_types"]["p2.8xlarge"]["docker"] = {
"worker_image": "p2.8x_image:latest",
"worker_run_options": ["p2.8x-run-options"]
}
config["available_node_types"]["p2.xlarge"]["docker"] = {
"worker_image": "p2x_image:nightly"
}
config["docker"]["worker_run_options"] = ["standard-run-options"]
config["docker"]["image"] = "default-image:nightly"
config["docker"]["worker_image"] = "default-image:nightly"
# Commenting out this line causes the test case to fail?!?!
config["min_workers"] = 0
config["max_workers"] = 10
config_path = self.write_config(config)
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
update_interval_s=0)
assert len(self.provider.non_terminated_nodes({})) == 0
autoscaler.update()
self.waitForNodes(0)
autoscaler.request_resources([{"CPU": 1}])
autoscaler.update()
self.waitForNodes(1)
assert self.provider.mock_nodes[0].node_type == "m4.large"
autoscaler.request_resources([{"GPU": 8}])
autoscaler.update()
self.waitForNodes(2)
assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
autoscaler.request_resources([{"GPU": 1}] * 9)
autoscaler.update()
self.waitForNodes(3)
assert self.provider.mock_nodes[2].node_type == "p2.xlarge"
autoscaler.update()
# Fill up m4, p2.8, p2 and request 2 more CPUs
autoscaler.request_resources([{
"CPU": 2
}, {
"CPU": 16
}, {
"CPU": 32
}, {
"CPU": 2
}])
autoscaler.update()
self.waitForNodes(4)
assert self.provider.mock_nodes[3].node_type == "m4.16xlarge"
autoscaler.update()
sleep(0.1)
runner.assert_has_call(self.provider.mock_nodes[1].internal_ip,
"p2.8x-run-options")
runner.assert_has_call(self.provider.mock_nodes[1].internal_ip,
"p2.8x_image:latest")
runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip,
"default-image:nightly")
runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip,
"standard-run-options")
runner.assert_has_call(self.provider.mock_nodes[2].internal_ip,
"p2x_image:nightly")
runner.assert_has_call(self.provider.mock_nodes[2].internal_ip,
"standard-run-options")
runner.assert_not_has_call(self.provider.mock_nodes[2].internal_ip,
"p2.8x-run-options")
runner.assert_has_call(self.provider.mock_nodes[3].internal_ip,
"default-image:nightly")
runner.assert_has_call(self.provider.mock_nodes[3].internal_ip,
"standard-run-options")
runner.assert_not_has_call(self.provider.mock_nodes[3].internal_ip,
"p2.8x-run-options")
runner.assert_not_has_call(self.provider.mock_nodes[3].internal_ip,
"p2x_image:nightly")
class Monitor:
"""Autoscaling monitor.
This process periodically collects stats from the GCS and triggers
autoscaler updates.
Attributes:
redis: A connection to the Redis server.
"""
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
# Initialize the gcs stub for getting all node resource usage.
gcs_address = self.redis.get("GcsServerAddress").decode("utf-8")
gcs_channel = grpc.insecure_channel(gcs_address)
self.gcs_node_resources_stub = \
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
if autoscaling_config:
self.autoscaler = StandardAutoscaler(
autoscaling_config,
self.load_metrics,
prefix_cluster_info=prefix_cluster_info,
event_summarizer=self.event_summarizer)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
logger.info("Monitor: Started")
def update_load_metrics(self):
"""Fetches resource usage data from GCS and updates load metrics."""
request = gcs_service_pb2.GetAllResourceUsageRequest()
response = self.gcs_node_resources_stub.GetAllResourceUsage(request,
timeout=4)
resources_batch_data = response.resource_usage_data
for resource_message in resources_batch_data.batch:
resource_load = dict(resource_message.resource_load)
total_resources = dict(resource_message.resources_total)
available_resources = dict(resource_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
resources_batch_data.resource_load_by_shape)
pending_placement_groups = list(
resources_batch_data.placement_group_load.placement_group_data)
ip = resource_message.node_manager_address
self.load_metrics.update(ip, total_resources, available_resources,
resource_load, waiting_bundles,
infeasible_bundles,
pending_placement_groups)
def update_resource_requests(self):
"""Fetches resource requests from the internal KV and updates load."""
if not _internal_kv_initialized():
return
data = _internal_kv_get(
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
if data:
try:
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
except Exception:
logger.exception("Error parsing resource requests")
def _run(self):
"""Run the monitor loop."""
while True:
self.update_load_metrics()
self.update_resource_requests()
self.update_event_summary()
status = {
"load_metrics_report": self.load_metrics.summary()._asdict()
}
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.autoscaler.update()
status["autoscaler_report"] = self.autoscaler.summary(
)._asdict()
for msg in self.event_summarizer.summary():
logger.info(":event_summary:{}".format(msg))
self.event_summarizer.clear()
as_json = json.dumps(status)
if _internal_kv_initialized():
_internal_kv_put(DEBUG_AUTOSCALING_STATUS,
as_json,
overwrite=True)
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def update_event_summary(self):
"""Report the current size of the cluster.
To avoid log spam, only cluster size changes (CPU or GPU count change)
are reported to the event summarizer. The event summarizer will report
only the latest cluster size per batch.
"""
avail_resources = self.load_metrics.resources_avail_summary()
if avail_resources != self.last_avail_resources:
self.event_summarizer.add(
"Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs.
quantity=avail_resources,
aggregate=lambda old, new: new)
self.last_avail_resources = avail_resources
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def run(self):
try:
self._run()
except Exception:
logger.exception("Error in monitor loop")
if self.autoscaler:
self.autoscaler.kill_workers()
raise
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False,
monitor_ip=None,
stop_event: Optional[Event] = None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(
redis_address, redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
if monitor_ip:
self.redis.set("AutoscalerMetricsAddress",
f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}")
(ip, port) = redis_address.split(":")
# Initialize the gcs stub for getting all node resource usage.
gcs_address = self.redis.get("GcsServerAddress").decode("utf-8")
options = (("grpc.enable_http_proxy", 0), )
gcs_channel = ray._private.utils.init_grpc_channel(
gcs_address, options)
# TODO: Use gcs client for this
self.gcs_node_resources_stub = \
gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)
self.gcs_node_info_stub = \
gcs_service_pb2_grpc.NodeInfoGcsServiceStub(gcs_channel)
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
gcs_client = GcsClient.create_from_redis(self.redis)
_initialize_internal_kv(gcs_client)
worker.mode = 0
head_node_ip = redis_address.split(":")[0]
self.redis_address = redis_address
self.redis_password = redis_password
if os.environ.get("RAY_FAKE_CLUSTER"):
self.load_metrics = LoadMetrics(local_ip=FAKE_HEAD_NODE_ID)
else:
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
self.last_avail_resources = None
self.event_summarizer = EventSummarizer()
self.prefix_cluster_info = prefix_cluster_info
# Can be used to signal graceful exit from monitor loop.
self.stop_event = stop_event # type: Optional[Event]
self.autoscaling_config = autoscaling_config
self.autoscaler = None
# If set, we are in a manually created cluster (non-autoscaling) and
# simply mirroring what the GCS tells us the cluster node types are.
self.readonly_config = None
self.prom_metrics = AutoscalerPrometheusMetrics()
if monitor_ip and prometheus_client:
# If monitor_ip wasn't passed in, then don't attempt to start the
# metric server to keep behavior identical to before metrics were
# introduced
try:
logger.info(
"Starting autoscaler metrics server on port {}".format(
AUTOSCALER_METRIC_PORT))
prometheus_client.start_http_server(
port=AUTOSCALER_METRIC_PORT,
addr="127.0.0.1" if head_node_ip == "127.0.0.1" else "",
registry=self.prom_metrics.registry)
except Exception:
logger.exception(
"An exception occurred while starting the metrics server.")
elif not prometheus_client:
logger.warning("`prometheus_client` not found, so metrics will "
"not be exported.")
logger.info("Monitor: Started")
class Simulator:
def __init__(
self,
config_path,
provider,
autoscaler_update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S,
node_startup_delay_s=120,
):
self.config_path = config_path
self.provider = provider
self.autoscaler_update_interval_s = autoscaler_update_interval_s
self.node_startup_delay_s = node_startup_delay_s
self._setup_autoscaler()
self._setup_simulator()
def _setup_autoscaler(self):
self.runner = MockProcessRunner()
self.config = yaml.safe_load(open(self.config_path).read())
self.provider.create_node(
{},
{
TAG_RAY_NODE_KIND: "head",
TAG_RAY_USER_NODE_TYPE: self.config["head_node_type"],
},
1,
)
self.head_ip = self.provider.non_terminated_node_ips({})[0]
self.load_metrics = LoadMetrics(local_ip=self.head_ip)
self.autoscaler = StandardAutoscaler(
self.config_path,
self.load_metrics,
# Don't let the autoscaler start any node launchers. Instead, we
# will launch nodes ourself after every update call.
max_concurrent_launches=0,
max_failures=0,
process_runner=self.runner,
update_interval_s=0,
)
# Manually create a node launcher. Note that we won't start it as a
# separate thread.
self.node_launcher = NodeLauncher(
provider=self.autoscaler.provider,
queue=self.autoscaler.launch_queue,
index=0,
pending=self.autoscaler.pending_launches,
node_types=self.autoscaler.available_node_types,
)
def _setup_simulator(self):
self.virtual_time = 0
self.ip_to_nodes = {}
self._update_cluster_state(join_immediately=True)
self.work_queue = []
self.event_queue = PriorityQueue()
self.event_queue.put(Event(0, SIMULATOR_EVENT_AUTOSCALER_UPDATE))
def _update_cluster_state(self, join_immediately=False):
nodes = self.provider.non_terminated_nodes(tag_filters={})
for node_id in nodes:
ip = self.provider.internal_ip(node_id)
if ip in self.ip_to_nodes:
continue
node_tags = self.provider.node_tags(node_id)
if TAG_RAY_USER_NODE_TYPE in node_tags:
node_type = node_tags[TAG_RAY_USER_NODE_TYPE]
resources = self.config["available_node_types"][node_type].get(
"resources", {})
node = Node(resources, join_immediately)
self.ip_to_nodes[ip] = node
if not join_immediately:
join_time = self.virtual_time + self.node_startup_delay_s
self.event_queue.put(
Event(join_time, SIMULATOR_EVEN_NODE_JOINED, node))
def submit(self, work):
if isinstance(work, list):
self.work_queue.extend(work)
else:
self.work_queue.append(work)
def _schedule_task(self, task):
for ip, node in self.ip_to_nodes.items():
if node.bundle_fits(task.resources):
node.allocate(task.resources)
task.node = node
task.start_time = self.virtual_time
end_time = self.virtual_time + task.duration
self.event_queue.put(
Event(end_time, SIMULATOR_EVENT_TASK_DONE, task))
if task.start_callback:
task.start_callback()
return True
return False
def schedule(self):
# TODO (Alex): Implement a more realistic scheduling algorithm.
new_work_queue = []
for work in self.work_queue:
if isinstance(work, Task):
scheduled = self._schedule_task(work)
if scheduled is False:
new_work_queue.append(work)
self.work_queue = new_work_queue
def _launch_nodes(self):
"""Launch all queued nodes. Since this will be run serially after
`autoscaler.update` there are no race conditions in checking if the
queue is empty.
"""
while not self.node_launcher.queue.empty():
config, count, node_type = self.node_launcher.queue.get()
try:
self.node_launcher._launch_node(config, count, node_type)
except Exception:
pass
finally:
self.node_launcher.pending.dec(node_type, count)
def _infeasible(self, bundle):
for node in self.ip_to_nodes.values():
if node.feasible(bundle):
return False
return True
def run_autoscaler(self):
waiting_bundles = []
infeasible_bundles = []
for work in self.work_queue:
if isinstance(work, Task):
shape = work.resources
if self._infeasible(shape):
infeasible_bundles.append(shape)
else:
waiting_bundles.append(shape)
for ip, node in self.ip_to_nodes.items():
if not node.in_cluster:
continue
self.load_metrics.update(
ip=ip,
static_resources=node.total_resources,
update_dynamic_resources=True,
dynamic_resources=node.available_resources,
update_resource_load=False,
resource_load={},
waiting_bundles=waiting_bundles,
infeasible_bundles=infeasible_bundles,
pending_placement_groups=[],
)
self.autoscaler.update()
self._launch_nodes()
self._update_cluster_state()
def process_event(self, event):
if event.event_type == SIMULATOR_EVENT_AUTOSCALER_UPDATE:
self.run_autoscaler()
next_update = self.virtual_time + self.autoscaler_update_interval_s
self.event_queue.put(
Event(next_update, SIMULATOR_EVENT_AUTOSCALER_UPDATE))
elif event.event_type == SIMULATOR_EVENT_TASK_DONE:
task = event.data
task.node.free(task.resources)
if task.done_callback:
task.done_callback()
elif event.event_type == SIMULATOR_EVEN_NODE_JOINED:
node = event.data
node.in_cluster = True
def step(self):
self.virtual_time = self.event_queue.queue[0].time
while self.event_queue.queue[0].time == self.virtual_time:
event = self.event_queue.get()
self.process_event(event)
self.schedule()
print(self.info_string())
return self.virtual_time
def info_string(self):
return (f"[t={self.virtual_time}] Nodes: {len(self.ip_to_nodes)} " +
f"Remaining requests: {len(self.work_queue)} ")
