def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) (ip, port) = redis_address.split(":") self.gcs_client = connect_to_gcs(ip, int(port), redis_password) # Initialize the gcs stub for getting all node resource usage. gcs_address = self.redis.get("GcsServerAddress").decode("utf-8") options = (("grpc.enable_http_proxy", 0), ) gcs_channel = grpc.insecure_channel(gcs_address, options=options) self.gcs_node_resources_stub = \ gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.gcs_client = self.gcs_client worker.mode = 0 head_node_ip = redis_address.split(":")[0] self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() self.prefix_cluster_info = prefix_cluster_info self.autoscaling_config = autoscaling_config self.autoscaler = None logger.info("Monitor: Started")
def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False, monitor_ip=None, stop_event: Optional[Event] = None): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) if monitor_ip: self.redis.set("AutoscalerMetricsAddress", f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}") (ip, port) = redis_address.split(":") self.gcs_client = connect_to_gcs(ip, int(port), redis_password) # Initialize the gcs stub for getting all node resource usage. gcs_address = self.redis.get("GcsServerAddress").decode("utf-8") options = (("grpc.enable_http_proxy", 0), ) gcs_channel = grpc.insecure_channel(gcs_address, options=options) self.gcs_node_resources_stub = \ gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.gcs_client = self.gcs_client worker.mode = 0 head_node_ip = redis_address.split(":")[0] self.redis_address = redis_address self.redis_password = redis_password self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() self.prefix_cluster_info = prefix_cluster_info # Can be used to signal graceful exit from monitor loop. self.stop_event = stop_event # type: Optional[Event] self.autoscaling_config = autoscaling_config self.autoscaler = None self.prom_metrics = AutoscalerPrometheusMetrics() if monitor_ip: # If monitor_ip wasn't passed in, then don't attempt to start the # metric server to keep behavior identical to before metrics were # introduced try: logger.info( "Starting autoscaler metrics server on port {}".format( AUTOSCALER_METRIC_PORT)) prometheus_client.start_http_server( AUTOSCALER_METRIC_PORT, registry=self.prom_metrics.registry) except Exception: logger.exception( "An exception occurred while starting the metrics server.") logger.info("Monitor: Started")
def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) self.global_state_accessor = GlobalStateAccessor( redis_address, redis_password, False) self.global_state_accessor.connect() # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.mode = 0 # Keep a mapping from raylet client ID to IP address to use # for updating the load metrics. self.raylet_id_to_ip_map = {} head_node_ip = redis_address.split(":")[0] self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() if autoscaling_config: self.autoscaler = StandardAutoscaler( autoscaling_config, self.load_metrics, prefix_cluster_info=prefix_cluster_info, event_summarizer=self.event_summarizer) self.autoscaling_config = autoscaling_config else: self.autoscaler = None self.autoscaling_config = None logger.info("Monitor: Started")
def _setup_autoscaler(self): self.runner = MockProcessRunner() self.config = yaml.safe_load(open(self.config_path).read()) self.provider.create_node( {}, { TAG_RAY_NODE_KIND: NODE_KIND_HEAD, TAG_RAY_USER_NODE_TYPE: self.config["head_node_type"], }, 1, ) self.head_ip = self.provider.non_terminated_node_ips({})[0] self.load_metrics = LoadMetrics() self.autoscaler = MockAutoscaler( self.config_path, self.load_metrics, MockNodeInfoStub(), # Don't let the autoscaler start any node launchers. Instead, we # will launch nodes ourself after every update call. max_concurrent_launches=0, max_failures=0, process_runner=self.runner, update_interval_s=0, ) # Manually create a node launcher. Note that we won't start it as a # separate thread. self.node_launcher = NodeLauncher( event_summarizer=EventSummarizer(), provider=self.autoscaler.provider, queue=self.autoscaler.launch_queue, index=0, pending=self.autoscaler.pending_launches, node_types=self.autoscaler.available_node_types, )
def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False, monitor_ip=None, stop_event: Optional[Event] = None): # Initialize the Redis clients. ray.state.state._initialize_global_state( redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) if monitor_ip: self.redis.set("AutoscalerMetricsAddress", f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}") (ip, port) = redis_address.split(":") # Initialize the gcs stub for getting all node resource usage. gcs_address = self.redis.get("GcsServerAddress").decode("utf-8") options = (("grpc.enable_http_proxy", 0), ) gcs_channel = ray._private.utils.init_grpc_channel( gcs_address, options) # TODO: Use gcs client for this self.gcs_node_resources_stub = \ gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel) self.gcs_node_info_stub = \ gcs_service_pb2_grpc.NodeInfoGcsServiceStub(gcs_channel) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis gcs_client = GcsClient.create_from_redis(self.redis) _initialize_internal_kv(gcs_client) worker.mode = 0 head_node_ip = redis_address.split(":")[0] self.redis_address = redis_address self.redis_password = redis_password if os.environ.get("RAY_FAKE_CLUSTER"): self.load_metrics = LoadMetrics(local_ip=FAKE_HEAD_NODE_ID) else: self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() self.prefix_cluster_info = prefix_cluster_info # Can be used to signal graceful exit from monitor loop. self.stop_event = stop_event # type: Optional[Event] self.autoscaling_config = autoscaling_config self.autoscaler = None # If set, we are in a manually created cluster (non-autoscaling) and # simply mirroring what the GCS tells us the cluster node types are. self.readonly_config = None self.prom_metrics = AutoscalerPrometheusMetrics() if monitor_ip and prometheus_client: # If monitor_ip wasn't passed in, then don't attempt to start the # metric server to keep behavior identical to before metrics were # introduced try: logger.info( "Starting autoscaler metrics server on port {}".format( AUTOSCALER_METRIC_PORT)) prometheus_client.start_http_server( port=AUTOSCALER_METRIC_PORT, addr="127.0.0.1" if head_node_ip == "127.0.0.1" else "", registry=self.prom_metrics.registry) except Exception: logger.exception( "An exception occurred while starting the metrics server.") elif not prometheus_client: logger.warning("`prometheus_client` not found, so metrics will " "not be exported.") logger.info("Monitor: Started")
class Monitor: """Autoscaling monitor. This process periodically collects stats from the GCS and triggers autoscaler updates. Attributes: redis: A connection to the Redis server. """ def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False, monitor_ip=None, stop_event: Optional[Event] = None): # Initialize the Redis clients. ray.state.state._initialize_global_state( redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) if monitor_ip: self.redis.set("AutoscalerMetricsAddress", f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}") (ip, port) = redis_address.split(":") # Initialize the gcs stub for getting all node resource usage. gcs_address = self.redis.get("GcsServerAddress").decode("utf-8") options = (("grpc.enable_http_proxy", 0), ) gcs_channel = ray._private.utils.init_grpc_channel( gcs_address, options) # TODO: Use gcs client for this self.gcs_node_resources_stub = \ gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel) self.gcs_node_info_stub = \ gcs_service_pb2_grpc.NodeInfoGcsServiceStub(gcs_channel) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis gcs_client = GcsClient.create_from_redis(self.redis) _initialize_internal_kv(gcs_client) worker.mode = 0 head_node_ip = redis_address.split(":")[0] self.redis_address = redis_address self.redis_password = redis_password if os.environ.get("RAY_FAKE_CLUSTER"): self.load_metrics = LoadMetrics(local_ip=FAKE_HEAD_NODE_ID) else: self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() self.prefix_cluster_info = prefix_cluster_info # Can be used to signal graceful exit from monitor loop. self.stop_event = stop_event # type: Optional[Event] self.autoscaling_config = autoscaling_config self.autoscaler = None # If set, we are in a manually created cluster (non-autoscaling) and # simply mirroring what the GCS tells us the cluster node types are. self.readonly_config = None self.prom_metrics = AutoscalerPrometheusMetrics() if monitor_ip and prometheus_client: # If monitor_ip wasn't passed in, then don't attempt to start the # metric server to keep behavior identical to before metrics were # introduced try: logger.info( "Starting autoscaler metrics server on port {}".format( AUTOSCALER_METRIC_PORT)) prometheus_client.start_http_server( port=AUTOSCALER_METRIC_PORT, addr="127.0.0.1" if head_node_ip == "127.0.0.1" else "", registry=self.prom_metrics.registry) except Exception: logger.exception( "An exception occurred while starting the metrics server.") elif not prometheus_client: logger.warning("`prometheus_client` not found, so metrics will " "not be exported.") logger.info("Monitor: Started") def _initialize_autoscaler(self): if self.autoscaling_config: autoscaling_config = self.autoscaling_config else: # This config mirrors the current setup of the manually created # cluster. Each node gets its own unique node type. self.readonly_config = BASE_READONLY_CONFIG # Note that the "available_node_types" of the config can change. def get_latest_readonly_config(): return self.readonly_config autoscaling_config = get_latest_readonly_config self.autoscaler = StandardAutoscaler( autoscaling_config, self.load_metrics, self.gcs_node_info_stub, prefix_cluster_info=self.prefix_cluster_info, event_summarizer=self.event_summarizer, prom_metrics=self.prom_metrics) def update_load_metrics(self): """Fetches resource usage data from GCS and updates load metrics.""" request = gcs_service_pb2.GetAllResourceUsageRequest() response = self.gcs_node_resources_stub.GetAllResourceUsage( request, timeout=60) resources_batch_data = response.resource_usage_data # Tell the readonly node provider what nodes to report. if self.readonly_config: new_nodes = [] for msg in list(resources_batch_data.batch): node_id = msg.node_id.hex() new_nodes.append((node_id, msg.node_manager_address)) self.autoscaler.provider._set_nodes(new_nodes) mirror_node_types = {} cluster_full = False for resource_message in resources_batch_data.batch: node_id = resource_message.node_id # Generate node type config based on GCS reported node list. if self.readonly_config: # Keep prefix in sync with ReadonlyNodeProvider. node_type = format_readonly_node_type(node_id.hex()) resources = {} for k, v in resource_message.resources_total.items(): resources[k] = v mirror_node_types[node_type] = { "resources": resources, "node_config": {}, "max_workers": 1, } if (hasattr(resource_message, "cluster_full_of_actors_detected") and resource_message.cluster_full_of_actors_detected): # Aggregate this flag across all batches. cluster_full = True resource_load = dict(resource_message.resource_load) total_resources = dict(resource_message.resources_total) available_resources = dict(resource_message.resources_available) waiting_bundles, infeasible_bundles = parse_resource_demands( resources_batch_data.resource_load_by_shape) pending_placement_groups = list( resources_batch_data.placement_group_load.placement_group_data) use_node_id_as_ip = (self.autoscaler is not None and self.autoscaler.config["provider"].get( "use_node_id_as_ip", False)) # "use_node_id_as_ip" is a hack meant to address situations in # which there's more than one Ray node residing at a given ip. # TODO (Dmitri): Stop using ips as node identifiers. # https://github.com/ray-project/ray/issues/19086 if use_node_id_as_ip: peloton_id = total_resources.get("NODE_ID_AS_RESOURCE") # Legacy support https://github.com/ray-project/ray/pull/17312 if peloton_id is not None: ip = str(int(peloton_id)) else: ip = node_id.hex() else: ip = resource_message.node_manager_address self.load_metrics.update(ip, node_id, total_resources, available_resources, resource_load, waiting_bundles, infeasible_bundles, pending_placement_groups, cluster_full) if self.readonly_config: self.readonly_config["available_node_types"].update( mirror_node_types) def update_resource_requests(self): """Fetches resource requests from the internal KV and updates load.""" if not _internal_kv_initialized(): return data = _internal_kv_get( ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL) if data: try: resource_request = json.loads(data) self.load_metrics.set_resource_requests(resource_request) except Exception: logger.exception("Error parsing resource requests") def _run(self): """Run the monitor loop.""" while True: if self.stop_event and self.stop_event.is_set(): break self.update_load_metrics() self.update_resource_requests() self.update_event_summary() status = { "load_metrics_report": self.load_metrics.summary()._asdict(), "time": time.time(), "monitor_pid": os.getpid() } # Process autoscaling actions if self.autoscaler: # Only used to update the load metrics for the autoscaler. self.autoscaler.update() status[ "autoscaler_report"] = self.autoscaler.summary()._asdict() for msg in self.event_summarizer.summary(): logger.info("{}{}".format( ray_constants.LOG_PREFIX_EVENT_SUMMARY, msg)) self.event_summarizer.clear() as_json = json.dumps(status) if _internal_kv_initialized(): _internal_kv_put( DEBUG_AUTOSCALING_STATUS, as_json, overwrite=True) # Wait for a autoscaler update interval before processing the next # round of messages. time.sleep(AUTOSCALER_UPDATE_INTERVAL_S) def update_event_summary(self): """Report the current size of the cluster. To avoid log spam, only cluster size changes (CPU or GPU count change) are reported to the event summarizer. The event summarizer will report only the latest cluster size per batch. """ avail_resources = self.load_metrics.resources_avail_summary() if (not self.readonly_config and avail_resources != self.last_avail_resources): self.event_summarizer.add( "Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs. quantity=avail_resources, aggregate=lambda old, new: new) self.last_avail_resources = avail_resources def destroy_autoscaler_workers(self): """Cleanup the autoscaler, in case of an exception in the run() method. We kill the worker nodes, but retain the head node in order to keep logs around, keeping costs minimal. This monitor process runs on the head node anyway, so this is more reliable.""" if self.autoscaler is None: return # Nothing to clean up. if self.autoscaling_config is None: # This is a logic error in the program. Can't do anything. logger.error( "Monitor: Cleanup failed due to lack of autoscaler config.") return logger.info("Monitor: Exception caught. Taking down workers...") clean = False while not clean: try: teardown_cluster( config_file=self.autoscaling_config, yes=True, # Non-interactive. workers_only=True, # Retain head node for logs. override_cluster_name=None, keep_min_workers=True, # Retain minimal amount of workers. ) clean = True logger.info("Monitor: Workers taken down.") except Exception: logger.error("Monitor: Cleanup exception. Trying again...") time.sleep(2) def _handle_failure(self, error): logger.exception("Error in monitor loop") if self.autoscaler is not None and \ os.environ.get("RAY_AUTOSCALER_FATESHARE_WORKERS", "") == "1": self.autoscaler.kill_workers() # Take down autoscaler workers if necessary. self.destroy_autoscaler_workers() # Something went wrong, so push an error to all current and future # drivers. message = f"The autoscaler failed with the following error:\n{error}" if _internal_kv_initialized(): _internal_kv_put(DEBUG_AUTOSCALING_ERROR, message, overwrite=True) redis_client = ray._private.services.create_redis_client( self.redis_address, password=self.redis_password) gcs_publisher = None if args.gcs_address: gcs_publisher = GcsPublisher(address=args.gcs_address) elif gcs_pubsub_enabled(): gcs_publisher = GcsPublisher( address=get_gcs_address_from_redis(redis_client)) from ray._private.utils import publish_error_to_driver publish_error_to_driver( ray_constants.MONITOR_DIED_ERROR, message, redis_client=redis_client, gcs_publisher=gcs_publisher) def _signal_handler(self, sig, frame): self._handle_failure(f"Terminated with signal {sig}\n" + "".join(traceback.format_stack(frame))) sys.exit(sig + 128) def run(self): # Register signal handlers for autoscaler termination. signal.signal(signal.SIGINT, self._signal_handler) signal.signal(signal.SIGTERM, self._signal_handler) try: if _internal_kv_initialized(): # Delete any previous autoscaling errors. _internal_kv_del(DEBUG_AUTOSCALING_ERROR) self._initialize_autoscaler() self._run() except Exception: self._handle_failure(traceback.format_exc()) raise
def __init__(self, config_path, load_metrics, max_launch_batch=AUTOSCALER_MAX_LAUNCH_BATCH, max_concurrent_launches=AUTOSCALER_MAX_CONCURRENT_LAUNCHES, max_failures=AUTOSCALER_MAX_NUM_FAILURES, process_runner=subprocess, update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S, prefix_cluster_info=False, event_summarizer=None): self.config_path = config_path # Prefix each line of info string with cluster name if True self.prefix_cluster_info = prefix_cluster_info # Keep this before self.reset (self.provider needs to be created # exactly once). self.provider = None self.resource_demand_scheduler = None self.reset(errors_fatal=True) self.head_node_ip = load_metrics.local_ip self.load_metrics = load_metrics self.max_failures = max_failures self.max_launch_batch = max_launch_batch self.max_concurrent_launches = max_concurrent_launches self.process_runner = process_runner self.event_summarizer = event_summarizer or EventSummarizer() # Map from node_id to NodeUpdater processes self.updaters = {} self.num_failed_updates = defaultdict(int) self.num_successful_updates = defaultdict(int) self.num_failures = 0 self.last_update_time = 0.0 self.update_interval_s = update_interval_s # Node launchers self.launch_queue = queue.Queue() self.pending_launches = ConcurrentCounter() max_batches = math.ceil(max_concurrent_launches / float(max_launch_batch)) for i in range(int(max_batches)): node_launcher = NodeLauncher( provider=self.provider, queue=self.launch_queue, index=i, pending=self.pending_launches, node_types=self.available_node_types, ) node_launcher.daemon = True node_launcher.start() # Expand local file_mounts to allow ~ in the paths. This can't be done # earlier when the config is written since we might be on different # platform and the expansion would result in wrong path. self.config["file_mounts"] = { remote: os.path.expanduser(local) for remote, local in self.config["file_mounts"].items() } for local_path in self.config["file_mounts"].values(): assert os.path.exists(local_path) logger.info("StandardAutoscaler: {}".format(self.config))
def __init__( self, config_path: str, load_metrics: LoadMetrics, max_launch_batch: int = AUTOSCALER_MAX_LAUNCH_BATCH, max_concurrent_launches: int = AUTOSCALER_MAX_CONCURRENT_LAUNCHES, max_failures: int = AUTOSCALER_MAX_NUM_FAILURES, process_runner: Any = subprocess, update_interval_s: int = AUTOSCALER_UPDATE_INTERVAL_S, prefix_cluster_info: bool = False, event_summarizer: Optional[EventSummarizer] = None, prom_metrics: Optional[AutoscalerPrometheusMetrics] = None): """Create a StandardAutoscaler. Args: config_path: Path to a Ray Autoscaler YAML. load_metrics: Provides metrics for the Ray cluster. max_launch_batch: Max number of nodes to launch in one request. max_concurrent_launches: Max number of nodes that can be concurrently launched. This value and `max_launch_batch` determine the number of batches that are used to launch nodes. max_failures: Number of failures that the autoscaler will tolerate before exiting. process_runner: Subprocess-like interface used by the CommandRunner. update_interval_s: Seconds between running the autoscaling loop. prefix_cluster_info: Whether to add the cluster name to info strings. event_summarizer: Utility to consolidate duplicated messages. prom_metrics: Prometheus metrics for autoscaler-related operations. """ self.config_path = config_path # Prefix each line of info string with cluster name if True self.prefix_cluster_info = prefix_cluster_info # Keep this before self.reset (self.provider needs to be created # exactly once). self.provider = None # Keep this before self.reset (if an exception occurs in reset # then prom_metrics must be instantitiated to increment the # exception counter) self.prom_metrics = prom_metrics or \ AutoscalerPrometheusMetrics() self.resource_demand_scheduler = None self.reset(errors_fatal=True) self.head_node_ip = load_metrics.local_ip self.load_metrics = load_metrics self.max_failures = max_failures self.max_launch_batch = max_launch_batch self.max_concurrent_launches = max_concurrent_launches self.process_runner = process_runner self.event_summarizer = event_summarizer or EventSummarizer() # Map from node_id to NodeUpdater threads self.updaters = {} self.num_failed_updates = defaultdict(int) self.num_successful_updates = defaultdict(int) self.num_failures = 0 self.last_update_time = 0.0 self.update_interval_s = update_interval_s # Tracks active worker nodes self.workers = [] # Tracks nodes scheduled for termination self.nodes_to_terminate = [] # Disable NodeUpdater threads if true. # Should be set to true in situations where another component, such as # a Kubernetes operator, is responsible for Ray setup on nodes. self.disable_node_updaters = self.config["provider"].get( "disable_node_updaters", False) # Node launchers self.launch_queue = queue.Queue() self.pending_launches = ConcurrentCounter() max_batches = math.ceil(max_concurrent_launches / float(max_launch_batch)) for i in range(int(max_batches)): node_launcher = NodeLauncher(provider=self.provider, queue=self.launch_queue, index=i, pending=self.pending_launches, node_types=self.available_node_types, prom_metrics=self.prom_metrics) node_launcher.daemon = True node_launcher.start() # NodeTracker maintains soft state to track the number of recently # failed nodes. It is best effort only. self.node_tracker = NodeTracker() # Expand local file_mounts to allow ~ in the paths. This can't be done # earlier when the config is written since we might be on different # platform and the expansion would result in wrong path. self.config["file_mounts"] = { remote: os.path.expanduser(local) for remote, local in self.config["file_mounts"].items() } for local_path in self.config["file_mounts"].values(): assert os.path.exists(local_path) logger.info("StandardAutoscaler: {}".format(self.config))
class Monitor: """Autoscaling monitor. This process periodically collects stats from the GCS and triggers autoscaler updates. Attributes: redis: A connection to the Redis server. """ def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False, monitor_ip=None, stop_event: Optional[Event] = None): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) if monitor_ip: self.redis.set("AutoscalerMetricsAddress", f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}") (ip, port) = redis_address.split(":") self.gcs_client = connect_to_gcs(ip, int(port), redis_password) # Initialize the gcs stub for getting all node resource usage. gcs_address = self.redis.get("GcsServerAddress").decode("utf-8") options = (("grpc.enable_http_proxy", 0), ) gcs_channel = grpc.insecure_channel(gcs_address, options=options) self.gcs_node_resources_stub = \ gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.gcs_client = self.gcs_client worker.mode = 0 head_node_ip = redis_address.split(":")[0] self.redis_address = redis_address self.redis_password = redis_password self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() self.prefix_cluster_info = prefix_cluster_info # Can be used to signal graceful exit from monitor loop. self.stop_event = stop_event # type: Optional[Event] self.autoscaling_config = autoscaling_config self.autoscaler = None self.prom_metrics = AutoscalerPrometheusMetrics() if monitor_ip: # If monitor_ip wasn't passed in, then don't attempt to start the # metric server to keep behavior identical to before metrics were # introduced try: logger.info( "Starting autoscaler metrics server on port {}".format( AUTOSCALER_METRIC_PORT)) prometheus_client.start_http_server( AUTOSCALER_METRIC_PORT, registry=self.prom_metrics.registry) except Exception: logger.exception( "An exception occurred while starting the metrics server.") logger.info("Monitor: Started") def __del__(self): disconnect_from_gcs(self.gcs_client) def _initialize_autoscaler(self): if self.autoscaling_config: self.autoscaler = StandardAutoscaler( self.autoscaling_config, self.load_metrics, prefix_cluster_info=self.prefix_cluster_info, event_summarizer=self.event_summarizer, prom_metrics=self.prom_metrics) def update_load_metrics(self): """Fetches resource usage data from GCS and updates load metrics.""" request = gcs_service_pb2.GetAllResourceUsageRequest() response = self.gcs_node_resources_stub.GetAllResourceUsage(request, timeout=4) resources_batch_data = response.resource_usage_data for resource_message in resources_batch_data.batch: resource_load = dict(resource_message.resource_load) total_resources = dict(resource_message.resources_total) available_resources = dict(resource_message.resources_available) waiting_bundles, infeasible_bundles = parse_resource_demands( resources_batch_data.resource_load_by_shape) pending_placement_groups = list( resources_batch_data.placement_group_load.placement_group_data) ip = resource_message.node_manager_address self.load_metrics.update(ip, total_resources, available_resources, resource_load, waiting_bundles, infeasible_bundles, pending_placement_groups) def update_resource_requests(self): """Fetches resource requests from the internal KV and updates load.""" if not _internal_kv_initialized(): return data = _internal_kv_get( ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL) if data: try: resource_request = json.loads(data) self.load_metrics.set_resource_requests(resource_request) except Exception: logger.exception("Error parsing resource requests") def _run(self): """Run the monitor loop.""" while True: if self.stop_event and self.stop_event.is_set(): break self.update_load_metrics() self.update_resource_requests() self.update_event_summary() status = { "load_metrics_report": self.load_metrics.summary()._asdict(), "time": time.time(), "monitor_pid": os.getpid() } # Process autoscaling actions if self.autoscaler: # Only used to update the load metrics for the autoscaler. self.autoscaler.update() status["autoscaler_report"] = self.autoscaler.summary( )._asdict() for msg in self.event_summarizer.summary(): logger.info(":event_summary:{}".format(msg)) self.event_summarizer.clear() as_json = json.dumps(status) if _internal_kv_initialized(): _internal_kv_put(DEBUG_AUTOSCALING_STATUS, as_json, overwrite=True) # Wait for a autoscaler update interval before processing the next # round of messages. time.sleep(AUTOSCALER_UPDATE_INTERVAL_S) def update_event_summary(self): """Report the current size of the cluster. To avoid log spam, only cluster size changes (CPU or GPU count change) are reported to the event summarizer. The event summarizer will report only the latest cluster size per batch. """ avail_resources = self.load_metrics.resources_avail_summary() if avail_resources != self.last_avail_resources: self.event_summarizer.add( "Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs. quantity=avail_resources, aggregate=lambda old, new: new) self.last_avail_resources = avail_resources def destroy_autoscaler_workers(self): """Cleanup the autoscaler, in case of an exception in the run() method. We kill the worker nodes, but retain the head node in order to keep logs around, keeping costs minimal. This monitor process runs on the head node anyway, so this is more reliable.""" if self.autoscaler is None: return # Nothing to clean up. if self.autoscaling_config is None: # This is a logic error in the program. Can't do anything. logger.error( "Monitor: Cleanup failed due to lack of autoscaler config.") return logger.info("Monitor: Exception caught. Taking down workers...") clean = False while not clean: try: teardown_cluster( config_file=self.autoscaling_config, yes=True, # Non-interactive. workers_only=True, # Retain head node for logs. override_cluster_name=None, keep_min_workers=True, # Retain minimal amount of workers. ) clean = True logger.info("Monitor: Workers taken down.") except Exception: logger.error("Monitor: Cleanup exception. Trying again...") time.sleep(2) def _handle_failure(self, error): logger.exception("Error in monitor loop") if self.autoscaler is not None and \ os.environ.get("RAY_AUTOSCALER_FATESHARE_WORKERS", "") == "1": self.autoscaler.kill_workers() # Take down autoscaler workers if necessary. self.destroy_autoscaler_workers() # Something went wrong, so push an error to all current and future # drivers. message = f"The autoscaler failed with the following error:\n{error}" if _internal_kv_initialized(): _internal_kv_put(DEBUG_AUTOSCALING_ERROR, message, overwrite=True) redis_client = ray._private.services.create_redis_client( self.redis_address, password=self.redis_password) from ray._private.utils import push_error_to_driver_through_redis push_error_to_driver_through_redis(redis_client, ray_constants.MONITOR_DIED_ERROR, message) def _signal_handler(self, sig, frame): self._handle_failure(f"Terminated with signal {sig}\n" + "".join(traceback.format_stack(frame))) sys.exit(sig + 128) def run(self): # Register signal handlers for autoscaler termination. signal.signal(signal.SIGINT, self._signal_handler) signal.signal(signal.SIGTERM, self._signal_handler) try: if _internal_kv_initialized(): # Delete any previous autoscaling errors. _internal_kv_del(DEBUG_AUTOSCALING_ERROR) self._initialize_autoscaler() self._run() except Exception: self._handle_failure(traceback.format_exc()) raise
def __init__( self, address: str, autoscaling_config: Union[str, Callable[[], Dict[str, Any]]], redis_password: Optional[str] = None, prefix_cluster_info: bool = False, monitor_ip: Optional[str] = None, stop_event: Optional[Event] = None, retry_on_failure: bool = True, ): gcs_address = address options = (("grpc.enable_http_proxy", 0), ) gcs_channel = ray._private.utils.init_grpc_channel( gcs_address, options) # TODO: Use gcs client for this self.gcs_node_resources_stub = ( gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel)) self.gcs_node_info_stub = gcs_service_pb2_grpc.NodeInfoGcsServiceStub( gcs_channel) if redis_password is not None: logger.warning("redis_password has been deprecated.") # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker gcs_client = GcsClient(address=gcs_address) if monitor_ip: monitor_addr = f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}" gcs_client.internal_kv_put(b"AutoscalerMetricsAddress", monitor_addr.encode(), True, None) _initialize_internal_kv(gcs_client) if monitor_ip: monitor_addr = f"{monitor_ip}:{AUTOSCALER_METRIC_PORT}" gcs_client.internal_kv_put(b"AutoscalerMetricsAddress", monitor_addr.encode(), True, None) worker.mode = 0 head_node_ip = gcs_address.split(":")[0] self.load_metrics = LoadMetrics() self.last_avail_resources = None self.event_summarizer = EventSummarizer() self.prefix_cluster_info = prefix_cluster_info # Can be used to signal graceful exit from monitor loop. self.stop_event = stop_event # type: Optional[Event] self.retry_on_failure = retry_on_failure self.autoscaling_config = autoscaling_config self.autoscaler = None # If set, we are in a manually created cluster (non-autoscaling) and # simply mirroring what the GCS tells us the cluster node types are. self.readonly_config = None self.prom_metrics = AutoscalerPrometheusMetrics() if monitor_ip and prometheus_client: # If monitor_ip wasn't passed in, then don't attempt to start the # metric server to keep behavior identical to before metrics were # introduced try: logger.info( "Starting autoscaler metrics server on port {}".format( AUTOSCALER_METRIC_PORT)) prometheus_client.start_http_server( port=AUTOSCALER_METRIC_PORT, addr="127.0.0.1" if head_node_ip == "127.0.0.1" else "", registry=self.prom_metrics.registry, ) except Exception: logger.exception( "An exception occurred while starting the metrics server.") elif not prometheus_client: logger.warning( "`prometheus_client` not found, so metrics will not be exported." ) logger.info("Monitor: Started")
class Monitor: """Autoscaling monitor. This process periodically collects stats from the GCS and triggers autoscaler updates. Attributes: redis: A connection to the Redis server. """ def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) # Initialize the gcs stub for getting all node resource usage. gcs_address = self.redis.get("GcsServerAddress").decode("utf-8") gcs_channel = grpc.insecure_channel(gcs_address) self.gcs_node_resources_stub = \ gcs_service_pb2_grpc.NodeResourceInfoGcsServiceStub(gcs_channel) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.mode = 0 head_node_ip = redis_address.split(":")[0] self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() if autoscaling_config: self.autoscaler = StandardAutoscaler( autoscaling_config, self.load_metrics, prefix_cluster_info=prefix_cluster_info, event_summarizer=self.event_summarizer) self.autoscaling_config = autoscaling_config else: self.autoscaler = None self.autoscaling_config = None logger.info("Monitor: Started") def update_load_metrics(self): """Fetches resource usage data from GCS and updates load metrics.""" request = gcs_service_pb2.GetAllResourceUsageRequest() response = self.gcs_node_resources_stub.GetAllResourceUsage(request, timeout=4) resources_batch_data = response.resource_usage_data for resource_message in resources_batch_data.batch: resource_load = dict(resource_message.resource_load) total_resources = dict(resource_message.resources_total) available_resources = dict(resource_message.resources_available) waiting_bundles, infeasible_bundles = parse_resource_demands( resources_batch_data.resource_load_by_shape) pending_placement_groups = list( resources_batch_data.placement_group_load.placement_group_data) ip = resource_message.node_manager_address self.load_metrics.update(ip, total_resources, available_resources, resource_load, waiting_bundles, infeasible_bundles, pending_placement_groups) def update_resource_requests(self): """Fetches resource requests from the internal KV and updates load.""" if not _internal_kv_initialized(): return data = _internal_kv_get( ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL) if data: try: resource_request = json.loads(data) self.load_metrics.set_resource_requests(resource_request) except Exception: logger.exception("Error parsing resource requests") def _run(self): """Run the monitor loop.""" while True: self.update_load_metrics() self.update_resource_requests() self.update_event_summary() status = { "load_metrics_report": self.load_metrics.summary()._asdict() } # Process autoscaling actions if self.autoscaler: # Only used to update the load metrics for the autoscaler. self.autoscaler.update() status["autoscaler_report"] = self.autoscaler.summary( )._asdict() for msg in self.event_summarizer.summary(): logger.info(":event_summary:{}".format(msg)) self.event_summarizer.clear() as_json = json.dumps(status) if _internal_kv_initialized(): _internal_kv_put(DEBUG_AUTOSCALING_STATUS, as_json, overwrite=True) # Wait for a autoscaler update interval before processing the next # round of messages. time.sleep(AUTOSCALER_UPDATE_INTERVAL_S) def update_event_summary(self): """Report the current size of the cluster. To avoid log spam, only cluster size changes (CPU or GPU count change) are reported to the event summarizer. The event summarizer will report only the latest cluster size per batch. """ avail_resources = self.load_metrics.resources_avail_summary() if avail_resources != self.last_avail_resources: self.event_summarizer.add( "Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs. quantity=avail_resources, aggregate=lambda old, new: new) self.last_avail_resources = avail_resources def destroy_autoscaler_workers(self): """Cleanup the autoscaler, in case of an exception in the run() method. We kill the worker nodes, but retain the head node in order to keep logs around, keeping costs minimal. This monitor process runs on the head node anyway, so this is more reliable.""" if self.autoscaler is None: return # Nothing to clean up. if self.autoscaling_config is None: # This is a logic error in the program. Can't do anything. logger.error( "Monitor: Cleanup failed due to lack of autoscaler config.") return logger.info("Monitor: Exception caught. Taking down workers...") clean = False while not clean: try: teardown_cluster( config_file=self.autoscaling_config, yes=True, # Non-interactive. workers_only=True, # Retain head node for logs. override_cluster_name=None, keep_min_workers=True, # Retain minimal amount of workers. ) clean = True logger.info("Monitor: Workers taken down.") except Exception: logger.error("Monitor: Cleanup exception. Trying again...") time.sleep(2) def run(self): try: self._run() except Exception: logger.exception("Error in monitor loop") if self.autoscaler: self.autoscaler.kill_workers() raise
class Monitor: """Autoscaling monitor. This process periodically collects stats from the GCS and triggers autoscaler updates. Attributes: redis: A connection to the Redis server. """ def __init__(self, redis_address, autoscaling_config, redis_password=None, prefix_cluster_info=False): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray._private.services.create_redis_client( redis_address, password=redis_password) self.global_state_accessor = GlobalStateAccessor( redis_address, redis_password, False) self.global_state_accessor.connect() # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.mode = 0 # Keep a mapping from raylet client ID to IP address to use # for updating the load metrics. self.raylet_id_to_ip_map = {} head_node_ip = redis_address.split(":")[0] self.load_metrics = LoadMetrics(local_ip=head_node_ip) self.last_avail_resources = None self.event_summarizer = EventSummarizer() if autoscaling_config: self.autoscaler = StandardAutoscaler( autoscaling_config, self.load_metrics, prefix_cluster_info=prefix_cluster_info, event_summarizer=self.event_summarizer) self.autoscaling_config = autoscaling_config else: self.autoscaler = None self.autoscaling_config = None logger.info("Monitor: Started") def __del__(self): """Destruct the monitor object.""" # We close the pubsub client to avoid leaking file descriptors. if self.global_state_accessor is not None: self.global_state_accessor.disconnect() self.global_state_accessor = None def update_load_metrics(self): """Fetches resource usage data from GCS and updates load metrics.""" all_resources = self.global_state_accessor.get_all_resource_usage() resources_batch_data = \ ray.gcs_utils.ResourceUsageBatchData.FromString(all_resources) for resource_message in resources_batch_data.batch: resource_load = dict(resource_message.resource_load) total_resources = dict(resource_message.resources_total) available_resources = dict(resource_message.resources_available) waiting_bundles, infeasible_bundles = parse_resource_demands( resources_batch_data.resource_load_by_shape) pending_placement_groups = list( resources_batch_data.placement_group_load.placement_group_data) # Update the load metrics for this raylet. node_id = ray.utils.binary_to_hex(resource_message.node_id) ip = self.raylet_id_to_ip_map.get(node_id) if ip: self.load_metrics.update(ip, total_resources, available_resources, resource_load, waiting_bundles, infeasible_bundles, pending_placement_groups) else: logger.warning( f"Monitor: could not find ip for node {node_id}") def update_resource_requests(self): """Fetches resource requests from the internal KV and updates load.""" if not _internal_kv_initialized(): return data = _internal_kv_get( ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL) if data: try: resource_request = json.loads(data) self.load_metrics.set_resource_requests(resource_request) except Exception: logger.exception("Error parsing resource requests") def update_raylet_map(self, _append_port=False): """Updates internal raylet map. Args: _append_port (bool): Defaults to False. Appending the port is useful in testing, as mock clusters have many nodes with the same IP and cannot be uniquely identified. """ all_raylet_nodes = ray.nodes() self.raylet_id_to_ip_map = {} for raylet_info in all_raylet_nodes: node_id = (raylet_info.get("DBClientID") or raylet_info["NodeID"]) ip_address = (raylet_info.get("AuxAddress") or raylet_info["NodeManagerAddress"]).split(":")[0] if _append_port: ip_address += ":" + str(raylet_info["NodeManagerPort"]) self.raylet_id_to_ip_map[node_id] = ip_address def _run(self): """Run the monitor loop.""" while True: self.update_raylet_map() self.update_load_metrics() self.update_resource_requests() self.update_event_summary() status = { "load_metrics_report": self.load_metrics.summary()._asdict() } # Process autoscaling actions if self.autoscaler: # Only used to update the load metrics for the autoscaler. self.autoscaler.update() status["autoscaler_report"] = self.autoscaler.summary( )._asdict() for msg in self.event_summarizer.summary(): logger.info(":event_summary:{}".format(msg)) self.event_summarizer.clear() as_json = json.dumps(status) if _internal_kv_initialized(): _internal_kv_put(DEBUG_AUTOSCALING_STATUS, as_json, overwrite=True) # Wait for a autoscaler update interval before processing the next # round of messages. time.sleep(AUTOSCALER_UPDATE_INTERVAL_S) def update_event_summary(self): """Report the current size of the cluster. To avoid log spam, only cluster size changes (CPU or GPU count change) are reported to the event summarizer. The event summarizer will report only the latest cluster size per batch. """ avail_resources = self.load_metrics.resources_avail_summary() if avail_resources != self.last_avail_resources: self.event_summarizer.add( "Resized to {}.", # e.g., Resized to 100 CPUs, 4 GPUs. quantity=avail_resources, aggregate=lambda old, new: new) self.last_avail_resources = avail_resources def destroy_autoscaler_workers(self): """Cleanup the autoscaler, in case of an exception in the run() method. We kill the worker nodes, but retain the head node in order to keep logs around, keeping costs minimal. This monitor process runs on the head node anyway, so this is more reliable.""" if self.autoscaler is None: return # Nothing to clean up. if self.autoscaling_config is None: # This is a logic error in the program. Can't do anything. logger.error( "Monitor: Cleanup failed due to lack of autoscaler config.") return logger.info("Monitor: Exception caught. Taking down workers...") clean = False while not clean: try: teardown_cluster( config_file=self.autoscaling_config, yes=True, # Non-interactive. workers_only=True, # Retain head node for logs. override_cluster_name=None, keep_min_workers=True, # Retain minimal amount of workers. ) clean = True logger.info("Monitor: Workers taken down.") except Exception: logger.error("Monitor: Cleanup exception. Trying again...") time.sleep(2) def run(self): try: self._run() except Exception: logger.exception("Error in monitor loop") if self.autoscaler: self.autoscaler.kill_workers() raise