def destroy_external_storage(self): object_spilling_config = self._config.get("object_spilling_config", {}) if object_spilling_config: object_spilling_config = json.loads(object_spilling_config) from ray import external_storage storage = external_storage.setup_external_storage( object_spilling_config) storage.destroy_external_storage()
def validate_external_storage(self): """Make sure we can setup the object spilling external storage. This will also fill up the default setting for object spilling if not specified. """ object_spilling_config = self._config.get("object_spilling_config", {}) automatic_spilling_enabled = self._config.get( "automatic_object_spilling_enabled", True) if not automatic_spilling_enabled: return # If the config is not specified, we fill up the default. if not object_spilling_config: object_spilling_config = json.dumps({ "type": "filesystem", "params": { "directory_path": self._session_dir } }) # Try setting up the storage. # Configure the proper system config. # We need to set both ray param's system config and self._config # because they could've been diverged at this point. deserialized_config = json.loads(object_spilling_config) self._ray_params._system_config["object_spilling_config"] = ( object_spilling_config) self._config["object_spilling_config"] = object_spilling_config is_external_storage_type_fs = ( deserialized_config["type"] == "filesystem") self._ray_params._system_config["is_external_storage_type_fs"] = ( is_external_storage_type_fs) self._config["is_external_storage_type_fs"] = ( is_external_storage_type_fs) # Validate external storage usage. from ray import external_storage external_storage.setup_external_storage(deserialized_config) external_storage.reset_external_storage()
mode = ray.UTIL_WORKER_MODE else: raise ValueError("Unknown worker type: " + args.worker_type) # NOTE(suquark): We must initialize the external storage before we # connect to raylet. Otherwise we may receive requests before the # external storage is intialized. if mode == ray.RESTORE_WORKER_MODE or mode == ray.SPILL_WORKER_MODE: from ray import external_storage if args.object_spilling_config: object_spilling_config = base64.b64decode( args.object_spilling_config) object_spilling_config = json.loads(object_spilling_config) else: object_spilling_config = {} external_storage.setup_external_storage(object_spilling_config) raylet_ip_address = args.raylet_ip_address if raylet_ip_address is None: raylet_ip_address = args.node_ip_address ray_params = RayParams( node_ip_address=args.node_ip_address, raylet_ip_address=raylet_ip_address, node_manager_port=args.node_manager_port, redis_address=args.redis_address, redis_password=args.redis_password, plasma_store_socket_name=args.object_store_name, raylet_socket_name=args.raylet_name, temp_dir=args.temp_dir, metrics_agent_port=args.metrics_agent_port,
def _check_usage(self): if self.worker_port_list is not None: for port_str in self.worker_port_list.split(","): try: port = int(port_str) except ValueError as e: raise ValueError( "worker_port_list must be a comma-separated " + "list of integers: {}".format(e)) from None if port < 1024 or port > 65535: raise ValueError( "Ports in worker_port_list must be " "between 1024 and 65535. Got: {}".format(port)) # Used primarily for testing. if os.environ.get("RAY_USE_RANDOM_PORTS", False): if self.min_worker_port is None and self.min_worker_port is None: self.min_worker_port = 0 self.max_worker_port = 0 if self.min_worker_port is not None: if self.min_worker_port != 0 and (self.min_worker_port < 1024 or self.min_worker_port > 65535): raise ValueError("min_worker_port must be 0 or an integer " "between 1024 and 65535.") if self.max_worker_port is not None: if self.min_worker_port is None: raise ValueError("If max_worker_port is set, min_worker_port " "must also be set.") elif self.max_worker_port != 0: if self.max_worker_port < 1024 or self.max_worker_port > 65535: raise ValueError( "max_worker_port must be 0 or an integer between " "1024 and 65535.") elif self.max_worker_port <= self.min_worker_port: raise ValueError("max_worker_port must be higher than " "min_worker_port.") if self.resources is not None: assert "CPU" not in self.resources, ( "'CPU' should not be included in the resource dictionary. Use " "num_cpus instead.") assert "GPU" not in self.resources, ( "'GPU' should not be included in the resource dictionary. Use " "num_gpus instead.") if self.redirect_worker_output is not None: raise DeprecationWarning( "The redirect_worker_output argument is deprecated. To " "control logging to the driver, use the 'log_to_driver' " "argument to 'ray.init()'") if self.redirect_output is not None: raise DeprecationWarning( "The redirect_output argument is deprecated.") # Parse the numpy version. numpy_version = np.__version__.split(".") numpy_major, numpy_minor = int(numpy_version[0]), int(numpy_version[1]) if numpy_major <= 1 and numpy_minor < 16: logger.warning("Using ray with numpy < 1.16.0 will result in slow " "serialization. Upgrade numpy if using with ray.") # Make sure object spilling configuration is applicable. object_spilling_config = self._system_config.get( "object_spilling_config", {}) if object_spilling_config: object_spilling_config = json.loads(object_spilling_config) from ray import external_storage # Validate external storage usage. external_storage.setup_external_storage(object_spilling_config) external_storage.reset_external_storage()
def main(args): ray.ray_logging.setup_logger(args.logging_level, args.logging_format) if args.worker_type == "WORKER": mode = ray.WORKER_MODE elif args.worker_type == "SPILL_WORKER": mode = ray.SPILL_WORKER_MODE elif args.worker_type == "RESTORE_WORKER": mode = ray.RESTORE_WORKER_MODE else: raise ValueError("Unknown worker type: " + args.worker_type) # NOTE(suquark): We must initialize the external storage before we # connect to raylet. Otherwise we may receive requests before the # external storage is intialized. if mode == ray.RESTORE_WORKER_MODE or mode == ray.SPILL_WORKER_MODE: from ray import external_storage if args.object_spilling_config: object_spilling_config = base64.b64decode( args.object_spilling_config) object_spilling_config = json.loads(object_spilling_config) else: object_spilling_config = {} external_storage.setup_external_storage(object_spilling_config) raylet_ip_address = args.raylet_ip_address if raylet_ip_address is None: raylet_ip_address = args.node_ip_address code_search_path = args.code_search_path if code_search_path is not None: for p in code_search_path.split(":"): if os.path.isfile(p): p = os.path.dirname(p) sys.path.append(p) ray_params = RayParams( node_ip_address=args.node_ip_address, raylet_ip_address=raylet_ip_address, node_manager_port=args.node_manager_port, redis_address=args.redis_address, redis_password=args.redis_password, plasma_store_socket_name=args.object_store_name, raylet_socket_name=args.raylet_name, temp_dir=args.temp_dir, load_code_from_local=args.load_code_from_local, metrics_agent_port=args.metrics_agent_port, ) node = ray.node.Node(ray_params, head=False, shutdown_at_exit=False, spawn_reaper=False, connect_only=True) ray.worker._global_node = node ray.worker.connect(node, mode=mode) # Redirect stdout and stderr to the default worker interceptor logger. # NOTE: We deprecated redirect_worker_output arg, # so we don't need to handle here. stdout_interceptor = StandardStreamInterceptor( setup_and_get_worker_interceptor_logger(is_for_stdout=True), intercept_stdout=True) stderr_interceptor = StandardStreamInterceptor( setup_and_get_worker_interceptor_logger(is_for_stdout=False), intercept_stdout=False) with redirect_stdout(stdout_interceptor): with redirect_stderr(stderr_interceptor): if mode == ray.WORKER_MODE: ray.worker.global_worker.main_loop() elif (mode == ray.RESTORE_WORKER_MODE or mode == ray.SPILL_WORKER_MODE): # It is handled by another thread in the C++ core worker. # We just need to keep the worker alive. while True: time.sleep(100000) else: raise ValueError(f"Unexcepted worker mode: {mode}")