def validate_external_storage(self): """Make sure we can setup the object spilling external storage. This will also fill up the default setting for object spilling if not specified. """ object_spilling_config = self._config.get("object_spilling_config", {}) automatic_spilling_enabled = self._config.get( "automatic_object_spilling_enabled", True) if not automatic_spilling_enabled: return # If the config is not specified, we fill up the default. if not object_spilling_config: object_spilling_config = json.dumps({ "type": "filesystem", "params": { "directory_path": self._session_dir } }) # Try setting up the storage. # Configure the proper system config. # We need to set both ray param's system config and self._config # because they could've been diverged at this point. deserialized_config = json.loads(object_spilling_config) self._ray_params._system_config["object_spilling_config"] = ( object_spilling_config) self._config["object_spilling_config"] = object_spilling_config is_external_storage_type_fs = ( deserialized_config["type"] == "filesystem") self._ray_params._system_config["is_external_storage_type_fs"] = ( is_external_storage_type_fs) self._config["is_external_storage_type_fs"] = ( is_external_storage_type_fs) # Validate external storage usage. from ray import external_storage external_storage.setup_external_storage(deserialized_config) external_storage.reset_external_storage()
def _check_usage(self): if self.worker_port_list is not None: for port_str in self.worker_port_list.split(","): try: port = int(port_str) except ValueError as e: raise ValueError( "worker_port_list must be a comma-separated " + "list of integers: {}".format(e)) from None if port < 1024 or port > 65535: raise ValueError( "Ports in worker_port_list must be " "between 1024 and 65535. Got: {}".format(port)) # Used primarily for testing. if os.environ.get("RAY_USE_RANDOM_PORTS", False): if self.min_worker_port is None and self.min_worker_port is None: self.min_worker_port = 0 self.max_worker_port = 0 if self.min_worker_port is not None: if self.min_worker_port != 0 and (self.min_worker_port < 1024 or self.min_worker_port > 65535): raise ValueError("min_worker_port must be 0 or an integer " "between 1024 and 65535.") if self.max_worker_port is not None: if self.min_worker_port is None: raise ValueError("If max_worker_port is set, min_worker_port " "must also be set.") elif self.max_worker_port != 0: if self.max_worker_port < 1024 or self.max_worker_port > 65535: raise ValueError( "max_worker_port must be 0 or an integer between " "1024 and 65535.") elif self.max_worker_port <= self.min_worker_port: raise ValueError("max_worker_port must be higher than " "min_worker_port.") if self.resources is not None: assert "CPU" not in self.resources, ( "'CPU' should not be included in the resource dictionary. Use " "num_cpus instead.") assert "GPU" not in self.resources, ( "'GPU' should not be included in the resource dictionary. Use " "num_gpus instead.") if self.redirect_worker_output is not None: raise DeprecationWarning( "The redirect_worker_output argument is deprecated. To " "control logging to the driver, use the 'log_to_driver' " "argument to 'ray.init()'") if self.redirect_output is not None: raise DeprecationWarning( "The redirect_output argument is deprecated.") # Parse the numpy version. numpy_version = np.__version__.split(".") numpy_major, numpy_minor = int(numpy_version[0]), int(numpy_version[1]) if numpy_major <= 1 and numpy_minor < 16: logger.warning("Using ray with numpy < 1.16.0 will result in slow " "serialization. Upgrade numpy if using with ray.") # Make sure object spilling configuration is applicable. object_spilling_config = self._system_config.get( "object_spilling_config", {}) if object_spilling_config: object_spilling_config = json.loads(object_spilling_config) from ray import external_storage # Validate external storage usage. external_storage.setup_external_storage(object_spilling_config) external_storage.reset_external_storage()