def prompt(self, msg: str, *args, **kwargs): """Prompt the user for some text input. Args: msg (str): The mesage to display to the user before the prompt. Returns: The string entered by the user. """ if self.old_style: return complete_str = cf.underlined(msg) rendered_message = _format_msg(complete_str, *args, **kwargs) # the rendered message ends with ascii coding if rendered_message and not msg.endswith("\n"): rendered_message += " " self._print(rendered_message, linefeed=False) res = "" try: ans = sys.stdin.readline() ans = ans.lower() res = ans.strip() except KeyboardInterrupt: self.newline() return res
def show(): """ Show the modifiers and colors """ # modifiers sys.stdout.write(colorful.bold('bold') + ' ') sys.stdout.write(colorful.dimmed('dimmed') + ' ') sys.stdout.write(colorful.italic('italic') + ' ') sys.stdout.write(colorful.underlined('underlined') + ' ') sys.stdout.write(colorful.inversed('inversed') + ' ') sys.stdout.write(colorful.concealed('concealed') + ' ') sys.stdout.write(colorful.struckthrough('struckthrough') + '\n') # foreground colors sys.stdout.write(colorful.red('red') + ' ') sys.stdout.write(colorful.green('green') + ' ') sys.stdout.write(colorful.yellow('yellow') + ' ') sys.stdout.write(colorful.blue('blue') + ' ') sys.stdout.write(colorful.magenta('magenta') + ' ') sys.stdout.write(colorful.cyan('cyan') + ' ') sys.stdout.write(colorful.white('white') + '\n') # background colors sys.stdout.write(colorful.on_red('red') + ' ') sys.stdout.write(colorful.on_green('green') + ' ') sys.stdout.write(colorful.on_yellow('yellow') + ' ') sys.stdout.write(colorful.on_blue('blue') + ' ') sys.stdout.write(colorful.on_magenta('magenta') + ' ') sys.stdout.write(colorful.on_cyan('cyan') + ' ') sys.stdout.write(colorful.on_white('white') + '\n')
def _render_token(self, key: Any, token: Any) -> str: """ Convert any token to a string representation. """ try: enum_key = StructlogKeywords(key) formatter_func = self._token_formatting_mapping[enum_key] token = formatter_func(token) # cannot cast to StructlogKeyword, it must be a log event keyword except ValueError: token = cf.underlined(token) except KeyError: pass # return as a string formatted token finally: str_token = f"{token}" return str_token
def create_or_update_cluster(config_file: str, override_min_workers: Optional[int], override_max_workers: Optional[int], no_restart: bool, restart_only: bool, yes: bool, override_cluster_name: Optional[str], no_config_cache: bool = False, redirect_command_output: bool = False, use_login_shells: bool = True) -> None: """Create or updates an autoscaling Ray cluster from a config json.""" set_using_login_shells(use_login_shells) if not use_login_shells: cmd_output_util.set_allow_interactive(False) if redirect_command_output is None: # Do not redirect by default. cmd_output_util.set_output_redirected(False) else: cmd_output_util.set_output_redirected(redirect_command_output) if use_login_shells: cli_logger.warning( "Commands running under a login shell can produce more " "output than special processing can handle.") cli_logger.warning( "Thus, the output from subcommands will be logged as is.") cli_logger.warning( "Consider using {}, {}.", cf.bold("--use-normal-shells"), cf.underlined("if you tested your workflow and it is compatible")) cli_logger.newline() def handle_yaml_error(e): cli_logger.error("Cluster config invalid") cli_logger.newline() cli_logger.error("Failed to load YAML file " + cf.bold("{}"), config_file) cli_logger.newline() with cli_logger.verbatim_error_ctx("PyYAML error:"): cli_logger.error(e) cli_logger.abort() try: config = yaml.safe_load(open(config_file).read()) except FileNotFoundError: cli_logger.abort( "Provided cluster configuration file ({}) does not exist", cf.bold(config_file)) raise except yaml.parser.ParserError as e: handle_yaml_error(e) raise except yaml.scanner.ScannerError as e: handle_yaml_error(e) raise # todo: validate file_mounts, ssh keys, etc. importer = NODE_PROVIDERS.get(config["provider"]["type"]) if not importer: cli_logger.abort( "Unknown provider type " + cf.bold("{}") + "\n" "Available providers are: {}", config["provider"]["type"], cli_logger.render_list([ k for k in NODE_PROVIDERS.keys() if NODE_PROVIDERS[k] is not None ])) raise NotImplementedError("Unsupported provider {}".format( config["provider"])) cli_logger.success("Cluster configuration valid") printed_overrides = False def handle_cli_override(key, override): if override is not None: if key in config: nonlocal printed_overrides printed_overrides = True cli_logger.warning( "`{}` override provided on the command line.\n" " Using " + cf.bold("{}") + cf.dimmed(" [configuration file has " + cf.bold("{}") + "]"), key, override, config[key]) config[key] = override handle_cli_override("min_workers", override_min_workers) handle_cli_override("max_workers", override_max_workers) handle_cli_override("cluster_name", override_cluster_name) if printed_overrides: cli_logger.newline() cli_logger.labeled_value("Cluster", config["cluster_name"]) # disable the cli_logger here if needed # because it only supports aws if config["provider"]["type"] != "aws": cli_logger.old_style = True cli_logger.newline() config = _bootstrap_config(config, no_config_cache=no_config_cache) try_logging_config(config) get_or_create_head_node(config, config_file, no_restart, restart_only, yes, override_cluster_name)
#!/usr/bin/env python # This is an executable script that runs an example of every single CliLogger # function for demonstration purposes. Primarily useful for tuning color and # other formatting. from ray.autoscaler._private.cli_logger import cli_logger import colorful as cf cli_logger.old_style = False cli_logger.verbosity = 999 cli_logger.detect_colors() cli_logger.print( cf.bold("Bold ") + cf.italic("Italic ") + cf.underlined("Underlined")) cli_logger.labeled_value("Label", "value") cli_logger.print("List: {}", cli_logger.render_list([1, 2, 3])) cli_logger.newline() cli_logger.very_verbose("Very verbose") cli_logger.verbose("Verbose") cli_logger.verbose_warning("Verbose warning") cli_logger.verbose_error("Verbose error") cli_logger.print("Info") cli_logger.success("Success") cli_logger.warning("Warning") cli_logger.error("Error") cli_logger.newline() try: cli_logger.abort("Abort") except Exception: pass
def confirm(self, yes: bool, msg: str, *args: Any, _abort: bool = False, _default: bool = False, **kwargs: Any): """Display a confirmation dialog. Valid answers are "y/yes/true/1" and "n/no/false/0". Args: yes (bool): If `yes` is `True` the dialog will default to "yes" and continue without waiting for user input. _abort (bool): If `_abort` is `True`, "no" means aborting the program. _default (bool): The default action to take if the user just presses enter with no input. """ if self.old_style: return should_abort = _abort default = _default if not self.interactive and not yes: # no formatting around --yes here since this is non-interactive self.error("This command requires user confirmation. " "When running non-interactively, supply --yes to skip.") raise ValueError("Non-interactive confirm without --yes.") if default: yn_str = "Y/n" else: yn_str = "y/N" confirm_str = cf.underlined("Confirm [" + yn_str + "]:") + " " rendered_message = _format_msg(msg, *args, **kwargs) # the rendered message ends with ascii coding if rendered_message and not msg.endswith("\n"): rendered_message += " " msg_len = len(rendered_message.split("\n")[-1]) complete_str = rendered_message + confirm_str if yes: self._print(complete_str + "y " + cf.dimmed("[automatic, due to --yes]")) return True self._print(complete_str, _linefeed=False) res = None yes_answers = ["y", "yes", "true", "1"] no_answers = ["n", "no", "false", "0"] try: while True: ans = sys.stdin.readline() ans = ans.lower() if ans == "\n": res = default break ans = ans.strip() if ans in yes_answers: res = True break if ans in no_answers: res = False break indent = " " * msg_len self.error("{}Invalid answer: {}. " "Expected {} or {}", indent, cf.bold(ans.strip()), self.render_list(yes_answers, "/"), self.render_list(no_answers, "/")) self._print(indent + confirm_str, _linefeed=False) except KeyboardInterrupt: self.newline() res = default if not res and should_abort: # todo: make sure we tell the user if they # need to do cleanup self._print("Exiting...") raise SilentClickException( "Exiting due to the response to confirm(should_abort=True).") return res
def handle_boto_error(exc, msg, *args, **kwargs): if cli_logger.old_style: # old-style logging doesn't do anything here # so we exit early return error_code = None error_info = None # todo: not sure if these exceptions always have response if hasattr(exc, "response"): error_info = exc.response.get("Error", None) if error_info is not None: error_code = error_info.get("Code", None) generic_message_args = [ "{}\n" "Error code: {}", msg.format(*args, **kwargs), cf.bold(error_code) ] # apparently # ExpiredTokenException # ExpiredToken # RequestExpired # are all the same pretty much credentials_expiration_codes = [ "ExpiredTokenException", "ExpiredToken", "RequestExpired" ] if error_code in credentials_expiration_codes: # "An error occurred (ExpiredToken) when calling the # GetInstanceProfile operation: The security token # included in the request is expired" # "An error occurred (RequestExpired) when calling the # DescribeKeyPairs operation: Request has expired." token_command = ( "aws sts get-session-token " "--serial-number arn:aws:iam::" + cf.underlined("ROOT_ACCOUNT_ID") + ":mfa/" + cf.underlined("AWS_USERNAME") + " --token-code " + cf.underlined("TWO_FACTOR_AUTH_CODE")) secret_key_var = ( "export AWS_SECRET_ACCESS_KEY = " + cf.underlined("REPLACE_ME") + " # found at Credentials.SecretAccessKey") session_token_var = ( "export AWS_SESSION_TOKEN = " + cf.underlined("REPLACE_ME") + " # found at Credentials.SessionToken") access_key_id_var = ( "export AWS_ACCESS_KEY_ID = " + cf.underlined("REPLACE_ME") + " # found at Credentials.AccessKeyId") # fixme: replace with a Github URL that points # to our repo aws_session_script_url = ("https://gist.github.com/maximsmol/" "a0284e1d97b25d417bd9ae02e5f450cf") cli_logger.verbose_error(*generic_message_args) cli_logger.verbose(vars(exc)) cli_logger.panic("Your AWS session has expired.") cli_logger.newline() cli_logger.panic("You can request a new one using") cli_logger.panic(cf.bold(token_command)) cli_logger.panic("then expose it to Ray by setting") cli_logger.panic(cf.bold(secret_key_var)) cli_logger.panic(cf.bold(session_token_var)) cli_logger.panic(cf.bold(access_key_id_var)) cli_logger.newline() cli_logger.panic("You can find a script that automates this at:") cli_logger.panic(cf.underlined(aws_session_script_url)) # Do not re-raise the exception here because it looks awful # and we already print all the info in verbose cli_logger.abort() # todo: any other errors that we should catch separately? cli_logger.panic(*generic_message_args) cli_logger.newline() with cli_logger.verbatim_error_ctx("Boto3 error:"): cli_logger.verbose("{}", str(vars(exc))) cli_logger.panic("{}", str(exc)) cli_logger.abort()
def confirm(self, yes, msg, *args, _abort=False, _default=False, **kwargs): """Display a confirmation dialog. Valid answers are "y/yes/true/1" and "n/no/false/0". Args: yes (bool): If `yes` is `True` the dialog will default to "yes" and continue without waiting for user input. _abort (bool): If `_abort` is `True`, "no" means aborting the program. _default (bool): The default action to take if the user just presses enter with no input. """ if self.old_style: return should_abort = _abort default = _default if default: yn_str = cf.green("Y") + "/" + cf.red("n") else: yn_str = cf.green("y") + "/" + cf.red("N") confirm_str = cf.underlined("Confirm [" + yn_str + "]:") + " " rendered_message = _format_msg(msg, *args, **kwargs) if rendered_message and rendered_message[-1] != "\n": rendered_message += " " msg_len = len(rendered_message.split("\n")[-1]) complete_str = rendered_message + confirm_str if yes: self._print(complete_str + "y " + cf.gray("[automatic, due to --yes]")) return True self._print(complete_str, linefeed=False) res = None yes_answers = ["y", "yes", "true", "1"] no_answers = ["n", "no", "false", "0"] try: while True: ans = sys.stdin.readline() ans = ans.lower() if ans == "\n": res = default break ans = ans.strip() if ans in yes_answers: res = True break if ans in no_answers: res = False break indent = " " * msg_len self.error("{}Invalid answer: {}. " "Expected {} or {}", indent, cf.bold(ans.strip()), self.render_list(yes_answers, "/"), self.render_list(no_answers, "/")) self._print(indent + confirm_str, linefeed=False) except KeyboardInterrupt: self.newline() res = default if not res and should_abort: # todo: make sure we tell the user if they # need to do cleanup self._print("Exiting...") raise SilentClickException( "Exiting due to the response to confirm(should_abort=True).") return res
def confirm(self, yes, msg, *args, _abort=False, _default=False, **kwargs): if self.old_style: return should_abort = _abort default = _default if default: yn_str = cf.green("Y") + "/" + cf.red("n") else: yn_str = cf.green("y") + "/" + cf.red("N") confirm_str = cf.underlined("Confirm [" + yn_str + "]:") + " " rendered_message = _format_msg(msg, *args, **kwargs) if rendered_message and rendered_message[-1] != "\n": rendered_message += " " msg_len = len(rendered_message.split("\n")[-1]) complete_str = rendered_message + confirm_str if yes: self._print(complete_str + "y " + cf.gray("[automatic, due to --yes]")) return True self._print(complete_str, linefeed=False) res = None yes_answers = ["y", "yes", "true", "1"] no_answers = ["n", "no", "false", "0"] try: while True: ans = sys.stdin.readline() ans = ans.lower() if ans == "\n": res = default break ans = ans.strip() if ans in yes_answers: res = True break if ans in no_answers: res = False break indent = " " * msg_len self.error("{}Invalid answer: {}. " "Expected {} or {}", indent, cf.bold(ans.strip()), self.render_list(yes_answers, "/"), self.render_list(no_answers, "/")) self._print(indent + confirm_str, linefeed=False) except KeyboardInterrupt: self.newline() res = default if not res and should_abort: # todo: make sure we tell the user if they # need to do cleanup self._print("Exiting...") raise SilentClickException( "Exiting due to the response to confirm(should_abort=True).") return res
def start(node_ip_address, address, port, redis_password, redis_shard_ports, object_manager_port, node_manager_port, gcs_server_port, min_worker_port, max_worker_port, memory, object_store_memory, redis_max_memory, num_cpus, num_gpus, resources, head, include_dashboard, dashboard_host, dashboard_port, block, plasma_directory, autoscaling_config, no_redirect_worker_output, no_redirect_output, plasma_store_socket_name, raylet_socket_name, temp_dir, java_worker_options, load_code_from_local, code_search_path, system_config, lru_evict, enable_object_reconstruction, metrics_export_port, log_style, log_color, verbose): """Start Ray processes manually on the local machine.""" cli_logger.configure(log_style, log_color, verbose) if gcs_server_port and not head: raise ValueError( "gcs_server_port can be only assigned when you specify --head.") # Convert hostnames to numerical IP address. if node_ip_address is not None: node_ip_address = services.address_to_ip(node_ip_address) redis_address = None if address is not None: (redis_address, redis_address_ip, redis_address_port) = services.validate_redis_address(address) try: resources = json.loads(resources) except Exception: cli_logger.error("`{}` is not a valid JSON string.", cf.bold("--resources")) cli_logger.abort( "Valid values look like this: `{}`", cf.bold("--resources='\"CustomResource3\": 1, " "\"CustomResource2\": 2}'")) raise Exception("Unable to parse the --resources argument using " "json.loads. Try using a format like\n\n" " --resources='{\"CustomResource1\": 3, " "\"CustomReseource2\": 2}'") redirect_worker_output = None if not no_redirect_worker_output else True redirect_output = None if not no_redirect_output else True ray_params = ray.parameter.RayParams( node_ip_address=node_ip_address, min_worker_port=min_worker_port, max_worker_port=max_worker_port, object_manager_port=object_manager_port, node_manager_port=node_manager_port, gcs_server_port=gcs_server_port, memory=memory, object_store_memory=object_store_memory, redis_password=redis_password, redirect_worker_output=redirect_worker_output, redirect_output=redirect_output, num_cpus=num_cpus, num_gpus=num_gpus, resources=resources, plasma_directory=plasma_directory, huge_pages=False, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir, include_dashboard=include_dashboard, dashboard_host=dashboard_host, dashboard_port=dashboard_port, java_worker_options=java_worker_options, load_code_from_local=load_code_from_local, code_search_path=code_search_path, _system_config=system_config, lru_evict=lru_evict, enable_object_reconstruction=enable_object_reconstruction, metrics_export_port=metrics_export_port) if head: num_redis_shards = None # Start Ray on the head node. if redis_shard_ports is not None: redis_shard_ports = redis_shard_ports.split(",") # Infer the number of Redis shards from the ports if the number is # not provided. num_redis_shards = len(redis_shard_ports) if redis_address is not None: cli_logger.abort( "`{}` starts a new Redis server, `{}` should not be set.", cf.bold("--head"), cf.bold("--address")) raise Exception("If --head is passed in, a Redis server will be " "started, so a Redis address should not be " "provided.") # Get the node IP address if one is not provided. ray_params.update_if_absent( node_ip_address=services.get_node_ip_address()) cli_logger.labeled_value("Local node IP", ray_params.node_ip_address) cli_logger.old_info(logger, "Using IP address {} for this node.", ray_params.node_ip_address) ray_params.update_if_absent( redis_port=port, redis_shard_ports=redis_shard_ports, redis_max_memory=redis_max_memory, num_redis_shards=num_redis_shards, redis_max_clients=None, autoscaling_config=autoscaling_config, ) node = ray.node.Node( ray_params, head=True, shutdown_at_exit=block, spawn_reaper=block) redis_address = node.redis_address # this is a noop if new-style is not set, so the old logger calls # are still in place cli_logger.newline() startup_msg = "Ray runtime started." cli_logger.success("-" * len(startup_msg)) cli_logger.success(startup_msg) cli_logger.success("-" * len(startup_msg)) cli_logger.newline() with cli_logger.group("Next steps"): cli_logger.print( "To connect to this Ray runtime from another node, run") cli_logger.print( cf.bold(" ray start --address='{}'{}"), redis_address, f" --redis-password='******'" if redis_password else "") cli_logger.newline() cli_logger.print("Alternatively, use the following Python code:") with cli_logger.indented(): with cf.with_style("monokai") as c: cli_logger.print("{} ray", c.magenta("import")) cli_logger.print( "ray{}init(address{}{}{})", c.magenta("."), c.magenta("="), c.yellow("'auto'"), ", _redis_password{}{}".format( c.magenta("="), c.yellow("'" + redis_password + "'")) if redis_password else "") cli_logger.newline() cli_logger.print( cf.underlined("If connection fails, check your " "firewall settings and " "network configuration.")) cli_logger.newline() cli_logger.print("To terminate the Ray runtime, run") cli_logger.print(cf.bold(" ray stop")) cli_logger.old_info( logger, "\nStarted Ray on this node. You can add additional nodes to " "the cluster by calling\n\n" " ray start --address='{}'{}\n\n" "from the node you wish to add. You can connect a driver to the " "cluster from Python by running\n\n" " import ray\n" " ray.init(address='auto'{})\n\n" "If you have trouble connecting from a different machine, check " "that your firewall is configured properly. If you wish to " "terminate the processes that have been started, run\n\n" " ray stop".format( redis_address, " --redis-password='******'" if redis_password else "", ", _redis_password='******'" if redis_password else "")) else: # Start Ray on a non-head node. if not (port is None): cli_logger.abort("`{}` should not be specified without `{}`.", cf.bold("--port"), cf.bold("--head")) raise Exception("If --head is not passed in, --port is not " "allowed.") if redis_shard_ports is not None: cli_logger.abort("`{}` should not be specified without `{}`.", cf.bold("--redis-shard-ports"), cf.bold("--head")) raise Exception("If --head is not passed in, --redis-shard-ports " "is not allowed.") if redis_address is None: cli_logger.abort("`{}` is required unless starting with `{}`.", cf.bold("--address"), cf.bold("--head")) raise Exception("If --head is not passed in, --address must " "be provided.") if include_dashboard: cli_logger.abort("`{}` should not be specified without `{}`.", cf.bold("--include-dashboard"), cf.bold("--head")) raise ValueError( "If --head is not passed in, the --include-dashboard" "flag is not relevant.") # Wait for the Redis server to be started. And throw an exception if we # can't connect to it. services.wait_for_redis_to_start( redis_address_ip, redis_address_port, password=redis_password) # Create a Redis client. redis_client = services.create_redis_client( redis_address, password=redis_password) # Check that the version information on this node matches the version # information that the cluster was started with. services.check_version_info(redis_client) # Get the node IP address if one is not provided. ray_params.update_if_absent( node_ip_address=services.get_node_ip_address(redis_address)) cli_logger.labeled_value("Local node IP", ray_params.node_ip_address) cli_logger.old_info(logger, "Using IP address {} for this node.", ray_params.node_ip_address) # Check that there aren't already Redis clients with the same IP # address connected with this Redis instance. This raises an exception # if the Redis server already has clients on this node. check_no_existing_redis_clients(ray_params.node_ip_address, redis_client) ray_params.update(redis_address=redis_address) node = ray.node.Node( ray_params, head=False, shutdown_at_exit=block, spawn_reaper=block) cli_logger.newline() startup_msg = "Ray runtime started." cli_logger.success("-" * len(startup_msg)) cli_logger.success(startup_msg) cli_logger.success("-" * len(startup_msg)) cli_logger.newline() cli_logger.print("To terminate the Ray runtime, run") cli_logger.print(cf.bold(" ray stop")) cli_logger.old_info( logger, "\nStarted Ray on this node. If you wish to terminate the " "processes that have been started, run\n\n" " ray stop") if block: cli_logger.newline() with cli_logger.group(cf.bold("--block")): cli_logger.print( "This command will now block until terminated by a signal.") cli_logger.print( "Runing subprocesses are monitored and a message will be " "printed if any of them terminate unexpectedly.") while True: time.sleep(1) deceased = node.dead_processes() if len(deceased) > 0: cli_logger.newline() cli_logger.error("Some Ray subprcesses exited unexpectedly:") cli_logger.old_error(logger, "Ray processes died unexpectedly:") with cli_logger.indented(): for process_type, process in deceased: cli_logger.error( "{}", cf.bold(str(process_type)), _tags={"exit code": str(process.returncode)}) cli_logger.old_error( logger, "\t{} died with exit code {}".format( process_type, process.returncode)) # shutdown_at_exit will handle cleanup. cli_logger.newline() cli_logger.error("Remaining processes will be killed.") cli_logger.old_error( logger, "Killing remaining processes and exiting...") sys.exit(1)