def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes,
log_style, log_color, verbose):
"""Upload specific files to a Ray cluster."""
cli_logger.configure(log_style, log_color, verbose)
rsync(
cluster_config_file,
source,
target,
cluster_name,
down=False,
all_nodes=all_nodes)
def rsync(cluster_config: Union[dict, str],
*,
source: Optional[str],
target: Optional[str],
down: bool,
ip_address: str = None,
use_internal_ip: bool = False,
no_config_cache: bool = False):
"""Rsyncs files to or from the cluster.
Args:
cluster_config (Union[str, dict]): Either the config dict of the
cluster, or a path pointing to a file containing the config.
source (str): rsync source argument.
target (str): rsync target argument.
down (bool): whether we're syncing remote -> local.
ip_address (str): Address of node.
use_internal_ip (bool): Whether the provided ip_address is
public or private.
no_config_cache (bool): Whether to disable the config cache and fully
resolve all environment settings from the Cloud provider again.
Raises:
RuntimeError if the cluster head node is not found.
"""
with _as_config_file(cluster_config) as config_file:
return commands.rsync(config_file=config_file,
source=source,
target=target,
override_cluster_name=None,
down=down,
ip_address=ip_address,
use_internal_ip=use_internal_ip,
no_config_cache=no_config_cache,
all_nodes=False)
def rsync(cluster_config: Union[dict, str],
*,
source: str,
target: str,
down: bool,
no_config_cache: bool = False):
"""Rsyncs files to or from the cluster.
Args:
cluster_config (Union[str, dict]): Either the config dict of the
cluster, or a path pointing to a file containing the config.
source (str): rsync source argument.
target (str): rsync target argument.
down (bool): whether we're syncing remote -> local.
no_config_cache (bool): Whether to disable the config cache and fully
resolve all environment settings from the Cloud provider again.
Raises:
RuntimeError if the cluster head node is not found.
"""
return commands.rsync(
config_file=_as_config_file(cluster_config),
source=source,
target=target,
override_cluster_name=None,
down=down,
no_config_cache=no_config_cache,
all_nodes=False)
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes,
log_style, log_color, verbose):
"""Upload specific files to a Ray cluster."""
cli_logger.configure(log_style, log_color, verbose)
if all_nodes:
cli_logger.warning(
"WARNING: the `all_nodes` option is deprecated and will be "
"removed in the future. "
"Rsync to worker nodes is not reliable since workers may be "
"added during autoscaling. Please use the `file_mounts` "
"feature instead for consistent file sync in autoscaling clusters")
rsync(
cluster_config_file,
source,
target,
cluster_name,
down=False,
all_nodes=all_nodes)
def submit(cluster_config_file, screen, tmux, stop, start, cluster_name,
no_config_cache, port_forward, script, args, script_args, log_style,
log_color, verbose):
"""Uploads and runs a script on the specified cluster.
The script is automatically synced to the following location:
os.path.join("~", os.path.basename(script))
Example:
>>> ray submit [CLUSTER.YAML] experiment.py -- --smoke-test
"""
cli_logger.configure(log_style, log_color, verbose)
cli_logger.doassert(not (screen and tmux),
"`{}` and `{}` are incompatible.", cf.bold("--screen"),
cf.bold("--tmux"))
cli_logger.doassert(
not (script_args and args),
"`{0}` and `{1}` are incompatible. Use only `{1}`.\n"
"Example: `{2}`", cf.bold("--args"), cf.bold("-- <args ...>"),
cf.bold("ray submit script.py -- --arg=123 --flag"))
assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
assert not (script_args and args), "Use -- --arg1 --arg2 for script args."
if args:
cli_logger.warning(
"`{}` is deprecated and will be removed in the future.",
cf.bold("--args"))
cli_logger.warning("Use `{}` instead. Example: `{}`.",
cf.bold("-- <args ...>"),
cf.bold("ray submit script.py -- --arg=123 --flag"))
cli_logger.newline()
if start:
create_or_update_cluster(
config_file=cluster_config_file,
override_min_workers=None,
override_max_workers=None,
no_restart=False,
restart_only=False,
yes=True,
override_cluster_name=cluster_name,
no_config_cache=no_config_cache,
redirect_command_output=False,
use_login_shells=True)
target = os.path.basename(script)
target = os.path.join("~", target)
rsync(
cluster_config_file,
script,
target,
cluster_name,
no_config_cache=no_config_cache,
down=False)
command_parts = ["python", target]
if script_args:
command_parts += list(script_args)
elif args is not None:
command_parts += [args]
port_forward = [(port, port) for port in list(port_forward)]
cmd = " ".join(command_parts)
exec_cluster(
cluster_config_file,
cmd=cmd,
run_env="docker",
screen=screen,
tmux=tmux,
stop=stop,
start=False,
override_cluster_name=cluster_name,
no_config_cache=no_config_cache,
port_forward=port_forward)
def rsync_down(cluster_config_file, source, target, cluster_name, log_style,
log_color, verbose):
"""Download specific files from a Ray cluster."""
cli_logger.configure(log_style, log_color, verbose)
rsync(cluster_config_file, source, target, cluster_name, down=True)
