def submit(cluster_config_file, docker, screen, tmux, stop, start,
cluster_name, port_forward, script, args):
"""Uploads and runs a script on the specified cluster.
The script is automatically synced to the following location:
os.path.join("~", os.path.basename(script))
Example:
>>> ray submit [CLUSTER.YAML] experiment.py --args="--smoke-test"
"""
assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
if start:
create_or_update_cluster(cluster_config_file, None, None, False, False,
True, cluster_name)
target = os.path.join("~", os.path.basename(script))
rsync(cluster_config_file, script, target, cluster_name, down=False)
command_parts = ["python", target]
if args is not None:
command_parts += [args]
cmd = " ".join(command_parts)
exec_cluster(cluster_config_file, cmd, docker, screen, tmux, stop, False,
cluster_name, port_forward)
def submit(cluster_config_file, screen, tmux, stop, start, cluster_name,
port_forward, script, script_args):
"""Uploads and runs a script on the specified cluster.
The script is automatically synced to the following location:
os.path.join("~", os.path.basename(script))
"""
assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
if start:
create_or_update_cluster(cluster_config_file, None, None, False, False,
True, cluster_name)
target = os.path.join("~", os.path.basename(script))
rsync(cluster_config_file, script, target, cluster_name, down=False)
cmd = " ".join(["python", target] + list(script_args))
exec_cluster(cluster_config_file, cmd, screen, tmux, stop, False,
cluster_name, port_forward)
if tmux or screen:
attach_command_parts = ["ray attach", cluster_config_file]
if cluster_name is not None:
attach_command_parts.append(
"--cluster-name={}".format(cluster_name))
if tmux:
attach_command_parts.append("--tmux")
elif screen:
attach_command_parts.append("--screen")
attach_command = " ".join(attach_command_parts)
attach_info = "Use `{}` to check on command status.".format(
attach_command)
logger.info(attach_info)
def setup_environment(self):
"""Set up the environment of the session."""
project_environment = self.project_definition.config.get(
"environment", {})
if "requirements" in project_environment:
requirements_txt = project_environment["requirements"]
# Create a temporary requirements_txt in the head node.
remote_requirements_txt = os.path.join(
ray.utils.get_user_temp_dir(),
"ray_project_requirements_txt_{}".format(time.time()))
rsync(
self.project_definition.cluster_yaml(),
source=requirements_txt,
target=remote_requirements_txt,
override_cluster_name=self.session_name,
down=False,
)
self.execute_command(
"pip install -r {}".format(remote_requirements_txt))
if "shell" in project_environment:
for cmd in project_environment["shell"]:
self.execute_command(cmd)
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes):
rsync(cluster_config_file,
source,
target,
cluster_name,
down=False,
all_nodes=all_nodes)
def setup_environment(self):
"""Set up the environment of the session."""
project_environment = self.project_definition.config["environment"]
if "requirements" in project_environment:
requirements_txt = project_environment["requirements"]
# Create a temporary requirements_txt in the head node.
remote_requirements_txt = (
"/tmp/" +
"ray_project_requirements_txt_{}".format(time.time()))
rsync(
self.project_definition.cluster_yaml(),
source=requirements_txt,
target=remote_requirements_txt,
override_cluster_name=None,
down=False,
)
self.execute_command(
"pip install -r {}".format(remote_requirements_txt))
if "shell" in project_environment:
for cmd in project_environment["shell"]:
self.execute_command(cmd)
def submit(cluster_config_file, screen, tmux, stop, start, cluster_name,
port_forward, script, script_args):
"""Uploads and runs a script on the specified cluster.
The script is automatically synced to the following location:
os.path.join("~", os.path.basename(script))
"""
assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
if start:
create_or_update_cluster(cluster_config_file, None, None, False, False,
True, cluster_name)
target = os.path.join("~", os.path.basename(script))
rsync(cluster_config_file, script, target, cluster_name, down=False)
cmd = " ".join(["python", target] + list(script_args))
exec_cluster(cluster_config_file, cmd, screen, tmux, stop, False,
cluster_name, port_forward)
if tmux or screen:
attach_command_parts = ["ray attach", cluster_config_file]
if cluster_name is not None:
attach_command_parts.append(
"--cluster-name={}".format(cluster_name))
if tmux:
attach_command_parts.append("--tmux")
elif screen:
attach_command_parts.append("--screen")
attach_command = " ".join(attach_command_parts)
attach_info = "Use `{}` to check on command status.".format(
attach_command)
logger.info(attach_info)
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes):
"""Upload specific files to a Ray cluster."""
rsync(cluster_config_file,
source,
target,
cluster_name,
down=False,
all_nodes=all_nodes)
def sync_files(self):
"""Synchronize files with the session."""
rsync(
self.project_definition.cluster_yaml(),
source=self.project_definition.root,
target=self.project_definition.working_directory(),
override_cluster_name=None,
down=False,
)
def start(command, args, shell):
project_definition = load_project_or_throw()
if shell:
command_to_run = command
elif command:
command_to_run = _get_command_to_run(command, project_definition, args)
else:
command_to_run = _get_command_to_run("default", project_definition,
args)
# Check for features we don't support right now
project_environment = project_definition["environment"]
need_docker = ("dockerfile" in project_environment
or "dockerimage" in project_environment)
if need_docker:
raise click.ClickException(
"Docker support in session is currently not implemented. "
"Please file an feature request at"
"https://github.com/ray-project/ray/issues")
cluster_yaml = project_definition["cluster"]
working_directory = project_definition["name"]
logger.info("[1/4] Creating cluster")
create_or_update_cluster(
config_file=cluster_yaml,
override_min_workers=None,
override_max_workers=None,
no_restart=False,
restart_only=False,
yes=True,
override_cluster_name=None,
)
logger.info("[2/4] Syncing the project")
project_root = ray.projects.find_root(os.getcwd())
# This is so that rsync syncs directly to the target directory, instead of
# nesting inside the target directory.
if not project_root.endswith("/"):
project_root += "/"
rsync(
cluster_yaml,
source=project_root,
target="~/{}/".format(working_directory),
override_cluster_name=None,
down=False,
)
logger.info("[3/4] Setting up environment")
_setup_environment(
cluster_yaml, project_definition["environment"], cwd=working_directory)
logger.info("[4/4] Running command")
logger.debug("Running {}".format(command))
session_exec_cluster(cluster_yaml, command_to_run, cwd=working_directory)
def start(command, args, shell):
project_definition = load_project_or_throw()
if shell:
command_to_run = command
else:
try:
command_to_run = project_definition.get_command_to_run(
command=command, args=args)
except ValueError as e:
raise click.ClickException(e)
# Check for features we don't support right now
project_environment = project_definition.config["environment"]
need_docker = ("dockerfile" in project_environment
or "dockerimage" in project_environment)
if need_docker:
raise click.ClickException(
"Docker support in session is currently not implemented. "
"Please file an feature request at"
"https://github.com/ray-project/ray/issues")
logger.info("[1/4] Creating cluster")
create_or_update_cluster(
config_file=project_definition.cluster_yaml(),
override_min_workers=None,
override_max_workers=None,
no_restart=False,
restart_only=False,
yes=True,
override_cluster_name=None,
)
logger.info("[2/4] Syncing the project")
rsync(
project_definition.cluster_yaml(),
source=project_definition.root,
target=project_definition.working_directory(),
override_cluster_name=None,
down=False,
)
logger.info("[3/4] Setting up environment")
_setup_environment(project_definition.cluster_yaml(),
project_environment,
cwd=project_definition.working_directory())
logger.info("[4/4] Running command")
logger.debug("Running {}".format(command))
session_exec_cluster(project_definition.cluster_yaml(),
command_to_run,
cwd=project_definition.working_directory())
def submit(cluster_config_file, docker, screen, tmux, stop, start,
cluster_name, port_forward, script, args, script_args):
"""Uploads and runs a script on the specified cluster.
The script is automatically synced to the following location:
os.path.join("~", os.path.basename(script))
Example:
>>> ray submit [CLUSTER.YAML] experiment.py -- --smoke-test
"""
assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
assert not (script_args and args), "Use -- --arg1 --arg2 for script args."
if args:
logger.warning(
"ray submit [yaml] [script.py] --args=... is deprecated and "
"will be removed in a future version of Ray. Use "
"`ray submit [yaml] script.py -- --arg1 --arg2` instead.")
if start:
create_or_update_cluster(cluster_config_file, None, None, False, False,
True, cluster_name)
target = os.path.join("~", os.path.basename(script))
rsync(cluster_config_file, script, target, cluster_name, down=False)
command_parts = ["python", target]
if script_args:
command_parts += list(script_args)
elif args is not None:
command_parts += [args]
port_forward = [(port, port) for port in list(port_forward)]
cmd = " ".join(command_parts)
exec_cluster(
cluster_config_file,
cmd,
docker,
screen,
tmux,
stop,
start=False,
override_cluster_name=cluster_name,
port_forward=port_forward)
def submit(cluster_config_file, screen, tmux, stop, start, cluster_name,
port_forward, script, script_args):
"""Uploads and runs a script on the specified cluster.
The script is automatically synced to the following location:
os.path.join("~", os.path.basename(script))
"""
assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
if start:
create_or_update_cluster(cluster_config_file, None, None, False, False,
True, cluster_name)
target = os.path.join("~", os.path.basename(script))
rsync(cluster_config_file, script, target, cluster_name, down=False)
cmd = " ".join(["python", target] + list(script_args))
exec_cluster(cluster_config_file, cmd, screen, tmux, stop, False,
cluster_name, port_forward)
def _setup_environment(cluster_yaml, project_environment, cwd):
if "requirements" in project_environment:
requirements_txt = project_environment["requirements"]
# Create a temporary requirements_txt in the head node.
remote_requirements_txt = (
"/tmp/" + "ray_project_requirements_txt_{}".format(time.time()))
rsync(
cluster_yaml,
source=requirements_txt,
target=remote_requirements_txt,
override_cluster_name=None,
down=False,
)
session_exec_cluster(
cluster_yaml,
"pip install -r {}".format(remote_requirements_txt),
cwd=cwd)
if "shell" in project_environment:
for cmd in project_environment["shell"]:
session_exec_cluster(cluster_yaml, cmd, cwd=cwd)
def rsync_down(cluster_config_file, source, target, cluster_name):
rsync(cluster_config_file, source, target, cluster_name, down=True)
def rsync_down(cluster_config_file, source, target, cluster_name):
"""Download specific files from a Ray cluster."""
rsync(cluster_config_file, source, target, cluster_name, down=True)
def rsync_up(cluster_config_file, source, target, cluster_name):
rsync(cluster_config_file, source, target, cluster_name, down=False)
def rsync_up(cluster_config_file, source, target, cluster_name):
rsync(cluster_config_file, source, target, cluster_name, down=False)
def rsync_down(cluster_config_file, source, target, cluster_name):
rsync(cluster_config_file, source, target, cluster_name, down=True)
