def submit(cluster_config_file, docker, screen, tmux, stop, start, cluster_name, port_forward, script, args): """Uploads and runs a script on the specified cluster. The script is automatically synced to the following location: os.path.join("~", os.path.basename(script)) Example: >>> ray submit [CLUSTER.YAML] experiment.py --args="--smoke-test" """ assert not (screen and tmux), "Can specify only one of `screen` or `tmux`." if start: create_or_update_cluster(cluster_config_file, None, None, False, False, True, cluster_name) target = os.path.join("~", os.path.basename(script)) rsync(cluster_config_file, script, target, cluster_name, down=False) command_parts = ["python", target] if args is not None: command_parts += [args] cmd = " ".join(command_parts) exec_cluster(cluster_config_file, cmd, docker, screen, tmux, stop, False, cluster_name, port_forward)
def submit(cluster_config_file, screen, tmux, stop, start, cluster_name, port_forward, script, script_args): """Uploads and runs a script on the specified cluster. The script is automatically synced to the following location: os.path.join("~", os.path.basename(script)) """ assert not (screen and tmux), "Can specify only one of `screen` or `tmux`." if start: create_or_update_cluster(cluster_config_file, None, None, False, False, True, cluster_name) target = os.path.join("~", os.path.basename(script)) rsync(cluster_config_file, script, target, cluster_name, down=False) cmd = " ".join(["python", target] + list(script_args)) exec_cluster(cluster_config_file, cmd, screen, tmux, stop, False, cluster_name, port_forward) if tmux or screen: attach_command_parts = ["ray attach", cluster_config_file] if cluster_name is not None: attach_command_parts.append( "--cluster-name={}".format(cluster_name)) if tmux: attach_command_parts.append("--tmux") elif screen: attach_command_parts.append("--screen") attach_command = " ".join(attach_command_parts) attach_info = "Use `{}` to check on command status.".format( attach_command) logger.info(attach_info)
def setup_environment(self): """Set up the environment of the session.""" project_environment = self.project_definition.config.get( "environment", {}) if "requirements" in project_environment: requirements_txt = project_environment["requirements"] # Create a temporary requirements_txt in the head node. remote_requirements_txt = os.path.join( ray.utils.get_user_temp_dir(), "ray_project_requirements_txt_{}".format(time.time())) rsync( self.project_definition.cluster_yaml(), source=requirements_txt, target=remote_requirements_txt, override_cluster_name=self.session_name, down=False, ) self.execute_command( "pip install -r {}".format(remote_requirements_txt)) if "shell" in project_environment: for cmd in project_environment["shell"]: self.execute_command(cmd)
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes): rsync(cluster_config_file, source, target, cluster_name, down=False, all_nodes=all_nodes)
def setup_environment(self): """Set up the environment of the session.""" project_environment = self.project_definition.config["environment"] if "requirements" in project_environment: requirements_txt = project_environment["requirements"] # Create a temporary requirements_txt in the head node. remote_requirements_txt = ( "/tmp/" + "ray_project_requirements_txt_{}".format(time.time())) rsync( self.project_definition.cluster_yaml(), source=requirements_txt, target=remote_requirements_txt, override_cluster_name=None, down=False, ) self.execute_command( "pip install -r {}".format(remote_requirements_txt)) if "shell" in project_environment: for cmd in project_environment["shell"]: self.execute_command(cmd)
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes): """Upload specific files to a Ray cluster.""" rsync(cluster_config_file, source, target, cluster_name, down=False, all_nodes=all_nodes)
def sync_files(self): """Synchronize files with the session.""" rsync( self.project_definition.cluster_yaml(), source=self.project_definition.root, target=self.project_definition.working_directory(), override_cluster_name=None, down=False, )
def start(command, args, shell): project_definition = load_project_or_throw() if shell: command_to_run = command elif command: command_to_run = _get_command_to_run(command, project_definition, args) else: command_to_run = _get_command_to_run("default", project_definition, args) # Check for features we don't support right now project_environment = project_definition["environment"] need_docker = ("dockerfile" in project_environment or "dockerimage" in project_environment) if need_docker: raise click.ClickException( "Docker support in session is currently not implemented. " "Please file an feature request at" "https://github.com/ray-project/ray/issues") cluster_yaml = project_definition["cluster"] working_directory = project_definition["name"] logger.info("[1/4] Creating cluster") create_or_update_cluster( config_file=cluster_yaml, override_min_workers=None, override_max_workers=None, no_restart=False, restart_only=False, yes=True, override_cluster_name=None, ) logger.info("[2/4] Syncing the project") project_root = ray.projects.find_root(os.getcwd()) # This is so that rsync syncs directly to the target directory, instead of # nesting inside the target directory. if not project_root.endswith("/"): project_root += "/" rsync( cluster_yaml, source=project_root, target="~/{}/".format(working_directory), override_cluster_name=None, down=False, ) logger.info("[3/4] Setting up environment") _setup_environment( cluster_yaml, project_definition["environment"], cwd=working_directory) logger.info("[4/4] Running command") logger.debug("Running {}".format(command)) session_exec_cluster(cluster_yaml, command_to_run, cwd=working_directory)
def start(command, args, shell): project_definition = load_project_or_throw() if shell: command_to_run = command else: try: command_to_run = project_definition.get_command_to_run( command=command, args=args) except ValueError as e: raise click.ClickException(e) # Check for features we don't support right now project_environment = project_definition.config["environment"] need_docker = ("dockerfile" in project_environment or "dockerimage" in project_environment) if need_docker: raise click.ClickException( "Docker support in session is currently not implemented. " "Please file an feature request at" "https://github.com/ray-project/ray/issues") logger.info("[1/4] Creating cluster") create_or_update_cluster( config_file=project_definition.cluster_yaml(), override_min_workers=None, override_max_workers=None, no_restart=False, restart_only=False, yes=True, override_cluster_name=None, ) logger.info("[2/4] Syncing the project") rsync( project_definition.cluster_yaml(), source=project_definition.root, target=project_definition.working_directory(), override_cluster_name=None, down=False, ) logger.info("[3/4] Setting up environment") _setup_environment(project_definition.cluster_yaml(), project_environment, cwd=project_definition.working_directory()) logger.info("[4/4] Running command") logger.debug("Running {}".format(command)) session_exec_cluster(project_definition.cluster_yaml(), command_to_run, cwd=project_definition.working_directory())
def submit(cluster_config_file, docker, screen, tmux, stop, start, cluster_name, port_forward, script, args, script_args): """Uploads and runs a script on the specified cluster. The script is automatically synced to the following location: os.path.join("~", os.path.basename(script)) Example: >>> ray submit [CLUSTER.YAML] experiment.py -- --smoke-test """ assert not (screen and tmux), "Can specify only one of `screen` or `tmux`." assert not (script_args and args), "Use -- --arg1 --arg2 for script args." if args: logger.warning( "ray submit [yaml] [script.py] --args=... is deprecated and " "will be removed in a future version of Ray. Use " "`ray submit [yaml] script.py -- --arg1 --arg2` instead.") if start: create_or_update_cluster(cluster_config_file, None, None, False, False, True, cluster_name) target = os.path.join("~", os.path.basename(script)) rsync(cluster_config_file, script, target, cluster_name, down=False) command_parts = ["python", target] if script_args: command_parts += list(script_args) elif args is not None: command_parts += [args] port_forward = [(port, port) for port in list(port_forward)] cmd = " ".join(command_parts) exec_cluster( cluster_config_file, cmd, docker, screen, tmux, stop, start=False, override_cluster_name=cluster_name, port_forward=port_forward)
def submit(cluster_config_file, screen, tmux, stop, start, cluster_name, port_forward, script, script_args): """Uploads and runs a script on the specified cluster. The script is automatically synced to the following location: os.path.join("~", os.path.basename(script)) """ assert not (screen and tmux), "Can specify only one of `screen` or `tmux`." if start: create_or_update_cluster(cluster_config_file, None, None, False, False, True, cluster_name) target = os.path.join("~", os.path.basename(script)) rsync(cluster_config_file, script, target, cluster_name, down=False) cmd = " ".join(["python", target] + list(script_args)) exec_cluster(cluster_config_file, cmd, screen, tmux, stop, False, cluster_name, port_forward)
def _setup_environment(cluster_yaml, project_environment, cwd): if "requirements" in project_environment: requirements_txt = project_environment["requirements"] # Create a temporary requirements_txt in the head node. remote_requirements_txt = ( "/tmp/" + "ray_project_requirements_txt_{}".format(time.time())) rsync( cluster_yaml, source=requirements_txt, target=remote_requirements_txt, override_cluster_name=None, down=False, ) session_exec_cluster( cluster_yaml, "pip install -r {}".format(remote_requirements_txt), cwd=cwd) if "shell" in project_environment: for cmd in project_environment["shell"]: session_exec_cluster(cluster_yaml, cmd, cwd=cwd)
def rsync_down(cluster_config_file, source, target, cluster_name): rsync(cluster_config_file, source, target, cluster_name, down=True)
def rsync_down(cluster_config_file, source, target, cluster_name): """Download specific files from a Ray cluster.""" rsync(cluster_config_file, source, target, cluster_name, down=True)
def rsync_up(cluster_config_file, source, target, cluster_name): rsync(cluster_config_file, source, target, cluster_name, down=False)