def stop(self, skip_containers: Optional[List[str]] = None): """Stop the Orchest application. Args: skip_containers: The names of the images of the containers for which the containers are not stopped. """ ids, running_containers = self.resource_manager.get_containers(state="running") if not utils.is_orchest_running(running_containers): utils.echo("Orchest is not running.") return # Exclude the orchest-ctl from shutting down itself. if skip_containers is None: skip_containers = [] skip_containers += ["orchest/orchest-ctl:latest"] utils.echo("Shutting down...") # This is necessary because some of our containers might spawn # other containers, leading to a possible race condition where # the listed running containers are not up to date with the # real state anymore. n = 2 for _ in range(n): id_containers = [ (id_, c) for id_, c in zip(ids, running_containers) if c not in skip_containers ] # It might be that there are no containers to shut down # after filtering through skip_containers. if not id_containers: break ids: Tuple[str] running_containers: Tuple[Optional[str]] ids, running_containers = list(zip(*id_containers)) logger.info("Shutting down containers:\n" + "\n".join(running_containers)) self.docker_client.remove_containers(ids) # This is a safeguard against the fact that docker might be # buffering the start of a container, which translates to # the fact that we could "miss" the container and leave it # dangling. See #239 for more info. time.sleep(2) ids, running_containers = self.resource_manager.get_containers( state="running" ) if not ids: break utils.echo("Shutdown successful.")
def status(self, ext=False): if self._is_restarting(): utils.echo("Orchest is currently restarting.") raise typer.Exit(code=4) if self._is_updating(): utils.echo("Orchest is currently updating.") raise typer.Exit(code=5) _, running_containers_names = self.resource_manager.get_containers( state="running") if not utils.is_orchest_running(running_containers_names): utils.echo("Orchest is not running.") raise typer.Exit(code=1) # Minimal set of containers to be running for Orchest to be in # a valid state. valid_set: Set[str] = reduce(lambda x, y: x.union(y), _on_start_images, set()) if valid_set - set(running_containers_names): utils.echo( "Orchest is running, but has reached an invalid state. Run:") utils.echo("\torchest restart") logger.warning( "Orchest has reached an invalid state. Running containers:\n" + "\n".join(running_containers_names)) raise typer.Exit(code=2) else: utils.echo("Orchest is running.") if ext: utils.echo("Performing extensive status checks...") no_issues = True for container, exit_code in health_check( self.resource_manager).items(): if exit_code != 0: no_issues = False utils.echo(f"{container} is not ready ({exit_code}).") if no_issues: utils.echo("All services are ready.") else: raise typer.Exit(code=3)
def update(language): typer.echo("[Update]: ...") # only start if it was running should_restart = utils.is_orchest_running() if should_restart: logging.info("[Shutdown]: ...") if config.UPDATE_MODE != "web": stop(trace="update") else: # Both nginx-proxy/update-server are left running # during the update to support _updateserver stop(skip_names=["nginx-proxy", "update-server"]) utils.clear_environment_images() # Update git repository to get the latest changes to the ``userdir`` # structure. logging.info("Updating repo ...") script_path = os.path.join( "/orchest", "services", "orchest-ctl", "app", "scripts", "git-update.sh" ) script_process = subprocess.Popen([script_path], cwd="/orchest-host", bufsize=0) return_code = script_process.wait() if return_code != 0: logging.error( "'git' repo update failed. Please make sure you don't have " "any commits that conflict with the main branch in the " "'orchest' repository. Cancelling update." ) else: logging.info("Pulling latest images ...") utils.install_images(language, force_pull=True) typer.echo("[Update]: success") if config.UPDATE_MODE != "web" and should_restart: start()
def update(self, mode=None, dev: bool = False): """Update Orchest. Args: mode: The mode in which to update Orchest. This is either ``None`` or ``"web"``, where the latter is used when update is invoked through the update-server. """ utils.echo("Updating...") _, running_containers = self.resource_manager.get_containers( state="running") if utils.is_orchest_running(running_containers): utils.echo( "Using Orchest whilst updating is NOT supported and will be shut" " down, killing all active pipeline runs and session. You have 2s" " to cancel the update operation.") # Give the user the option to cancel the update operation # using a keyboard interrupt. time.sleep(2) # It is possible to pull new image whilst the older versions # of those images are running. self.stop(skip_containers=[ "orchest/update-server:latest", "orchest/auth-server:latest", "orchest/nginx-proxy:latest", "postgres:13.1", ]) # Update the Orchest git repo to get the latest changes to the # "userdir/" structure. if not dev: exit_code = utils.update_git_repo() if exit_code != 0: utils.echo("Cancelling update...") utils.echo( "It seems like you have unstaged changes in the 'orchest'" " repository. Please commit or stash them as 'orchest update'" " pulls the newest changes to the 'userdir/' using a rebase.", ) logger.error("Failed update due to unstaged changes.") return # Get all installed images and pull new versions. The pulled # images are checked to make sure optional images, e.g. lang # specific images, are updated as well. pulled_images = self.resource_manager.get_images() to_pull_images = set(ORCHEST_IMAGES["minimal"]) | set(pulled_images) logger.info("Updating images:\n" + "\n".join(to_pull_images)) self.docker_client.pull_images(to_pull_images, prog_bar=True, force=True) # Delete user-built environment images to avoid the issue of # having environments with mismatching Orchest SDK versions. logger.info("Deleting user-built environment images.") self.resource_manager.remove_env_build_imgs() # Delete user-built Jupyter image to make sure the Jupyter # server is updated to the latest version of Orchest. logger.info("Deleting user-built Jupyter image.") self.resource_manager.remove_jupyter_build_imgs() # Delete Orchest dangling images. self.resource_manager.remove_orchest_dangling_imgs() # We invoke the Orchest restart from the webserver ui-updater. # Hence we do not show the message to restart manually. if mode == "web": utils.echo("Update completed.") else: # Let the user know they need to restart the application # for the changes to take effect. NOTE: otherwise Orchest # might also be running a mix of containers on different # versions. utils.echo( "Don't forget to restart Orchest for the changes to take effect:" ) utils.echo("\torchest restart")
def update(self, mode=None, dev: bool = False): """Update Orchest. Args: mode: The mode in which to update Orchest. This is either ``None`` or ``"web"``, where the latter is used when update is invoked through the update-server. """ utils.echo("Updating...") _, running_containers = self.resource_manager.get_containers(state="running") if utils.is_orchest_running(running_containers): if mode != "web": # In the web updater it is not possible to cancel the # update once started. So there is no value in showing # this message or sleeping. utils.echo( "Using Orchest whilst updating is NOT supported and will be shut" " down, killing all active pipeline runs and session. You have 2s" " to cancel the update operation." ) # Give the user the option to cancel the update # operation using a keyboard interrupt. time.sleep(2) skip_containers = [] if mode == "web": # It is possible to pull new images whilst the older # versions of those images are running. We will invoke # Orchest restart from the webserver ui-updater. skip_containers = [ "orchest/update-server:latest", "orchest/auth-server:latest", "orchest/nginx-proxy:latest", "postgres:13.1", ] self.stop(skip_containers=skip_containers) # Update the Orchest git repo to get the latest changes to the # "userdir/" structure. if not dev: exit_code = utils.update_git_repo() if exit_code == 0: logger.info("Successfully updated git repo during update.") elif exit_code == 21: utils.echo("Cancelling update...") utils.echo( "Make sure you have the master branch checked out before updating." ) logger.error( "Failed update due to master branch not being checked out." ) return else: utils.echo("Cancelling update...") utils.echo( "It seems like you have unstaged changes in the 'orchest'" " repository. Please commit or stash them as 'orchest update'" " pulls the newest changes to the 'userdir/' using a rebase.", ) logger.error("Failed update due to unstaged changes.") return # Get all installed images and pull new versions. The pulled # images are checked to make sure optional images, e.g. lang # specific images, are updated as well. pulled_images = self.resource_manager.get_images() to_pull_images = set(ORCHEST_IMAGES["minimal"]) | set(pulled_images) logger.info("Updating images:\n" + "\n".join(to_pull_images)) self.docker_client.pull_images(to_pull_images, prog_bar=True, force=True) # Delete user-built environment images to avoid the issue of # having environments with mismatching Orchest SDK versions. logger.info("Deleting user-built environment images.") self.resource_manager.remove_env_build_imgs() # Delete user-built Jupyter image to make sure the Jupyter # server is updated to the latest version of Orchest. logger.info("Deleting user-built Jupyter image.") self.resource_manager.remove_jupyter_build_imgs() # Delete Orchest dangling images. self.resource_manager.remove_orchest_dangling_imgs() if mode == "web": utils.echo("Update completed.") else: utils.echo("Update completed. To start Orchest again, run:") utils.echo("\torchest start") utils.echo( "Checking whether all containers are running the same version of Orchest." ) self.version(ext=True)