def stop(self, skip_containers: Optional[List[str]] = None):
"""Stop the Orchest application.
Args:
skip_containers: The names of the images of the containers
for which the containers are not stopped.
"""
ids, running_containers = self.resource_manager.get_containers(state="running")
if not utils.is_orchest_running(running_containers):
utils.echo("Orchest is not running.")
return
# Exclude the orchest-ctl from shutting down itself.
if skip_containers is None:
skip_containers = []
skip_containers += ["orchest/orchest-ctl:latest"]
utils.echo("Shutting down...")
# This is necessary because some of our containers might spawn
# other containers, leading to a possible race condition where
# the listed running containers are not up to date with the
# real state anymore.
n = 2
for _ in range(n):
id_containers = [
(id_, c)
for id_, c in zip(ids, running_containers)
if c not in skip_containers
]
# It might be that there are no containers to shut down
# after filtering through skip_containers.
if not id_containers:
break
ids: Tuple[str]
running_containers: Tuple[Optional[str]]
ids, running_containers = list(zip(*id_containers))
logger.info("Shutting down containers:\n" + "\n".join(running_containers))
self.docker_client.remove_containers(ids)
# This is a safeguard against the fact that docker might be
# buffering the start of a container, which translates to
# the fact that we could "miss" the container and leave it
# dangling. See #239 for more info.
time.sleep(2)
ids, running_containers = self.resource_manager.get_containers(
state="running"
)
if not ids:
break
utils.echo("Shutdown successful.")
def status(self, ext=False):
if self._is_restarting():
utils.echo("Orchest is currently restarting.")
raise typer.Exit(code=4)
if self._is_updating():
utils.echo("Orchest is currently updating.")
raise typer.Exit(code=5)
_, running_containers_names = self.resource_manager.get_containers(
state="running")
if not utils.is_orchest_running(running_containers_names):
utils.echo("Orchest is not running.")
raise typer.Exit(code=1)
# Minimal set of containers to be running for Orchest to be in
# a valid state.
valid_set: Set[str] = reduce(lambda x, y: x.union(y), _on_start_images,
set())
if valid_set - set(running_containers_names):
utils.echo(
"Orchest is running, but has reached an invalid state. Run:")
utils.echo("\torchest restart")
logger.warning(
"Orchest has reached an invalid state. Running containers:\n" +
"\n".join(running_containers_names))
raise typer.Exit(code=2)
else:
utils.echo("Orchest is running.")
if ext:
utils.echo("Performing extensive status checks...")
no_issues = True
for container, exit_code in health_check(
self.resource_manager).items():
if exit_code != 0:
no_issues = False
utils.echo(f"{container} is not ready ({exit_code}).")
if no_issues:
utils.echo("All services are ready.")
else:
raise typer.Exit(code=3)
def update(language):
typer.echo("[Update]: ...")
# only start if it was running
should_restart = utils.is_orchest_running()
if should_restart:
logging.info("[Shutdown]: ...")
if config.UPDATE_MODE != "web":
stop(trace="update")
else:
# Both nginx-proxy/update-server are left running
# during the update to support _updateserver
stop(skip_names=["nginx-proxy", "update-server"])
utils.clear_environment_images()
# Update git repository to get the latest changes to the ``userdir``
# structure.
logging.info("Updating repo ...")
script_path = os.path.join(
"/orchest", "services", "orchest-ctl", "app", "scripts", "git-update.sh"
)
script_process = subprocess.Popen([script_path], cwd="/orchest-host", bufsize=0)
return_code = script_process.wait()
if return_code != 0:
logging.error(
"'git' repo update failed. Please make sure you don't have "
"any commits that conflict with the main branch in the "
"'orchest' repository. Cancelling update."
)
else:
logging.info("Pulling latest images ...")
utils.install_images(language, force_pull=True)
typer.echo("[Update]: success")
if config.UPDATE_MODE != "web" and should_restart:
start()
def update(self, mode=None, dev: bool = False):
"""Update Orchest.
Args:
mode: The mode in which to update Orchest. This is either
``None`` or ``"web"``, where the latter is used when
update is invoked through the update-server.
"""
utils.echo("Updating...")
_, running_containers = self.resource_manager.get_containers(
state="running")
if utils.is_orchest_running(running_containers):
utils.echo(
"Using Orchest whilst updating is NOT supported and will be shut"
" down, killing all active pipeline runs and session. You have 2s"
" to cancel the update operation.")
# Give the user the option to cancel the update operation
# using a keyboard interrupt.
time.sleep(2)
# It is possible to pull new image whilst the older versions
# of those images are running.
self.stop(skip_containers=[
"orchest/update-server:latest",
"orchest/auth-server:latest",
"orchest/nginx-proxy:latest",
"postgres:13.1",
])
# Update the Orchest git repo to get the latest changes to the
# "userdir/" structure.
if not dev:
exit_code = utils.update_git_repo()
if exit_code != 0:
utils.echo("Cancelling update...")
utils.echo(
"It seems like you have unstaged changes in the 'orchest'"
" repository. Please commit or stash them as 'orchest update'"
" pulls the newest changes to the 'userdir/' using a rebase.",
)
logger.error("Failed update due to unstaged changes.")
return
# Get all installed images and pull new versions. The pulled
# images are checked to make sure optional images, e.g. lang
# specific images, are updated as well.
pulled_images = self.resource_manager.get_images()
to_pull_images = set(ORCHEST_IMAGES["minimal"]) | set(pulled_images)
logger.info("Updating images:\n" + "\n".join(to_pull_images))
self.docker_client.pull_images(to_pull_images,
prog_bar=True,
force=True)
# Delete user-built environment images to avoid the issue of
# having environments with mismatching Orchest SDK versions.
logger.info("Deleting user-built environment images.")
self.resource_manager.remove_env_build_imgs()
# Delete user-built Jupyter image to make sure the Jupyter
# server is updated to the latest version of Orchest.
logger.info("Deleting user-built Jupyter image.")
self.resource_manager.remove_jupyter_build_imgs()
# Delete Orchest dangling images.
self.resource_manager.remove_orchest_dangling_imgs()
# We invoke the Orchest restart from the webserver ui-updater.
# Hence we do not show the message to restart manually.
if mode == "web":
utils.echo("Update completed.")
else:
# Let the user know they need to restart the application
# for the changes to take effect. NOTE: otherwise Orchest
# might also be running a mix of containers on different
# versions.
utils.echo(
"Don't forget to restart Orchest for the changes to take effect:"
)
utils.echo("\torchest restart")
def update(self, mode=None, dev: bool = False):
"""Update Orchest.
Args:
mode: The mode in which to update Orchest. This is either
``None`` or ``"web"``, where the latter is used when
update is invoked through the update-server.
"""
utils.echo("Updating...")
_, running_containers = self.resource_manager.get_containers(state="running")
if utils.is_orchest_running(running_containers):
if mode != "web":
# In the web updater it is not possible to cancel the
# update once started. So there is no value in showing
# this message or sleeping.
utils.echo(
"Using Orchest whilst updating is NOT supported and will be shut"
" down, killing all active pipeline runs and session. You have 2s"
" to cancel the update operation."
)
# Give the user the option to cancel the update
# operation using a keyboard interrupt.
time.sleep(2)
skip_containers = []
if mode == "web":
# It is possible to pull new images whilst the older
# versions of those images are running. We will invoke
# Orchest restart from the webserver ui-updater.
skip_containers = [
"orchest/update-server:latest",
"orchest/auth-server:latest",
"orchest/nginx-proxy:latest",
"postgres:13.1",
]
self.stop(skip_containers=skip_containers)
# Update the Orchest git repo to get the latest changes to the
# "userdir/" structure.
if not dev:
exit_code = utils.update_git_repo()
if exit_code == 0:
logger.info("Successfully updated git repo during update.")
elif exit_code == 21:
utils.echo("Cancelling update...")
utils.echo(
"Make sure you have the master branch checked out before updating."
)
logger.error(
"Failed update due to master branch not being checked out."
)
return
else:
utils.echo("Cancelling update...")
utils.echo(
"It seems like you have unstaged changes in the 'orchest'"
" repository. Please commit or stash them as 'orchest update'"
" pulls the newest changes to the 'userdir/' using a rebase.",
)
logger.error("Failed update due to unstaged changes.")
return
# Get all installed images and pull new versions. The pulled
# images are checked to make sure optional images, e.g. lang
# specific images, are updated as well.
pulled_images = self.resource_manager.get_images()
to_pull_images = set(ORCHEST_IMAGES["minimal"]) | set(pulled_images)
logger.info("Updating images:\n" + "\n".join(to_pull_images))
self.docker_client.pull_images(to_pull_images, prog_bar=True, force=True)
# Delete user-built environment images to avoid the issue of
# having environments with mismatching Orchest SDK versions.
logger.info("Deleting user-built environment images.")
self.resource_manager.remove_env_build_imgs()
# Delete user-built Jupyter image to make sure the Jupyter
# server is updated to the latest version of Orchest.
logger.info("Deleting user-built Jupyter image.")
self.resource_manager.remove_jupyter_build_imgs()
# Delete Orchest dangling images.
self.resource_manager.remove_orchest_dangling_imgs()
if mode == "web":
utils.echo("Update completed.")
else:
utils.echo("Update completed. To start Orchest again, run:")
utils.echo("\torchest start")
utils.echo(
"Checking whether all containers are running the same version of Orchest."
)
self.version(ext=True)
