def install_network(self) -> None:
"""Installs the Orchest Docker network."""
# Don't install the network again if it is already installed
# because that will create the another network with the same
# name but with another ID. Thereby, breaking Orchest.
try:
is_installed = self.is_network_installed()
except docker.errors.APIError:
# TODO: reraise the error but with a helpful message that
# helps the user fix the issue.
raise
if not is_installed:
# We only want to print this message to the user once. The
# best bet is that if the Orchest network has not yet been
# installed, then most likely the user has not seen this
# message before.
utils.echo(
"Orchest sends anonymized telemetry to analytics.orchestapp.com."
" To disable it, please refer to:", )
utils.echo(
"\thttps://orchest.readthedocs.io/en/stable/user_guide/other.html#configuration", # noqa: E501, W505
wrap=100,
)
self.docker_client.install_network(self.network)
async def _pull_images(
self,
images: Iterable[str],
prog_bar: bool = True,
force: bool = False,
):
pulls = [self._pull_image(image, force=force) for image in images]
if prog_bar:
pulls = tqdm.as_completed(
pulls,
total=len(pulls),
ncols=WRAP_LINES,
desc="Pulling images",
ascii=True,
position=0,
leave=True,
bar_format="{desc}: {n}/{total}|{bar}|",
)
for pull in pulls:
await pull
if prog_bar:
pulls.close() # type: ignore
# Makes the next echo start on the line underneath the
# status bar instead of after.
await asyncio.sleep(0.05)
# TODO: Check whether we can use print(flush=True) here to
# make this class not dependend on utils.
utils.echo()
await self.close_aclient()
def update_container_config_with_cloud(container_config: dict,
env: Optional[dict] = None) -> None:
"""Updates the container config to run with --cloud.
Args:
container_config: An existing container config, to be updated in
place.
env: Refer to :meth:`get_container_config`.
"""
if env is None:
env = utils.get_env()
utils.echo(
"Starting Orchest with --cloud. Some GUI functionality is altered.")
cloud_inject = {
"orchest-webserver": {
"Env": [
"CLOUD=true",
],
},
"update-server": {
"Env": [
"CLOUD=true",
],
},
"auth-server": {
"Env": [
"CLOUD=true",
],
},
}
inject_dict(container_config, cloud_inject, overwrite=False)
def adduser(
username: str = typer.Argument(..., help="Name of the user to add."),
set_token: bool = typer.Option(
False,
"--set-token",
help="True if the token of the user is to be setup."),
is_admin: bool = typer.Option(
False,
show_default="--no-is-admin",
help=("If the newly created user should be an admin or not. An admin"
" user cannot be deleted."),
),
non_interactive: bool = typer.Option(
False,
hidden=True,
help="Use non interactive mode for password and token."),
non_interactive_password: Optional[str] = typer.Option(
None, hidden=True, help="User password in non interactive mode."),
non_interactive_token: Optional[str] = typer.Option(
None, hidden=True, help="User token in non interactive mode."),
):
"""
Add user to Orchest.
"""
if non_interactive:
password = non_interactive_password
token = non_interactive_token
else:
if non_interactive_password:
echo("Cannot use non_interactive_password in interactive mode.",
err=True)
raise typer.Exit(code=1)
if non_interactive_token:
echo("Cannot use non_interactive_token in interactive mode.",
err=True)
raise typer.Exit(code=1)
password = typer.prompt("Password",
hide_input=True,
confirmation_prompt=True)
if set_token:
token = typer.prompt("Token",
hide_input=True,
confirmation_prompt=True)
else:
token = None
if password is None:
echo("Password must be specified.", err=True)
raise typer.Exit(code=1)
elif not password:
echo("Password cannot be empty.", err=True)
raise typer.Exit(code=1)
if token is not None and not token:
echo("Token cannot be empty.", err=True)
raise typer.Exit(code=1)
app.add_user(username, password, token, is_admin)
def install(self, language: str, gpu: bool = False):
"""Installs Orchest for the given language.
Pulls all the Orchest containers necessary to run all the
features for the given `language`.
"""
self.resource_manager.install_network()
# Check whether the install is complete.
pulled_images = self.resource_manager.get_images()
req_images = get_required_images(language, gpu)
missing_images = set(req_images) - set(pulled_images)
if not missing_images:
utils.echo("Installation is already complete. Did you mean to run:")
utils.echo("\torchest update")
return
utils.echo("Installing Orchest...")
logger.info("Pulling images:\n" + "\n".join(missing_images))
self.docker_client.pull_images(missing_images, prog_bar=True)
utils.echo(
"Checking whether all containers are running the same version of Orchest."
)
self.version(ext=True)
def stop(self, skip_containers: Optional[List[str]] = None):
"""Stop the Orchest application.
Args:
skip_containers: The names of the images of the containers
for which the containers are not stopped.
"""
ids, running_containers = self.resource_manager.get_containers(
state="running")
if not self.is_running(running_containers):
utils.echo("Orchest is not running.")
return
# Exclude the orchest-ctl from shutting down itself.
if skip_containers is None:
skip_containers = []
skip_containers += ["orchest/orchest-ctl:latest"]
ids: Tuple[str]
running_containers: Tuple[Optional[str]]
ids, running_containers = list(
zip(*[(id_, c) for id_, c in zip(ids, running_containers)
if c not in skip_containers]))
utils.echo("Shutting down...")
logger.info("Shutting down containers:\n" +
"\n".join(running_containers))
self.docker_client.remove_containers(ids)
utils.echo("Shutdown successful.")
def stop(self, skip_containers: Optional[List[str]] = None):
"""Stop the Orchest application.
Args:
skip_containers: The names of the images of the containers
for which the containers are not stopped.
"""
ids, running_containers = self.resource_manager.get_containers(state="running")
if not utils.is_orchest_running(running_containers):
utils.echo("Orchest is not running.")
return
# Exclude the orchest-ctl from shutting down itself.
if skip_containers is None:
skip_containers = []
skip_containers += ["orchest/orchest-ctl:latest"]
utils.echo("Shutting down...")
# This is necessary because some of our containers might spawn
# other containers, leading to a possible race condition where
# the listed running containers are not up to date with the
# real state anymore.
n = 2
for _ in range(n):
id_containers = [
(id_, c)
for id_, c in zip(ids, running_containers)
if c not in skip_containers
]
# It might be that there are no containers to shut down
# after filtering through skip_containers.
if not id_containers:
break
ids: Tuple[str]
running_containers: Tuple[Optional[str]]
ids, running_containers = list(zip(*id_containers))
logger.info("Shutting down containers:\n" + "\n".join(running_containers))
self.docker_client.remove_containers(ids)
# This is a safeguard against the fact that docker might be
# buffering the start of a container, which translates to
# the fact that we could "miss" the container and leave it
# dangling. See #239 for more info.
time.sleep(2)
ids, running_containers = self.resource_manager.get_containers(
state="running"
)
if not ids:
break
utils.echo("Shutdown successful.")
def status(self):
_, running_containers = self.resource_manager.get_containers(state="running")
if not self.is_running(running_containers):
utils.echo("Orchest is not running.")
return
# Minimal set of containers to be running for Orchest to be in
# a valid state.
valid_set: Set[str] = reduce(
lambda x, y: x.union(y), self.on_start_images, set()
)
if valid_set - set(running_containers):
utils.echo("Orchest is running, but has reached an invalid state. Run:")
utils.echo("\torchest restart")
logger.warning(
"Orchest has reached an invalid state. Running containers:\n"
+ "\n".join(running_containers)
)
else:
utils.echo("Orchest is running.")
def add_user(self, username, password, token, is_admin):
"""Adds a new user to Orchest.
Args:
username:
password:
token:
is_admin:
"""
ids, running_containers = self.resource_manager.get_containers(
state="running")
auth_server_id = None
database_running = False
for id, container in zip(ids, running_containers):
if "postgres" in container:
database_running = True
if "auth-server" in container:
auth_server_id = id
if not database_running:
utils.echo("The orchest-database service needs to be running.",
err=True)
raise typer.Exit(code=1)
if auth_server_id is None:
utils.echo("The auth-server service needs to be running.",
err=True)
raise typer.Exit(code=1)
cmd = f"python add_user.py {username} {password}"
if token:
cmd += f" --token {token}"
if is_admin:
cmd += " --is_admin"
exit_code = self.docker_client.exec_runs([(auth_server_id, cmd)])[0]
if exit_code != 0:
utils.echo(
("Non zero exit code whilst trying to add a user to "
f"the auth-server: {exit_code}."),
err=True,
)
raise typer.Exit(code=exit_code)
def status(self, ext=False):
_, running_containers_names = self.resource_manager.get_containers(
state="running")
if not self.is_running(running_containers_names):
utils.echo("Orchest is not running.")
raise typer.Exit(code=1)
# Minimal set of containers to be running for Orchest to be in
# a valid state.
valid_set: Set[str] = reduce(lambda x, y: x.union(y), _on_start_images,
set())
if valid_set - set(running_containers_names):
utils.echo(
"Orchest is running, but has reached an invalid state. Run:")
utils.echo("\torchest restart")
logger.warning(
"Orchest has reached an invalid state. Running containers:\n" +
"\n".join(running_containers_names))
raise typer.Exit(code=2)
else:
utils.echo("Orchest is running.")
if ext:
utils.echo("Performing extensive status checks...")
no_issues = True
for container, exit_code in health_check(
self.resource_manager).items():
if exit_code != 0:
no_issues = False
utils.echo(f"{container} is not ready ({exit_code}).")
if no_issues:
utils.echo("All services are ready.")
else:
raise typer.Exit(code=3)
def debug_dump(ext: bool, compress: bool) -> None:
debug_dump_path = "/tmp/debug-dump"
os.mkdir(debug_dump_path)
rmanager = OrchestResourceManager()
errors = []
for name, func, args in [
("configuration", orchest_config_dump, (debug_dump_path, )),
(
"containers version",
containers_version_dump,
(
rmanager,
debug_dump_path,
),
),
("containers logs", containers_logs_dump, (rmanager, debug_dump_path,
ext)),
(
"running containers",
running_containers_dump,
(
rmanager,
debug_dump_path,
),
),
(
"health check",
health_check_dump,
(
rmanager,
debug_dump_path,
),
),
(
"database",
database_debug_dump,
(
rmanager,
debug_dump_path,
),
),
("celery", celery_debug_dump, (rmanager, debug_dump_path, ext)),
(
"webserver",
webserver_debug_dump,
(
rmanager,
debug_dump_path,
),
),
]:
try:
utils.echo(f"Generating debug data: {name}.")
func(*args)
except Exception as e:
utils.echo(f"\tError during generation of debug data: {name}.")
errors.append((name, e))
with open(os.path.join(debug_dump_path, "errors.txt"), "w") as file:
file.write(
"This is a log of errors that happened during the dump, if any.\n")
for name, exc in errors:
file.write(f"{name}: {exc}\n")
if compress:
os.system(f"tar -zcf {debug_dump_path}.tar.gz -C {debug_dump_path} .")
debug_dump_path = f"{debug_dump_path}.tar.gz"
# relative to orchest repo directory
host_path = "debug-dump.tar.gz"
os.system(f"cp {debug_dump_path} /orchest-host/{host_path}")
else:
# This is to account for the behaviour of cp when it comes to
# already exising directory. Otherwise, if "t" already exists,
# the data we want to copy will become a subdirectory of "t",
# instead of overwriting its files.
host_path = "debug-dump/"
t = "/orchest-host/" + host_path
os.system(f"mkdir -p {t} && cp -r {debug_dump_path}/* {t}")
utils.echo(f"Complete! Wrote debug dump to: ./{host_path}")
def start(self, container_config: dict):
"""Starts Orchest.
Raises:
ValueError: If the `container_config` does not contain a
configuration for every image that is supposed to run
on start.
"""
# Check whether the minimal set of images is present for Orchest
# to be started.
pulled_images = self.resource_manager.get_images()
req_images: Set[str] = reduce(lambda x, y: x.union(y),
_on_start_images, set())
missing_images = req_images - set(pulled_images)
if missing_images or not self.resource_manager.is_network_installed():
utils.echo(
"Before starting Orchest, make sure Orchest is installed. Run:"
)
utils.echo("\torchest install")
return
# Check whether the container config contains the set of
# required images.
present_imgs = set(config["Image"]
for config in container_config.values())
if present_imgs < req_images: # proper subset
raise ValueError(
"The container_config does not contain a configuration for "
" every required image: " + ", ".join(req_images))
# Orchest is already running
ids, running_containers = self.resource_manager.get_containers(
state="running")
if not (req_images - set(running_containers)):
# TODO: Ideally this would print the port on which Orchest
# is running. (Was started before and so we do not
# simply know.)
utils.echo("Orchest is already running...")
return
# Orchest is partially running and thus in an inconsistent
# state. Possibly the start command was issued whilst Orchest
# is still shutting down.
if running_containers:
utils.echo(
"Orchest seems to be partially running. Before attempting to start"
" Orchest, shut the application down first:", )
utils.echo("\torchest stop")
return
# Remove old lingering containers.
ids, exited_containers = self.resource_manager.get_containers(
state="exited")
self.docker_client.remove_containers(ids)
utils.fix_userdir_permissions()
logger.info("Fixing permissions on the 'userdir/'.")
utils.echo("Starting Orchest...")
logger.info("Starting containers:\n" + "\n".join(req_images))
# Start the containers in the correct order, keeping in mind
# dependencies between containers.
for i, to_start_imgs in enumerate(_on_start_images):
filter_ = {"Image": to_start_imgs}
config = spec.filter_container_config(container_config,
filter=filter_)
stdouts = self.docker_client.run_containers(config,
use_name=True,
detach=True)
# TODO: Abstract version of when the next set of images can
# be started. In case the `on_start_images` has more
# stages.
if i == 0:
utils.wait_for_zero_exitcode(
self.docker_client,
stdouts["orchest-database"]["id"],
"pg_isready --username postgres",
)
utils.wait_for_zero_exitcode(
self.docker_client,
stdouts["rabbitmq-server"]["id"],
('su rabbitmq -c "/opt/rabbitmq/sbin/rabbitmq-diagnostics '
'-q check_port_connectivity"'),
)
# Get the port on which Orchest is running.
nginx_proxy = container_config.get("nginx-proxy")
if nginx_proxy is not None:
for port, port_binding in nginx_proxy["HostConfig"][
"PortBindings"].items():
exposed_port = port_binding[0]["HostPort"]
utils.echo(
f"Orchest is running at: http://localhost:{exposed_port}")
def install(self, language: str, gpu: bool = False):
"""Installs Orchest for the given language.
Pulls all the Orchest containers necessary to run all the
features for the given `language`.
"""
self.resource_manager.install_network()
# Check whether the install is complete.
pulled_images = self.resource_manager.get_images()
req_images = get_required_images(language, gpu)
missing_images = set(req_images) - set(pulled_images)
if not missing_images:
utils.echo(
"Installation is already complete. Did you mean to run:")
utils.echo("\torchest update")
return
# The installation is not yet complete, but some images were
# already pulled before.
if pulled_images:
utils.echo(
"Some images have been pulled before. Don't forget to run:")
utils.echo("\torchest update")
utils.echo(
"after the installation is finished to ensure that all images are"
" running the same version of Orchest.", )
utils.echo("Installing Orchest...")
logger.info("Pulling images:\n" + "\n".join(missing_images))
self.docker_client.pull_images(missing_images, prog_bar=True)
def version(self, ext=False):
"""Returns the version of Orchest.
Args:
ext: If True return the extensive version of Orchest.
Meaning that the version of every pulled image is
checked.
"""
if not ext:
version = os.getenv("ORCHEST_VERSION")
utils.echo(f"Orchest version: {version}")
return
utils.echo("Getting versions of all containers...")
stdouts = self.resource_manager.containers_version()
stdout_values = set()
for img, stdout in stdouts.items():
stdout_values.add(stdout)
utils.echo(f"{img:<44}: {stdout}")
# If not all versions are the same.
if len(stdout_values) > 1:
utils.echo(
"Not all containers are running on the same version of Orchest, which"
" can lead to the application crashing. You can fix this by running:",
)
utils.echo("\torchest update")
utils.echo("To get all containers on the same version again.")
def update(self, mode=None):
"""Update Orchest.
Args:
mode: The mode in which to update Orchest. This is either
``None`` or ``"web"``, where the latter is used when
update is invoked through the update-server.
"""
# Get all installed containers.
pulled_images = self.resource_manager.get_images()
# Pull images. It is possible to pull new image whilst the older
# versions of those images are running.
# TODO: remove the warning from the orchest.sh script that
# containers will be shut down.
utils.echo("Updating...")
_, running_containers = self.resource_manager.get_containers(
state="running")
if self.is_running(running_containers):
utils.echo("Using Orchest whilst updating is NOT recommended.")
# Update the Orchest git repo to get the latest changes to the
# "userdir/" structure.
exit_code = utils.update_git_repo()
if exit_code != 0:
utils.echo("Cancelling update...")
utils.echo(
"It seems like you have unstaged changes in the 'orchest'"
" repository. Please commit or stash them as 'orchest update'"
" pulls the newest changes to the 'userdir/' using a rebase.",
)
logger.error("Failed update due to unstaged changes.")
return
logger.info("Updating images:\n" + "\n".join(pulled_images))
self.docker_client.pull_images(pulled_images,
prog_bar=True,
force=True)
# Delete user-built environment images to avoid the issue of
# having environments with mismatching Orchest SDK versions.
logger.info("Deleting user-built environment images.")
self.resource_manager.remove_env_build_imgs()
# Delete user-built Jupyter image to make sure the Jupyter
# server is updated to the latest version of Orchest.
logger.info("Deleting user-built Jupyter image.")
self.resource_manager.remove_jupyter_build_imgs()
# We invoke the Orchest restart from the webserver ui-updater.
# Hence we do not show the message to restart manually.
if mode == "web":
utils.echo("Update completed.")
else:
# Let the user know they need to restart the application
# for the changes to take effect. NOTE: otherwise Orchest
# might also be running a mix of containers on different
# versions.
utils.echo(
"Don't forget to restart Orchest for the changes to take effect:"
)
utils.echo("\torchest restart")
def update(self, mode=None, dev: bool = False):
"""Update Orchest.
Args:
mode: The mode in which to update Orchest. This is either
``None`` or ``"web"``, where the latter is used when
update is invoked through the update-server.
"""
utils.echo("Updating...")
_, running_containers = self.resource_manager.get_containers(state="running")
if utils.is_orchest_running(running_containers):
if mode != "web":
# In the web updater it is not possible to cancel the
# update once started. So there is no value in showing
# this message or sleeping.
utils.echo(
"Using Orchest whilst updating is NOT supported and will be shut"
" down, killing all active pipeline runs and session. You have 2s"
" to cancel the update operation."
)
# Give the user the option to cancel the update
# operation using a keyboard interrupt.
time.sleep(2)
skip_containers = []
if mode == "web":
# It is possible to pull new images whilst the older
# versions of those images are running. We will invoke
# Orchest restart from the webserver ui-updater.
skip_containers = [
"orchest/update-server:latest",
"orchest/auth-server:latest",
"orchest/nginx-proxy:latest",
"postgres:13.1",
]
self.stop(skip_containers=skip_containers)
# Update the Orchest git repo to get the latest changes to the
# "userdir/" structure.
if not dev:
exit_code = utils.update_git_repo()
if exit_code == 0:
logger.info("Successfully updated git repo during update.")
elif exit_code == 21:
utils.echo("Cancelling update...")
utils.echo(
"Make sure you have the master branch checked out before updating."
)
logger.error(
"Failed update due to master branch not being checked out."
)
return
else:
utils.echo("Cancelling update...")
utils.echo(
"It seems like you have unstaged changes in the 'orchest'"
" repository. Please commit or stash them as 'orchest update'"
" pulls the newest changes to the 'userdir/' using a rebase.",
)
logger.error("Failed update due to unstaged changes.")
return
# Get all installed images and pull new versions. The pulled
# images are checked to make sure optional images, e.g. lang
# specific images, are updated as well.
pulled_images = self.resource_manager.get_images()
to_pull_images = set(ORCHEST_IMAGES["minimal"]) | set(pulled_images)
logger.info("Updating images:\n" + "\n".join(to_pull_images))
self.docker_client.pull_images(to_pull_images, prog_bar=True, force=True)
# Delete user-built environment images to avoid the issue of
# having environments with mismatching Orchest SDK versions.
logger.info("Deleting user-built environment images.")
self.resource_manager.remove_env_build_imgs()
# Delete user-built Jupyter image to make sure the Jupyter
# server is updated to the latest version of Orchest.
logger.info("Deleting user-built Jupyter image.")
self.resource_manager.remove_jupyter_build_imgs()
# Delete Orchest dangling images.
self.resource_manager.remove_orchest_dangling_imgs()
if mode == "web":
utils.echo("Update completed.")
else:
utils.echo("Update completed. To start Orchest again, run:")
utils.echo("\torchest start")
utils.echo(
"Checking whether all containers are running the same version of Orchest."
)
self.version(ext=True)
def version(self, ext=False):
"""Returns the version of Orchest.
Args:
ext: If True return the extensive version of Orchest.
Meaning that the version of every pulled image is
checked.
"""
if not ext:
version = os.getenv("ORCHEST_VERSION")
utils.echo(f"Orchest version: {version}")
return
utils.echo("Getting versions of all containers...")
pulled_images = self.resource_manager.get_images(orchest_owned=True)
configs = {}
for img in pulled_images:
configs[img] = {
"Image": img,
"Entrypoint": ["printenv", "ORCHEST_VERSION"],
}
stdouts = self.docker_client.run_containers(configs, detach=False)
stdout_values = set()
for img, info in stdouts.items():
stdout = info["stdout"]
# stdout = ['v0.4.1-58-g3f4bc64\n']
stdout = stdout[0].rstrip()
stdout_values.add(stdout)
utils.echo(f"{img:<44}: {stdout}")
# If not all versions are the same.
if len(stdout_values) > 1:
utils.echo(
"Not all containers are running on the same version of Orchest, which"
" can lead to the application crashing. You can fix this by running:",
wrap=WRAP_LINES,
)
utils.echo("\torchest update")
utils.echo("To get all containers on the same version again.")
def update(self, mode=None, dev: bool = False):
"""Update Orchest.
Args:
mode: The mode in which to update Orchest. This is either
``None`` or ``"web"``, where the latter is used when
update is invoked through the update-server.
"""
utils.echo("Updating...")
_, running_containers = self.resource_manager.get_containers(
state="running")
if utils.is_orchest_running(running_containers):
utils.echo(
"Using Orchest whilst updating is NOT supported and will be shut"
" down, killing all active pipeline runs and session. You have 2s"
" to cancel the update operation.")
# Give the user the option to cancel the update operation
# using a keyboard interrupt.
time.sleep(2)
# It is possible to pull new image whilst the older versions
# of those images are running.
self.stop(skip_containers=[
"orchest/update-server:latest",
"orchest/auth-server:latest",
"orchest/nginx-proxy:latest",
"postgres:13.1",
])
# Update the Orchest git repo to get the latest changes to the
# "userdir/" structure.
if not dev:
exit_code = utils.update_git_repo()
if exit_code != 0:
utils.echo("Cancelling update...")
utils.echo(
"It seems like you have unstaged changes in the 'orchest'"
" repository. Please commit or stash them as 'orchest update'"
" pulls the newest changes to the 'userdir/' using a rebase.",
)
logger.error("Failed update due to unstaged changes.")
return
# Get all installed images and pull new versions. The pulled
# images are checked to make sure optional images, e.g. lang
# specific images, are updated as well.
pulled_images = self.resource_manager.get_images()
to_pull_images = set(ORCHEST_IMAGES["minimal"]) | set(pulled_images)
logger.info("Updating images:\n" + "\n".join(to_pull_images))
self.docker_client.pull_images(to_pull_images,
prog_bar=True,
force=True)
# Delete user-built environment images to avoid the issue of
# having environments with mismatching Orchest SDK versions.
logger.info("Deleting user-built environment images.")
self.resource_manager.remove_env_build_imgs()
# Delete user-built Jupyter image to make sure the Jupyter
# server is updated to the latest version of Orchest.
logger.info("Deleting user-built Jupyter image.")
self.resource_manager.remove_jupyter_build_imgs()
# Delete Orchest dangling images.
self.resource_manager.remove_orchest_dangling_imgs()
# We invoke the Orchest restart from the webserver ui-updater.
# Hence we do not show the message to restart manually.
if mode == "web":
utils.echo("Update completed.")
else:
# Let the user know they need to restart the application
# for the changes to take effect. NOTE: otherwise Orchest
# might also be running a mix of containers on different
# versions.
utils.echo(
"Don't forget to restart Orchest for the changes to take effect:"
)
utils.echo("\torchest restart")
