def use_cluster_credentials(cluster_name):
"""
Quickly gain command-line access to a cluster by updating the current
kubeconfig file to include the deployer's access credentials for the named
cluster and mark it as the cluster to work against by default.
This function is to be used with the `use-cluster-credentials` CLI
command only - it is not used by the rest of the deployer codebase.
"""
validate_cluster_config(cluster_name)
config_file_path = find_absolute_path_to_cluster_file(cluster_name)
with open(config_file_path) as f:
cluster = Cluster(yaml.load(f), config_file_path.parent)
# Cluster.auth() method has the context manager decorator so cannot call
# it like a normal function
with cluster.auth():
# This command will spawn a new shell with all the env vars (including
# KUBECONFIG) inherited, and once you quit that shell the python program
# will resume as usual.
# TODO: Figure out how to change the PS1 env var of the spawned shell
# to change the prompt to f"cluster-{cluster.spec['name']}". This will
# make it visually clear that the user is now operating in a different
# shell.
subprocess.check_call([os.environ["SHELL"], "-l"])
def exec_homes_shell(cluster_name, hub_name):
"""
Pop a shell with the home directories of the given hub mounted
Homes will be mounter under /home
"""
config_file_path = find_absolute_path_to_cluster_file(cluster_name)
with open(config_file_path) as f:
cluster = Cluster(yaml.load(f), config_file_path.parent)
with cluster.auth():
hubs = cluster.hubs
hub = next((hub for hub in hubs if hub.spec["name"] == hub_name), None)
hub.exec_homes_shell()
def deploy(cluster_name, hub_name, config_path, dask_gateway_version):
"""
Deploy one or more hubs in a given cluster
"""
validate_cluster_config(cluster_name)
validate_hub_config(cluster_name, hub_name)
assert_single_auth_method_enabled(cluster_name, hub_name)
with get_decrypted_file(config_path) as decrypted_file_path:
with open(decrypted_file_path) as f:
config = yaml.load(f)
# Most of our hubs use Auth0 for Authentication. This lets us programmatically
# determine what auth provider each hub uses - GitHub, Google, etc. Without
# this, we'd have to manually generate credentials for each hub - and we
# don't want to do that. Auth0 domains are tied to a account, and
# this is our auth0 domain for the paid account that 2i2c has.
auth0 = config["auth0"]
k = KeyProvider(auth0["domain"], auth0["client_id"],
auth0["client_secret"])
# Each hub needs a unique proxy.secretToken. However, we don't want
# to manually generate & save it. We also don't want it to change with
# each deploy - that causes a pod restart with downtime. So instead,
# we generate it based on a single secret key (`PROXY_SECRET_KEY`)
# combined with the name of each hub. This way, we get unique,
# cryptographically secure proxy.secretTokens without having to
# keep much state. We can rotate them by changing `PROXY_SECRET_KEY`.
# However, if `PROXY_SECRET_KEY` leaks, that means all the hub's
# proxy.secretTokens have leaked. So let's be careful with that!
SECRET_KEY = bytes.fromhex(config["secret_key"])
config_file_path = find_absolute_path_to_cluster_file(cluster_name)
with open(config_file_path) as f:
cluster = Cluster(yaml.load(f), config_file_path.parent)
with cluster.auth():
hubs = cluster.hubs
if hub_name:
hub = next((hub for hub in hubs if hub.spec["name"] == hub_name),
None)
print_colour(f"Deploying hub {hub.spec['name']}...")
hub.deploy(k, SECRET_KEY, dask_gateway_version)
else:
for i, hub in enumerate(hubs):
print_colour(
f"{i+1} / {len(hubs)}: Deploying hub {hub.spec['name']}..."
)
hub.deploy(k, SECRET_KEY, dask_gateway_version)
def deploy_support(cluster_name, cert_manager_version):
"""Deploy support components to a cluster
Args:
cluster_name (str): The name of the cluster to deploy support components to
cert_manager_version (str): The version of cert-manager to deploy to the
cluster, in the form vX.Y.Z. where X.Y.Z is valid SemVer.
"""
validate_cluster_config(cluster_name)
validate_support_config(cluster_name)
config_file_path = find_absolute_path_to_cluster_file(cluster_name)
with open(config_file_path) as f:
cluster = Cluster(yaml.load(f), config_file_path.parent)
if cluster.support:
with cluster.auth():
cluster.deploy_support(cert_manager_version=cert_manager_version)
def run_hub_health_check(cluster_name, hub_name, check_dask_scaling=False):
"""Run a health check on a given hub on a given cluster. Optionally check scaling
of dask workers if the hub is a daskhub.
Args:
cluster_name (str): The name of the cluster where the hub is deployed
hub_name (str): The name of the hub to run a health check for
check_dask_scaling (bool, optional): If true, run an additional check that dask
workers can scale. Only applies to daskhubs. Defaults to False.
Returns
exit_code (int): The exit code of the pytest process. 0 for pass, any other
integer number greater than 0 for failure.
"""
# Read in the cluster.yaml file
config_file_path = find_absolute_path_to_cluster_file(cluster_name)
with open(config_file_path) as f:
cluster = Cluster(yaml.load(f), config_file_path.parent)
# Find the hub's config
hub_indx = [
indx for (indx, h) in enumerate(cluster.hubs)
if h.spec["name"] == hub_name
]
if len(hub_indx) == 1:
hub = cluster.hubs[hub_indx[0]]
elif len(hub_indx) > 1:
print_colour("ERROR: More than one hub with this name found!")
sys.exit(1)
elif len(hub_indx) == 0:
print_colour("ERROR: No hubs with this name found!")
sys.exit(1)
print_colour(f"Running hub health check for {hub.spec['name']}...")
# Check if this hub has a domain override file. If yes, apply override.
if "domain_override_file" in hub.spec.keys():
domain_override_file = hub.spec["domain_override_file"]
with get_decrypted_file(
hub.cluster.config_path.joinpath(
domain_override_file)) as decrypted_path:
with open(decrypted_path) as f:
domain_override_config = yaml.load(f)
hub.spec["domain"] = domain_override_config["domain"]
# Retrieve hub's URL
hub_url = f'https://{hub.spec["domain"]}'
# Read in the service api token from a k8s Secret in the k8s cluster
with cluster.auth():
try:
service_api_token_b64encoded = subprocess.check_output(
[
"kubectl",
"get",
"secrets",
"hub",
f"--namespace={hub.spec['name']}",
r"--output=jsonpath={.data['hub\.services\.hub-health\.apiToken']}",
],
text=True,
)
except subprocess.CalledProcessError as e:
raise ValueError(
f"Failed to acquire a JupyterHub API token for the hub-health service: {e.stdout}"
)
service_api_token = base64.b64decode(
service_api_token_b64encoded).decode()
# On failure, pytest prints out params to the test that failed.
# This can contain sensitive info - so we hide stderr
# FIXME: Don't use pytest - just call a function instead
#
# Show errors locally but redirect on CI
gh_ci = os.environ.get("CI", "false")
pytest_args = [
"-q",
"deployer/tests",
f"--hub-url={hub_url}",
f"--api-token={service_api_token}",
f"--hub-type={hub.spec['helm_chart']}",
]
if (hub.spec["helm_chart"] == "daskhub") and check_dask_scaling:
pytest_args.append("--check-dask-scaling")
if gh_ci == "true":
print_colour("Testing on CI, not printing output")
with open(os.devnull,
"w") as dn, redirect_stderr(dn), redirect_stdout(dn):
exit_code = pytest.main(pytest_args)
else:
print_colour("Testing locally, do not redirect output")
exit_code = pytest.main(pytest_args)
if exit_code != 0:
print("Health check failed!", file=sys.stderr)
sys.exit(exit_code)
else:
print_colour("Health check succeeded!")
return exit_code
