def pod_exec(self, pod_name, exec_command, timeout=60):
"""work as the command (with timeout): kubectl exec 'pod_name' 'exec_command'"""
try:
logging.info("Exec on pod {}: {}".format(pod_name, exec_command))
client = stream(
self.v1.connect_get_namespaced_pod_exec,
name=pod_name,
namespace=self.namespace,
command=exec_command,
stderr=True,
stdin=False,
stdout=True,
tty=False,
_preload_content=False,
)
client.run_forever(timeout=timeout)
err = yaml.full_load(client.read_channel(ERROR_CHANNEL))
if err is None:
return [-1, "Timeout"]
if err["status"] == "Success":
status_code = 0
else:
logging.debug("Exec on pod {} failed. cmd: {}, err: {}.".format(pod_name, exec_command, err))
status_code = int(err["details"]["causes"][0]["message"])
output = client.read_all()
logging.info("Exec on pod {}, status: {}, cmd: {}, output: {}".format(pod_name, status_code, exec_command, output))
return [status_code, output]
except ApiException as err:
logging.error("Exec on pod {} error. cmd: {}, err: {}.".format(pod_name, exec_command, err), exc_info=True)
return [-1, err.message]
def ps_pod(pod, userid=1000):
"""Get ps output from a single pod"""
kube = get_kube()
try:
client = stream(
kube.connect_get_namespaced_pod_exec,
pod["metadata"]["name"],
namespace=pod["metadata"]["namespace"],
command=["ps", "aux"],
stderr=True,
stdin=False,
stdout=True,
_preload_content=False,
)
client.run_forever(timeout=60)
stderr = client.read_stderr()
if stderr.strip():
print(f"err! {stderr}", file=sys.stderr)
stdout = client.read_stdout()
returncode = client.returncode
if returncode:
raise RuntimeError(f"stdout={stdout}\nstderr={stderr}")
return stdout
except Exception as e:
return f"Error reporting on ps in {pod['metadata']['name']}: {e}"
def pod_exec(pod_name, exec_command, timeout=60):
"""work as the command (with timeout): kubectl exec 'pod_name' 'exec_command'"""
try:
logger.debug("Exec on pod %s: %s", pod_name, exec_command)
client = stream(
k8s_core_api.connect_get_namespaced_pod_exec,
name=pod_name,
namespace="default",
command=exec_command,
stderr=True,
stdin=False,
stdout=True,
tty=False,
_preload_content=False,
)
client.run_forever(timeout=timeout)
err = yaml.full_load(client.read_channel(ERROR_CHANNEL))
if err is None:
return [-1, "Timeout"]
if err["status"] == "Success":
status_code = 0
else:
logger.debug("Exec on pod %s failed. cmd: %s, err: %s.", pod_name,
exec_command, err)
status_code = int(err["details"]["causes"][0]["message"])
output = client.read_all()
logger.info("Exec on pod %s, status: %s, cmd: %s, output: %s", pod_name,
status_code, exec_command, output)
return [status_code, output]
except ApiException as err:
logger.exception("Exec on pod %s error. cmd: %s, err: %s.", pod_name,
exec_command, err)
return [-1, err.message]
def _run_pod_exec(
ns: str,
cluster_name: str,
orchest_service: str,
command: t.List[str],
check_gate=True,
) -> None:
"""Runs `command` inside the pod defined by `orchest_service`.
Raises:
RuntimeError: If something went wrong when trying to run the
command in the pod, or when the command did not return with
a zero exit code.
"""
def passes_gate(ns: str, cluster_name: str) -> t.Tuple[bool, str]:
"""Returns whether the Orchest Cluster is in a valid state.
Returns:
True, "": if Cluster is in a valid state.
False, reason: if Cluster is in an invalid state.
"""
try:
status = _get_orchest_cluster_status(ns, cluster_name)
except CRObjectNotFound as e:
return False, str(e)
if status != ClusterStatus.RUNNING:
reason = (
"The Orchest Cluster state is "
" '{'unknown' if status is None else status.value}', whereas it needs"
" to be '{ClusterStatus.RUNNING.value}'. Check:"
"\n\torchest status")
return False, reason
return True, ""
if check_gate:
passed_gate, reason = passes_gate(ns, cluster_name)
if not passed_gate:
raise RuntimeError(f"Failed to pass gate: {reason}")
pods = CORE_API.list_namespaced_pod(
ns,
label_selector=(f"{orchest_service}={orchest_service},"
"contoller.orchest.io/part-of=orchest,"
f"controller.orchest.io={cluster_name}"),
)
if not pods:
raise RuntimeError(
f"Orchest Cluster is in an invalid state: no '{orchest_service}' found."
)
elif len(pods.items) > 1:
raise RuntimeError("Orchest Cluster is in an invalid state: multiple "
f" '{orchest_service}' found.")
else:
pod = pods.items[0]
client = stream.stream(
CORE_API.connect_get_namespaced_pod_exec,
name=pod.metadata.name,
namespace=ns,
command=command,
stderr=True,
stdin=False,
stdout=True,
tty=False,
# Gives us a WS client so we can get the return code.
_preload_content=False,
)
# NOTE: Timeout shouldn't be needed, but we don't want to keep the
# WS connection open indefinitely if something goes wrong when
# running the command.
client.run_forever(timeout=20)
if client.returncode != 0:
raise RuntimeError(client.read_all())