def get_notebook_state():
"""
Get notebook server state.
Returns
-------
bool
Raises
------
ApiException
"""
load_kube_config()
v1 = client.CoreV1Api()
try:
pod = v1.read_namespaced_pod(
name=NOTEBOOK_POD_NAME,
namespace=NOTEBOOK_NAMESPACE,
_request_timeout=5,
)
if pod.status.phase == "Running" \
and all([c.state.running for c in pod.status.container_statuses]):
return True
return False
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
f"Error while trying to get notebook server state: {message}")
def load_kube_config():
"""
Loads authentication and cluster information from Load kube-config file.
Raises
------
InternalServerError
When the connection is not successfully established.
Notes
-----
Default file location is `~/.kube/config`.
"""
try:
config.load_kube_config()
success = True
except Exception:
success = False
if success:
return
try:
config.load_incluster_config()
except Exception:
raise InternalServerError("Failed to connect to cluster.")
def get_container_logs(pod, container):
"""
Returns latest logs of the specified container.
Parameters
----------
pod : str
container : str
Returns
-------
str
Container's logs.
Raises
------
InternalServerError
While trying to query Kubernetes API.
"""
load_kube_config()
core_api = client.CoreV1Api()
try:
logs = core_api.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=KF_PIPELINES_NAMESPACE,
container=container.name,
pretty="true",
tail_lines=512,
timestamps=True,
)
return logs
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
if pod.status.reason in IGNORABLE_STATUSES_REASONS:
return None
for ignorable_messages in IGNORABLE_MESSAGES_KEYTEXTS:
if ignorable_messages in message:
return None
raise InternalServerError(
code="CannotRetrieveContainerLogs",
message=
f"Error while trying to retrieve container's log: {message}",
)
def get_container_logs(pod, container):
"""
Returns latest logs of the specified container.
Parameters
----------
pod : str
container : str
Returns
-------
str
Container's logs.
Raises
------
InternalServerError
While trying to query Kubernetes API.
"""
load_kube_config()
core_api = client.CoreV1Api()
try:
logs = core_api.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=KF_PIPELINES_NAMESPACE,
container=container.name,
pretty="true",
tail_lines=512,
timestamps=True,
)
return logs
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
if "ContainerCreating" in message:
return None
raise InternalServerError(
f"Error while trying to retrive container's log: {message}")
def delete_task(self, task_id: str):
"""
Delete a task in our database.
Parameters
----------
task_id : str
Returns
-------
projects.schemas.message.Message
Raises
------
NotFound
When task_id does not exist.
"""
task = self.session.query(models.Task).get(task_id)
if task is None:
raise NOT_FOUND
if task.operator:
raise Forbidden(code="TaskProtectedFromDeletion",
message="Task related to an operator")
# remove the volume for the task in the notebook server
self.session.delete(task)
self.session.commit()
all_tasks = self.session.query(models.Task).all()
try:
make_task_deletion_job(
task=task,
all_tasks=all_tasks,
namespace=KF_PIPELINES_NAMESPACE,
)
except Exception as e:
raise InternalServerError(
code="DeletionJobError",
message=
f"Error while trying to make deletion container job: {e}",
)
return schemas.Message(message="Task deleted")
def create_prediction(self, project_id: str, deployment_id: str, file: bytes = None):
"""
POST a prediction file to seldon deployment.
Parameters
----------
project_id : str
deployment_id : str
file : bytes
File buffer.
Returns
-------
dict
"""
url = get_seldon_deployment_url(deployment_id, external_url=False)
request = parse_file_buffer_to_seldon_request(file)
response = requests.post(url, json=request)
try:
return json.loads(response._content)
except json.decoder.JSONDecodeError:
raise InternalServerError(response._content)
def get_volume_from_pod(volume_name, namespace, experiment_id):
"""
Get volume content zipped on base64.
Parameters
----------
volume_name: str
namespace: str
experiment_id: str
Returns
-------
str
Volume content in base64.
"""
load_kube_config()
api_instance = client.CoreV1Api()
pod_name = f"download-{experiment_id}"
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": pod_name
},
"spec": {
"containers": [{
"image":
"alpine",
"name":
"main",
"command": ["/bin/sh"],
"args": ["-c", "while true;do date;sleep 5; done;apk add zip"],
"volumeMounts": [{
"mountPath": "/tmp/data",
"name": "vol-tmp-data"
}],
}],
"volumes": [{
"name": "vol-tmp-data",
"persistentVolumeClaim": {
"claimName": volume_name
},
}],
},
}
try:
api_instance.create_namespaced_pod(body=pod_manifest,
namespace=namespace)
except ApiException as e:
# status 409: AlreadyExists
if e.status != 409:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
code="CannotCreateNamespacedPod",
message=f"Error while trying to create pod: {message}",
)
while True:
resp = api_instance.read_namespaced_pod(name=pod_name,
namespace=namespace)
if resp.status.phase != "Pending":
break
time.sleep(1)
exec_command = [
"/bin/sh", "-c", "apk add zip -q && zip -q -r - /tmp/data | base64"
]
container_stream = stream.stream(
api_instance.connect_get_namespaced_pod_exec,
name=pod_name,
namespace=namespace,
command=exec_command,
container="main",
stderr=True,
stdin=False,
stdout=True,
tty=False,
_preload_content=False,
)
zip_file_content = ""
while container_stream.is_open():
container_stream.update(timeout=10)
if container_stream.peek_stdout():
zip_file_content += container_stream.read_stdout()
container_stream.close()
api_instance.delete_namespaced_pod(name=pod_name, namespace=namespace)
# the stdout string contains \n character, we must remove
clean_zip_file_content = zip_file_content.replace("\n", "")
return clean_zip_file_content
def create_persistent_volume_claim(name, mount_path):
"""
Creates a persistent volume claim and mounts it in the default notebook server.
Parameters
----------
name : str
mount_path : str
"""
load_kube_config()
v1 = client.CoreV1Api()
custom_api = client.CustomObjectsApi(api_client=ApiClientForJsonPatch())
try:
body = {
"metadata": {
"name": name,
},
"spec": {
"accessModes": [
"ReadWriteOnce",
],
"resources": {
"requests": {
"storage": "10Gi",
},
}
},
}
v1.create_namespaced_persistent_volume_claim(
namespace=NOTEBOOK_NAMESPACE,
body=body,
)
body = [
{
"op": "add",
"path": "/spec/template/spec/volumes/-",
"value": {
"name": name,
"persistentVolumeClaim": {
"claimName": name,
},
},
},
{
"op": "add",
"path": "/spec/template/spec/containers/0/volumeMounts/-",
"value": {
"mountPath": mount_path,
"name": name,
},
},
]
custom_api.patch_namespaced_custom_object(
group=NOTEBOOK_GROUP,
version="v1",
namespace=NOTEBOOK_NAMESPACE,
plural="notebooks",
name=NOTEBOOK_NAME,
body=body,
)
# Wait for the pod to be ready and have all containers running
while True:
try:
pod = v1.read_namespaced_pod(
name=NOTEBOOK_POD_NAME,
namespace=NOTEBOOK_NAMESPACE,
_request_timeout=5,
)
if pod.status.phase == "Running" \
and all([c.state.running for c in pod.status.container_statuses]) \
and any([v for v in pod.spec.volumes if v.name == f"{name}"]):
warnings.warn(f"Mounted volume {name} in notebook server!")
break
except ApiException:
pass
finally:
warnings.warn(NOTEBOOK_WAITING_MSG)
time.sleep(5)
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
f"Error while trying to patch notebook server: {message}")
def remove_persistent_volume_claim(name, mount_path):
"""
Remove a persistent volume claim in the default notebook server.
Parameters
----------
name : str
mount_path : str
"""
load_kube_config()
v1 = client.CoreV1Api()
custom_api = client.CustomObjectsApi(api_client=ApiClientForJsonPatch())
try:
pod = v1.read_namespaced_pod(
name=NOTEBOOK_POD_NAME,
namespace=NOTEBOOK_NAMESPACE,
_request_timeout=5,
)
pod_vols = enumerate(pod.spec.volumes)
vol_index = next((i for i, v in pod_vols if v.name == f"{name}"), -1)
if vol_index == -1:
raise InternalServerError(f"Not found volume: {name}")
v1.delete_namespaced_persistent_volume_claim(
name=name,
namespace=NOTEBOOK_NAMESPACE,
)
body = [
{
"op": "remove",
"path": f"/spec/template/spec/volumes/{vol_index}",
},
{
"op":
"remove",
"path":
f"/spec/template/spec/containers/0/volumeMounts/{vol_index}",
},
]
custom_api.patch_namespaced_custom_object(
group=NOTEBOOK_GROUP,
version="v1",
namespace=NOTEBOOK_NAMESPACE,
plural="notebooks",
name=NOTEBOOK_NAME,
body=body,
)
# Wait for the pod to be ready and have all containers running
while True:
try:
pod = v1.read_namespaced_pod(
name=NOTEBOOK_POD_NAME,
namespace=NOTEBOOK_NAMESPACE,
_request_timeout=5,
)
if pod.status.phase == "Running" \
and all([c.state.running for c in pod.status.container_statuses]) \
and not [v for v in pod.spec.volumes if v.name == f"{name}"]:
warnings.warn(f"Removed volume {name} in notebook server!")
break
except ApiException:
pass
finally:
warnings.warn(NOTEBOOK_WAITING_MSG)
time.sleep(5)
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
f"Error while trying to patch notebook server: {message}")
def create_persistent_volume_claim(name, mount_path):
"""
Creates a persistent volume claim and mounts it in the default notebook server.
Parameters
----------
name : str
mount_path : str
"""
load_kube_config()
v1 = client.CoreV1Api()
custom_api = client.CustomObjectsApi(api_client=ApiClientForJsonPatch())
try:
body = {
"metadata": {
"name": name,
},
"spec": {
"accessModes": [
"ReadWriteOnce",
],
"resources": {
"requests": {
"storage": "10Gi",
},
},
},
}
v1.create_namespaced_persistent_volume_claim(
namespace=NOTEBOOK_NAMESPACE,
body=body,
)
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
code="CannotCreatePersistentVolumeClaim",
message=
f"Error while trying to create persistent volume claim: {message}",
)
try:
notebook = custom_api.get_namespaced_custom_object(
group=NOTEBOOK_GROUP,
version="v1",
namespace=NOTEBOOK_NAMESPACE,
plural="notebooks",
name=NOTEBOOK_NAME,
_request_timeout=5,
)
# Prevents the creation of duplicate mountPath
pod_vols = enumerate(notebook["spec"]["template"]["spec"]["containers"]
[0]["volumeMounts"])
vol_index = next(
(i for i, v in pod_vols if v["mountPath"] == mount_path), -1)
if vol_index > -1:
warnings.warn(
f"Notebook server already has a task at: {mount_path}. Skipping the mount of volume {name}"
)
return
except ApiException as e:
if e.status == 404:
warnings.warn(
f"Notebook server does not exist. Skipping the mount of volume {name}"
)
return
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
code="CannotPatchNotebookServer",
message=f"Error while trying to patch notebook server: {message}",
)
try:
body = [
{
"op": "add",
"path": "/spec/template/spec/volumes/-",
"value": {
"name": name,
"persistentVolumeClaim": {
"claimName": name,
},
},
},
{
"op": "add",
"path": "/spec/template/spec/containers/0/volumeMounts/-",
"value": {
"mountPath": mount_path,
"name": name,
},
},
]
custom_api.patch_namespaced_custom_object(
group=NOTEBOOK_GROUP,
version="v1",
namespace=NOTEBOOK_NAMESPACE,
plural="notebooks",
name=NOTEBOOK_NAME,
body=body,
)
# Wait for the pod to be ready and have all containers running
while True:
try:
pod = v1.read_namespaced_pod(
name=NOTEBOOK_POD_NAME,
namespace=NOTEBOOK_NAMESPACE,
_request_timeout=5,
)
pod_is_running = pod.status.phase == "Running"
containers_are_running = all(
[c.state.running for c in pod.status.container_statuses])
# ToDo
# volume_is_mounted = any(
# [v for v in pod.spec.volumes if v.name == f"{name}"]
# )
volume_is_mounted = True
if pod_is_running and containers_are_running and volume_is_mounted:
warnings.warn(f"Mounted volume {name} in notebook server!")
break
except ApiException:
pass
finally:
warnings.warn(NOTEBOOK_WAITING_MSG)
time.sleep(5)
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
code="CannotPatchNotebookServer",
message=f"Error while trying to patch notebook server: {message}",
)
def update_persistent_volume_claim(name, mount_path):
"""
Update a persistent volume mount in the default notebook server.
Parameters
----------
name : str
mount_path : str
"""
load_kube_config()
v1 = client.CoreV1Api()
custom_api = client.CustomObjectsApi(api_client=ApiClientForJsonPatch())
try:
notebook = custom_api.get_namespaced_custom_object(
group=NOTEBOOK_GROUP,
version="v1",
namespace=NOTEBOOK_NAMESPACE,
plural="notebooks",
name=NOTEBOOK_NAME,
_request_timeout=5,
)
except ApiException as e:
if e.status == 404:
warnings.warn(
f"Notebook server does not exist. Skipping update volume mount path {name}"
)
return
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
code="CannotPatchNotebookServer",
message=f"Error while trying to patch notebook server: {message}",
)
try:
pod_vols = enumerate(notebook["spec"]["template"]["spec"]["volumes"])
vol_index = next((i for i, v in pod_vols if v["name"] == f"{name}"),
-1)
if vol_index == -1:
warnings.warn(f"Volume mount path not found: {name}")
return
body = [
{
"op": "replace",
"path":
f"/spec/template/spec/containers/0/volumeMounts/{vol_index}/mountPath",
"value": mount_path,
},
]
custom_api.patch_namespaced_custom_object(
group=NOTEBOOK_GROUP,
version="v1",
namespace=NOTEBOOK_NAMESPACE,
plural="notebooks",
name=NOTEBOOK_NAME,
body=body,
)
# Wait for the pod to be ready and have all containers running
while True:
try:
pod = v1.read_namespaced_pod(
name=NOTEBOOK_POD_NAME,
namespace=NOTEBOOK_NAMESPACE,
_request_timeout=5,
)
pod_volume_mounts = pod.spec.containers[0].volume_mounts
if (pod.status.phase == "Running" and all(
[c.state.running for c in pod.status.container_statuses])
and any([
vm for vm in pod_volume_mounts
if vm.mount_path == f"{mount_path}"
])):
warnings.warn(
f"Updated volume mount path {name} in notebook server!"
)
break
except ApiException:
pass
finally:
warnings.warn(NOTEBOOK_WAITING_MSG)
time.sleep(5)
except ApiException as e:
body = literal_eval(e.body)
message = body["message"]
raise InternalServerError(
code="CannotPatchNotebookServer",
message=f"Error while trying to patch notebook server: {message}",
)