async def aw():
await fw.add(V1Service(metadata=V1ObjectMeta(name="rejected")))
await fw.delete(V1Pod(metadata=V1ObjectMeta(name="foo")))
await fw.modify(
V1Pod(metadata=V1ObjectMeta(name="bar", resource_version="55"))
)
await fw.add(
V1Pod(metadata=V1ObjectMeta(name="baz", resource_version="32"))
)
await fw.stop()
def test_get_names_in_job():
pod_list = V1PodList(items=[
V1Pod(metadata=V1ObjectMeta(name="foo")),
V1Pod(metadata=V1ObjectMeta(name="bar"))
])
mock_client = create_mocked_client()
mock_client.core_api.list_namespaced_pod.side_effect = [pod_list]
assert mock_client.get_pod_names_in_job("job",
"namespace") == ["foo", "bar"]
def test_get_names_in_job():
pod_list = V1PodList(items=[
V1Pod(metadata=V1ObjectMeta(name='foo')),
V1Pod(metadata=V1ObjectMeta(name='bar'))
])
mock_client = create_mocked_client()
mock_client.core_api.list_namespaced_pod.side_effect = [pod_list]
assert mock_client.get_pod_names_in_job('job',
'namespace') == ['foo', 'bar']
async def test_listener_resync_periods(self):
source = fake_controller_source.FakeControllerSource()
await source.add(V1Pod(metadata=V1ObjectMeta(name="pod1")))
await source.add(V1Pod(metadata=V1ObjectMeta(name="pod2")))
informer = new_shared_informer(source, V1Pod, 1)
clock_ = clock.FakeClock(time.time())
informer._clock = clock_
informer._processor._clock = clock_
listener1 = TestListener("listener1", 0, "pod1", "pod2")
await informer.add_event_handler(
listener1, resync_period=listener1._resync_period)
listener2 = TestListener("listener2", 2, "pod1", "pod2")
await informer.add_event_handler(
listener2, resync_period=listener2._resync_period)
listener3 = TestListener("listener3", 3, "pod1", "pod2")
await informer.add_event_handler(
listener3, resync_period=listener3._resync_period)
listeners = [listener1, listener2, listener3]
try:
task = asyncio.ensure_future(informer.run())
for listener in listeners:
self.assertTrue(await listener._ok())
for listener in listeners:
listener._received_item_names = []
await clock_.step(2)
self.assertTrue(await listener2._ok())
await asyncio.sleep(1)
self.assertEqual(len(listener1._received_item_names), 0)
self.assertEqual(len(listener3._received_item_names), 0)
for listener in listeners:
listener._received_item_names = []
await clock_.step(1)
self.assertTrue(await listener3._ok())
await asyncio.sleep(1)
self.assertEqual(len(listener1._received_item_names), 0)
self.assertEqual(len(listener2._received_item_names), 0)
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
def test_wait_for_statuses_then_success():
mock_client = create_mocked_client()
single_no_status_pod = V1PodList(items=[V1Pod(status=V1PodStatus())])
single_ready_running_pod = _pod_list_for_container_status(
_ready_running_status())
mock_client.core_api.list_namespaced_pod.side_effect = [
single_no_status_pod,
single_ready_running_pod,
]
pod_name = "a_pod"
mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace")
assert_logger_calls(
mock_client.logger,
[
'Waiting for pod "%s"' % pod_name,
"Waiting for pod container status to be set by kubernetes...",
'Pod "%s" is ready, done waiting' % pod_name,
],
)
# slept only once
assert len(mock_client.sleeper.mock_calls) == 1
def get_or_create_pod(self, pod_request_obj: k8s.V1Pod, context):
if self.reattach_on_restart:
pod = self.find_pod(self.namespace or pod_request_obj.metadata.namespace, context=context)
if pod:
return pod
self.log.debug("Starting pod:\n%s", yaml.safe_dump(pod_request_obj.to_dict()))
self.pod_manager.create_pod(pod=pod_request_obj)
return pod_request_obj
async def test_get_index_func_values(self):
index = store.new_indexer(store.meta_namespace_key_func,
Indexers(testmodes=test_index_func))
pod1 = V1Pod(metadata=V1ObjectMeta(name="one", labels={"foo": "bar"}))
pod2 = V1Pod(metadata=V1ObjectMeta(name="two", labels={"foo": "bar"}))
pod3 = V1Pod(metadata=V1ObjectMeta(name="tre", labels={"foo": "biz"}))
await index.add(pod1)
await index.add(pod2)
await index.add(pod3)
keys = await index.list_index_func_values("testmodes")
self.assertEqual(len(keys), 2)
for key in keys:
self.assertIn(key, ("bar", "biz"))
def mocked_k8s_CoreV1Api(mocker):
mocked_coreV1Api_class = mocker.patch('kubernetes.client.CoreV1Api')
mocker.patch('kubernetes.client.ApiClient')
coreV1API_instance = mocked_coreV1Api_class.return_value
pods_mock = MagicMock()
pods_mock.items = [
MagicMock(spec=V1Pod),
MagicMock(spec=V1Pod),
MagicMock(spec=V1Pod)
]
coreV1API_instance.list_pod_for_all_namespaces.return_value = pods_mock
services_mock = MagicMock()
services_mock.items = [
MagicMock(spec=V1Service),
MagicMock(spec=V1Service),
MagicMock(spec=V1Service)
]
coreV1API_instance.list_service_for_all_namespaces.return_value = services_mock
v1_namespace = V1Namespace()
v1_metadata_namespace = V1ObjectMeta(name=test_namespace)
v1_namespace.metadata = v1_metadata_namespace
v1_namespace_status = V1NamespaceStatus(phase=NamespaceStatus.ACTIVE.value)
v1_namespace.status = v1_namespace_status
coreV1API_instance.read_namespace.return_value = v1_namespace
coreV1API_instance.delete_namespace.return_value = V1Status(
status="{'phase': 'Terminating'}")
v1_config_map = V1ConfigMap(data=test_config_map_data())
coreV1API_instance.read_namespaced_config_map.return_value = v1_config_map
secret_data = {"token": TEST_TOKEN}
v1_metadata_secret = V1ObjectMeta(name="default-token")
v1_secret = V1Secret(metadata=v1_metadata_secret, data=secret_data)
v1_secret_list = V1SecretList(items=[v1_secret])
coreV1API_instance.list_namespaced_secret.return_value = v1_secret_list
v1_pod_status = V1PodStatus(phase=K8S_RUNNING_POD_STATUS)
v1_pod = V1Pod(status=v1_pod_status)
v1_pod_lists = V1PodList(items=[v1_pod])
coreV1API_instance.list_namespaced_pod.return_value = v1_pod_lists
v1_metadata_event = V1ObjectMeta(name="default-name")
v1_object = V1ObjectReference(name="pod_name")
v1_event = V1Event(message="Insufficient cpu",
involved_object=v1_object,
metadata=v1_metadata_event)
v1_event_list = V1EventList(items=[v1_event])
coreV1API_instance.list_namespaced_event.return_value = v1_event_list
return coreV1API_instance
async def test_watch_handler(self):
s = store.new_store(store.meta_namespace_key_func)
g = Reflector(TestLW.__new__(TestLW), V1Pod, s, 0)
fw = watch.new_fake()
await s.add(V1Pod(metadata=V1ObjectMeta(name="foo")))
await s.add(V1Pod(metadata=V1ObjectMeta(name="bar")))
async def aw():
await fw.add(V1Service(metadata=V1ObjectMeta(name="rejected")))
await fw.delete(V1Pod(metadata=V1ObjectMeta(name="foo")))
await fw.modify(
V1Pod(metadata=V1ObjectMeta(name="bar", resource_version="55"))
)
await fw.add(
V1Pod(metadata=V1ObjectMeta(name="baz", resource_version="32"))
)
await fw.stop()
asyncio.ensure_future(aw())
options = {}
await g._watch_handler(fw, options, asyncio.Queue())
def mk_pod(id_, rv):
return V1Pod(metadata=V1ObjectMeta(name=id_, resource_version=rv))
table = [
{"pod": mk_pod("foo", ""), "exists": False},
{"pod": mk_pod("rejected", ""), "exists": False},
{"pod": mk_pod("bar", "55"), "exists": True},
{"pod": mk_pod("baz", "32"), "exists": True},
]
for item in table:
obj = s.get(item["pod"])
exists = obj is not None
self.assertIs(exists, item["exists"])
if not exists:
continue
self.assertEqual(
obj.metadata.resource_version, item["pod"].metadata.resource_version
)
self.assertEqual(options["resource_version"], "32")
self.assertEqual(g.last_sync_resource_version(), "32")
async def test_multi_index_keys(self):
index = store.new_indexer(store.meta_namespace_key_func,
Indexers(by_user=test_users_index_func))
pod1 = V1Pod(metadata=V1ObjectMeta(
name="one", annotations={"users": "ernie,bert"}))
pod2 = V1Pod(metadata=V1ObjectMeta(
name="two", annotations={"users": "bert,oscar"}))
pod3 = V1Pod(metadata=V1ObjectMeta(
name="tre", annotations={"users": "ernie,elmo"}))
await index.add(pod1)
await index.add(pod2)
await index.add(pod3)
expected = {
"ernie": {"one", "tre"},
"bert": {"one", "two"},
"elmo": {"tre"},
"oscar": {"two"},
}
for k, v in expected.items():
index_results = await index.by_index("by_user", k)
found = {item.metadata.name for item in index_results}
self.assertEqual(found, v)
await index.delete(pod3)
ernie_pods = await index.by_index("by_user", "ernie")
self.assertEqual(len(ernie_pods), 1)
for ernie_pod in ernie_pods:
self.assertEqual(ernie_pod.metadata.name, "one")
elmo_pods = await index.by_index("by_user", "elmo")
self.assertEqual(len(elmo_pods), 0)
copy_of_pod2 = copy.deepcopy(pod2)
copy_of_pod2.metadata.annotations["users"] = "oscar"
await index.update(copy_of_pod2)
bert_pods = await index.by_index("by_user", "bert")
self.assertEqual(len(bert_pods), 1)
for bert_pod in bert_pods:
self.assertEqual(bert_pod.metadata.name, "one")
async def test_run_until(self):
s = store.new_store(store.meta_namespace_key_func)
fw = watch.new_fake()
def list_func(options):
return V1PodList(metadata=V1ListMeta(resource_version="1"), items=[])
lister_watcher = TestLW(list_func, lambda _: fw)
r = Reflector(lister_watcher, V1Pod, s, 0)
task = asyncio.ensure_future(r.run())
await fw.add(V1Pod(metadata=V1ObjectMeta(name="bar")))
task.cancel()
async def aw():
async for _ in fw:
self.fail("Watch left open after stopping the watch")
await asyncio.wait_for(aw(), wait.FOREVER_TEST_TIMEOUT)
async def test_close_watch_on_error(self):
pod = V1Pod(metadata=V1ObjectMeta(name="bar"))
fw = watch.new_fake()
def list_func(options):
return V1PodList(metadata=V1ListMeta(resource_version="1"), items=[])
lister_watcher = TestLW(list_func, lambda _: fw)
r = Reflector(
lister_watcher, V1Pod, store.new_store(store.meta_namespace_key_func), 0
)
asyncio.ensure_future(r.list_and_watch())
await fw.error(pod)
async def aw():
async for _ in fw:
self.fail("Watch left open after cancellation")
await asyncio.wait_for(aw(), wait.FOREVER_TEST_TIMEOUT)
async def test_list_and_watch(self):
created_fakes = asyncio.Queue()
expected_rvs = ["1", "3"]
def list_func(options):
return V1PodList(metadata=V1ListMeta(resource_version="1"), items=[])
def watch_func(options):
nonlocal expected_rvs
rv = options["resource_version"]
fw = watch.new_fake()
self.assertEqual(rv, expected_rvs[0])
expected_rvs = expected_rvs[1:]
asyncio.ensure_future(created_fakes.put(fw))
return fw
lw = TestLW(list_func, watch_func)
s = fifo.FIFO(store.meta_namespace_key_func)
r = Reflector(lw, V1Pod, s, 0)
asyncio.ensure_future(r.list_and_watch())
ids = ["foo", "bar", "baz", "qux", "zoo"]
fw = None
for i, id_ in enumerate(ids):
if not fw:
fw = await created_fakes.get()
sending_rv = str(i + 2)
await fw.add(
V1Pod(metadata=V1ObjectMeta(name=id_, resource_version=sending_rv))
)
if sending_rv == "3":
await fw.stop()
fw = None
for i, id_ in enumerate(ids):
pod = await pop(s)
self.assertEqual(pod.metadata.name, id_)
self.assertEqual(pod.metadata.resource_version, str(i + 2))
self.assertEqual(len(expected_rvs), 0)
async def task():
current_names = set()
for _ in range(100):
if not current_names or not random.randrange(3):
name = "".join(
random.choice(string.ascii_letters) for _ in range(16)
)
is_new = True
else:
name = random.choice(list(current_names))
is_new = False
# TODO: fuzz
pod = V1Pod(metadata=V1ObjectMeta(name=name, namespace="default"))
if is_new:
current_names.add(name)
await source.add(pod)
continue
if random.randrange(2):
current_names.add(name)
await source.modify(pod)
else:
current_names.remove(name)
await source.delete(pod)
def make_task(operator: str, task_params: Dict[str, Any]) -> BaseOperator:
"""
Takes an operator and params and creates an instance of that operator.
:returns: instance of operator object
"""
try:
# class is a Callable https://stackoverflow.com/a/34578836/3679900
operator_obj: Callable[..., BaseOperator] = import_string(operator)
except Exception as err:
raise Exception(f"Failed to import operator: {operator}") from err
try:
if operator_obj in [PythonOperator, BranchPythonOperator]:
if not task_params.get(
"python_callable_name") and not task_params.get(
"python_callable_file"):
raise Exception(
"Failed to create task. PythonOperator and BranchPythonOperator requires \
`python_callable_name` and `python_callable_file` parameters."
)
task_params[
"python_callable"]: Callable = utils.get_python_callable(
task_params["python_callable_name"],
task_params["python_callable_file"],
)
# remove dag-factory specific parameters
# Airflow 2.0 doesn't allow these to be passed to operator
del task_params["python_callable_name"]
del task_params["python_callable_file"]
# Check for the custom success and failure callables in SqlSensor. These are considered
# optional, so no failures in case they aren't found. Note: there's no reason to
# declare both a callable file and a lambda function for success/failure parameter.
# If both are found the object will not throw and error, instead callable file will
# take precedence over the lambda function
if operator_obj in [SqlSensor]:
# Success checks
if task_params.get("success_check_file") and task_params.get(
"success_check_name"):
task_params[
"success"]: Callable = utils.get_python_callable(
task_params["success_check_name"],
task_params["success_check_file"],
)
del task_params["success_check_name"]
del task_params["success_check_file"]
elif task_params.get("success_check_lambda"):
task_params[
"success"]: Callable = utils.get_python_callable_lambda(
task_params["success_check_lambda"])
del task_params["success_check_lambda"]
# Failure checks
if task_params.get("failure_check_file") and task_params.get(
"failure_check_name"):
task_params[
"failure"]: Callable = utils.get_python_callable(
task_params["failure_check_name"],
task_params["failure_check_file"],
)
del task_params["failure_check_name"]
del task_params["failure_check_file"]
elif task_params.get("failure_check_lambda"):
task_params[
"failure"]: Callable = utils.get_python_callable_lambda(
task_params["failure_check_lambda"])
del task_params["failure_check_lambda"]
if operator_obj in [HttpSensor]:
if not (task_params.get("response_check_name")
and task_params.get("response_check_file")
) and not task_params.get("response_check_lambda"):
raise Exception(
"Failed to create task. HttpSensor requires \
`response_check_name` and `response_check_file` parameters \
or `response_check_lambda` parameter.")
if task_params.get("response_check_file"):
task_params[
"response_check"]: Callable = utils.get_python_callable(
task_params["response_check_name"],
task_params["response_check_file"],
)
# remove dag-factory specific parameters
# Airflow 2.0 doesn't allow these to be passed to operator
del task_params["response_check_name"]
del task_params["response_check_file"]
else:
task_params[
"response_check"]: Callable = utils.get_python_callable_lambda(
task_params["response_check_lambda"])
# remove dag-factory specific parameters
# Airflow 2.0 doesn't allow these to be passed to operator
del task_params["response_check_lambda"]
# KubernetesPodOperator
if operator_obj == KubernetesPodOperator:
task_params["secrets"] = ([
Secret(**v) for v in task_params.get("secrets")
] if task_params.get("secrets") is not None else None)
task_params["ports"] = ([
Port(**v) for v in task_params.get("ports")
] if task_params.get("ports") is not None else None)
task_params["volume_mounts"] = ([
VolumeMount(**v) for v in task_params.get("volume_mounts")
] if task_params.get("volume_mounts") is not None else None)
task_params["volumes"] = ([
Volume(**v) for v in task_params.get("volumes")
] if task_params.get("volumes") is not None else None)
task_params["pod_runtime_info_envs"] = ([
PodRuntimeInfoEnv(**v)
for v in task_params.get("pod_runtime_info_envs")
] if task_params.get("pod_runtime_info_envs") is not None else
None)
task_params["full_pod_spec"] = (
V1Pod(**task_params.get("full_pod_spec"))
if task_params.get("full_pod_spec") is not None else None)
task_params["init_containers"] = ([
V1Container(**v)
for v in task_params.get("init_containers")
] if task_params.get("init_containers") is not None else None)
if utils.check_dict_key(task_params, "execution_timeout_secs"):
task_params["execution_timeout"]: timedelta = timedelta(
seconds=task_params["execution_timeout_secs"])
del task_params["execution_timeout_secs"]
if utils.check_dict_key(task_params, "sla_secs"):
task_params["sla"]: timedelta = timedelta(
seconds=task_params["sla_secs"])
del task_params["sla_secs"]
if utils.check_dict_key(task_params, "execution_delta_secs"):
task_params["execution_delta"]: timedelta = timedelta(
seconds=task_params["execution_delta_secs"])
del task_params["execution_delta_secs"]
if utils.check_dict_key(
task_params,
"execution_date_fn_name") and utils.check_dict_key(
task_params, "execution_date_fn_file"):
task_params[
"execution_date_fn"]: Callable = utils.get_python_callable(
task_params["execution_date_fn_name"],
task_params["execution_date_fn_file"],
)
del task_params["execution_date_fn_name"]
del task_params["execution_date_fn_file"]
# on_execute_callback is an Airflow 2.0 feature
if utils.check_dict_key(
task_params, "on_execute_callback"
) and version.parse(AIRFLOW_VERSION) >= version.parse("2.0.0"):
task_params["on_execute_callback"]: Callable = import_string(
task_params["on_execute_callback"])
if utils.check_dict_key(task_params, "on_failure_callback"):
task_params["on_failure_callback"]: Callable = import_string(
task_params["on_failure_callback"])
if utils.check_dict_key(task_params, "on_success_callback"):
task_params["on_success_callback"]: Callable = import_string(
task_params["on_success_callback"])
if utils.check_dict_key(task_params, "on_retry_callback"):
task_params["on_retry_callback"]: Callable = import_string(
task_params["on_retry_callback"])
# use variables as arguments on operator
if utils.check_dict_key(task_params, "variables_as_arguments"):
variables: List[Dict[str, str]] = task_params.get(
"variables_as_arguments")
for variable in variables:
if Variable.get(variable["variable"],
default_var=None) is not None:
task_params[variable["attribute"]] = Variable.get(
variable["variable"], default_var=None)
del task_params["variables_as_arguments"]
task: BaseOperator = operator_obj(**task_params)
except Exception as err:
raise Exception(f"Failed to create {operator_obj} task") from err
return task
def mk_pod(id_, rv):
return V1Pod(metadata=V1ObjectMeta(name=id_, resource_version=rv))
def _pod_list_for_container_status(container_status):
return V1PodList(items=[
V1Pod(status=V1PodStatus(container_statuses=[container_status]))
])
def pod(name, check, final):
labels = {"check": check}
if final:
labels["final"] = "true"
return V1Pod(metadata=V1ObjectMeta(name=name, labels=labels))
def make_pod(
name,
cmd,
port,
image_spec,
image_pull_policy,
image_pull_secret=None,
node_selector=None,
run_as_uid=None,
run_as_gid=None,
fs_gid=None,
supplemental_gids=None,
run_privileged=False,
env=None,
working_dir=None,
volumes=None,
volume_mounts=None,
labels=None,
annotations=None,
cpu_limit=None,
cpu_guarantee=None,
mem_limit=None,
mem_guarantee=None,
extra_resource_limits=None,
extra_resource_guarantees=None,
lifecycle_hooks=None,
init_containers=None,
service_account=None,
extra_container_config=None,
extra_pod_config=None,
extra_containers=None,
scheduler_name=None,
tolerations=None,
node_affinity_preferred=None,
node_affinity_required=None,
pod_affinity_preferred=None,
pod_affinity_required=None,
pod_anti_affinity_preferred=None,
pod_anti_affinity_required=None,
priority_class_name=None,
logger=None,
userdir={},
):
"""
Make a k8s pod specification for running a user notebook.
Parameters
----------
name:
Name of pod. Must be unique within the namespace the object is
going to be created in. Must be a valid DNS label.
image_spec:
Image specification - usually a image name and tag in the form
of image_name:tag. Same thing you would use with docker commandline
arguments
image_pull_policy:
Image pull policy - one of 'Always', 'IfNotPresent' or 'Never'. Decides
when kubernetes will check for a newer version of image and pull it when
running a pod.
image_pull_secret:
Image pull secret - Default is None -- set to your secret name to pull
from private docker registry.
port:
Port the notebook server is going to be listening on
cmd:
The command used to execute the singleuser server.
node_selector:
Dictionary Selector to match nodes where to launch the Pods
run_as_uid:
The UID used to run single-user pods. The default is to run as the user
specified in the Dockerfile, if this is set to None.
run_as_gid:
The GID used to run single-user pods. The default is to run as the primary
group of the user specified in the Dockerfile, if this is set to None.
fs_gid
The gid that will own any fresh volumes mounted into this pod, if using
volume types that support this (such as GCE). This should be a group that
the uid the process is running as should be a member of, so that it can
read / write to the volumes mounted.
supplemental_gids:
A list of GIDs that should be set as additional supplemental groups to
the user that the container runs as. You may have to set this if you are
deploying to an environment with RBAC/SCC enforced and pods run with a
'restricted' SCC which results in the image being run as an assigned
user ID. The supplemental group IDs would need to include the
corresponding group ID of the user ID the image normally would run as.
The image must setup all directories/files any application needs access
to, as group writable.
run_privileged:
Whether the container should be run in privileged mode.
env:
Dictionary of environment variables.
volumes:
List of dictionaries containing the volumes of various types this pod
will be using. See k8s documentation about volumes on how to specify
these
volume_mounts:
List of dictionaries mapping paths in the container and the volume(
specified in volumes) that should be mounted on them. See the k8s
documentaiton for more details
working_dir:
String specifying the working directory for the notebook container
labels:
Labels to add to the spawned pod.
annotations:
Annotations to add to the spawned pod.
cpu_limit:
Float specifying the max number of CPU cores the user's pod is
allowed to use.
cpu_guarentee:
Float specifying the max number of CPU cores the user's pod is
guaranteed to have access to, by the scheduler.
mem_limit:
String specifying the max amount of RAM the user's pod is allowed
to use. String instead of float/int since common suffixes are allowed
mem_guarantee:
String specifying the max amount of RAM the user's pod is guaranteed
to have access to. String ins loat/int since common suffixes
are allowed
lifecycle_hooks:
Dictionary of lifecycle hooks
init_containers:
List of initialization containers belonging to the pod.
service_account:
Service account to mount on the pod. None disables mounting
extra_container_config:
Extra configuration (e.g. envFrom) for notebook container which is not covered by parameters above.
extra_pod_config:
Extra configuration (e.g. tolerations) for pod which is not covered by parameters above.
extra_containers:
Extra containers besides notebook container. Used for some housekeeping jobs (e.g. crontab).
scheduler_name:
The pod's scheduler explicitly named.
tolerations:
Tolerations can allow a pod to schedule or execute on a tainted node. To
learn more about pod tolerations, see
https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/.
Pass this field an array of "Toleration" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core
node_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PreferredSchedulingTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#preferredschedulingterm-v1-core
node_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "NodeSelectorTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core
pod_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core
pod_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core
pod_anti_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core
pod_anti_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core
priority_class_name:
The name of the PriorityClass to be assigned the pod. This feature is Beta available in K8s 1.11.
"""
pod = V1Pod()
pod.kind = "Pod"
pod.api_version = "v1"
pod.metadata = V1ObjectMeta(name=name,
labels=(labels or {}).copy(),
annotations=(annotations or {}).copy())
pod.spec = V1PodSpec(containers=[])
pod.spec.restartPolicy = 'Never'
security_context = V1PodSecurityContext()
if fs_gid is not None:
security_context.fs_group = int(fs_gid)
if supplemental_gids is not None and supplemental_gids:
security_context.supplemental_groups = [
int(gid) for gid in supplemental_gids
]
if run_as_uid is not None:
security_context.run_as_user = int(run_as_uid)
if run_as_gid is not None:
security_context.run_as_group = int(run_as_gid)
pod.spec.security_context = security_context
if image_pull_secret is not None:
pod.spec.image_pull_secrets = []
image_secret = V1LocalObjectReference()
image_secret.name = image_pull_secret
pod.spec.image_pull_secrets.append(image_secret)
if node_selector:
pod.spec.node_selector = node_selector
if lifecycle_hooks:
lifecycle_hooks = get_k8s_model(V1Lifecycle, lifecycle_hooks)
notebook_container = V1Container(
name='notebook',
image=image_spec,
working_dir=working_dir,
ports=[V1ContainerPort(name='notebook-port', container_port=port)],
env=[V1EnvVar(k, v) for k, v in (env or {}).items()],
args=cmd,
image_pull_policy=image_pull_policy,
lifecycle=lifecycle_hooks,
resources=V1ResourceRequirements(),
volume_mounts=[
get_k8s_model(V1VolumeMount, obj) for obj in (volume_mounts or [])
],
)
if service_account is None:
# This makes sure that we don't accidentally give access to the whole
# kubernetes API to the users in the spawned pods.
pod.spec.automount_service_account_token = False
else:
pod.spec.service_account_name = service_account
if run_privileged:
notebook_container.security_context = V1SecurityContext(
privileged=True)
notebook_container.resources.requests = {}
if userdir.get('jp_cpu_request'):
notebook_container.resources.requests['cpu'] = userdir.get(
'jp_cpu_request')
elif cpu_guarantee:
notebook_container.resources.requests['cpu'] = cpu_guarantee
if userdir.get('jp_mem_request'):
notebook_container.resources.requests['memory'] = userdir.get(
'jp_mem_request')
elif mem_guarantee:
notebook_container.resources.requests['memory'] = mem_guarantee
if extra_resource_guarantees:
notebook_container.resources.requests.update(extra_resource_guarantees)
notebook_container.resources.limits = {}
if userdir.get('jp_cpu_limit'):
notebook_container.resources.limits['cpu'] = userdir.get(
'jp_cpu_limit')
elif cpu_limit:
notebook_container.resources.limits['cpu'] = cpu_limit
if userdir.get('jp_mem_limit'):
notebook_container.resources.limits['memory'] = userdir.get(
'jp_mem_limit')
elif mem_limit:
notebook_container.resources.limits['memory'] = mem_limit
if extra_resource_limits:
notebook_container.resources.limits.update(extra_resource_limits)
if userdir.get("jp_gpu_enable", False):
notebook_container.resources.limits[r'nvidia.com/gpu'] = userdir.get(
'jp_gpu_number', 0)
if extra_container_config:
notebook_container = update_k8s_model(
target=notebook_container,
changes=extra_container_config,
logger=logger,
target_name="notebook_container",
changes_name="extra_container_config",
)
pod.spec.containers.append(notebook_container)
if extra_containers:
pod.spec.containers.extend(
[get_k8s_model(V1Container, obj) for obj in extra_containers])
if tolerations:
pod.spec.tolerations = [
get_k8s_model(V1Toleration, obj) for obj in tolerations
]
if init_containers:
pod.spec.init_containers = [
get_k8s_model(V1Container, obj) for obj in init_containers
]
if volumes:
pod.spec.volumes = [get_k8s_model(V1Volume, obj) for obj in volumes]
else:
# Keep behaving exactly like before by not cleaning up generated pod
# spec by setting the volumes field even though it is an empty list.
pod.spec.volumes = []
if scheduler_name:
pod.spec.scheduler_name = scheduler_name
node_affinity = None
if node_affinity_preferred or node_affinity_required:
node_selector = None
if node_affinity_required:
node_selector = V1NodeSelector(node_selector_terms=[
get_k8s_model(V1NodeSelectorTerm, obj)
for obj in node_affinity_required
], )
preferred_scheduling_terms = None
if node_affinity_preferred:
preferred_scheduling_terms = [
get_k8s_model(V1PreferredSchedulingTerm, obj)
for obj in node_affinity_preferred
]
node_affinity = V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=
preferred_scheduling_terms,
required_during_scheduling_ignored_during_execution=node_selector,
)
pod_affinity = None
if pod_affinity_preferred or pod_affinity_required:
weighted_pod_affinity_terms = None
if pod_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_affinity_preferred
]
pod_affinity_terms = None
if pod_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_affinity_required
]
pod_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=
weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=
pod_affinity_terms,
)
pod_anti_affinity = None
if pod_anti_affinity_preferred or pod_anti_affinity_required:
weighted_pod_affinity_terms = None
if pod_anti_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_anti_affinity_preferred
]
pod_affinity_terms = None
if pod_anti_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_anti_affinity_required
]
pod_anti_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=
weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=
pod_affinity_terms,
)
affinity = None
if (node_affinity or pod_affinity or pod_anti_affinity):
affinity = V1Affinity(
node_affinity=node_affinity,
pod_affinity=pod_affinity,
pod_anti_affinity=pod_anti_affinity,
)
if affinity:
pod.spec.affinity = affinity
if priority_class_name:
pod.spec.priority_class_name = priority_class_name
if extra_pod_config:
pod.spec = update_k8s_model(
target=pod.spec,
changes=extra_pod_config,
logger=logger,
target_name="pod.spec",
changes_name="extra_pod_config",
)
return pod
def make_pod(
name,
cmd,
port,
image,
image_pull_policy,
image_pull_secrets=None,
node_selector=None,
uid=None,
gid=None,
fs_gid=None,
supplemental_gids=None,
privileged=False,
allow_privilege_escalation=True,
container_security_context=None,
pod_security_context=None,
env=None,
working_dir=None,
volumes=None,
volume_mounts=None,
labels=None,
annotations=None,
cpu_limit=None,
cpu_guarantee=None,
mem_limit=None,
mem_guarantee=None,
extra_resource_limits=None,
extra_resource_guarantees=None,
lifecycle_hooks=None,
init_containers=None,
service_account=None,
automount_service_account_token=None,
extra_container_config=None,
extra_pod_config=None,
extra_containers=None,
scheduler_name=None,
tolerations=None,
node_affinity_preferred=None,
node_affinity_required=None,
pod_affinity_preferred=None,
pod_affinity_required=None,
pod_anti_affinity_preferred=None,
pod_anti_affinity_required=None,
priority_class_name=None,
ssl_secret_name=None,
ssl_secret_mount_path=None,
logger=None,
):
"""
Make a k8s pod specification for running a user notebook.
Parameters
----------
name:
Name of pod. Must be unique within the namespace the object is
going to be created in. Must be a valid DNS label.
image:
Image specification - usually a image name and tag in the form
of image_name:tag. Same thing you would use with docker commandline
arguments
image_pull_policy:
Image pull policy - one of 'Always', 'IfNotPresent' or 'Never'. Decides
when kubernetes will check for a newer version of image and pull it when
running a pod.
image_pull_secrets:
Image pull secrets - a list of references to Kubernetes Secret resources
with credentials to pull images from image registries. This list can
either have strings in it or objects with the string value nested under
a name field.
port:
Port the notebook server is going to be listening on
cmd:
The command used to execute the singleuser server.
node_selector:
Dictionary Selector to match nodes where to launch the Pods
uid:
The UID used to run single-user pods. The default is to run as the user
specified in the Dockerfile, if this is set to None.
gid:
The GID used to run single-user pods. The default is to run as the primary
group of the user specified in the Dockerfile, if this is set to None.
Setting this parameter requires that *feature-gate* **RunAsGroup** be enabled,
otherwise the effective GID of the pod will be 0 (root). In addition, not
setting `gid` once feature-gate RunAsGroup is enabled will also
result in an effective GID of 0 (root).
fs_gid
The gid that will own any fresh volumes mounted into this pod, if using
volume types that support this (such as GCE). This should be a group that
the uid the process is running as should be a member of, so that it can
read / write to the volumes mounted.
supplemental_gids:
A list of GIDs that should be set as additional supplemental groups to
the user that the container runs as. You may have to set this if you are
deploying to an environment with RBAC/SCC enforced and pods run with a
'restricted' SCC which results in the image being run as an assigned
user ID. The supplemental group IDs would need to include the
corresponding group ID of the user ID the image normally would run as.
The image must setup all directories/files any application needs access
to, as group writable.
privileged:
Whether the container should be run in privileged mode.
allow_privilege_escalation:
Controls whether a process can gain more privileges than its parent process.
container_security_context:
A kubernetes securityContext to apply to the container.
pod_security_context:
A kubernetes securityContext to apply to the pod.
env:
Dictionary of environment variables.
volumes:
List of dictionaries containing the volumes of various types this pod
will be using. See k8s documentation about volumes on how to specify
these
volume_mounts:
List of dictionaries mapping paths in the container and the volume(
specified in volumes) that should be mounted on them. See the k8s
documentaiton for more details
working_dir:
String specifying the working directory for the notebook container
labels:
Labels to add to the spawned pod.
annotations:
Annotations to add to the spawned pod.
cpu_limit:
Float specifying the max number of CPU cores the user's pod is
allowed to use.
cpu_guarantee:
Float specifying the max number of CPU cores the user's pod is
guaranteed to have access to, by the scheduler.
mem_limit:
String specifying the max amount of RAM the user's pod is allowed
to use. String instead of float/int since common suffixes are allowed
mem_guarantee:
String specifying the max amount of RAM the user's pod is guaranteed
to have access to. String ins loat/int since common suffixes
are allowed
lifecycle_hooks:
Dictionary of lifecycle hooks
init_containers:
List of initialization containers belonging to the pod.
service_account:
Service account to mount on the pod. None disables mounting
extra_container_config:
Extra configuration (e.g. envFrom) for notebook container which is not covered by parameters above.
extra_pod_config:
Extra configuration (e.g. tolerations) for pod which is not covered by parameters above.
extra_containers:
Extra containers besides notebook container. Used for some housekeeping jobs (e.g. crontab).
scheduler_name:
The pod's scheduler explicitly named.
tolerations:
Tolerations can allow a pod to schedule or execute on a tainted node. To
learn more about pod tolerations, see
https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/.
Pass this field an array of "Toleration" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#toleration-v1-core
node_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PreferredSchedulingTerm" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20#preferredschedulingterm-v1-core
node_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "NodeSelectorTerm" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#nodeselectorterm-v1-core
pod_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#weightedpodaffinityterm-v1-core
pod_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#podaffinityterm-v1-core
pod_anti_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#weightedpodaffinityterm-v1-core
pod_anti_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#podaffinityterm-v1-core
priority_class_name:
The name of the PriorityClass to be assigned the pod. This feature is Beta available in K8s 1.11 and GA in 1.14.
ssl_secret_name:
Specifies the name of the ssl secret
ssl_secret_mount_path:
Specifies the name of the ssl secret mount path for the pod
"""
pod = V1Pod()
pod.kind = "Pod"
pod.api_version = "v1"
pod.metadata = V1ObjectMeta(
name=name,
labels=(labels or {}).copy(),
annotations=(annotations or {}).copy(),
)
pod.spec = V1PodSpec(containers=[])
pod.spec.restart_policy = 'OnFailure'
if image_pull_secrets is not None:
# image_pull_secrets as received by the make_pod function should always
# be a list, but it is allowed to have "a-string" elements or {"name":
# "a-string"} elements.
pod.spec.image_pull_secrets = [
V1LocalObjectReference(name=secret_ref)
if type(secret_ref) == str
else get_k8s_model(V1LocalObjectReference, secret_ref)
for secret_ref in image_pull_secrets
]
if ssl_secret_name and ssl_secret_mount_path:
if not volumes:
volumes = []
volumes.append(
{
'name': 'jupyterhub-internal-certs',
'secret': {'secretName': ssl_secret_name, 'defaultMode': 511},
}
)
env['JUPYTERHUB_SSL_KEYFILE'] = ssl_secret_mount_path + "ssl.key"
env['JUPYTERHUB_SSL_CERTFILE'] = ssl_secret_mount_path + "ssl.crt"
env['JUPYTERHUB_SSL_CLIENT_CA'] = (
ssl_secret_mount_path + "notebooks-ca_trust.crt"
)
if not volume_mounts:
volume_mounts = []
volume_mounts.append(
{
'name': 'jupyterhub-internal-certs',
'mountPath': ssl_secret_mount_path,
}
)
if node_selector:
pod.spec.node_selector = node_selector
if lifecycle_hooks:
lifecycle_hooks = get_k8s_model(V1Lifecycle, lifecycle_hooks)
# Security contexts can be configured on Pod and Container level. The
# Dedicated KubeSpawner API will bootstraps the container_security_context
# except for if can only be configured on the Pod level, then it bootstraps
# pod_security_context.
#
# The pod|container_security_context configuration is given a higher
# priority than the dedicated KubeSpawner API options.
#
# Note that validation against the Python kubernetes-client isn't made as
# the security contexts has evolved significantly and kubernetes-client is
# too outdated.
#
# | Dedicated KubeSpawner API | Kubernetes API | Security contexts |
# | -------------------------- | ------------------------ | ----------------- |
# | supplemental_gids | supplementalGroups | Pod only |
# | fs_gid | fsGroup | Pod only |
# | - | fsGroupChangePolicy | Pod only |
# | - | sysctls | Pod only |
# | privileged | privileged | Container only |
# | allow_privilege_escalation | allowPrivilegeEscalation | Container only |
# | - | capabilities | Container only |
# | - | procMount | Container only |
# | - | readOnlyRootFilesystem | Container only |
# | uid | runAsUser | Pod and Container |
# | gid | runAsGroup | Pod and Container |
# | - | runAsNonRoot | Pod and Container |
# | - | seLinuxOptions | Pod and Container |
# | - | seccompProfile | Pod and Container |
# | - | windowsOptions | Pod and Container |
#
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
# ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#securitycontext-v1-core (container)
# ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#podsecuritycontext-v1-core (pod)
#
psc = {}
# populate with fs_gid / supplemental_gids
if fs_gid is not None:
psc["fsGroup"] = int(fs_gid)
if supplemental_gids:
psc["supplementalGroups"] = [int(gid) for gid in supplemental_gids]
if pod_security_context:
for key in pod_security_context.keys():
if "_" in key:
raise ValueError(
f"pod_security_context's keys should have k8s camelCase names, got '{key}'"
)
psc.update(pod_security_context)
if not psc:
psc = None
pod.spec.security_context = psc
csc = {}
# populate with uid / gid / privileged / allow_privilege_escalation
if uid is not None:
csc["runAsUser"] = int(uid)
if gid is not None:
csc["runAsGroup"] = int(gid)
if privileged: # false as default
csc["privileged"] = True
if not allow_privilege_escalation: # true as default
csc["allowPrivilegeEscalation"] = False
if container_security_context:
for key in container_security_context.keys():
if "_" in key:
raise ValueError(
f"container_security_context's keys should have k8s camelCase names, got '{key}'"
)
csc.update(container_security_context)
if not csc:
csc = None
def _get_env_var_deps(env):
# only consider env var objects with an explicit string value
if not env.value:
return set()
# $(MY_ENV) pattern: $( followed by non-)-characters to be captured, followed by )
re_k8s_env_reference_pattern = r"\$\(([^\)]+)\)"
deps = set(re.findall(re_k8s_env_reference_pattern, env.value))
return deps - {env.name}
unsorted_env = {}
for key, env in (env or {}).items():
# Normalize KubeSpawners env input to valid Kubernetes EnvVar Python
# representations. They should have a "name" field as well as either a
# "value" field or "value_from" field. For examples see the
# test_make_pod_with_env function.
if type(env) == dict:
if not "name" in env:
env["name"] = key
env = get_k8s_model(V1EnvVar, env)
else:
env = V1EnvVar(name=key, value=env)
# Extract information about references to other envs as we want to use
# those to make an intelligent sorting before we render this into a list
# with an order that matters.
unsorted_env[env.name] = {
"deps": _get_env_var_deps(env),
"key": key,
"env": env,
}
# We sort environment variables in a way that allows dependencies to other
# env to resolve as much as possible. There could be circular dependencies
# so we will just do our best and settle with that.
#
# Algorithm description:
#
# - loop step:
# - pop all unsorted_env entries with dependencies in sorted_env
# - sort popped env based on key and extend the sorted_env list
# - loop exit:
# - exit if loop step didn't pop anything from unsorted_env
# - before exit, sort what remains and extending the sorted_env list
#
sorted_env = []
while True:
already_resolved_env_names = [e.name for e in sorted_env]
extracted_env = {}
for k, v in unsorted_env.copy().items():
if v["deps"].issubset(already_resolved_env_names):
extracted_env[k] = unsorted_env.pop(k)
if extracted_env:
extracted_env = [
d["env"]
for d in sorted(extracted_env.values(), key=operator.itemgetter("key"))
]
sorted_env.extend(extracted_env)
else:
remaining_env = [
d["env"]
for d in sorted(unsorted_env.values(), key=operator.itemgetter("key"))
]
sorted_env.extend(remaining_env)
break
notebook_container = V1Container(
name='notebook',
image=image,
working_dir=working_dir,
ports=[V1ContainerPort(name='notebook-port', container_port=port)],
env=sorted_env,
args=cmd,
image_pull_policy=image_pull_policy,
lifecycle=lifecycle_hooks,
resources=V1ResourceRequirements(),
volume_mounts=[
get_k8s_model(V1VolumeMount, obj) for obj in (volume_mounts or [])
],
security_context=csc,
)
if service_account is not None:
pod.spec.service_account_name = service_account
if automount_service_account_token is None:
if service_account is None:
# This makes sure that we don't accidentally give access to the whole
# kubernetes API to the users in the spawned pods.
pod.spec.automount_service_account_token = False
else:
pod.spec.automount_service_account_token = automount_service_account_token
notebook_container.resources.requests = {}
if cpu_guarantee:
notebook_container.resources.requests['cpu'] = cpu_guarantee
if mem_guarantee:
notebook_container.resources.requests['memory'] = mem_guarantee
if extra_resource_guarantees:
notebook_container.resources.requests.update(extra_resource_guarantees)
notebook_container.resources.limits = {}
if cpu_limit:
notebook_container.resources.limits['cpu'] = cpu_limit
if mem_limit:
notebook_container.resources.limits['memory'] = mem_limit
if extra_resource_limits:
notebook_container.resources.limits.update(extra_resource_limits)
if extra_container_config:
notebook_container = update_k8s_model(
target=notebook_container,
changes=extra_container_config,
logger=logger,
target_name="notebook_container",
changes_name="extra_container_config",
)
pod.spec.containers.append(notebook_container)
if extra_containers:
pod.spec.containers.extend(
[get_k8s_model(V1Container, obj) for obj in extra_containers]
)
if tolerations:
pod.spec.tolerations = [get_k8s_model(V1Toleration, obj) for obj in tolerations]
if init_containers:
pod.spec.init_containers = [
get_k8s_model(V1Container, obj) for obj in init_containers
]
if volumes:
pod.spec.volumes = [get_k8s_model(V1Volume, obj) for obj in volumes]
else:
# Keep behaving exactly like before by not cleaning up generated pod
# spec by setting the volumes field even though it is an empty list.
pod.spec.volumes = []
if scheduler_name:
pod.spec.scheduler_name = scheduler_name
node_affinity = None
if node_affinity_preferred or node_affinity_required:
node_selector = None
if node_affinity_required:
node_selector = V1NodeSelector(
node_selector_terms=[
get_k8s_model(V1NodeSelectorTerm, obj)
for obj in node_affinity_required
],
)
preferred_scheduling_terms = None
if node_affinity_preferred:
preferred_scheduling_terms = [
get_k8s_model(V1PreferredSchedulingTerm, obj)
for obj in node_affinity_preferred
]
node_affinity = V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=preferred_scheduling_terms,
required_during_scheduling_ignored_during_execution=node_selector,
)
pod_affinity = None
if pod_affinity_preferred or pod_affinity_required:
weighted_pod_affinity_terms = None
if pod_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_affinity_preferred
]
pod_affinity_terms = None
if pod_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj) for obj in pod_affinity_required
]
pod_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=pod_affinity_terms,
)
pod_anti_affinity = None
if pod_anti_affinity_preferred or pod_anti_affinity_required:
weighted_pod_affinity_terms = None
if pod_anti_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_anti_affinity_preferred
]
pod_affinity_terms = None
if pod_anti_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_anti_affinity_required
]
pod_anti_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=pod_affinity_terms,
)
affinity = None
if node_affinity or pod_affinity or pod_anti_affinity:
affinity = V1Affinity(
node_affinity=node_affinity,
pod_affinity=pod_affinity,
pod_anti_affinity=pod_anti_affinity,
)
if affinity:
pod.spec.affinity = affinity
if priority_class_name:
pod.spec.priority_class_name = priority_class_name
if extra_pod_config:
pod.spec = update_k8s_model(
target=pod.spec,
changes=extra_pod_config,
logger=logger,
target_name="pod.spec",
changes_name="extra_pod_config",
)
return pod
def make_pod_spec(self, tls_secret, worker_name=None):
annotations = self.common_annotations
env = self.get_env()
if worker_name is not None:
# Worker
name = "dask-gateway-worker-%s" % worker_name
container_name = "dask-gateway-worker"
labels = self.get_labels_for("dask-gateway-worker",
worker_name=worker_name)
mem_req = self.worker_memory
mem_lim = self.worker_memory_limit
cpu_req = self.worker_cores
cpu_lim = self.worker_cores_limit
env["DASK_GATEWAY_WORKER_NAME"] = worker_name
cmd = self.worker_command
extra_pod_config = self.worker_extra_pod_config
extra_container_config = self.worker_extra_container_config
else:
# Scheduler
name = "dask-gateway-scheduler-%s" % self.cluster_name
container_name = "dask-gateway-scheduler"
labels = self.get_labels_for("dask-gateway-scheduler")
mem_req = self.scheduler_memory
mem_lim = self.scheduler_memory_limit
cpu_req = self.scheduler_cores
cpu_lim = self.scheduler_cores_limit
cmd = self.scheduler_command
extra_pod_config = self.scheduler_extra_pod_config
extra_container_config = self.scheduler_extra_container_config
volume = V1Volume(name="dask-credentials",
secret=V1SecretVolumeSource(secret_name=tls_secret))
container = V1Container(
name=container_name,
image=self.image,
args=cmd,
env=[V1EnvVar(k, v) for k, v in env.items()],
image_pull_policy=self.image_pull_policy,
resources=V1ResourceRequirements(
requests={
"cpu": cpu_req,
"memory": mem_req
},
limits={
"cpu": cpu_lim,
"memory": mem_lim
},
),
volume_mounts=[
V1VolumeMount(
name=volume.name,
mount_path="/etc/dask-credentials/",
read_only=True,
)
],
)
if extra_container_config:
container = merge_kube_objects(container, extra_container_config)
pod = V1Pod(
kind="Pod",
api_version="v1",
metadata=V1ObjectMeta(name=name,
labels=labels,
annotations=annotations),
spec=V1PodSpec(containers=[container],
volumes=[volume],
restart_policy="Never"),
)
# Ensure we don't accidentally give access to the kubernetes API
pod.spec.automount_service_account_token = False
if extra_pod_config:
pod.spec = merge_kube_objects(pod.spec, extra_pod_config)
return pod
def make_pod(name,
cmd,
port,
image_spec,
image_pull_policy,
image_pull_secret=None,
node_selector=None,
run_as_uid=None,
run_as_gid=None,
fs_gid=None,
supplemental_gids=None,
run_privileged=False,
env={},
working_dir=None,
volumes=[],
volume_mounts=[],
labels={},
annotations={},
cpu_limit=None,
cpu_guarantee=None,
mem_limit=None,
mem_guarantee=None,
extra_resource_limits=None,
extra_resource_guarantees=None,
lifecycle_hooks=None,
init_containers=None,
service_account=None,
extra_container_config=None,
extra_pod_config=None,
extra_containers=None,
scheduler_name=None):
"""
Make a k8s pod specification for running a user notebook.
Parameters
----------
name:
Name of pod. Must be unique within the namespace the object is
going to be created in. Must be a valid DNS label.
image_spec:
Image specification - usually a image name and tag in the form
of image_name:tag. Same thing you would use with docker commandline
arguments
image_pull_policy:
Image pull policy - one of 'Always', 'IfNotPresent' or 'Never'. Decides
when kubernetes will check for a newer version of image and pull it when
running a pod.
image_pull_secret:
Image pull secret - Default is None -- set to your secret name to pull
from private docker registry.
port:
Port the notebook server is going to be listening on
cmd:
The command used to execute the singleuser server.
node_selector:
Dictionary Selector to match nodes where to launch the Pods
run_as_uid:
The UID used to run single-user pods. The default is to run as the user
specified in the Dockerfile, if this is set to None.
run_as_gid:
The GID used to run single-user pods. The default is to run as the primary
group of the user specified in the Dockerfile, if this is set to None.
fs_gid
The gid that will own any fresh volumes mounted into this pod, if using
volume types that support this (such as GCE). This should be a group that
the uid the process is running as should be a member of, so that it can
read / write to the volumes mounted.
supplemental_gids:
A list of GIDs that should be set as additional supplemental groups to
the user that the container runs as. You may have to set this if you are
deploying to an environment with RBAC/SCC enforced and pods run with a
'restricted' SCC which results in the image being run as an assigned
user ID. The supplemental group IDs would need to include the
corresponding group ID of the user ID the image normally would run as.
The image must setup all directories/files any application needs access
to, as group writable.
run_privileged:
Whether the container should be run in privileged mode.
env:
Dictionary of environment variables.
volumes:
List of dictionaries containing the volumes of various types this pod
will be using. See k8s documentation about volumes on how to specify
these
volume_mounts:
List of dictionaries mapping paths in the container and the volume(
specified in volumes) that should be mounted on them. See the k8s
documentaiton for more details
working_dir:
String specifying the working directory for the notebook container
labels:
Labels to add to the spawned pod.
annotations:
Annotations to add to the spawned pod.
cpu_limit:
Float specifying the max number of CPU cores the user's pod is
allowed to use.
cpu_guarentee:
Float specifying the max number of CPU cores the user's pod is
guaranteed to have access to, by the scheduler.
mem_limit:
String specifying the max amount of RAM the user's pod is allowed
to use. String instead of float/int since common suffixes are allowed
mem_guarantee:
String specifying the max amount of RAM the user's pod is guaranteed
to have access to. String ins loat/int since common suffixes
are allowed
lifecycle_hooks:
Dictionary of lifecycle hooks
init_containers:
List of initialization containers belonging to the pod.
service_account:
Service account to mount on the pod. None disables mounting
extra_container_config:
Extra configuration (e.g. envFrom) for notebook container which is not covered by parameters above.
extra_pod_config:
Extra configuration (e.g. tolerations) for pod which is not covered by parameters above.
extra_containers:
Extra containers besides notebook container. Used for some housekeeping jobs (e.g. crontab).
scheduler_name:
A custom scheduler's name.
"""
pod = V1Pod()
pod.kind = "Pod"
pod.api_version = "v1"
pod.metadata = V1ObjectMeta(name=name,
labels=labels.copy(),
annotations=annotations.copy())
pod.spec = V1PodSpec(containers=[])
pod.spec.restartPolicy = 'Never'
security_context = V1PodSecurityContext()
if fs_gid is not None:
security_context.fs_group = int(fs_gid)
if supplemental_gids is not None and supplemental_gids:
security_context.supplemental_groups = [
int(gid) for gid in supplemental_gids
]
if run_as_uid is not None:
security_context.run_as_user = int(run_as_uid)
if run_as_gid is not None:
security_context.run_as_group = int(run_as_gid)
pod.spec.security_context = security_context
if image_pull_secret is not None:
pod.spec.image_pull_secrets = []
image_secret = V1LocalObjectReference()
image_secret.name = image_pull_secret
pod.spec.image_pull_secrets.append(image_secret)
if node_selector:
pod.spec.node_selector = node_selector
notebook_container = V1Container(
name='notebook',
image=image_spec,
working_dir=working_dir,
ports=[V1ContainerPort(name='notebook-port', container_port=port)],
env=[V1EnvVar(k, v) for k, v in env.items()],
args=cmd,
image_pull_policy=image_pull_policy,
lifecycle=lifecycle_hooks,
resources=V1ResourceRequirements())
if service_account is None:
# Add a hack to ensure that no service accounts are mounted in spawned pods
# This makes sure that we don"t accidentally give access to the whole
# kubernetes API to the users in the spawned pods.
# Note: We don't simply use `automountServiceAccountToken` here since we wanna be compatible
# with older kubernetes versions too for now.
hack_volume = V1Volume(name='no-api-access-please', empty_dir={})
hack_volumes = [hack_volume]
hack_volume_mount = V1VolumeMount(
name='no-api-access-please',
mount_path="/var/run/secrets/kubernetes.io/serviceaccount",
read_only=True)
hack_volume_mounts = [hack_volume_mount]
# Non-hacky way of not mounting service accounts
pod.spec.automount_service_account_token = False
else:
hack_volumes = []
hack_volume_mounts = []
pod.spec.service_account_name = service_account
if run_privileged:
notebook_container.security_context = V1SecurityContext(
privileged=True)
notebook_container.resources.requests = {}
if cpu_guarantee:
notebook_container.resources.requests['cpu'] = cpu_guarantee
if mem_guarantee:
notebook_container.resources.requests['memory'] = mem_guarantee
if extra_resource_guarantees:
for k in extra_resource_guarantees:
notebook_container.resources.requests[
k] = extra_resource_guarantees[k]
notebook_container.resources.limits = {}
if cpu_limit:
notebook_container.resources.limits['cpu'] = cpu_limit
if mem_limit:
notebook_container.resources.limits['memory'] = mem_limit
if extra_resource_limits:
for k in extra_resource_limits:
notebook_container.resources.limits[k] = extra_resource_limits[k]
notebook_container.volume_mounts = volume_mounts + hack_volume_mounts
pod.spec.containers.append(notebook_container)
if extra_container_config:
for key, value in extra_container_config.items():
setattr(notebook_container,
_map_attribute(notebook_container.attribute_map, key),
value)
if extra_pod_config:
for key, value in extra_pod_config.items():
setattr(pod.spec, _map_attribute(pod.spec.attribute_map, key),
value)
if extra_containers:
pod.spec.containers.extend(extra_containers)
pod.spec.init_containers = init_containers
pod.spec.volumes = volumes + hack_volumes
if scheduler_name:
pod.spec.scheduler_name = scheduler_name
return pod
def test_pod_resources(self):
cases = [
{
"pod":
V1Pod(spec=V1PodSpec(
init_containers=[
V1Container(name="1",
resources=V1ResourceRequirements(
limits={}, requests={}))
],
containers=[
V1Container(name="1",
resources=V1ResourceRequirements(
limits={}, requests={})),
V1Container(name="2",
resources=V1ResourceRequirements(
limits={}, requests={}))
])),
# req_cpu, req_memory, lim_cpu, lim_memory, gpu_spec
"expected": (0, 0, 0, 0, '')
},
{
"pod":
V1Pod(
spec=V1PodSpec(
init_containers=[
V1Container(name="1",
resources=V1ResourceRequirements(
limits={
"cpu": "1",
"memory": "512Mi"
},
requests={
"cpu": "0.5",
"memory": "256Mi"
}))
],
containers=[
V1Container(name="1",
resources=V1ResourceRequirements(
limits={
"cpu": "6",
"memory": "6Gi"
},
requests={
"cpu": "3",
"memory": "0.5Gi"
})),
V1Container(name="2",
resources=V1ResourceRequirements(
limits={
"cpu": "2",
"memory": "2Gi"
},
requests={
"cpu": "1.5",
"memory": "1Gi"
}))
])),
# req_cpu, req_memory, lim_cpu, lim_memory, gpu_spec
"expected": (4.5, 1.5, 8.0, 8.0, '')
}
]
for case in cases:
got = k8s_pod_resource_requirements(case['pod'])
self.assertEqual(got, case['expected'])
def make_pod(
name,
cmd,
port,
image,
image_pull_policy,
image_pull_secrets=None,
node_selector=None,
run_as_uid=None,
run_as_gid=None,
fs_gid=None,
supplemental_gids=None,
run_privileged=False,
allow_privilege_escalation=True,
env=None,
working_dir=None,
volumes=None,
volume_mounts=None,
labels=None,
annotations=None,
cpu_limit=None,
cpu_guarantee=None,
mem_limit=None,
mem_guarantee=None,
extra_resource_limits=None,
extra_resource_guarantees=None,
lifecycle_hooks=None,
init_containers=None,
service_account=None,
extra_container_config=None,
extra_pod_config=None,
extra_containers=None,
scheduler_name=None,
tolerations=None,
node_affinity_preferred=None,
node_affinity_required=None,
pod_affinity_preferred=None,
pod_affinity_required=None,
pod_anti_affinity_preferred=None,
pod_anti_affinity_required=None,
priority_class_name=None,
logger=None,
):
"""
Make a k8s pod specification for running a user notebook.
Parameters
----------
name:
Name of pod. Must be unique within the namespace the object is
going to be created in. Must be a valid DNS label.
image:
Image specification - usually a image name and tag in the form
of image_name:tag. Same thing you would use with docker commandline
arguments
image_pull_policy:
Image pull policy - one of 'Always', 'IfNotPresent' or 'Never'. Decides
when kubernetes will check for a newer version of image and pull it when
running a pod.
image_pull_secrets:
Image pull secrets - a list of references to Kubernetes Secret resources
with credentials to pull images from image registries. This list can
either have strings in it or objects with the string value nested under
a name field.
port:
Port the notebook server is going to be listening on
cmd:
The command used to execute the singleuser server.
node_selector:
Dictionary Selector to match nodes where to launch the Pods
run_as_uid:
The UID used to run single-user pods. The default is to run as the user
specified in the Dockerfile, if this is set to None.
run_as_gid:
The GID used to run single-user pods. The default is to run as the primary
group of the user specified in the Dockerfile, if this is set to None.
Setting this parameter requires that *feature-gate* **RunAsGroup** be enabled,
otherwise the effective GID of the pod will be 0 (root). In addition, not
setting `run_as_gid` once feature-gate RunAsGroup is enabled will also
result in an effective GID of 0 (root).
fs_gid
The gid that will own any fresh volumes mounted into this pod, if using
volume types that support this (such as GCE). This should be a group that
the uid the process is running as should be a member of, so that it can
read / write to the volumes mounted.
supplemental_gids:
A list of GIDs that should be set as additional supplemental groups to
the user that the container runs as. You may have to set this if you are
deploying to an environment with RBAC/SCC enforced and pods run with a
'restricted' SCC which results in the image being run as an assigned
user ID. The supplemental group IDs would need to include the
corresponding group ID of the user ID the image normally would run as.
The image must setup all directories/files any application needs access
to, as group writable.
run_privileged:
Whether the container should be run in privileged mode.
allow_privilege_escalation:
Controls whether a process can gain more privileges than its parent process.
env:
Dictionary of environment variables.
volumes:
List of dictionaries containing the volumes of various types this pod
will be using. See k8s documentation about volumes on how to specify
these
volume_mounts:
List of dictionaries mapping paths in the container and the volume(
specified in volumes) that should be mounted on them. See the k8s
documentaiton for more details
working_dir:
String specifying the working directory for the notebook container
labels:
Labels to add to the spawned pod.
annotations:
Annotations to add to the spawned pod.
cpu_limit:
Float specifying the max number of CPU cores the user's pod is
allowed to use.
cpu_guarentee:
Float specifying the max number of CPU cores the user's pod is
guaranteed to have access to, by the scheduler.
mem_limit:
String specifying the max amount of RAM the user's pod is allowed
to use. String instead of float/int since common suffixes are allowed
mem_guarantee:
String specifying the max amount of RAM the user's pod is guaranteed
to have access to. String ins loat/int since common suffixes
are allowed
lifecycle_hooks:
Dictionary of lifecycle hooks
init_containers:
List of initialization containers belonging to the pod.
service_account:
Service account to mount on the pod. None disables mounting
extra_container_config:
Extra configuration (e.g. envFrom) for notebook container which is not covered by parameters above.
extra_pod_config:
Extra configuration (e.g. tolerations) for pod which is not covered by parameters above.
extra_containers:
Extra containers besides notebook container. Used for some housekeeping jobs (e.g. crontab).
scheduler_name:
The pod's scheduler explicitly named.
tolerations:
Tolerations can allow a pod to schedule or execute on a tainted node. To
learn more about pod tolerations, see
https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/.
Pass this field an array of "Toleration" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core
node_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PreferredSchedulingTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#preferredschedulingterm-v1-core
node_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "NodeSelectorTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#nodeselectorterm-v1-core
pod_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core
pod_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core
pod_anti_affinity_preferred:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "WeightedPodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#weightedpodaffinityterm-v1-core
pod_anti_affinity_required:
Affinities describe where pods prefer or require to be scheduled, they
may prefer or require a node to have a certain label or be in proximity
/ remoteness to another pod. To learn more visit
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
Pass this field an array of "PodAffinityTerm" objects.*
* https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#podaffinityterm-v1-core
priority_class_name:
The name of the PriorityClass to be assigned the pod. This feature is Beta available in K8s 1.11.
"""
pod = V1Pod()
pod.kind = "Pod"
pod.api_version = "v1"
pod.metadata = V1ObjectMeta(name=name,
labels=(labels or {}).copy(),
annotations=(annotations or {}).copy())
pod.spec = V1PodSpec(containers=[])
pod.spec.restart_policy = 'OnFailure'
if image_pull_secrets is not None:
# image_pull_secrets as received by the make_pod function should always
# be a list, but it is allowed to have "a-string" elements or {"name":
# "a-string"} elements.
pod.spec.image_pull_secrets = [
V1LocalObjectReference(name=secret_ref) if type(secret_ref) == str
else get_k8s_model(V1LocalObjectReference, secret_ref)
for secret_ref in image_pull_secrets
]
if node_selector:
pod.spec.node_selector = node_selector
if lifecycle_hooks:
lifecycle_hooks = get_k8s_model(V1Lifecycle, lifecycle_hooks)
# There are security contexts both on the Pod level or the Container level.
# The security settings that you specify for a Pod apply to all Containers
# in the Pod, but settings on the container level can override them.
#
# We configure the pod to be spawned on the container level unless the
# option is only available on the pod level, such as for those relating to
# the volumes as compared to the running user of the container. Volumes
# belong to the pod and are only mounted by containers after all.
#
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
# ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#securitycontext-v1-core (container)
# ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#podsecuritycontext-v1-core (pod)
pod_security_context = V1PodSecurityContext()
if fs_gid is not None:
pod_security_context.fs_group = int(fs_gid)
if supplemental_gids is not None and supplemental_gids:
pod_security_context.supplemental_groups = [
int(gid) for gid in supplemental_gids
]
# Only clutter pod spec with actual content
if not all([e is None for e in pod_security_context.to_dict().values()]):
pod.spec.security_context = pod_security_context
container_security_context = V1SecurityContext()
if run_as_uid is not None:
container_security_context.run_as_user = int(run_as_uid)
if run_as_gid is not None:
container_security_context.run_as_group = int(run_as_gid)
if run_privileged:
container_security_context.privileged = True
if not allow_privilege_escalation:
container_security_context.allow_privilege_escalation = False
# Only clutter container spec with actual content
if all([e is None for e in container_security_context.to_dict().values()]):
container_security_context = None
# Transform a dict into valid Kubernetes EnvVar Python representations. This
# representation shall always have a "name" field as well as either a
# "value" field or "value_from" field. For examples see the
# test_make_pod_with_env function.
prepared_env = []
for k, v in (env or {}).items():
if type(v) == dict:
if not "name" in v:
v["name"] = k
prepared_env.append(get_k8s_model(V1EnvVar, v))
else:
prepared_env.append(V1EnvVar(name=k, value=v))
notebook_container = V1Container(
name='notebook',
image=image,
working_dir=working_dir,
ports=[V1ContainerPort(name='notebook-port', container_port=port)],
env=prepared_env,
args=cmd,
image_pull_policy=image_pull_policy,
lifecycle=lifecycle_hooks,
resources=V1ResourceRequirements(),
volume_mounts=[
get_k8s_model(V1VolumeMount, obj) for obj in (volume_mounts or [])
],
security_context=container_security_context,
)
if service_account is None:
# This makes sure that we don't accidentally give access to the whole
# kubernetes API to the users in the spawned pods.
pod.spec.automount_service_account_token = False
else:
pod.spec.service_account_name = service_account
notebook_container.resources.requests = {}
if cpu_guarantee:
notebook_container.resources.requests['cpu'] = cpu_guarantee
if mem_guarantee:
notebook_container.resources.requests['memory'] = mem_guarantee
if extra_resource_guarantees:
notebook_container.resources.requests.update(extra_resource_guarantees)
notebook_container.resources.limits = {}
if cpu_limit:
notebook_container.resources.limits['cpu'] = cpu_limit
if mem_limit:
notebook_container.resources.limits['memory'] = mem_limit
if extra_resource_limits:
notebook_container.resources.limits.update(extra_resource_limits)
if extra_container_config:
notebook_container = update_k8s_model(
target=notebook_container,
changes=extra_container_config,
logger=logger,
target_name="notebook_container",
changes_name="extra_container_config",
)
pod.spec.containers.append(notebook_container)
if extra_containers:
pod.spec.containers.extend(
[get_k8s_model(V1Container, obj) for obj in extra_containers])
if tolerations:
pod.spec.tolerations = [
get_k8s_model(V1Toleration, obj) for obj in tolerations
]
if init_containers:
pod.spec.init_containers = [
get_k8s_model(V1Container, obj) for obj in init_containers
]
if volumes:
pod.spec.volumes = [get_k8s_model(V1Volume, obj) for obj in volumes]
else:
# Keep behaving exactly like before by not cleaning up generated pod
# spec by setting the volumes field even though it is an empty list.
pod.spec.volumes = []
if scheduler_name:
pod.spec.scheduler_name = scheduler_name
node_affinity = None
if node_affinity_preferred or node_affinity_required:
node_selector = None
if node_affinity_required:
node_selector = V1NodeSelector(node_selector_terms=[
get_k8s_model(V1NodeSelectorTerm, obj)
for obj in node_affinity_required
], )
preferred_scheduling_terms = None
if node_affinity_preferred:
preferred_scheduling_terms = [
get_k8s_model(V1PreferredSchedulingTerm, obj)
for obj in node_affinity_preferred
]
node_affinity = V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=
preferred_scheduling_terms,
required_during_scheduling_ignored_during_execution=node_selector,
)
pod_affinity = None
if pod_affinity_preferred or pod_affinity_required:
weighted_pod_affinity_terms = None
if pod_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_affinity_preferred
]
pod_affinity_terms = None
if pod_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_affinity_required
]
pod_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=
weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=
pod_affinity_terms,
)
pod_anti_affinity = None
if pod_anti_affinity_preferred or pod_anti_affinity_required:
weighted_pod_affinity_terms = None
if pod_anti_affinity_preferred:
weighted_pod_affinity_terms = [
get_k8s_model(V1WeightedPodAffinityTerm, obj)
for obj in pod_anti_affinity_preferred
]
pod_affinity_terms = None
if pod_anti_affinity_required:
pod_affinity_terms = [
get_k8s_model(V1PodAffinityTerm, obj)
for obj in pod_anti_affinity_required
]
pod_anti_affinity = V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=
weighted_pod_affinity_terms,
required_during_scheduling_ignored_during_execution=
pod_affinity_terms,
)
affinity = None
if (node_affinity or pod_affinity or pod_anti_affinity):
affinity = V1Affinity(
node_affinity=node_affinity,
pod_affinity=pod_affinity,
pod_anti_affinity=pod_anti_affinity,
)
if affinity:
pod.spec.affinity = affinity
if priority_class_name:
pod.spec.priority_class_name = priority_class_name
if extra_pod_config:
pod.spec = update_k8s_model(
target=pod.spec,
changes=extra_pod_config,
logger=logger,
target_name="pod.spec",
changes_name="extra_pod_config",
)
return pod
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from kubernetes.client.models import V1ObjectMeta, V1Pod, V1PodStatus
from util.k8s.k8s_info import PodStatus
from util.k8s.k8s_statistics import get_highest_usage
CPU_USER_NAME = "cpu_user_name"
MEM_USER_NAME = "mem_user_name"
PODS = [
V1Pod(metadata=V1ObjectMeta(name="cpu_first_pod", namespace=CPU_USER_NAME),
status=V1PodStatus(phase=PodStatus.RUNNING.value)),
V1Pod(metadata=V1ObjectMeta(name="mem_first_pod", namespace=MEM_USER_NAME),
status=V1PodStatus(phase=PodStatus.RUNNING.value)),
V1Pod(metadata=V1ObjectMeta(name="cpu_second_pod",
namespace=CPU_USER_NAME),
status=V1PodStatus(phase=PodStatus.RUNNING.value)),
V1Pod(metadata=V1ObjectMeta(name="mem_second_pod",
namespace=MEM_USER_NAME),
status=V1PodStatus(phase=PodStatus.RUNNING.value)),
V1Pod(metadata=V1ObjectMeta(name="tech_pod", namespace="kube-system"),
status=V1PodStatus(phase=PodStatus.RUNNING.value))
]
TOP_RESULTS = [("3m", "200Ki"), ("2m", "400Ki"), ("3m", "200Ki"),
("2m", "400Ki")]
def pod(name):
return V1Pod(metadata=V1ObjectMeta(name=name))
def make_task(operator: str, task_params: Dict[str, Any]) -> BaseOperator:
"""
Takes an operator and params and creates an instance of that operator.
:returns: instance of operator object
"""
try:
# class is a Callable https://stackoverflow.com/a/34578836/3679900
operator_obj: Callable[..., BaseOperator] = import_string(operator)
except Exception as err:
raise f"Failed to import operator: {operator}" from err
try:
if operator_obj == PythonOperator:
if not task_params.get(
"python_callable_name") and not task_params.get(
"python_callable_file"):
raise Exception(
"Failed to create task. PythonOperator requires `python_callable_name` \
and `python_callable_file` parameters.")
task_params[
"python_callable"]: Callable = utils.get_python_callable(
task_params["python_callable_name"],
task_params["python_callable_file"],
)
# KubernetesPodOperator
if operator_obj == KubernetesPodOperator:
task_params["secrets"] = ([
Secret(**v) for v in task_params.get("secrets")
] if task_params.get("secrets") is not None else None)
task_params["ports"] = ([
Port(**v) for v in task_params.get("ports")
] if task_params.get("ports") is not None else None)
task_params["volume_mounts"] = ([
VolumeMount(**v) for v in task_params.get("volume_mounts")
] if task_params.get("volume_mounts") is not None else None)
task_params["volumes"] = ([
Volume(**v) for v in task_params.get("volumes")
] if task_params.get("volumes") is not None else None)
task_params["pod_runtime_info_envs"] = ([
PodRuntimeInfoEnv(**v)
for v in task_params.get("pod_runtime_info_envs")
] if task_params.get("pod_runtime_info_envs") is not None else
None)
task_params["full_pod_spec"] = (
V1Pod(**task_params.get("full_pod_spec"))
if task_params.get("full_pod_spec") is not None else None)
task_params["init_containers"] = ([
V1Container(**v)
for v in task_params.get("init_containers")
] if task_params.get("init_containers") is not None else None)
if utils.check_dict_key(task_params, "execution_timeout_secs"):
task_params["execution_timeout"]: timedelta = timedelta(
seconds=task_params["execution_timeout_secs"])
del task_params["execution_timeout_secs"]
# use variables as arguments on operator
if utils.check_dict_key(task_params, "variables_as_arguments"):
variables: List[Dict[str, str]] = task_params.get(
"variables_as_arguments")
for variable in variables:
if Variable.get(variable["variable"],
default_var=None) is not None:
task_params[variable["attribute"]] = Variable.get(
variable["variable"], default_var=None)
del task_params["variables_as_arguments"]
task: BaseOperator = operator_obj(**task_params)
except Exception as err:
raise f"Failed to create {operator_obj} task" from err
return task
def make_task(operator: str, task_params: Dict[str, Any],
af_vars: Dict[str, Any]) -> BaseOperator:
"""
Takes an operator and params and creates an instance of that operator.
:returns: instance of operator object
"""
try:
# class is a Callable https://stackoverflow.com/a/34578836/3679900
operator_obj: Callable[..., BaseOperator] = import_string(operator)
except Exception as err:
raise Exception(f"Failed to import operator: {operator}") from err
try:
if operator_obj in [
PythonOperator, BranchPythonOperator, PythonSensor
]:
if (not task_params.get("python_callable")
and not task_params.get("python_callable_name")
and not task_params.get("python_callable_file")):
# pylint: disable=line-too-long
raise Exception(
"Failed to create task. PythonOperator, BranchPythonOperator and PythonSensor requires \
`python_callable_name` and `python_callable_file` "
"parameters.\nOptionally you can load python_callable "
"from a file. with the special pyyaml notation:\n"
" python_callable_file: !!python/name:my_module.my_func"
)
if not task_params.get("python_callable"):
task_params[
"python_callable"]: Callable = utils.get_python_callable(
task_params["python_callable_name"],
task_params["python_callable_file"],
)
# remove dag-factory specific parameters
# Airflow 2.0 doesn't allow these to be passed to operator
del task_params["python_callable_name"]
del task_params["python_callable_file"]
# Check for the custom success and failure callables in SqlSensor. These are considered
# optional, so no failures in case they aren't found. Note: there's no reason to
# declare both a callable file and a lambda function for success/failure parameter.
# If both are found the object will not throw and error, instead callable file will
# take precedence over the lambda function
if operator_obj in [SqlSensor]:
# Success checks
if task_params.get("success_check_file") and task_params.get(
"success_check_name"):
task_params[
"success"]: Callable = utils.get_python_callable(
task_params["success_check_name"],
task_params["success_check_file"],
)
del task_params["success_check_name"]
del task_params["success_check_file"]
elif task_params.get("success_check_lambda"):
task_params[
"success"]: Callable = utils.get_python_callable_lambda(
task_params["success_check_lambda"])
del task_params["success_check_lambda"]
# Failure checks
if task_params.get("failure_check_file") and task_params.get(
"failure_check_name"):
task_params[
"failure"]: Callable = utils.get_python_callable(
task_params["failure_check_name"],
task_params["failure_check_file"],
)
del task_params["failure_check_name"]
del task_params["failure_check_file"]
elif task_params.get("failure_check_lambda"):
task_params[
"failure"]: Callable = utils.get_python_callable_lambda(
task_params["failure_check_lambda"])
del task_params["failure_check_lambda"]
if operator_obj in [HttpSensor]:
if not (task_params.get("response_check_name")
and task_params.get("response_check_file")
) and not task_params.get("response_check_lambda"):
raise Exception(
"Failed to create task. HttpSensor requires \
`response_check_name` and `response_check_file` parameters \
or `response_check_lambda` parameter.")
if task_params.get("response_check_file"):
task_params[
"response_check"]: Callable = utils.get_python_callable(
task_params["response_check_name"],
task_params["response_check_file"],
)
# remove dag-factory specific parameters
# Airflow 2.0 doesn't allow these to be passed to operator
del task_params["response_check_name"]
del task_params["response_check_file"]
else:
task_params[
"response_check"]: Callable = utils.get_python_callable_lambda(
task_params["response_check_lambda"])
# remove dag-factory specific parameters
# Airflow 2.0 doesn't allow these to be passed to operator
del task_params["response_check_lambda"]
# KubernetesPodOperator
if operator_obj == KubernetesPodOperator:
task_params["secrets"] = ([
Secret(**v) for v in task_params.get("secrets")
] if task_params.get("secrets") is not None else None)
task_params["ports"] = ([
Port(**v) for v in task_params.get("ports")
] if task_params.get("ports") is not None else None)
task_params["volume_mounts"] = ([
VolumeMount(**v) for v in task_params.get("volume_mounts")
] if task_params.get("volume_mounts") is not None else None)
task_params["volumes"] = ([
Volume(**v) for v in task_params.get("volumes")
] if task_params.get("volumes") is not None else None)
task_params["pod_runtime_info_envs"] = ([
PodRuntimeInfoEnv(**v)
for v in task_params.get("pod_runtime_info_envs")
] if task_params.get("pod_runtime_info_envs") is not None else
None)
task_params["full_pod_spec"] = (
V1Pod(**task_params.get("full_pod_spec"))
if task_params.get("full_pod_spec") is not None else None)
task_params["init_containers"] = ([
V1Container(**v)
for v in task_params.get("init_containers")
] if task_params.get("init_containers") is not None else None)
if operator_obj == DockerOperator:
if task_params.get("environment") is not None:
task_params["environment"] = {
k: os.environ.get(v, v)
for k, v in task_params["environment"].items()
}
if operator_obj == EcsOperator:
for c in task_params["overrides"]["containerOverrides"]:
if c.get('environment') is not None:
for env in c['environment']:
env['value'] = os.environ.get(
env['value'], env['value'])
if 'ECS_SECURITY_GROUPS' in af_vars and 'network_configuration' in task_params:
task_params["network_configuration"]["awsvpcConfiguration"]['securityGroups'] \
= af_vars['ECS_SECURITY_GROUPS']
if 'ECS_SUBNETS' in af_vars and 'network_configuration' in task_params:
task_params['network_configuration'][
"awsvpcConfiguration"]["subnets"] = af_vars[
"ECS_SUBNETS"]
if af_vars.get('ECS_CLUSTER'):
task_params['cluster'] = af_vars["ECS_CLUSTER"]
task_params['task_definition'] = (
af_vars.get('ECS_CLUSTER') + '_' +
task_params['task_definition']).lower()
task_params['awslogs_group'] = \
task_params['awslogs_group'] + '/' + af_vars.get('ECS_CLUSTER').lower()
if utils.check_dict_key(task_params, "execution_timeout_secs"):
task_params["execution_timeout"]: timedelta = timedelta(
seconds=task_params["execution_timeout_secs"])
del task_params["execution_timeout_secs"]
if utils.check_dict_key(task_params, "sla_secs"):
task_params["sla"]: timedelta = timedelta(
seconds=task_params["sla_secs"])
del task_params["sla_secs"]
if utils.check_dict_key(task_params, "execution_delta_secs"):
task_params["execution_delta"]: timedelta = timedelta(
seconds=task_params["execution_delta_secs"])
del task_params["execution_delta_secs"]
if utils.check_dict_key(
task_params,
"execution_date_fn_name") and utils.check_dict_key(
task_params, "execution_date_fn_file"):
task_params[
"execution_date_fn"]: Callable = utils.get_python_callable(
task_params["execution_date_fn_name"],
task_params["execution_date_fn_file"],
)
del task_params["execution_date_fn_name"]
del task_params["execution_date_fn_file"]
# on_execute_callback is an Airflow 2.0 feature
if utils.check_dict_key(
task_params, "on_execute_callback"
) and version.parse(AIRFLOW_VERSION) >= version.parse("2.0.0"):
task_params["on_execute_callback"]: Callable = import_string(
task_params["on_execute_callback"])
if utils.check_dict_key(task_params, "on_failure_callback"):
task_params["on_failure_callback"]: Callable = import_string(
task_params["on_failure_callback"])
if utils.check_dict_key(task_params, "on_success_callback"):
task_params["on_success_callback"]: Callable = import_string(
task_params["on_success_callback"])
if utils.check_dict_key(task_params, "on_retry_callback"):
task_params["on_retry_callback"]: Callable = import_string(
task_params["on_retry_callback"])
# use variables as arguments on operator
if utils.check_dict_key(task_params, "variables_as_arguments"):
variables: List[Dict[str, str]] = task_params.get(
"variables_as_arguments")
for variable in variables:
if Variable.get(variable["variable"],
default_var=None) is not None:
task_params[variable["attribute"]] = Variable.get(
variable["variable"], default_var=None)
del task_params["variables_as_arguments"]
# use variables as arguments on operator
if utils.check_dict_key(task_params, "af_vars_as_arguments"):
variables: List[Dict[str, str]] = task_params.get(
"af_vars_as_arguments")
for variable in variables:
if af_vars.get(variable["variable"], None) is not None:
task_params[variable["attribute"]] = af_vars.get(
variable["variable"], None)
del task_params["af_vars_as_arguments"]
task: BaseOperator = operator_obj(**task_params)
except Exception as err:
raise Exception(f"Failed to create {operator_obj} task") from err
return task
