def test_make_pod_git_sync_ssh_without_known_hosts(self):
# Tests the pod created with git-sync SSH authentication option is correct without known hosts
self.kube_config.airflow_configmap = 'airflow-configmap'
self.kube_config.git_ssh_key_secret_name = 'airflow-secrets'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", str(uuid.uuid4()),
"test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1,
"bash -c 'ls /'", kube_executor_config)
init_containers = worker_config._get_init_containers()
git_ssh_key_file = next((x['value'] for x in init_containers[0]['env']
if x['name'] == 'GIT_SSH_KEY_FILE'), None)
volume_mount_ssh_key = next(
(x['mountPath'] for x in init_containers[0]['volumeMounts']
if x['name'] == worker_config.git_sync_ssh_secret_volume_name),
None)
self.assertTrue(git_ssh_key_file)
self.assertTrue(volume_mount_ssh_key)
self.assertEqual(65533, pod.security_context['fsGroup'])
self.assertEqual(
git_ssh_key_file, volume_mount_ssh_key,
'The location where the git ssh secret is mounted'
' needs to be the same as the GIT_SSH_KEY_FILE path')
def test_make_pod_git_sync_rev(self):
# Tests the pod created with git_sync_credentials_secret will get into the init container
self.kube_config.git_sync_rev = 'sampletag'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
kube_executor_config)
rev_env = {
'name': 'GIT_SYNC_REV',
'value': self.kube_config.git_sync_rev
}
self.assertIn(rev_env, pod.init_containers[0]["env"],
'The git_sync_rev env did not get into the init container')
def test_set_airflow_local_settings_configmap(self):
"""
Test that airflow_local_settings.py can be set via configmap by
checking volume & volume-mounts are set correctly.
"""
self.kube_config.airflow_home = '/usr/local/airflow'
self.kube_config.airflow_configmap = 'airflow-configmap'
self.kube_config.airflow_local_settings_configmap = 'airflow-configmap'
self.kube_config.dags_folder = '/workers/path/to/dags'
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
kube_executor_config)
airflow_config_volume = [
volume for volume in pod.volumes if volume["name"] == 'airflow-config'
]
# Test that volume_name is found
self.assertEqual(1, len(airflow_config_volume))
# Test that config map exists
self.assertEqual(
{'configMap': {'name': 'airflow-configmap'}, 'name': 'airflow-config'},
airflow_config_volume[0]
)
# Test that 2 Volume Mounts exists and has 2 different mount-paths
# One for airflow.cfg
# Second for airflow_local_settings.py
volume_mounts = [
volume_mount for volume_mount in pod.volume_mounts
if volume_mount['name'] == 'airflow-config'
]
self.assertEqual(2, len(volume_mounts))
six.assertCountEqual(
self,
[
{
'mountPath': '/usr/local/airflow/airflow.cfg',
'name': 'airflow-config',
'readOnly': True,
'subPath': 'airflow.cfg',
},
{
'mountPath': '/usr/local/airflow/config/airflow_local_settings.py',
'name': 'airflow-config',
'readOnly': True,
'subPath': 'airflow_local_settings.py',
}
],
volume_mounts
)
def test_make_pod_git_sync_credentials_secret(self):
# Tests the pod created with git_sync_credentials_secret will get into the init container
self.kube_config.git_sync_credentials_secret = 'airflow-git-creds-secret'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", str(uuid.uuid4()),
"test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1,
"bash -c 'ls /'", kube_executor_config)
username_env = {
'name': 'GIT_SYNC_USERNAME',
'valueFrom': {
'secretKeyRef': {
'name': self.kube_config.git_sync_credentials_secret,
'key': 'GIT_SYNC_USERNAME'
}
}
}
password_env = {
'name': 'GIT_SYNC_PASSWORD',
'valueFrom': {
'secretKeyRef': {
'name': self.kube_config.git_sync_credentials_secret,
'key': 'GIT_SYNC_PASSWORD'
}
}
}
self.assertIn(
username_env, pod.init_containers[0]["env"],
'The username env for git credentials did not get into the init container'
)
self.assertIn(
password_env, pod.init_containers[0]["env"],
'The password env for git credentials did not get into the init container'
)
def test_make_pod_run_as_user_0(self):
# Tests the pod created with run-as-user 0 actually gets that in it's config
self.kube_config.worker_run_as_user = 0
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
kube_executor_config)
self.assertEqual(0, pod.security_context['runAsUser'])
def test_make_pod_with_executor_config(self):
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(
affinity=self.affinity_config,
tolerations=self.tolerations_config,
annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", str(uuid.uuid4()),
"test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1,
"bash -c 'ls /'", kube_executor_config)
self.assertTrue(pod.affinity['podAntiAffinity'] is not None)
self.assertEqual(
'app', pod.affinity['podAntiAffinity']
['requiredDuringSchedulingIgnoredDuringExecution'][0]
['labelSelector']['matchExpressions'][0]['key'])
self.assertEqual(2, len(pod.tolerations))
self.assertEqual('prod', pod.tolerations[1]['key'])
def test_make_pod_assert_labels(self):
# Tests the pod created has all the expected labels set
self.kube_config.dags_folder = 'dags'
worker_config = WorkerConfiguration(self.kube_config)
kube_executor_config = KubernetesExecutorConfig(annotations=[],
volumes=[],
volume_mounts=[])
pod = worker_config.make_pod("default", "sample-uuid", "test_pod_id", "test_dag_id",
"test_task_id", "2019-11-21 11:08:22.920875", 1, "bash -c 'ls /'",
kube_executor_config)
expected_labels = {
'airflow-worker': 'sample-uuid',
'airflow_version': airflow_version.replace('+', '-'),
'dag_id': 'test_dag_id',
'execution_date': '2019-11-21 11:08:22.920875',
'kubernetes_executor': 'True',
'my_label': 'label_id',
'task_id': 'test_task_id',
'try_number': '1'
}
self.assertEqual(pod.labels, expected_labels)