def test_make_pod_git_sync_ssh_without_known_hosts(self):
# Tests the pod created with git-sync SSH authentication option is correct without known hosts
self.kube_config.airflow_configmap = 'airflow-configmap'
self.kube_config.git_ssh_key_secret_name = 'airflow-secrets'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
init_containers = worker_config._get_init_containers()
git_ssh_key_file = next((x.value for x in init_containers[0].env
if x.name == 'GIT_SSH_KEY_FILE'), None)
volume_mount_ssh_key = next((x.mount_path for x in init_containers[0].volume_mounts
if x.name == worker_config.git_sync_ssh_secret_volume_name),
None)
self.assertTrue(git_ssh_key_file)
self.assertTrue(volume_mount_ssh_key)
self.assertEqual(65533, pod.spec.security_context.fs_group)
self.assertEqual(git_ssh_key_file,
volume_mount_ssh_key,
'The location where the git ssh secret is mounted'
' needs to be the same as the GIT_SSH_KEY_FILE path')
def test_make_pod_assert_labels(self):
# Tests the pod created has all the expected labels set
self.kube_config.dags_folder = 'dags'
worker_config = WorkerConfiguration(self.kube_config)
execution_date = parser.parse('2019-11-21 11:08:22.920875')
pod = PodGenerator.construct_pod(
"test_dag_id",
"test_task_id",
"test_pod_id",
1,
execution_date,
["bash -c 'ls /'"],
None,
worker_config.as_pod(),
"default",
"sample-uuid",
)
expected_labels = {
'airflow-worker': 'sample-uuid',
'airflow_version': airflow_version.replace('+', '-'),
'dag_id': 'test_dag_id',
'execution_date': datetime_to_label_safe_datestring(execution_date),
'kubernetes_executor': 'True',
'my_label': 'label_id',
'task_id': 'test_task_id',
'try_number': '1'
}
self.assertEqual(pod.metadata.labels, expected_labels)
def test_pod_template_file(self):
fixture = sys.path[0] + '/tests/kubernetes/pod.yaml'
self.kube_config.pod_template_file = fixture
worker_config = WorkerConfiguration(self.kube_config)
result = worker_config.as_pod()
expected = PodGenerator.deserialize_model_file(fixture)
expected.metadata.name = ANY
self.assertEqual(expected, result)
def test_set_airflow_local_settings_configmap(self):
"""
Test that airflow_local_settings.py can be set via configmap by
checking volume & volume-mounts are set correctly.
"""
self.kube_config.airflow_home = '/usr/local/airflow'
self.kube_config.airflow_configmap = 'airflow-configmap'
self.kube_config.airflow_local_settings_configmap = 'airflow-configmap'
self.kube_config.dags_folder = '/workers/path/to/dags'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
pod_spec_dict = pod.spec.to_dict()
airflow_config_volume = [
volume for volume in pod_spec_dict['volumes'] if volume["name"] == 'airflow-config'
]
# Test that volume_name is found
self.assertEqual(1, len(airflow_config_volume))
# Test that config map exists
self.assertEqual(
{'default_mode': None, 'items': None, 'name': 'airflow-configmap', 'optional': None},
airflow_config_volume[0]['config_map']
)
# Test that 2 Volume Mounts exists and has 2 different mount-paths
# One for airflow.cfg
# Second for airflow_local_settings.py
volume_mounts = [
volume_mount for volume_mount in pod_spec_dict['containers'][0]['volume_mounts']
if volume_mount['name'] == 'airflow-config'
]
self.assertEqual(2, len(volume_mounts))
self.assertEqual(
[
{
'mount_path': '/usr/local/airflow/airflow.cfg',
'mount_propagation': None,
'name': 'airflow-config',
'read_only': True,
'sub_path': 'airflow.cfg',
'sub_path_expr': None
},
{
'mount_path': '/usr/local/airflow/config/airflow_local_settings.py',
'mount_propagation': None,
'name': 'airflow-config',
'read_only': True,
'sub_path': 'airflow_local_settings.py',
'sub_path_expr': None
}
],
volume_mounts
)
def test_make_pod_with_image_pull_secrets(self):
# Tests the pod created with image_pull_secrets actually gets that in it's config
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
self.kube_config.image_pull_secrets = 'image_pull_secret1,image_pull_secret2'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
self.assertEqual(2, len(pod.spec.image_pull_secrets))
def test_make_pod_run_as_user_0(self):
# Tests the pod created with run-as-user 0 actually gets that in it's config
self.kube_config.worker_run_as_user = 0
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
self.assertEqual(0, pod.spec.security_context.run_as_user)
def test_make_pod_with_empty_executor_config(self):
self.kube_config.kube_affinity = self.affinity_config
self.kube_config.kube_tolerations = self.tolerations_config
self.kube_config.kube_annotations = self.worker_annotations_config
self.kube_config.dags_folder = 'dags'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
self.assertTrue(pod.spec.affinity['podAntiAffinity'] is not None)
self.assertEqual(
'app', pod.spec.affinity['podAntiAffinity']
['requiredDuringSchedulingIgnoredDuringExecution'][0]
['labelSelector']['matchExpressions'][0]['key'])
self.assertEqual(2, len(pod.spec.tolerations))
self.assertEqual('prod', pod.spec.tolerations[1]['key'])
self.assertEqual('role-arn',
pod.metadata.annotations['iam.amazonaws.com/role'])
self.assertEqual('value', pod.metadata.annotations['other/annotation'])
def test_make_pod_with_executor_config(self):
self.kube_config.dags_folder = 'dags'
worker_config = WorkerConfiguration(self.kube_config)
config_pod = PodGenerator(
image='',
affinity=self.affinity_config,
tolerations=self.tolerations_config,
).gen_pod()
pod = worker_config.as_pod()
result = PodGenerator.reconcile_pods(pod, config_pod)
self.assertTrue(result.spec.affinity['podAntiAffinity'] is not None)
self.assertEqual(
'app', result.spec.affinity['podAntiAffinity']
['requiredDuringSchedulingIgnoredDuringExecution'][0]
['labelSelector']['matchExpressions'][0]['key'])
self.assertEqual(2, len(result.spec.tolerations))
self.assertEqual('prod', result.spec.tolerations[1]['key'])
def test_make_pod_git_sync_rev(self):
# Tests the pod created with git_sync_credentials_secret will get into the init container
self.kube_config.git_sync_rev = 'sampletag'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
rev_env = k8s.V1EnvVar(
name='GIT_SYNC_REV',
value=self.kube_config.git_sync_rev,
)
self.assertIn(rev_env, pod.spec.init_containers[0].env,
'The git_sync_rev env did not get into the init container')
def test_make_pod_git_sync_credentials_secret(self):
# Tests the pod created with git_sync_credentials_secret will get into the init container
self.kube_config.git_sync_credentials_secret = 'airflow-git-creds-secret'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
self.kube_config.base_log_folder = '/logs'
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.as_pod()
username_env = k8s.V1EnvVar(
name='GIT_SYNC_USERNAME',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.kube_config.git_sync_credentials_secret,
key='GIT_SYNC_USERNAME')))
password_env = k8s.V1EnvVar(
name='GIT_SYNC_PASSWORD',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.kube_config.git_sync_credentials_secret,
key='GIT_SYNC_PASSWORD')))
self.assertIn(
username_env, pod.spec.init_containers[0].env,
'The username env for git credentials did not get into the init container'
)
self.assertIn(
password_env, pod.spec.init_containers[0].env,
'The password env for git credentials did not get into the init container'
)
