예제 #1
0
    def test_make_pod_git_sync_ssh_without_known_hosts(self):
        # Tests the pod created with git-sync SSH authentication option is correct without known hosts
        self.kube_config.airflow_configmap = 'airflow-configmap'
        self.kube_config.git_ssh_key_secret_name = 'airflow-secrets'
        self.kube_config.dags_volume_claim = None
        self.kube_config.dags_volume_host = None
        self.kube_config.dags_in_image = None
        self.kube_config.worker_fs_group = None

        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(annotations=[],
                                                        volumes=[],
                                                        volume_mounts=[])

        pod = worker_config.make_pod("default", str(uuid.uuid4()),
                                     "test_pod_id", "test_dag_id",
                                     "test_task_id", str(datetime.utcnow()), 1,
                                     "bash -c 'ls /'", kube_executor_config)

        init_containers = worker_config._get_init_containers()
        git_ssh_key_file = next((x['value'] for x in init_containers[0]['env']
                                 if x['name'] == 'GIT_SSH_KEY_FILE'), None)
        volume_mount_ssh_key = next(
            (x['mountPath'] for x in init_containers[0]['volumeMounts']
             if x['name'] == worker_config.git_sync_ssh_secret_volume_name),
            None)
        self.assertTrue(git_ssh_key_file)
        self.assertTrue(volume_mount_ssh_key)
        self.assertEqual(65533, pod.security_context['fsGroup'])
        self.assertEqual(
            git_ssh_key_file, volume_mount_ssh_key,
            'The location where the git ssh secret is mounted'
            ' needs to be the same as the GIT_SSH_KEY_FILE path')
    def test_make_pod_git_sync_rev(self):
        # Tests the pod created with git_sync_credentials_secret will get into the init container
        self.kube_config.git_sync_rev = 'sampletag'
        self.kube_config.dags_volume_claim = None
        self.kube_config.dags_volume_host = None
        self.kube_config.dags_in_image = None
        self.kube_config.worker_fs_group = None
        self.kube_config.git_dags_folder_mount_point = 'dags'
        self.kube_config.git_sync_dest = 'repo'
        self.kube_config.git_subpath = 'path'

        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(annotations=[],
                                                        volumes=[],
                                                        volume_mounts=[])

        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
                                     kube_executor_config)

        rev_env = {
            'name': 'GIT_SYNC_REV',
            'value': self.kube_config.git_sync_rev
        }

        self.assertIn(rev_env, pod.init_containers[0]["env"],
                      'The git_sync_rev env did not get into the init container')
    def test_set_airflow_local_settings_configmap(self):
        """
        Test that airflow_local_settings.py can be set via configmap by
        checking volume & volume-mounts are set correctly.
        """
        self.kube_config.airflow_home = '/usr/local/airflow'
        self.kube_config.airflow_configmap = 'airflow-configmap'
        self.kube_config.airflow_local_settings_configmap = 'airflow-configmap'
        self.kube_config.dags_folder = '/workers/path/to/dags'

        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(annotations=[],
                                                        volumes=[],
                                                        volume_mounts=[])

        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
                                     kube_executor_config)

        airflow_config_volume = [
            volume for volume in pod.volumes if volume["name"] == 'airflow-config'
        ]
        # Test that volume_name is found
        self.assertEqual(1, len(airflow_config_volume))

        # Test that config map exists
        self.assertEqual(
            {'configMap': {'name': 'airflow-configmap'}, 'name': 'airflow-config'},
            airflow_config_volume[0]
        )

        # Test that 2 Volume Mounts exists and has 2 different mount-paths
        # One for airflow.cfg
        # Second for airflow_local_settings.py
        volume_mounts = [
            volume_mount for volume_mount in pod.volume_mounts
            if volume_mount['name'] == 'airflow-config'
        ]
        self.assertEqual(2, len(volume_mounts))

        six.assertCountEqual(
            self,
            [
                {
                    'mountPath': '/usr/local/airflow/airflow.cfg',
                    'name': 'airflow-config',
                    'readOnly': True,
                    'subPath': 'airflow.cfg',
                },
                {
                    'mountPath': '/usr/local/airflow/config/airflow_local_settings.py',
                    'name': 'airflow-config',
                    'readOnly': True,
                    'subPath': 'airflow_local_settings.py',
                }
            ],
            volume_mounts
        )
    def test_make_pod_git_sync_credentials_secret(self):
        # Tests the pod created with git_sync_credentials_secret will get into the init container
        self.kube_config.git_sync_credentials_secret = 'airflow-git-creds-secret'
        self.kube_config.dags_volume_claim = None
        self.kube_config.dags_volume_host = None
        self.kube_config.dags_in_image = None
        self.kube_config.worker_fs_group = None

        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(annotations=[],
                                                        volumes=[],
                                                        volume_mounts=[])

        pod = worker_config.make_pod("default", str(uuid.uuid4()),
                                     "test_pod_id", "test_dag_id",
                                     "test_task_id", str(datetime.utcnow()), 1,
                                     "bash -c 'ls /'", kube_executor_config)

        username_env = {
            'name': 'GIT_SYNC_USERNAME',
            'valueFrom': {
                'secretKeyRef': {
                    'name': self.kube_config.git_sync_credentials_secret,
                    'key': 'GIT_SYNC_USERNAME'
                }
            }
        }
        password_env = {
            'name': 'GIT_SYNC_PASSWORD',
            'valueFrom': {
                'secretKeyRef': {
                    'name': self.kube_config.git_sync_credentials_secret,
                    'key': 'GIT_SYNC_PASSWORD'
                }
            }
        }

        self.assertIn(
            username_env, pod.init_containers[0]["env"],
            'The username env for git credentials did not get into the init container'
        )

        self.assertIn(
            password_env, pod.init_containers[0]["env"],
            'The password env for git credentials did not get into the init container'
        )
    def test_make_pod_run_as_user_0(self):
        # Tests the pod created with run-as-user 0 actually gets that in it's config
        self.kube_config.worker_run_as_user = 0
        self.kube_config.dags_volume_claim = None
        self.kube_config.dags_volume_host = None
        self.kube_config.dags_in_image = None
        self.kube_config.worker_fs_group = None

        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(annotations=[],
                                                        volumes=[],
                                                        volume_mounts=[])

        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
                                     kube_executor_config)

        self.assertEqual(0, pod.security_context['runAsUser'])
예제 #6
0
    def test_make_pod_with_executor_config(self):
        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(
            affinity=self.affinity_config,
            tolerations=self.tolerations_config,
            annotations=[],
            volumes=[],
            volume_mounts=[])

        pod = worker_config.make_pod("default", str(uuid.uuid4()),
                                     "test_pod_id", "test_dag_id",
                                     "test_task_id", str(datetime.utcnow()), 1,
                                     "bash -c 'ls /'", kube_executor_config)

        self.assertTrue(pod.affinity['podAntiAffinity'] is not None)
        self.assertEqual(
            'app', pod.affinity['podAntiAffinity']
            ['requiredDuringSchedulingIgnoredDuringExecution'][0]
            ['labelSelector']['matchExpressions'][0]['key'])

        self.assertEqual(2, len(pod.tolerations))
        self.assertEqual('prod', pod.tolerations[1]['key'])
    def test_make_pod_assert_labels(self):
        # Tests the pod created has all the expected labels set
        self.kube_config.dags_folder = 'dags'

        worker_config = WorkerConfiguration(self.kube_config)
        kube_executor_config = KubernetesExecutorConfig(annotations=[],
                                                        volumes=[],

                                                        volume_mounts=[])
        pod = worker_config.make_pod("default", "sample-uuid", "test_pod_id", "test_dag_id",
                                     "test_task_id", "2019-11-21 11:08:22.920875", 1, "bash -c 'ls /'",
                                     kube_executor_config)
        expected_labels = {
            'airflow-worker': 'sample-uuid',
            'airflow_version': airflow_version.replace('+', '-'),
            'dag_id': 'test_dag_id',
            'execution_date': '2019-11-21 11:08:22.920875',
            'kubernetes_executor': 'True',
            'my_label': 'label_id',
            'task_id': 'test_task_id',
            'try_number': '1'
        }
        self.assertEqual(pod.labels, expected_labels)