def test_make_pod_git_sync_ssh_with_known_hosts(self): # Tests the pod created with git-sync SSH authentication option is correct with known hosts self.kube_config.airflow_configmap = 'airflow-configmap' self.kube_config.git_ssh_secret_name = 'airflow-secrets' self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None worker_config = WorkerConfiguration(self.kube_config) init_containers = worker_config._get_init_containers() git_ssh_known_hosts_file = next( (x['value'] for x in init_containers[0]['env'] if x['name'] == 'GIT_SSH_KNOWN_HOSTS_FILE'), None) volume_mount_ssh_known_hosts_file = next( (x['mountPath'] for x in init_containers[0]['volumeMounts'] if x['name'] == worker_config.git_sync_ssh_known_hosts_volume_name ), None) self.assertTrue(git_ssh_known_hosts_file) self.assertTrue(volume_mount_ssh_known_hosts_file) self.assertEqual( git_ssh_known_hosts_file, volume_mount_ssh_known_hosts_file, 'The location where the git known hosts file is mounted' ' needs to be the same as the GIT_SSH_KNOWN_HOSTS_FILE path')
def test_init_environment_using_git_sync_ssh_with_known_hosts(self): # Tests the init environment created with git-sync SSH authentication option is correct # with known hosts file self.kube_config.airflow_configmap = 'airflow-configmap' self.kube_config.git_ssh_key_secret_name = 'airflow-secrets' self.kube_config.git_ssh_known_hosts_configmap_name = 'airflow-configmap' self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None worker_config = WorkerConfiguration(self.kube_config) init_containers = worker_config._get_init_containers() self.assertTrue(init_containers) # check not empty env = init_containers[0]['env'] self.assertTrue({ 'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh' } in env) self.assertTrue({'name': 'GIT_KNOWN_HOSTS', 'value': 'true'} in env) self.assertTrue({ 'name': 'GIT_SSH_KNOWN_HOSTS_FILE', 'value': '/etc/git-secret/known_hosts' } in env) self.assertTrue({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
def test_make_pod_git_sync_ssh_without_known_hosts(self): # Tests the pod created with git-sync SSH authentication option is correct without known hosts self.kube_config.airflow_configmap = 'airflow-configmap' self.kube_config.git_ssh_key_secret_name = 'airflow-secrets' self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None self.kube_config.worker_fs_group = None worker_config = WorkerConfiguration(self.kube_config) kube_executor_config = KubernetesExecutorConfig(annotations=[], volumes=[], volume_mounts=[]) pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id", "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'", kube_executor_config) init_containers = worker_config._get_init_containers() git_ssh_key_file = next((x['value'] for x in init_containers[0]['env'] if x['name'] == 'GIT_SSH_KEY_FILE'), None) volume_mount_ssh_key = next( (x['mountPath'] for x in init_containers[0]['volumeMounts'] if x['name'] == worker_config.git_sync_ssh_secret_volume_name), None) self.assertTrue(git_ssh_key_file) self.assertTrue(volume_mount_ssh_key) self.assertEqual(65533, pod.security_context['fsGroup']) self.assertEqual( git_ssh_key_file, volume_mount_ssh_key, 'The location where the git ssh secret is mounted' ' needs to be the same as the GIT_SSH_KEY_FILE path')
def test_worker_git_dags(self): # Tests persistence volume config created when `git_repo` is set self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_folder = '/usr/local/airflow/dags' self.kube_config.worker_dags_folder = '/usr/local/airflow/dags' self.kube_config.git_sync_container_repository = 'gcr.io/google-containers/git-sync-amd64' self.kube_config.git_sync_container_tag = 'v2.0.5' self.kube_config.git_sync_container = 'gcr.io/google-containers/git-sync-amd64:v2.0.5' self.kube_config.git_sync_init_container_name = 'git-sync-clone' self.kube_config.git_subpath = 'dags_folder' self.kube_config.git_sync_root = '/git' self.kube_config.git_sync_run_as_user = 65533 self.kube_config.git_dags_folder_mount_point = '/usr/local/airflow/dags/repo/dags_folder' worker_config = WorkerConfiguration(self.kube_config) volumes, volume_mounts = worker_config._get_volumes_and_mounts() dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags'] dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags'] self.assertTrue('emptyDir' in dag_volume[0]) self.assertEqual(self.kube_config.git_dags_folder_mount_point, dag_volume_mount[0]['mountPath']) self.assertTrue(dag_volume_mount[0]['readOnly']) init_container = worker_config._get_init_containers()[0] init_container_volume_mount = [mount for mount in init_container['volumeMounts'] if mount['name'] == 'airflow-dags'] self.assertEqual('git-sync-clone', init_container['name']) self.assertEqual('gcr.io/google-containers/git-sync-amd64:v2.0.5', init_container['image']) self.assertEqual(1, len(init_container_volume_mount)) self.assertFalse(init_container_volume_mount[0]['readOnly']) self.assertEqual(65533, init_container['securityContext']['runAsUser'])
def test_worker_git_dags(self): # Tests persistence volume config created when `git_repo` is set self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_folder = '/usr/local/airflow/dags' self.kube_config.worker_dags_folder = '/usr/local/airflow/dags' self.kube_config.git_sync_container_repository = 'gcr.io/google-containers/git-sync-amd64' self.kube_config.git_sync_container_tag = 'v2.0.5' self.kube_config.git_sync_container = 'gcr.io/google-containers/git-sync-amd64:v2.0.5' self.kube_config.git_sync_init_container_name = 'git-sync-clone' self.kube_config.git_subpath = 'dags_folder' self.kube_config.git_sync_root = '/git' self.kube_config.git_dags_folder_mount_point = '/usr/local/airflow/dags/repo/dags_folder' worker_config = WorkerConfiguration(self.kube_config) volumes, volume_mounts = worker_config.init_volumes_and_mounts() dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags'] dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags'] self.assertTrue('emptyDir' in dag_volume[0]) self.assertEqual(self.kube_config.git_dags_folder_mount_point, dag_volume_mount[0]['mountPath']) self.assertTrue(dag_volume_mount[0]['readOnly']) init_container = worker_config._get_init_containers(volume_mounts)[0] init_container_volume_mount = [mount for mount in init_container['volumeMounts'] if mount['name'] == 'airflow-dags'] self.assertEqual('git-sync-clone', init_container['name']) self.assertEqual('gcr.io/google-containers/git-sync-amd64:v2.0.5', init_container['image']) self.assertEqual(1, len(init_container_volume_mount)) self.assertFalse(init_container_volume_mount[0]['readOnly'])
def test_make_pod_git_sync_ssh_without_known_hosts(self): # Tests the pod created with git-sync SSH authentication option is correct without known hosts self.kube_config.airflow_configmap = 'airflow-configmap' self.kube_config.git_ssh_key_secret_name = 'airflow-secrets' self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None self.kube_config.worker_fs_group = None worker_config = WorkerConfiguration(self.kube_config) kube_executor_config = KubernetesExecutorConfig(annotations=[], volumes=[], volume_mounts=[]) pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id", "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'", kube_executor_config) init_containers = worker_config._get_init_containers() git_ssh_key_file = next((x['value'] for x in init_containers[0]['env'] if x['name'] == 'GIT_SSH_KEY_FILE'), None) volume_mount_ssh_key = next((x['mountPath'] for x in init_containers[0]['volumeMounts'] if x['name'] == worker_config.git_sync_ssh_secret_volume_name), None) self.assertTrue(git_ssh_key_file) self.assertTrue(volume_mount_ssh_key) self.assertEqual(65533, pod.security_context['fsGroup']) self.assertEqual(git_ssh_key_file, volume_mount_ssh_key, 'The location where the git ssh secret is mounted' ' needs to be the same as the GIT_SSH_KEY_FILE path')
def test_init_environment_using_git_sync_user_with_known_hosts(self): # Tests the init environment created with git-sync User authentication option is correct # with known hosts file self.kube_config.airflow_configmap = 'airflow-configmap' self.kube_config.git_user = '******' self.kube_config.git_password = '******' self.kube_config.git_ssh_known_hosts_configmap_name = 'airflow-configmap' self.kube_config.git_ssh_key_secret_name = None self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None worker_config = WorkerConfiguration(self.kube_config) init_containers = worker_config._get_init_containers() self.assertTrue(init_containers) # check not empty env = init_containers[0]['env'] self.assertFalse({'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh'} in env) self.assertTrue({'name': 'GIT_SYNC_USERNAME', 'value': 'git_user'} in env) self.assertTrue({'name': 'GIT_SYNC_PASSWORD', 'value': 'git_password'} in env) self.assertTrue({'name': 'GIT_KNOWN_HOSTS', 'value': 'true'} in env) self.assertTrue({'name': 'GIT_SSH_KNOWN_HOSTS_FILE', 'value': '/etc/git-secret/known_hosts'} in env) self.assertFalse({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
def test_init_environment_using_git_sync_run_as_user_empty(self): # Tests if git_syn_run_as_user is none, then no securityContext created in init container self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None self.kube_config.git_sync_run_as_user = '' worker_config = WorkerConfiguration(self.kube_config) init_containers = worker_config._get_init_containers() self.assertTrue(init_containers) # check not empty self.assertNotIn('securityContext', init_containers[0], "securityContext shouldn't be defined")
def test_worker_container_dags(self): # Tests that the 'airflow-dags' persistence volume is NOT created when `dags_in_image` is set self.kube_config.dags_in_image = True worker_config = WorkerConfiguration(self.kube_config) volumes, volume_mounts = worker_config._get_volumes_and_mounts() dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags'] dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags'] init_containers = worker_config._get_init_containers() self.assertEqual(0, len(dag_volume)) self.assertEqual(0, len(dag_volume_mount)) self.assertEqual(0, len(init_containers))
def test_worker_container_dags(self): # Tests that the 'airflow-dags' persistence volume is NOT created when `dags_in_image` is set self.kube_config.dags_in_image = True worker_config = WorkerConfiguration(self.kube_config) volumes, volume_mounts = worker_config.init_volumes_and_mounts() dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags'] dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags'] init_containers = worker_config._get_init_containers(volume_mounts) self.assertEqual(0, len(dag_volume)) self.assertEqual(0, len(dag_volume_mount)) self.assertEqual(0, len(init_containers))
def test_worker_pvc_dags(self): # Tests persistence volume config created when `dags_volume_claim` is set self.kube_config.dags_volume_claim = 'airflow-dags' worker_config = WorkerConfiguration(self.kube_config) volumes, volume_mounts = worker_config._get_volumes_and_mounts() init_containers = worker_config._get_init_containers() dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags'] dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags'] self.assertEqual('airflow-dags', dag_volume[0]['persistentVolumeClaim']['claimName']) self.assertEqual(1, len(dag_volume_mount)) self.assertTrue(dag_volume_mount[0]['readOnly']) self.assertEqual(0, len(init_containers))
def test_worker_pvc_dags(self): # Tests persistence volume config created when `dags_volume_claim` is set self.kube_config.dags_volume_claim = 'airflow-dags' worker_config = WorkerConfiguration(self.kube_config) volumes, volume_mounts = worker_config.init_volumes_and_mounts() init_containers = worker_config._get_init_containers(volume_mounts) dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags'] dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags'] self.assertEqual('airflow-dags', dag_volume[0]['persistentVolumeClaim']['claimName']) self.assertEqual(1, len(dag_volume_mount)) self.assertTrue(dag_volume_mount[0]['readOnly']) self.assertEqual(0, len(init_containers))
def test_kubernetes_environment_variables_for_init_container(self): self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None # Tests the kubernetes environment variables get copied into the worker pods input_environment = { 'ENVIRONMENT': 'prod', 'LOG_LEVEL': 'warning' } self.kube_config.kube_env_vars = input_environment worker_config = WorkerConfiguration(self.kube_config) env = worker_config._get_environment() for key in input_environment: self.assertIn(key, env) self.assertIn(input_environment[key], env.values()) init_containers = worker_config._get_init_containers() self.assertTrue(init_containers) # check not empty env = init_containers[0]['env'] self.assertTrue({'name': 'ENVIRONMENT', 'value': 'prod'} in env) self.assertTrue({'name': 'LOG_LEVEL', 'value': 'warning'} in env)
def test_make_pod_git_sync_ssh_with_known_hosts(self): # Tests the pod created with git-sync SSH authentication option is correct with known hosts self.kube_config.airflow_configmap = 'airflow-configmap' self.kube_config.git_ssh_secret_name = 'airflow-secrets' self.kube_config.dags_volume_claim = None self.kube_config.dags_volume_host = None self.kube_config.dags_in_image = None worker_config = WorkerConfiguration(self.kube_config) init_containers = worker_config._get_init_containers() git_ssh_known_hosts_file = next((x['value'] for x in init_containers[0]['env'] if x['name'] == 'GIT_SSH_KNOWN_HOSTS_FILE'), None) volume_mount_ssh_known_hosts_file = next( (x['mountPath'] for x in init_containers[0]['volumeMounts'] if x['name'] == worker_config.git_sync_ssh_known_hosts_volume_name), None) self.assertTrue(git_ssh_known_hosts_file) self.assertTrue(volume_mount_ssh_known_hosts_file) self.assertEqual(git_ssh_known_hosts_file, volume_mount_ssh_known_hosts_file, 'The location where the git known hosts file is mounted' ' needs to be the same as the GIT_SSH_KNOWN_HOSTS_FILE path')