def test_extract_env_and_secrets(self): # Test when secrets and envs are not empty secrets = [ Secret('env', None, 's1'), Secret('volume', 'KEY2', 's2', 'key-2'), Secret('env', None, 's3') ] envs = { 'ENV1': 'val1', 'ENV2': 'val2' } configmaps = ['configmap_a', 'configmap_b'] pod_runtime_envs = [PodRuntimeInfoEnv("ENV3", "status.podIP")] pod = Pod( image='v3.14', envs=envs, cmds=[], secrets=secrets, configmaps=configmaps, pod_runtime_info_envs=pod_runtime_envs) self.expected['spec']['containers'][0]['env'] = [ {'name': 'ENV1', 'value': 'val1'}, {'name': 'ENV2', 'value': 'val2'}, { 'name': 'ENV3', 'valueFrom': { 'fieldRef': { 'fieldPath': 'status.podIP' } } } ] self.expected['spec']['containers'][0]['envFrom'] = [{ 'secretRef': { 'name': 's1' } }, { 'secretRef': { 'name': 's3' } }, { 'configMapRef': { 'name': 'configmap_a' } }, { 'configMapRef': { 'name': 'configmap_b' } }] KubernetesRequestFactory.extract_env_and_secrets(pod, self.input_req) self.input_req['spec']['containers'][0]['env'].sort(key=lambda x: x['name']) self.assertEqual(self.input_req, self.expected)
def handle_container_environment_variables( env_vars: List[V1EnvVar], ) -> Tuple[Dict[str, str], List[Secret], List[str], List[PodRuntimeInfoEnv]]: secrets = [] plain_env_vars = {} config_maps = [] runtime_env_vars = [] for env_var in env_vars or []: value_from: V1EnvVarSource = env_var.value_from if value_from: if value_from.resource_field_ref: # not handled for now continue if value_from.field_ref: field_ref: V1ObjectFieldSelector = value_from.field_ref runtime_env_vars.append( PodRuntimeInfoEnv( field_path=field_ref.field_path, name=env_var.name ) ) continue if value_from.config_map_key_ref: key_ref: V1ConfigMapKeySelector = value_from.config_map_key_ref config_maps.append(key_ref.name) continue if value_from.secret_key_ref: key_ref: V1SecretKeySelector = value_from.secret_key_ref secrets.append( Secret( deploy_type="env", deploy_target=env_var.name, secret=key_ref.name, key=key_ref.key, ) ) continue plain_env_vars[env_var.name] = env_var.value return plain_env_vars, secrets, config_maps, runtime_env_vars
def test_extract_env_and_secret_order(self): envs = { 'ENV': 'val1', } pod_runtime_envs = [PodRuntimeInfoEnv('RUNTIME_ENV', 'status.podIP')] pod = Pod(image='v3.14', envs=envs, cmds=[], pod_runtime_info_envs=pod_runtime_envs) self.expected['spec']['containers'][0]['env'] = [{ 'name': 'RUNTIME_ENV', 'valueFrom': { 'fieldRef': { 'fieldPath': 'status.podIP' } } }, { 'name': 'ENV', 'value': 'val1' }] KubernetesRequestFactory.extract_env_and_secrets(pod, self.input_req) self.assertEqual(self.input_req, self.expected)
DAG_NAME = "spark_batch_job_distributed_mode" ENV = os.environ.get("ENV") docker_image = "dcr.flix.tech/data/flux/k8s-spark-example:latest" envs = { "SERVICE_NAME": DAG_NAME, "CONTAINER_IMAGE": docker_image, "SPARK_DRIVER_PORT": "35000", "PY_FILES": "/workspace/dist/libs.zip,/workspace/dist/dependencies.zip", "PYTHON_FILE": "/workspace/python/pi.py", } pod_runtime_info_envs = [ PodRuntimeInfoEnv('MY_POD_NAMESPACE', 'metadata.namespace'), PodRuntimeInfoEnv('MY_POD_NAME', 'metadata.name'), PodRuntimeInfoEnv('MY_POD_IP', 'status.podIP') ] args = {'owner': 'Airflow', 'start_date': airflow.utils.dates.days_ago(2)} # base path returned zip dag path base_path = os.path.split(__file__)[0] plain_txt = read_packaged_file(f"{base_path}/plain_files/plain.txt") with DAG(dag_id=DAG_NAME, default_args=args, schedule_interval='30 0 * * *') as dag: # Use the zip binary, which is only found in this special docker image read_local_file = BashOperator(task_id='read_local_file', bash_command=f"echo {plain_txt}")