def test_env_vars(self):
# WHEN
env_vars = [
k8s.V1EnvVar(name="ENV1", value="val1"),
k8s.V1EnvVar(name="ENV2", value="val2"),
k8s.V1EnvVar(
name="ENV3",
value_from=k8s.V1EnvVarSource(field_ref=k8s.V1ObjectFieldSelector(field_path="status.podIP")),
),
]
k = KubernetesPodOperator(
namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo 10"],
env_vars=env_vars,
labels={"foo": "bar"},
name="test-" + str(random.randint(0, 1000000)),
task_id="task" + self.get_current_task_name(),
in_cluster=False,
do_xcom_push=False,
)
context = create_context(k)
k.execute(context)
# THEN
actual_pod = self.api_client.sanitize_for_serialization(k.pod)
self.expected_pod['spec']['containers'][0]['env'] = [
{'name': 'ENV1', 'value': 'val1'},
{'name': 'ENV2', 'value': 'val2'},
{'name': 'ENV3', 'valueFrom': {'fieldRef': {'fieldPath': 'status.podIP'}}},
]
assert self.expected_pod == actual_pod
def to_k8s_client_obj(self) -> k8s.V1EnvVar:
""":return: kubernetes.client.models.V1EnvVar"""
return k8s.V1EnvVar(
name=self.name,
value_from=k8s.V1EnvVarSource(field_ref=k8s.V1ObjectFieldSelector(
field_path=self.field_path)),
)
def to_env_secret(self) -> k8s.V1EnvVar:
"""Stores es environment secret"""
return k8s.V1EnvVar(
name=self.deploy_target,
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(name=self.secret,
key=self.key)),
)
def test_to_env_secret(self):
secret = Secret('env', 'name', 'secret', 'key')
self.assertEqual(
secret.to_env_secret(),
k8s.V1EnvVar(name='NAME',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name='secret', key='key'))))
def get_kubeadmin_password(release: OpenshiftRelease):
return k8s.V1EnvVar(
name="KUBEADMIN_PASSWORD",
value_from=k8s.V1EnvVarSource(
secret_key_ref= k8s.V1SecretKeySelector(
name=f"{release.get_release_name()}-kubeadmin",
key="KUBEADMIN_PASSWORD"
)
)
)
def get_kubeadmin_password(release: OpenshiftRelease, task_group):
prefix=f"{task_group}-"
return k8s.V1EnvVar(
name="KUBEADMIN_PASSWORD",
value_from=k8s.V1EnvVarSource(
secret_key_ref= k8s.V1SecretKeySelector(
name=f"{release.get_release_name()}-{prefix if 'hosted' in task_group else ''}kubeadmin",
key="KUBEADMIN_PASSWORD"
)
)
)
def test_make_pod_git_sync_credentials_secret(self):
# Tests the pod created with git_sync_credentials_secret will get into the init container
self.kube_config.git_sync_credentials_secret = 'airflow-git-creds-secret'
self.kube_config.dags_volume_claim = None
self.kube_config.dags_volume_host = None
self.kube_config.dags_in_image = None
self.kube_config.worker_fs_group = None
self.kube_config.git_dags_folder_mount_point = 'dags'
self.kube_config.git_sync_dest = 'repo'
self.kube_config.git_subpath = 'path'
worker_config = WorkerConfiguration(self.kube_config)
pod = worker_config.make_pod("default", str(uuid.uuid4()),
"test_pod_id", "test_dag_id",
"test_task_id", str(datetime.utcnow()), 1,
"bash -c 'ls /'")
username_env = k8s.V1EnvVar(
name='GIT_SYNC_USERNAME',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.kube_config.git_sync_credentials_secret,
key='GIT_SYNC_USERNAME')))
password_env = k8s.V1EnvVar(
name='GIT_SYNC_PASSWORD',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.kube_config.git_sync_credentials_secret,
key='GIT_SYNC_PASSWORD')))
self.assertIn(
username_env, pod.spec.init_containers[0].env,
'The username env for git credentials did not get into the init container'
)
self.assertIn(
password_env, pod.spec.init_containers[0].env,
'The password env for git credentials did not get into the init container'
)
def handleEnv(self, key: str, value: T.Dict[str, T.Any],
full_path: str) -> None:
env = []
for k, v in value.items():
if "secret" in v:
if not self.jobSecret:
# logging.critical("No secret found")
return
keyRef = self.jobSecret.metadata.name
# TODO: find ref from secrets metadata
key_selector = K.V1SecretKeySelector(key=v["secret"],
name=keyRef)
env_var_src = K.V1EnvVarSource(secret_key_ref=key_selector)
env.append(K.V1EnvVar(name=k, value_from=env_var_src))
else:
env.append(K.V1EnvVar(name=k, value=v))
self.container.env = env
def setUp(self):
self.static_uuid = uuid.UUID('cf4a56d2-8101-4217-b027-2af6216feb48')
self.deserialize_result = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {'name': 'memory-demo', 'namespace': 'mem-example'},
'spec': {
'containers': [
{
'args': ['--vm', '1', '--vm-bytes', '150M', '--vm-hang', '1'],
'command': ['stress'],
'image': 'apache/airflow:stress-2020.07.10-1.0.4',
'name': 'memory-demo-ctr',
'resources': {'limits': {'memory': '200Mi'}, 'requests': {'memory': '100Mi'}},
}
]
},
}
self.envs = {'ENVIRONMENT': 'prod', 'LOG_LEVEL': 'warning'}
self.secrets = [
# This should be a secretRef
Secret('env', None, 'secret_a'),
# This should be a single secret mounted in volumeMounts
Secret('volume', '/etc/foo', 'secret_b'),
# This should produce a single secret mounted in env
Secret('env', 'TARGET', 'secret_b', 'source_b'),
]
self.execution_date = parser.parse('2020-08-24 00:00:00.000000')
self.execution_date_label = datetime_to_label_safe_datestring(self.execution_date)
self.dag_id = 'dag_id'
self.task_id = 'task_id'
self.try_number = 3
self.labels = {
'airflow-worker': 'uuid',
'dag_id': self.dag_id,
'execution_date': self.execution_date_label,
'task_id': self.task_id,
'try_number': str(self.try_number),
'airflow_version': __version__.replace('+', '-'),
'kubernetes_executor': 'True',
}
self.annotations = {
'dag_id': self.dag_id,
'task_id': self.task_id,
'execution_date': self.execution_date.isoformat(),
'try_number': str(self.try_number),
}
self.metadata = {
'labels': self.labels,
'name': 'pod_id-' + self.static_uuid.hex,
'namespace': 'namespace',
'annotations': self.annotations,
}
self.resources = k8s.V1ResourceRequirements(
requests={
"cpu": 1,
"memory": "1Gi",
"ephemeral-storage": "2Gi",
},
limits={"cpu": 2, "memory": "2Gi", "ephemeral-storage": "4Gi", 'nvidia.com/gpu': 1},
)
self.k8s_client = ApiClient()
self.expected = k8s.V1Pod(
api_version="v1",
kind="Pod",
metadata=k8s.V1ObjectMeta(
namespace="default",
name='myapp-pod-' + self.static_uuid.hex,
labels={'app': 'myapp'},
),
spec=k8s.V1PodSpec(
containers=[
k8s.V1Container(
name='base',
image='busybox',
command=['sh', '-c', 'echo Hello Kubernetes!'],
env=[
k8s.V1EnvVar(name='ENVIRONMENT', value='prod'),
k8s.V1EnvVar(
name="LOG_LEVEL",
value='warning',
),
k8s.V1EnvVar(
name='TARGET',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(name='secret_b', key='source_b')
),
),
],
env_from=[
k8s.V1EnvFromSource(config_map_ref=k8s.V1ConfigMapEnvSource(name='configmap_a')),
k8s.V1EnvFromSource(config_map_ref=k8s.V1ConfigMapEnvSource(name='configmap_b')),
k8s.V1EnvFromSource(secret_ref=k8s.V1SecretEnvSource(name='secret_a')),
],
ports=[k8s.V1ContainerPort(name="foo", container_port=1234)],
resources=k8s.V1ResourceRequirements(
requests={'memory': '100Mi'},
limits={
'memory': '200Mi',
},
),
)
],
security_context=k8s.V1PodSecurityContext(
fs_group=2000,
run_as_user=1000,
),
host_network=True,
image_pull_secrets=[
k8s.V1LocalObjectReference(name="pull_secret_a"),
k8s.V1LocalObjectReference(name="pull_secret_b"),
],
),
)
def get_kubeadmin_password(version, platform, profile):
return k8s.V1EnvVar(
name="KUBEADMIN_PASSWORD",
value_from=k8s.V1EnvVarSource(secret_key_ref=k8s.V1SecretKeySelector(
name=f"{version}-{platform}-{profile}-kubeadmin",
key="KUBEADMIN_PASSWORD")))
def _get_init_containers(self) -> List[k8s.V1Container]:
"""When using git to retrieve the DAGs, use the GitSync Init Container"""
# If we're using volume claims to mount the dags, no init container is needed
if self.kube_config.dags_volume_claim or \
self.kube_config.dags_volume_host or self.kube_config.dags_in_image:
return []
# Otherwise, define a git-sync init container
init_environment = [
k8s.V1EnvVar(name='GIT_SYNC_REPO',
value=self.kube_config.git_repo),
k8s.V1EnvVar(name='GIT_SYNC_BRANCH',
value=self.kube_config.git_branch),
k8s.V1EnvVar(name='GIT_SYNC_ROOT',
value=self.kube_config.git_sync_root),
k8s.V1EnvVar(name='GIT_SYNC_DEST',
value=self.kube_config.git_sync_dest),
k8s.V1EnvVar(name='GIT_SYNC_REV',
value=self.kube_config.git_sync_rev),
k8s.V1EnvVar(name='GIT_SYNC_DEPTH', value='1'),
k8s.V1EnvVar(name='GIT_SYNC_ONE_TIME', value='true')
]
if self.kube_config.git_user:
init_environment.append(
k8s.V1EnvVar(name='GIT_SYNC_USERNAME',
value=self.kube_config.git_user))
if self.kube_config.git_password:
init_environment.append(
k8s.V1EnvVar(name='GIT_SYNC_PASSWORD',
value=self.kube_config.git_password))
volume_mounts = [
k8s.V1VolumeMount(mount_path=self.kube_config.git_sync_root,
name=self.dags_volume_name,
read_only=False)
]
if self.kube_config.git_sync_credentials_secret:
init_environment.extend([
k8s.V1EnvVar(
name='GIT_SYNC_USERNAME',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.kube_config.git_sync_credentials_secret,
key='GIT_SYNC_USERNAME'))),
k8s.V1EnvVar(
name='GIT_SYNC_PASSWORD',
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.kube_config.git_sync_credentials_secret,
key='GIT_SYNC_PASSWORD')))
])
if self.kube_config.git_ssh_key_secret_name:
volume_mounts.append(
k8s.V1VolumeMount(name=self.git_sync_ssh_secret_volume_name,
mount_path='/etc/git-secret/ssh',
sub_path='ssh'))
init_environment.extend([
k8s.V1EnvVar(name='GIT_SSH_KEY_FILE',
value='/etc/git-secret/ssh'),
k8s.V1EnvVar(name='GIT_SYNC_SSH', value='true')
])
if self.kube_config.git_ssh_known_hosts_configmap_name:
volume_mounts.append(
k8s.V1VolumeMount(
name=self.git_sync_ssh_known_hosts_volume_name,
mount_path='/etc/git-secret/known_hosts',
sub_path='known_hosts'))
init_environment.extend([
k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'),
k8s.V1EnvVar(name='GIT_SSH_KNOWN_HOSTS_FILE',
value='/etc/git-secret/known_hosts')
])
else:
init_environment.append(
k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'))
init_containers = k8s.V1Container(
name=self.kube_config.git_sync_init_container_name,
image=self.kube_config.git_sync_container,
env=init_environment,
volume_mounts=volume_mounts)
if self.kube_config.git_sync_run_as_user != "":
init_containers.security_context = k8s.V1SecurityContext(
run_as_user=self.kube_config.git_sync_run_as_user
) # git-sync user
return [init_containers]
max_active_tasks=10,
max_active_runs=10) as dag:
load_resources = V1ResourceRequirements(requests={"memory": "18Gi"},
limits={"memory": "18Gi"})
node_selector = {"loader-node": "true"}
image_pull_secrets = [k8s.V1LocalObjectReference('falkonry-pull-secret')]
envs = list()
load_file = "{{ dag_run.conf['load_file'] }}"
compact_file = "{{ dag_run.conf['compact_file'] }}"
task_id = "{{ dag_run.conf['task_id'] }}"
envs.append(
k8s.V1EnvVar(name="AWS_SECRET_ACCESS_KEY",
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
key="secret-id", name="aws-key-prod"))))
envs.append(
k8s.V1EnvVar(name="AWS_DEFAULT_REGION",
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
key="region-w2", name="aws-key-prod"))))
envs.append(
k8s.V1EnvVar(name="AWS_REGION",
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
key="region-w2", name="aws-key-prod"))))
envs.append(
k8s.V1EnvVar(name="AWS_ACCESS_KEY_ID",
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
def to_env_secret(self) -> k8s.V1EnvVar:
return k8s.V1EnvVar(name=self.deploy_target,
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.secret, key=self.key)))
def to_env_secret(self):
import kubernetes.client.models as k8s
return k8s.V1EnvVar(name=self.deploy_target,
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name=self.secret, key=self.key)))
def test_convert_to_airflow_pod(self):
input_pod = k8s.V1Pod(
metadata=k8s.V1ObjectMeta(name="foo", namespace="bar"),
spec=k8s.V1PodSpec(
init_containers=[
k8s.V1Container(name="init-container",
volume_mounts=[
k8s.V1VolumeMount(mount_path="/tmp",
name="init-secret")
])
],
containers=[
k8s.V1Container(
name="base",
command=["foo"],
image="myimage",
env=[
k8s.V1EnvVar(
name="AIRFLOW_SECRET",
value_from=k8s.V1EnvVarSource(
secret_key_ref=k8s.V1SecretKeySelector(
name="ai", key="secret_key")))
],
ports=[
k8s.V1ContainerPort(
name="myport",
container_port=8080,
)
],
volume_mounts=[
k8s.V1VolumeMount(name="myvolume",
mount_path="/tmp/mount",
read_only="True"),
k8s.V1VolumeMount(name='airflow-config',
mount_path='/config',
sub_path='airflow.cfg',
read_only=True),
k8s.V1VolumeMount(name="airflow-secret",
mount_path="/opt/mount",
read_only=True)
])
],
security_context=k8s.V1PodSecurityContext(
run_as_user=0,
fs_group=0,
),
volumes=[
k8s.V1Volume(name="myvolume"),
k8s.V1Volume(
name="airflow-config",
config_map=k8s.V1ConfigMap(data="airflow-data")),
k8s.V1Volume(name="airflow-secret",
secret=k8s.V1SecretVolumeSource(
secret_name="secret-name", )),
k8s.V1Volume(name="init-secret",
secret=k8s.V1SecretVolumeSource(
secret_name="init-secret", ))
]))
result_pod = _convert_to_airflow_pod(input_pod)
expected = Pod(
name="foo",
namespace="bar",
envs={},
init_containers=[{
'name':
'init-container',
'volumeMounts': [{
'mountPath': '/tmp',
'name': 'init-secret'
}]
}],
cmds=["foo"],
image="myimage",
ports=[Port(name="myport", container_port=8080)],
volume_mounts=[
VolumeMount(name="myvolume",
mount_path="/tmp/mount",
sub_path=None,
read_only="True"),
VolumeMount(name="airflow-config",
read_only=True,
mount_path="/config",
sub_path="airflow.cfg"),
VolumeMount(name="airflow-secret",
mount_path="/opt/mount",
sub_path=None,
read_only=True)
],
secrets=[Secret("env", "AIRFLOW_SECRET", "ai", "secret_key")],
security_context={
'fsGroup': 0,
'runAsUser': 0
},
volumes=[
Volume(name="myvolume", configs={'name': 'myvolume'}),
Volume(name="airflow-config",
configs={
'configMap': {
'data': 'airflow-data'
},
'name': 'airflow-config'
}),
Volume(name='airflow-secret',
configs={
'name': 'airflow-secret',
'secret': {
'secretName': 'secret-name'
}
}),
Volume(name='init-secret',
configs={
'name': 'init-secret',
'secret': {
'secretName': 'init-secret'
}
})
],
)
expected_dict = expected.as_dict()
result_dict = result_pod.as_dict()
print(result_pod.volume_mounts)
parsed_configs = self.pull_out_volumes(result_dict)
result_dict['volumes'] = parsed_configs
self.assertEqual(result_dict['secrets'], expected_dict['secrets'])
self.assertDictEqual(expected_dict, result_dict)
