class PodDefaults:
"""
Static defaults for Pods
"""
XCOM_MOUNT_PATH = '/airflow/xcom'
SIDECAR_CONTAINER_NAME = 'airflow-xcom-sidecar'
XCOM_CMD = 'trap "exit 0" INT; while true; do sleep 30; done;'
VOLUME_MOUNT = k8s.V1VolumeMount(name='xcom', mount_path=XCOM_MOUNT_PATH)
VOLUME = k8s.V1Volume(name='xcom', empty_dir=k8s.V1EmptyDirVolumeSource())
SIDECAR_CONTAINER = k8s.V1Container(
name=SIDECAR_CONTAINER_NAME,
command=['sh', '-c', XCOM_CMD],
image='alpine',
volume_mounts=[VOLUME_MOUNT],
resources=k8s.V1ResourceRequirements(requests={
"cpu": "1m",
}),
)
class PodDefaults:
"""
Static defaults for the PodGenerator
"""
XCOM_MOUNT_PATH = '/airflow/xcom'
SIDECAR_CONTAINER_NAME = 'airflow-xcom-sidecar'
XCOM_CMD = """import time
while True:
try:
time.sleep(3600)
except KeyboardInterrupt:
exit(0)
"""
VOLUME_MOUNT = k8s.V1VolumeMount(name='xcom', mount_path=XCOM_MOUNT_PATH)
VOLUME = k8s.V1Volume(name='xcom', empty_dir=k8s.V1EmptyDirVolumeSource())
SIDECAR_CONTAINER = k8s.V1Container(name=SIDECAR_CONTAINER_NAME,
command=['python', '-c', XCOM_CMD],
image='python:3.5-alpine',
volume_mounts=[VOLUME_MOUNT])
k8s.V1Container(
name="sidecar",
image="ubuntu",
args=[
"echo \"retrieved from mount\" > /shared/test.txt"
],
command=["bash", "-cx"],
volume_mounts=[
k8s.V1VolumeMount(mount_path="/shared/",
name="shared-empty-dir")
],
),
],
volumes=[
k8s.V1Volume(name="shared-empty-dir",
empty_dir=k8s.V1EmptyDirVolumeSource()),
],
)),
}
@task(executor_config=executor_config_sidecar)
def test_sharedvolume_mount():
"""
Tests whether the volume has been mounted.
"""
for i in range(5):
try:
return_code = os.system("cat /shared/test.txt")
if return_code != 0:
raise ValueError(
f"Error when checking volume mount. Return code {return_code}"
def get_empty_dir_volume():
return k8s.V1Volume(name="tmpdir", empty_dir=k8s.V1EmptyDirVolumeSource())
volume_mounts=[
k8s.V1VolumeMount(mount_path="/shared/", name="shared-empty-dir")
],
),
k8s.V1Container(
name="sidecar",
image="ubuntu",
args=["echo \"retrieved from mount\" > /shared/test.txt"],
command=["bash", "-cx"],
volume_mounts=[
k8s.V1VolumeMount(mount_path="/shared/", name="shared-empty-dir")
],
),
],
volumes=[
k8s.V1Volume(name="shared-empty-dir", empty_dir=k8s.V1EmptyDirVolumeSource()),
],
)
),
},
)
# [END task_with_sidecar]
# Test that we can add labels to pods
third_task = PythonOperator(
task_id="non_root_task",
python_callable=print_stuff,
executor_config={
"pod_override": k8s.V1Pod(metadata=k8s.V1ObjectMeta(labels={"release": "stable"}))
},
)
