class PodDefaults: """ Static defaults for Pods """ XCOM_MOUNT_PATH = '/airflow/xcom' SIDECAR_CONTAINER_NAME = 'airflow-xcom-sidecar' XCOM_CMD = 'trap "exit 0" INT; while true; do sleep 30; done;' VOLUME_MOUNT = k8s.V1VolumeMount(name='xcom', mount_path=XCOM_MOUNT_PATH) VOLUME = k8s.V1Volume(name='xcom', empty_dir=k8s.V1EmptyDirVolumeSource()) SIDECAR_CONTAINER = k8s.V1Container( name=SIDECAR_CONTAINER_NAME, command=['sh', '-c', XCOM_CMD], image='alpine', volume_mounts=[VOLUME_MOUNT], resources=k8s.V1ResourceRequirements(requests={ "cpu": "1m", }), )
class PodDefaults: """ Static defaults for the PodGenerator """ XCOM_MOUNT_PATH = '/airflow/xcom' SIDECAR_CONTAINER_NAME = 'airflow-xcom-sidecar' XCOM_CMD = """import time while True: try: time.sleep(3600) except KeyboardInterrupt: exit(0) """ VOLUME_MOUNT = k8s.V1VolumeMount(name='xcom', mount_path=XCOM_MOUNT_PATH) VOLUME = k8s.V1Volume(name='xcom', empty_dir=k8s.V1EmptyDirVolumeSource()) SIDECAR_CONTAINER = k8s.V1Container(name=SIDECAR_CONTAINER_NAME, command=['python', '-c', XCOM_CMD], image='python:3.5-alpine', volume_mounts=[VOLUME_MOUNT])
k8s.V1Container( name="sidecar", image="ubuntu", args=[ "echo \"retrieved from mount\" > /shared/test.txt" ], command=["bash", "-cx"], volume_mounts=[ k8s.V1VolumeMount(mount_path="/shared/", name="shared-empty-dir") ], ), ], volumes=[ k8s.V1Volume(name="shared-empty-dir", empty_dir=k8s.V1EmptyDirVolumeSource()), ], )), } @task(executor_config=executor_config_sidecar) def test_sharedvolume_mount(): """ Tests whether the volume has been mounted. """ for i in range(5): try: return_code = os.system("cat /shared/test.txt") if return_code != 0: raise ValueError( f"Error when checking volume mount. Return code {return_code}"
def get_empty_dir_volume(): return k8s.V1Volume(name="tmpdir", empty_dir=k8s.V1EmptyDirVolumeSource())
volume_mounts=[ k8s.V1VolumeMount(mount_path="/shared/", name="shared-empty-dir") ], ), k8s.V1Container( name="sidecar", image="ubuntu", args=["echo \"retrieved from mount\" > /shared/test.txt"], command=["bash", "-cx"], volume_mounts=[ k8s.V1VolumeMount(mount_path="/shared/", name="shared-empty-dir") ], ), ], volumes=[ k8s.V1Volume(name="shared-empty-dir", empty_dir=k8s.V1EmptyDirVolumeSource()), ], ) ), }, ) # [END task_with_sidecar] # Test that we can add labels to pods third_task = PythonOperator( task_id="non_root_task", python_callable=print_stuff, executor_config={ "pod_override": k8s.V1Pod(metadata=k8s.V1ObjectMeta(labels={"release": "stable"})) }, )