def test_volume_mount(): with mock.patch.object(PodLauncher, 'log') as mock_logger: volume_mount = VolumeMount('test-volume', mount_path='/root/mount_file', sub_path=None, read_only=True) volume_config = { 'persistentVolumeClaim': { 'claimName': 'test-volume' } } volume = Volume(name='test-volume', configs=volume_config) k = KubernetesPodOperator( namespace='default', image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["cat /root/mount_file/test.txt"], labels={"foo": "bar"}, volume_mounts=[volume_mount], volumes=[volume], name="test", task_id="task") k.execute(None) mock_logger.info.assert_any_call(b"retrieved from mount\n")
def convert_volume_mounts(container: V1Container) -> List[VolumeMount]: volume_mounts: List[V1VolumeMount] = container.volume_mounts return [ VolumeMount( name=vm.name, mount_path=vm.mount_path, sub_path=vm.sub_path, read_only=vm.read_only, ) for vm in volume_mounts or [] ]
'ga-service-account-json', 'ga-service-account.json') client_secret_secret_file = Secret('volume', '/etc/ga_client_secret', 'ga-client-secret-json', 'ga-client-secret.json') script_root_path = '/tmp/scripts' get_user_ids_r_script_path = "Services/ELT/DA/getUserIDs.R" get_user_ids_r_script_whole_path = script_root_path + \ "/" + get_user_ids_r_script_path retrieve_user_session_activity_r_script_path = "Services/ELT/DA/getUserIDs.R" retrieve_user_session_activity_r_script_whole_path = script_root_path + \ "/" + retrieve_user_session_activity_r_script_path volume_mount = VolumeMount('git-root-path', mount_path=script_root_path, sub_path=None, read_only=False) volume_config = { "hostPath": { "path": "/home/DA_git_master/DataAnalysis.git", "type": "Directory" } } volume = Volume(name='git-root-path', configs=volume_config) start_date = "{{ ds }}" end_date = "{{ macros.ds_add(next_ds, -1) }}" get_user_ids_task = KubernetesPodOperator( namespace='default', image="bowenkuo/dump-ga-to-bq:1.0.1",
"owner": "airflow", "on_failure_callback": mm_failed_task, "retries": 0, "retry_delay": timedelta(minutes=1), "sla": timedelta(hours=8), "start_date": datetime(2019, 1, 1, 0, 0, 0), } # Create the DAG dag = DAG("blapi", default_args=default_args, schedule_interval="0 * * * *") volume_config = {"persistentVolumeClaim": {"claimName": "pipelinewise-pv"}} volume = Volume(name="pipelinewise-volume", configs=volume_config) volume_mount = VolumeMount( "pipelinewise-volume", mount_path="/app/.pipelinewise", sub_path=None, read_only=False, ) if "cmds" in pod_defaults: del pod_defaults["cmds"] blapi = KubernetesPodOperator( **pod_defaults, image=PIPELINEWISE_IMAGE, task_id="blapi-import", name="blapi-import", secrets=[ PIPELINEWISE_SECRETS, ], arguments=["run_tap", "--tap", "blapi", "--target", "snowflake"],
from datetime import timedelta from airflow import DAG from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount from airflow.operators.dummy_operator import DummyOperator from airflow.operators.bash_operator import BashOperator from airflow.utils.dates import days_ago ########################### ###### Set up volume ###### ########################### volume_mount = VolumeMount('results-volume', mount_path='/outputs', sub_path=None, read_only=False) volume_config= { 'persistentVolumeClaim': { 'claimName': 'results-claim' } } volume = Volume(name='results-volume', configs=volume_config) ############################ ####### Generate DAG ####### ############################
import datetime from airflow import models from airflow.contrib.operators import kubernetes_pod_operator from airflow.operators import dummy_operator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) volume_mount = VolumeMount('test-volume', mount_path='/root/mount_file', sub_path=None, read_only=True) volume_config = {'persistentVolumeClaim': {'claimName': 'test-volume'}} volume = Volume(name='test-volume', configs=volume_config) with models.DAG(dag_id='airflow-demo', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) as dag: task1 = kubernetes_pod_operator.KubernetesPodOperator( task_id='t1', name='task1', namespace='airflow', image='eu.gcr.io/taiyo-239217/dag:fae4887', arguments=["AlphaVantage()"], volume=[], volume_mounts=[], in_cluster=True, xcom_push=True, is_delete_operator_pod=True)
import os import json import requests from datetime import datetime from airflow import DAG from airflow.operators.python_operator import PythonOperator from airflow.operators import WeatherFileSensor from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator from airflow.contrib.kubernetes.volume_mount import VolumeMount from airflow.contrib.kubernetes.volume import Volume from kubernetes import client, config volume_mount = VolumeMount('test-dir', mount_path='/data', sub_path=None, read_only=False) volume_config = {'persistentVolumeClaim': {'claimName': 'airflow'}} volume = Volume(name='test-dir', configs=volume_config) NAM_BASE_DIR = '/data/weatherdata/nam' EXECUTE_DIR = '/tmp' POD_PREFIX = 'run-ungrib' def clean_completed_pods(**context): config.load_kube_config() v1 = client.CoreV1Api() body = client.V1DeleteOptions() ret = v1.list_namespaced_pod('airflow')
from airflow.contrib.kubernetes.volume_mount import VolumeMount ## # Persistent Volume Configuration ## ## Input Volume volume_config = { 'persistentVolumeClaim': { 'claimName': 'pvc-competitions-airflow2' } } volume = Volume(name='input-mount', configs=volume_config) volume_mount = VolumeMount('input-mount', mount_path='/mnt/azure', sub_path=None, read_only=True) ## Output Volume volume2_config = { 'persistentVolumeClaim': { 'claimName': 'pvc-competitions-airflow3' } } volume2 = Volume(name='output-mount', configs=volume2_config) volume2_mount = VolumeMount('output-mount', mount_path='/mnt/azure2', sub_path=None, read_only=False)
# KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from __future__ import print_function import airflow from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator from airflow.contrib.kubernetes.volume_mount import VolumeMount from airflow.contrib.kubernetes.volume import Volume from airflow.models import DAG import os args = {'owner': '{{ owner }}', 'start_date': airflow.utils.dates.days_ago(2)} dag = DAG(dag_id='{{ ID }}', default_args=args, schedule_interval=None) volume_mount = VolumeMount(name='test-volume', mount_path='/root/runtime', sub_path='{{ subpath }}', read_only=False) volume_config = {'persistentVolumeClaim': {'claimName': 'test-volume'}} volume = Volume(name='test-volume', configs=volume_config) t1 = KubernetesPodOperator(task_id="run_script", dag=dag, in_cluster=True, volume_mounts=[volume_mount], namespace='default', name="{}-runner".format(ID), volumes=[volume], image='{{ image }}', arguments=['python', '/root/runtime/run_job.py'])
from airflow import DAG from airflow.utils.dates import days_ago from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount ## # Persistent Volume Configuration ## ## Reference Volume input_ref_config = {'persistentVolumeClaim': {'claimName': 'pvc-references'}} input_ref_volume = Volume(name='reference-mount', configs=input_ref_config) input_ref_mount = VolumeMount(name='reference-mount', mount_path='/rnaseq/ref', sub_path='ref', read_only=True) # Input Data Volume input_data_config = {'persistentVolumeClaim': {'claimName': 'pvc-input'}} input_data_volume = Volume(name='input-mount', configs=input_data_config) input_data_mount = VolumeMount(name='input-mount', mount_path='/rnaseq/data', sub_path=None, read_only=True) ### Output Volume output_config = {'persistentVolumeClaim': {'claimName': 'pvc-output'}} output_volume = Volume(name='output-mount', configs=output_config)
import os from airflow import DAG, utils as airflow_utils from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount with DAG( dag_id="chapter11_movielens_kubernetes", description="Fetches ratings from the Movielens API using kubernetes.", start_date=airflow_utils.dates.days_ago(3), schedule_interval="@daily", ) as dag: volume_mount = VolumeMount( "data-volume", mount_path="/data", sub_path=None, read_only=False ) volume_config = {"persistentVolumeClaim": {"claimName": "data-volume"}} volume = Volume(name="data-volume", configs=volume_config) fetch_ratings = KubernetesPodOperator( task_id="fetch_ratings", image="airflowbook/movielens-fetch", cmds=["fetch_ratings.py"], arguments=[ "--start_date", "{{ds}}", "--end_date", "{{next_ds}}", "--output_path",
from datetime import timedelta from airflow.utils.dates import days_ago from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount from airflow.operators.bash_operator import BashOperator from airflow.operators.dummy_operator import DummyOperator ## # Persistent Volume Configuration ## ## Reference Volume input_ref_mount = VolumeMount(name='reference-mount', mount_path='/mnt/references', sub_path=None, read_only=True) input_ref_volume = Volume(name='reference-mount', configs={'persistentVolumeClaim':{'claimName': 'pvc-references'}}) # Input Data Volume input_data_mount = VolumeMount(name='input-mount', mount_path='/mnt/data', sub_path=None, read_only=True) input_data_volume = Volume(name='input-mount', configs={'persistentVolumeClaim':{'claimName': 'pvc-input'}}) # Temp Data Volume temp_data_mount = VolumeMount(name='temp-mount', mount_path='/mnt/temp', sub_path=None, read_only=False)
dag = DAG( dag_id='talend_kubernetes_example', default_args=args, schedule_interval=None) tolerations = [ { 'key': "key", 'operator': 'Equal', 'value': 'value' } ] # Input File Volume volume_in = VolumeMount('inpath', mount_path='/opt/talend/input_files/', sub_path=None, read_only=False) volume_config_in= { 'hostPath': { 'path': '/home/osboxes/talend_kub_airflow/talend/input_files/' } } volume1 = Volume(name='inpath', configs=volume_config_in) # Output File Volume volume_out = VolumeMount('outpath', mount_path='/opt/talend/out_files/', sub_path=None,
} docker_image_tag = 'client5' default_args = { 'owner': 'airflow', 'start_date': datetime(2017, 11, 25), 'depends_on_past': False, 'email_on_failure': False, 'email_on_retry': False, 'retries': 0, 'retry_delay': timedelta(minutes=5), 'provide_context': True } dags_volume_mount = VolumeMount('airflow-dags', mount_path='/root/airflow/dags', sub_path=None, read_only=True) dags_volume_config = {'persistentVolumeClaim': {'claimName': 'airflow-dags'}} dags_volume = Volume(name='airflow-dags', configs=dags_volume_config) logs_volume_mount = VolumeMount('airflow-logs', mount_path='/root/airflow/logs', sub_path=None, read_only=True) logs_volume_config = {'persistentVolumeClaim': {'claimName': 'airflow-logs'}} logs_volume = Volume(name='airflow-logs', configs=logs_volume_config) dag = DAG(dag_id='{}'.format(PARENT_DAG_NAME), catchup=True,
# the nodepool's name. 'key': 'cloud.google.com/gke-nodepool', 'operator': 'In', # The label key's value that pods can be scheduled # on. 'values': [ 'pool-1', ] }] }] } } } volume_mount_code = VolumeMount('my-volume', mount_path='/home/git/', sub_path=None, read_only=True) volume_mount_key = VolumeMount('google-cloud-key', mount_path='/var/secrets/google', sub_path=None, read_only=False) volume_config_key = {'secret': {'secretName': 'casper-worker-key'}} volume_config = {'persistentVolumeClaim': {'claimName': 'my-vol'}} volume_code = Volume(name='my-volume', configs=volume_config) volume_key = Volume(name='google-cloud-key', configs=volume_config_key) default_dag_args = { 'start_date': datetime(2018, 1, 1),
from datetime import datetime, timedelta default_args = { 'owner': 'airflow', 'depends_on_past': False, 'start_date': datetime(2018, 7, 26), 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, 'retries': 1, 'provide_context': True, 'retry_delay': timedelta(minutes=5) } volume_mount = VolumeMount('cadc-volume', mount_path='/data/cadc', sub_path=None, read_only=False) volume_config = {'persistentVolumeClaim': {'claimName': 'cadc-volume'}} volume = Volume(name='cadc-volume', configs=volume_config) http_conn_id = 'test_netrc' redis_conn_id = 'redis_default' output_format = 'csv' collection = 'VLASS' dag_id = 'vlass_processing' datetime_format = '%Y-%m-%d %H:%M:%S' # TODO - when deploying to have this actually run, catchup=True!!!!! # and schedule_interval=timedelta(hours=1)
'depends_on_past': False, 'start_date': datetime.utcnow(), 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, 'retries': 1, 'retry_delay': timedelta(minutes=5) } dag = DAG( 'kubernetes_test_podv3', default_args=default_args, schedule_interval=timedelta(minutes=10)) start = DummyOperator(task_id='run_this_first', dag=dag) volume_mount = VolumeMount('airflow-dags', mount_path='/dags', sub_path='dags', read_only=True) volume_config = { 'persistentVolumeClaim': { 'claimName': 'airflow-dags' } } volume = Volume(name='airflow-dags', configs=volume_config) file_path = "/root/kubeconfig/kubeconfig" passing = KubernetesPodOperator(namespace='airflow', image="python:3.6", cmds=["python", "/dags/test-python.py"],
Arguments: claimName (string): Name of the PVC claim in kubernetes """ volume_config = {'persistentVolumeClaim': {'claimName': 'airflow1data'}} volume = Volume(name='airflow1data', configs=volume_config) """ Configuration for Volume Mounting location from PVC Arguments: name (string): Name of the PVC volume request mount_path (string): Mount directory in the pod sub_path (string): Sub path based on the mount directory read_only (boolean): If the mount is read only or not """ volume_mount = VolumeMount('airflow1data', mount_path='/mnt/azure', sub_path=None, read_only=False) args = {'owner': 'airflow', 'start_date': days_ago(2)} ## # hello ## with DAG( dag_id='hello_world', default_args=args, schedule_interval=None, tags=['example'], ) as dag: """ Example Task using KubernetesPodOperator
#secret_file = Secret('volume', '/etc/sql_conn', 'airflow-secrets', 'sql_alchemy_conn') #secret_env = Secret('env', 'SQL_CONN', 'airflow-secrets', 'sql_alchemy_conn') #secret_all_keys = Secret('env', None, 'airflow-secrets-2') port = Port('http', 80) configmaps = ['test-configmap-1', 'test-configmap-2'] volume_config = { 'persistentVolumeClaim': { 'claimName': 'pvc-test-volume-alamano' } } volume = Volume(name='pv-test-volume', configs=volume_config) volume_mount = VolumeMount('pv-test-volume', mount_path='/root/mount_file', sub_path=None, read_only=False) affinity = { 'nodeAffinity': { 'preferredDuringSchedulingIgnoredDuringExecution': [{ "weight": 1, "preference": { "matchExpressions": [ # Added fix from original code { "key": "disktype", "operator": "In", "values": ["ssd"] } ] }
'claimName': 'pvc-blobfuse-flexvol' } } volume = Volume(name='flexvol-mount', configs=volume_config) """ Configuration for Volume Mounting location from PVC Arguments: name (string): Name of the PVC volume request mount_path (string): Mount directory in the pod sub_path (string): Sub path based on the mount directory read_only (boolean): If the mount is read only or not """ volume_mount = VolumeMount('flexvol-mount', mount_path='/mnt/azure', sub_path=None, read_only=True) args = { 'owner': 'airflow', 'start_date': days_ago(2) } ## # Secret Configuration ## """ Secrets pull secret variables and their contents from Kubernetes. You do this to protect things like database credentials. You can do this with files, tokens, or variables. Arguments: deploy_type (string): How you want to deploy this secret inside the container
# If memory limit is exceeded the Pod goes up for termination, if no # limit is specified there is no upper bound on the amount of memory it can # use. You can also specify a default memory limit on a per-namespace basis limit_memory='100Mi', # If cpu request exceeds your node's capacity, it will fail to ever get # scheduled. The m suffix stands for milli-cpus, therefore .5 cpu and 500m # cpu are equivalent. limit_cpu='500m') # Creates a volume of type emptyDir without any configs volumes = [Volume(name='empty-vol', configs={'emptyDir', {}})] # Used to mount pod level volumes to a running container volume_mounts = [ VolumeMount(name='test-vol-mount', mount_path='/root/mount_file', sub_path='None', read_only=False) ] # Any task you create within the context manager is automatically added to the # DAG object. with models.DAG(dag_id='kubernetes-example', default_args=default_dag_args) as dag: # Only name, namespace, and image are required to create a # KubernetesPodOperator. This operator defaults to using the config file found # at `~/.kube/config` if no `config_file` parameter is specified. kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task task_id='pi_task_full', # Name of task you want to run, used to generate pod id name='pi',
'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, 'retries': 0, 'retry_delay': timedelta(minutes=5), } dag = DAG('random_number_generator', default_args=default_args, schedule_interval='*/5 * * * *', catchup=False) volume_config = {'persistentVolumeClaim': {'claimName': 'file-store'}} volume = Volume(name='file-store', configs=volume_config) volume_mount = VolumeMount('file-store', mount_path='/mnt/file-store', sub_path=None, read_only=False) create_file = KubernetesPodOperator( namespace='airflow', image="debian:9.4", cmds=["bash", "-c"], arguments=[ 'echo $(($RANDOM%10000+10000)) >> /mnt/file-store/random_number-$(date "+%Y.%m.%d-%H.%M.%S").txt' ], name="generate-random-number", task_id="generate-random-number", volumes=[volume], volume_mounts=[volume_mount], in_cluster=True, get_logs=True,
'owner': 'prem', 'depends_on_past': False, 'start_date': datetime.utcnow(), 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, 'retries': 1, 'retry_delay': timedelta(minutes=5) } dag = DAG('3_Step_Model_flow', default_args=default_args, schedule_interval=timedelta(minutes=10)) input_volume_mount = VolumeMount('data-volume', mount_path='/input-dataset', sub_path=None, read_only=False) output_volume_mount = VolumeMount('data-volume', mount_path='/output-dataset', sub_path=None, read_only=False) volume_config = { 'persistentVolumeClaim': { 'claimName': 'airflow-dags' # uses the persistentVolumeClaim given in the Kube yaml } } in_volume = Volume(name='input-dataset', configs=volume_config) out_volume = Volume(name='output-dataset', configs=volume_config)