def create_dag(data_marker_config, default_args): dag = DAG( dag_id="data_marker_pipeline", schedule_interval=datetime.timedelta(days=1), default_args=default_args, start_date=YESTERDAY, ) with dag: before_start = PythonOperator( task_id="data_marking_start", python_callable=data_marking_start, op_kwargs={}, ) before_start for source in data_marker_config.keys(): filter_by_config = data_marker_config.get(source) language = filter_by_config.get("language").lower() print(f"Language for source is {language}") data_marker_task = kubernetes_pod_operator.KubernetesPodOperator( task_id=f"data-marker-{source}", name="data-marker", cmds=[ "python", "invocation_script.py", "-b", bucket_name, "-a", "data_marking", "-rc", f"data/audiotospeech/config/config.yaml", "-as", source, "-fb", json.dumps(filter_by_config), "-l", language, ], namespace=composer_namespace, startup_timeout_seconds=300, secrets=[secret_file], image= f"us.gcr.io/ekstepspeechrecognition/ekstep_data_pipelines:{env_name}_1.0.0", image_pull_policy="Always", ) before_start >> data_marker_task return dag
def gen_kubernets_operator_python(input_path, output_path, exec_file_name, requirement_file_path, task_name): return kubernetes_pod_operator.KubernetesPodOperator( task_id=task_name, name=task_name, namespace='airflowsys', in_cluster=True, env_vars={ 'INPUT_PATH': str(input_path), 'OUTPUT_PATH': output_path, 'EXEC_FILE_NAME': str(exec_file_name), 'REQUIREMENT_FILE_PATH': str(requirement_file_path) }, is_delete_operator_pod=True, image="reg.sfai.ml/library/airflow/python:3.7-slim-buster")
def create_dag(dag_id, dag_number, default_args, args, batch_count): dag = DAG( dag_id, schedule_interval=datetime.timedelta(days=1), default_args=default_args, start_date=YESTERDAY, ) with dag: audio_format = args.get("audio_format") language = args.get("language") print(args) print(f"Language for source is {language}") source_path_for_snr_set = interpolate_language_paths(language) get_file_path_from_gcp_bucket = PythonOperator( task_id=dag_id + "_get_file_path", python_callable=get_file_path_from_bucket, op_kwargs={ "source": dag_id, "source_landing_path": source_path_for_snr_set, "batch_count": batch_count, "audio_format": audio_format, "meta_file_extension": meta_file_extension, "bucket_name": bucket_name, }, dag_number=dag_number, ) get_file_path_from_gcp_bucket parallelism = args.get("parallelism") file_path_list = json.loads(Variable.get("audiofilelist"))[dag_id] if len(file_path_list) > 0: chunk_size = math.ceil(len(file_path_list) / parallelism) batches = [ file_path_list[i:i + chunk_size] for i in range(0, len(file_path_list), chunk_size) ] data_prep_cataloguer = kubernetes_pod_operator.KubernetesPodOperator( task_id="data-catalogure", name="data-catalogure", cmds=[ "python", "invocation_script.py", "-b", bucket_name, "-a", "audio_cataloguer", "-rc", f"data/audiotospeech/config/config.yaml", ], namespace=composer_namespace, startup_timeout_seconds=300, secrets=[secret_file], image= f"us.gcr.io/ekstepspeechrecognition/ekstep_data_pipelines:{env_name}_1.0.0", image_pull_policy="Always", resources=resource_limits, ) else: batches = [] for batch_file_path_list in batches: data_prep_task = kubernetes_pod_operator.KubernetesPodOperator( task_id=dag_id + "_data_snr_" + batch_file_path_list[0], name="data-prep-snr", cmds=[ "python", "invocation_script.py", "-b", bucket_name, "-a", "audio_processing", "-rc", f"data/audiotospeech/config/config.yaml", "-fl", ",".join(batch_file_path_list), "-af", args.get("audio_format"), "-as", dag_id, "-l", language, ], namespace=composer_namespace, startup_timeout_seconds=300, secrets=[secret_file], image= f"us.gcr.io/ekstepspeechrecognition/ekstep_data_pipelines:{env_name}_1.0.0", image_pull_policy="Always", resources=resource_limits, ) get_file_path_from_gcp_bucket >> data_prep_task >> data_prep_cataloguer return dag
# no `config_file` parameter is specified. By default it will contain the # credentials for Cloud Composer's Google Kubernetes Engine cluster that is # created upon environment creation. # [START composer_kubernetespodoperator_minconfig] kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id='pod-ex-minimum', # Name of task you want to run, used to generate Pod ID. name='pod-ex-minimum', # Entrypoint of the container, if not specified the Docker container's # entrypoint is used. The cmds parameter is templated. cmds=['echo'], # The namespace to run within Kubernetes, default namespace is # `default`. There is the potential for the resource starvation of # Airflow workers and scheduler within the Cloud Composer environment, # the recommended solution is to increase the amount of nodes in order # to satisfy the computing requirements. Alternatively, launching pods # into a custom namespace will stop fighting over resources. namespace='default', # Docker image specified. Defaults to hub.docker.com, but any fully # qualified URLs will point to a custom repository. Supports private # gcr.io images if the Composer Environment is under the same # project-id as the gcr.io images and the service account that Composer # uses has permission to access the Google Container Registry # (the default service account has permission) image='gcr.io/gcp-runtimes/ubuntu_18_0_4') # [END composer_kubernetespodoperator_minconfig] # [START composer_kubernetespodoperator_templateconfig] kubenetes_template_ex = kubernetes_pod_operator.KubernetesPodOperator( task_id='ex-kube-templates', name='ex-kube-templates',
dag_id="kube-pod-operator-cluster", start_date=airflow.utils.dates.days_ago(2), schedule_interval="@daily", ) start_kube_process = BashOperator( task_id="start_kube_process", bash_command="echo upload to s3", dag=dag, ) in_cluster = True kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='pod-ex-minimum', name='pod-ex-minimum', cmds=['echo'], namespace='default', image='ubuntu:latest', in_cluster=in_cluster, executor_config={"LocalExecutor": {}}) run_another_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='run-another-pod', name='run-another-pod', cmds=['echo'], namespace='default', image='ubuntu:latest', in_cluster=in_cluster, executor_config={"LocalExecutor": {}}) start_kube_process >> kubernetes_min_pod >> run_another_pod
# If a Pod fails to launch, or has an error occur in the container, Airflow # will show the task as failed, as well as contain all of the task logs # required to debug. with models.DAG( dag_id="data_prep_cataloguer_pipeline", schedule_interval=datetime.timedelta(days=1), default_args=default_args, start_date=YESTERDAY, ) as dag: kubernetes_list_bucket_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id="data-normalizer", name="data-normalizer", cmds=[ "python", "invocation_script.py", "-b", bucket_name, "-a", "audio_cataloguer", "-rc", f"data/audiotospeech/config/config.yaml", ], namespace=composer_namespace, startup_timeout_seconds=300, secrets=[secret_file], image= f"us.gcr.io/ekstepspeechrecognition/ekstep_data_pipelines:{env_name}_1.0.0", image_pull_policy="Always", )
# created upon environment creation. # [START composer_kubernetespodoperator_minconfig] kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id='pod-ex-minimum', # Name of task you want to run, used to generate Pod ID. name='pod-ex-minimum', # Entrypoint of the container, if not specified the Docker container's # entrypoint is used. The cmds parameter is templated. cmds=["bash", "-cx"], arguments=["echo", "10"], # The namespace to run within Kubernetes, default namespace is # `default`. There is the potential for the resource starvation of # Airflow workers and scheduler within the Cloud Composer environment, # the recommended solution is to increase the amount of nodes in order # to satisfy the computing requirements. Alternatively, launching pods # into a custom namespace will stop fighting over resources. namespace='default', # Docker image specified. Defaults to hub.docker.com, but any fully # qualified URLs will point to a custom repository. Supports private # gcr.io images if the Composer Environment is under the same # project-id as the gcr.io images and the service account that Composer # uses has permission to access the Google Container Registry # (the default service account has permission) is_delete_operator_pod=True, image_pull_policy="Always", get_logs=True, image='ubuntu:16.04') # [END composer_kubernetespodoperator_minconfig] # [END composer_kubernetespodoperator]
bash_command="echo upload to s3", dag=dag, ) kube_files_volume_config = {'name': 'minikube'} kube_files_volume = k8s.V1Volume(name='kube-files-volume', config_map=kube_files_volume_config) kube_files_volume_mount = k8s.V1VolumeMount(name='kube-files-volume', mount_path='/tmp/k8s') in_cluster = False kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='pod-ex-minimum', name='pod-ex-minimum', cmds=['echo'], namespace='default', image='ubuntu:latest', in_cluster=in_cluster, config_file='/tmp/k8s/minikube/config', volumes=[kube_files_volume], volume_mounts=[kube_files_volume_mount]) run_another_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='run-another-pod', name='run-another-pod', cmds=['echo'], namespace='default', image='ubuntu:latest', in_cluster=in_cluster, config_file='/tmp/k8s/minikube/config') start_kube_process >> kubernetes_min_pod >> run_another_pod
"name": "EOD_DATE", "value": "{{ ds }}" }] k8s_spark_launcher = kubernetes_pod_operator.KubernetesPodOperator( task_id='k8s-spark-launcher', name='k8s-spark-launcher', cmds=['/opt/spark/bin/spark-submit'], arguments=[ '--master', 'k8s://https://kubernetes.default.svc.cluster.local:443', '--deploy-mode', 'cluster', '--name', 'hello', '--class', 'demo.Count', '--conf', 'spark.executor.instances=2', '--conf', 'spark.kubernetes.executor.request.cores=0.4', '--conf', 'spark.kubernetes.driver.request.cores=0.2', '--conf', 'spark.kubernetes.container.image=skhatri/spark-k8s-hello:1.0.7', '--conf', 'spark.jars.ivy=/tmp/.ivy', '--conf', 'spark.app.name=hello', '--conf', 'spark.kubernetes.authenticate.driver.serviceAccountName=job-trigger-sa', '--conf', 'spark.authenticate=false', 'local:///tmp/jars/spark-k8s-hello.jar' ], namespace='default', image='skhatri/spark:v3.0.1', in_cluster=in_cluster, env_vars=spark_env_vars, service_account_name='job-trigger-sa', executor_config={"LocalExecutor": {}}) k8s_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='k8s-pod', name='k8s-pod',
"retry_delay": timedelta(minutes=5) } slack_token = BaseHook.get_connection("slack_conn").password with DAG(dag_id="not_so_dangerous_dag", schedule_interval="@daily", start_date=datetime(2020, 11, 1), default_args=default_args, catchup=True) as dag: tasks = [ kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id=f'kubernetes_task_{i}', # Name of task you want to run, used to generate Pod ID. name=f'kubernetes_task_{i}', image='busybox', cmds=['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 30'], in_cluster=True, is_delete_operator_pod=True, namespace='default') for i in range(10) ] sending_slack_notification = SlackWebhookOperator( task_id='sending_slack', http_conn_id='slack_conn', webhook_token=slack_token, message="Esta todo bien desde Kubernetes! \n Ahora toma un gatito! " "https://www.youtube.com/watch?v=J---aiyznGQ", username='******', icon_url='https://raw.githubusercontent.com/apache/' 'airflow/master/airflow/www/static/pin_100.png',
from airflow.operators.bash_operator import BashOperator from airflow.contrib.operators import kubernetes_pod_operator from airflow.utils.dates import days_ago with models.DAG( "example_pod_operator_xcom", schedule_interval=None, # Override to match your needs start_date=days_ago(1), tags=['example'], ) as dag: # [START howto_operator_gke_start_pod_xcom] pod_task_xcom = kubernetes_pod_operator.KubernetesPodOperator( task_id="pod_task_xcom", do_xcom_push=True, namespace="default", image="alpine", cmds=["sh", "-c", 'mkdir -p /airflow/xcom/;echo \'[1,2,3,4]\' > /airflow/xcom/return.json'], name="test-pod-xcom", ) # [END howto_operator_gke_start_pod_xcom] # [START howto_operator_gke_xcom_result] pod_task_xcom_result = BashOperator( bash_command="echo \"{{ task_instance.xcom_pull('pod_task_xcom')[0] }}\"", task_id="pod_task_xcom_result", ) # [END howto_operator_gke_xcom_result] pod_task_xcom >> pod_task_xcom_result
with DAG(dag_name, default_args=default_args, schedule_interval=datetime.timedelta(days=1)) as dag: start = BashOperator( task_id='start', bash_command='date', ) alphavantage_path = dag_name + '/alphavantage.json' alphavantage = kubernetes_pod_operator.KubernetesPodOperator( task_id='alphavantage', name='alphavantage', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; AlphaVantageData(BUCKET_PATH_PUSH='{}')" .format(alphavantage_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) fred_path = dag_name + '/fred.json' fred = kubernetes_pod_operator.KubernetesPodOperator( task_id='fred', name='fred', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; FRED(BUCKET_PATH_PUSH='{}')". format(fred_path)
# -*- coding: utf-8 -*- import datetime from airflow import models from airflow.contrib.kubernetes import secret from airflow.contrib.operators import kubernetes_pod_operator from utils.kubernetes import Tolerations, Affinity YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) with models.DAG(dag_id='composer_kubernetes_pod_simple', schedule_interval=None, start_date=YESTERDAY) as dag: kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='pod-workshop-simple', name='pod-workshop-simple', cmds=['echo', '"Hello world"'], namespace='default', resources={ 'request_memory': '128Mi', 'request_cpu': '500m', 'limit_memory': '500Mi', 'limit_cpu': 1 }, image='gcr.io/gcp-runtimes/ubuntu_18_0_4', tolerations=Tolerations.default, affinity=Affinity.memory_heavy)
from airflow import DAG from datetime import datetime, timedelta from airflow.contrib.operators import kubernetes_pod_operator default_args = { 'owner': 'Damavis', 'start_date': datetime(2020, 5, 5), 'retries': 1, 'retry_delay': timedelta(seconds=5) } with DAG('etl_dag', default_args=default_args, schedule_interval=None) as dag: extract_tranform = kubernetes_pod_operator.KubernetesPodOperator( namespace='airflow', image="python:3.7-slim", cmds=["echo"], arguments=["This can be the extract part of an ETL"], labels={"foo": "bar"}, name="extract-tranform", task_id="extract-tranform", get_logs=True ) extract_tranform
configuring_init_container = k8s.V1Container( name='secret-resolver', image='ubuntu:latest', command=['sh', '-c', 'echo {"user": "******", "password": "******"}|tee /tmp/out/hello'], volume_mounts=[ data_write_secrets_mount ], ) data_read_for_secrets_mount=k8s.V1VolumeMount(name='data-shared-dir', mount_path='/tmp/secrets') k8s_pod_with_init = kubernetes_pod_operator.KubernetesPodOperator( task_id='k8s-pod-with-init', name='k8s-pod', cmds=['sh', '-c', 'cat /tmp/secrets/hello'], namespace='default', image='ubuntu:latest', in_cluster=in_cluster, config_file='/tmp/k8s/minikube/config', init_containers=[configuring_init_container], volumes=[data_share_volume], volume_mounts=[ data_read_for_secrets_mount ] ) k8s_init_pod_dag_start >> k8s_pod_with_init >> k8s_init_pod_dag_finish
kubernetes_full_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='ex-all-configs', name='pi', namespace='airflow', image='perl', # Entrypoint of the container, if not specified the Docker container's # entrypoint is used. The cmds parameter is templated. cmds=['perl'], # Arguments to the entrypoint. The docker image's CMD is used if this # is not provided. The arguments parameter is templated. arguments=['-Mbignum=bpi', '-wle', 'print bpi(2000)'], # The secrets to pass to Pod, the Pod will fail to create if the # secrets you specify in a Secret object do not exist in Kubernetes. secrets=[], # Labels to apply to the Pod. labels={'pod-label': 'label-name'}, # Timeout to start up the Pod, default is 120. startup_timeout_seconds=120, # The environment variables to be initialized in the container # env_vars are templated. env_vars={'EXAMPLE_VAR': '/example/value'}, # If true, logs stdout output of container. Defaults to True. get_logs=True, # Determines when to pull a fresh image, if 'IfNotPresent' will cause # the Kubelet to skip pulling an image if it already exists. If you # want to always pull a new image, set it to 'Always'. image_pull_policy='Always', # Annotations are non-identifying metadata you can attach to the Pod. # Can be a large range of data, and can include characters that are not # permitted by labels. annotations={'key1': 'value1'}, # Resource specifications for Pod, this will allow you to set both cpu # and memory limits and requirements. #resources=pod.Resources(), # Specifies path to kubernetes config. If no config is specified will # default to '~/.kube/config'. The config_file is templated. #config_file='/home/airflow/composer_kube_config', # If true, the content of /airflow/xcom/return.json from container will # also be pushed to an XCom when the container ends. xcom_push=False, in_cluster=True, # List of Volume objects to pass to the Pod. # List of VolumeMount objects to pass to the Pod. # Affinity determines which nodes the Pod can run on based on the # config. For more information see: # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ dag=dag)
from airflow import models from airflow.contrib.operators import kubernetes_pod_operator from airflow.operators import dummy_operator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) volume_mount = VolumeMount('test-volume', mount_path='/root/mount_file', sub_path=None, read_only=True) volume_config = {'persistentVolumeClaim': {'claimName': 'test-volume'}} volume = Volume(name='test-volume', configs=volume_config) with models.DAG(dag_id='airflow-demo', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) as dag: task1 = kubernetes_pod_operator.KubernetesPodOperator( task_id='t1', name='task1', namespace='airflow', image='eu.gcr.io/taiyo-239217/dag:fae4887', arguments=["AlphaVantage()"], volume=[], volume_mounts=[], in_cluster=True, xcom_push=True, is_delete_operator_pod=True)
) t6 = BashOperator( task_id='t6', depends_on_past=False, bash_command='echo start t6', dag=dag, ) t7 = BashOperator( task_id='t7', depends_on_past=False, bash_command='echo start t7', dag=dag, ) t8 = kubernetes_pod_operator.KubernetesPodOperator( task_id="t8", name="podtest", cmds=['echo','hello'], namespace='default', in_cluster=True, image='gcr.io/gcp-runtimes/ubuntu_18_0_4', dag=dag, ) t1 >> [t2, t3] t2 >> t4 t3 >> [t5, t6] [t6, t4] >> t7 [t5, t7] >> t8
from airflow import models from airflow.contrib.kubernetes import secret from airflow.contrib.operators import kubernetes_pod_operator from airflow.contrib.kubernetes.pod import Resources YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) with models.DAG(dag_id='demo_3', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) as dag: meetup_munge = kubernetes_pod_operator.KubernetesPodOperator( task_id='meetupmunge', name='meetupmuge', cmds=['python', 'munge.py'], namespace='default', image='brandonwatts/rvade-meetupmunge:latest', image_pull_policy='Always', xcom_push=True, get_logs=True) pscreatewebsite = kubernetes_pod_operator.KubernetesPodOperator( task_id='pscreatewebsite', name='pscreatewebsite', cmds=[ 'pwsh', './create_site.ps1', "{{ task_instance.xcom_pull(task_ids='meetupmunge', key='return_value')}}" ], namespace='default', image='brandonwatts/rvade-pscreatewebsite:latest', image_pull_policy='Always',
YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) default_args = { 'owner': 'Trigger', 'depends_on_past': False, 'email': [''], 'email_on_failure': False, 'email_on_retry': False, 'retries': 1, 'retry_delay': datetime.timedelta(minutes=5), } # If a Pod fails to launch, or has an error occur in the container, Airflow # will show the task as failed, as well as contain all of the task logs # required to debug. with models.DAG( dag_id='k8s_pod_operator_grid', default_args=default_args, #schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY, schedule_interval=None, ) as dag: kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='pod-grid', name='pod-grid', namespace='default', image='rodriguesflavio/poc-pubsub-ok4')
import datetime from airflow import models from airflow.contrib.operators import kubernetes_pod_operator from airflow.operators import dummy_operator YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) with models.DAG(dag_id='demo', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) as dag: task1 = kubernetes_pod_operator.KubernetesPodOperator( task_id='t1', name='task1', namespace='airflow', image='eu.gcr.io/taiyo-239217/dag:fae4886', arguments=["AlphaVantage()"], in_cluster=True, xcom_push=True, is_delete_operator_pod=True) task2 = kubernetes_pod_operator.KubernetesPodOperator( task_id='t2', name='task2', namespace='airflow', image='eu.gcr.io/taiyo-239217/dag:fae4886', arguments=["FRED()"], in_cluster=True, xcom_push=True, is_delete_operator_pod=True)
import datetime from airflow import models from airflow.contrib.operators import kubernetes_pod_operator from airflow.contrib.kubernetes.volume import Volume from airflow.contrib.kubernetes.volume_mount import VolumeMount YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) dag = models.DAG(dag_id='k8s-pod-ex-minimum', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id='pod-ex-minimum', # Name of task you want to run, used to generate Pod ID. dag=dag, name='pod-ex-minimum', cmds=['echo'], config_file="/usr/local/airflow/etc/kube.config", namespace='airflow', image='ubuntu:16.04')
dag_id='composer_sample_kubernetes_pod', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) as dag: # Only name, namespace, image, and task_id are required to create a # KubernetesPodOperator. In Cloud Composer, currently the operator defaults # to using the config file found at `/home/airflow/composer_kube_config if # no `config_file` parameter is specified. By default it will contain the # credentials for Cloud Composer's Google Kubernetes Engine cluster that is # created upon environment creation. kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id='pod-ex-minimum', # Name of task you want to run, used to generate Pod ID. name='pod-ex-minimum', # Entrypoint of the container, if not specified the Docker container's # entrypoint is used. The cmds parameter is templated. cmds=['echo'], # The namespace to run within Kubernetes, default namespace is # `default`. There is the potential for the resource starvation of # Airflow workers and scheduler within the Cloud Composer environment, # the recommended solution is to increase the amount of nodes in order # to satisfy the computing requirements. Alternatively, launching pods # into a custom namespace will stop fighting over resources. namespace='default', # Docker image specified. Defaults to hub.docker.com, but any fully # qualified URLs will point to a custom repository. Supports private # gcr.io images if the Composer Environment is under the same # project-id as the gcr.io images. image='gcr.io/gcp-runtimes/ubuntu_16_0_4')
source_2_task = BashOperator( bash_command="echo \"Pretending to be source 2\"; sleep 20", task_id="source_2", ) processing_tasks = [] for i in range(1, 11): processing_tasks.append( kubernetes_pod_operator.KubernetesPodOperator( task_id='3Gi-task_{}'.format(i), name='3Gi-task-{}'.format(i), cmds=[ "sh", "-c", 'echo \'Sleeping..\'; sleep 120; echo \'Done!\'' ], namespace='default', resources={ 'request_memory': '3Gi', 'request_cpu': '200m', 'limit_memory': '3Gi', 'limit_cpu': 1 }, image='gcr.io/gcp-runtimes/ubuntu_18_0_4', tolerations=Tolerations.default, affinity=Affinity.memory_heavy, startup_timeout_seconds=300)) start_task >> source_1_task >> start_processing start_task >> source_2_task >> start_processing for task in processing_tasks: start_processing >> task >> end_task
def create_dag(dag_name, config_path): global default_args dag = DAG(dag_name, default_args=default_args, schedule_interval=None) with dag: start = BashOperator( task_id='start', bash_command='date', ) alphavantage_path = dag_name + '/alphavantage.json' alphavantage = kubernetes_pod_operator.KubernetesPodOperator( task_id='alphavantage', name='alphavantage', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; AlphaVantageData(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}')" .format(config_path, alphavantage_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) fred_path = dag_name + '/fred.json' fred = kubernetes_pod_operator.KubernetesPodOperator( task_id='fred', name='fred', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; FRED(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}')" .format(config_path, fred_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) techfeatures_path = dag_name + '/techfeatures.json' techfeatures = kubernetes_pod_operator.KubernetesPodOperator( task_id='techfeatures', name='techfeatures', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; TechFeatures(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL='{}')" .format(config_path, techfeatures_path, alphavantage_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) dataaggregation_path = dag_name + '/dataaggregation.json' dataaggregation = kubernetes_pod_operator.KubernetesPodOperator( task_id='dataaggregation', name='dataaggregation', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; DataAggregation(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL_1='{}', BUCKET_PATH_PULL_2='{}')" .format(config_path, dataaggregation_path, techfeatures_path, fred_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) inference_path = dag_name + '/inference.json' inference = kubernetes_pod_operator.KubernetesPodOperator( task_id='inference', name='inference', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; Inference(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL='{}')" .format(config_path, inference_path, dataaggregation_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) uncertainitybounds_path = dag_name + '/uncertainitybounds.json' uncertainitybounds = kubernetes_pod_operator.KubernetesPodOperator( task_id='uncertainitybounds', name='uncertainitybounds', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; UncertainityBounds(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL='{}')" .format(config_path, uncertainitybounds_path, inference_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) resultsgen_path = dag_name + '/resultsgen.json' resultsgen = kubernetes_pod_operator.KubernetesPodOperator( task_id='resultsgen', name='resultsgen', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; ResultsGen(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL='{}')" .format(config_path, resultsgen_path, inference_path) ], startup_timeout_seconds=900, is_delete_operator_pod=True) tradecards_path = dag_name + '/tradecards.json' tradecards = kubernetes_pod_operator.KubernetesPodOperator( task_id='tradecards', name='tradecards', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; TradeCards(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL_1='{}', BUCKET_PATH_PULL_2='{}', RunTime='{}')" .format(config_path, tradecards_path, uncertainitybounds_path, resultsgen_path, run_time) ], startup_timeout_seconds=900, is_delete_operator_pod=True) mrm_path = dag_name + '/mrm.json' mrm = kubernetes_pod_operator.KubernetesPodOperator( task_id='mrm', name='mrm', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; MRM(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL='{}', RunTime='{}')" .format(config_path, mrm_path, resultsgen_path, run_time) ], startup_timeout_seconds=900, is_delete_operator_pod=True) simtables_path = dag_name + '/simtables.json' dirtables_path = dag_name + '/dirtables.json' clsmatrix_path = dag_name + '/clsmatrix.json' simtables = kubernetes_pod_operator.KubernetesPodOperator( task_id='simtables', name='simtables', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; SimTables(CONFIG_PATH='{}', BUCKET_PATH_PUSH_1='{}', BUCKET_PATH_PUSH_2='{}', BUCKET_PATH_PUSH_3='{}', BUCKET_PATH_PULL='{}', RunTime='{}')" .format(config_path, simtables_path, dirtables_path, clsmatrix_path, resultsgen_path, run_time) ], startup_timeout_seconds=900, is_delete_operator_pod=True) mts_path = dag_name + '/mts.json' mts = kubernetes_pod_operator.KubernetesPodOperator( task_id='mts', name='mts', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; MTS(CONFIG_PATH='{}', BUCKET_PATH_PUSH='{}', BUCKET_PATH_PULL='{}', RunTime='{}')" .format(config_path, mts_path, resultsgen_path, run_time) ], startup_timeout_seconds=900, is_delete_operator_pod=True) publishpostgress = kubernetes_pod_operator.KubernetesPodOperator( task_id='publishpostgress', name='publishpostgress', namespace=namespace, image=image, arguments=[ "from pipeline_abstractions import *; PublishPostgress()" ], startup_timeout_seconds=900, is_delete_operator_pod=True) end = BashOperator( task_id='end', bash_command='date', ) start >> [fred, alphavantage] techfeatures << [fred, alphavantage] dataaggregation << techfeatures inference << dataaggregation [uncertainitybounds, resultsgen] << inference tradecards << [uncertainitybounds, resultsgen] mrm << resultsgen simtables << resultsgen mts << resultsgen publishpostgress << [tradecards, mrm, simtables, mts] end << publishpostgress return dag
import datetime from airflow import models from airflow.contrib.operators import kubernetes_pod_operator YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1) dag = models.DAG(dag_id='a_pod-ex-minimum', schedule_interval=datetime.timedelta(days=1), start_date=YESTERDAY) kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator( task_id='pod-ex-minimum', dag=dag, name='pod-ex-minimum', cmds=['echo', 'hello'], #config_file="/var/airflow/secrets/kubeconfig/kube.config", namespace='airflow', image='ubuntu:16.04')
# setattr(pod_res, 'limit_cpu', None) # An instance of an operator is called a task. In this case, the # hello_python task calls the "greeting" Python function. scale_down = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id='node-scale_down', # Name of task you want to run, used to generate Pod ID. name='scale-down', # resources=pod_res, # Entrypoint of the container, if not specified the Docker container's # entrypoint is used. The cmds parameter is templated. cmds=["echo", "I am here to scale down"], resources=pod_res, # The namespace to run within Kubernetes, default namespace is # `default`. There is the potential for the resource starvation of # Airflow workers and scheduler within the Cloud Composer environment, # the recommended solution is to increase the amount of nodes in order # to satisfy the computing requirements. Alternatively, launching pods # into a custom namespace will stop fighting over resources. namespace='bs4-app', is_delete_operator_pod=True, affinity=affinity_values, config_file='/home/airflow/composer_kube_config', # Docker image specified. Defaults to hub.docker.com, but any fully # qualified URLs will point to a custom repository. Supports private # gcr.io images if the Composer Environment is under the same # project-id as the gcr.io images. image='alpine:latest') scale_down
"email_on_retry": False, "email": "*****@*****.**", "retries": 1, "retry_delay": timedelta(minutes=5) } slack_token = BaseHook.get_connection("slack_conn").password with DAG(dag_id="kubernetes_dag", schedule_interval="*/5 * * * *", default_args=default_args, catchup=False) as dag: say_hello_to_my_little_friend = kubernetes_pod_operator.KubernetesPodOperator( # The ID specified for the task. task_id='kubernetes_task', # Name of task you want to run, used to generate Pod ID. name='kubernetes_task', image='busybox', is_delete_operator_pod=True, cmds=['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 30'], in_cluster=True, namespace='default' ) sending_slack_notification = SlackWebhookOperator( task_id='sending_slack', http_conn_id='slack_conn', webhook_token=slack_token, message="Esta todo bien desde Kubernetes! \n Ahora toma un gatito! " "https://www.youtube.com/watch?v=J---aiyznGQ", username='******', icon_url='https://raw.githubusercontent.com/apache/' 'airflow/master/airflow/www/static/pin_100.png',
def build_kubernetes_pod_operator(operator_ref, dag_ref): """ Builds a DAG operator of type: KubernetesPodOperator. Args: operator_ref (string): the definition of the operator dag_ref (string): the reference to the dag to associate this operator """ op = kubernetes_pod_operator.KubernetesPodOperator( task_id=operator_ref['task_id'], name=operator_ref['name'], image=operator_ref['image'], namespace=operator_ref['namespace'] if 'namespace' in operator_ref else 'default', dag=dag_ref) # populate non-default operator values if 'cmds' in operator_ref: op.cmds = operator_ref['cmds'] if 'arguments' in operator_ref: op.arguments = operator_ref['arguments'] if 'env_vars' in operator_ref: op.env_vars = operator_ref['env_vars'] if 'labels' in operator_ref: op.env_vars = operator_ref['labels'] if 'startup_timeout_seconds' in operator_ref: op.startup_timeout_seconds = operator_ref['startup_timeout_seconds'] if 'ports' in operator_ref: op.ports = operator_ref['ports'] if 'params' in operator_ref: op.params = operator_ref['params'] if 'node_selectors' in operator_ref: op.node_selectors = operator_ref['node_selectors'] if 'resources' in operator_ref: op.resources = operator_ref['resources'] if 'config_file' in operator_ref: op.config_file = operator_ref['config_file'] if 'annotations' in operator_ref: op.annotations = operator_ref['annotations'] if 'volumes' in operator_ref: op.volumes = operator_ref['volumes'] if 'volume_mounts' in operator_ref: op.volumes = operator_ref['volume_mounts'] if 'affinity' in operator_ref: op.affinity = operator_ref['affinity'] if 'configmaps' in operator_ref: op.configmaps = operator_ref['configmaps'] # define pod secrets pod_secrets = [] if 'pod_secret_refs' in operator_ref: for pod_secret in operator_ref['pod_secret_refs']: if not list(find_key_in_dict('kubernetes_secrets', payload)): raise ValueError( f"Pod {operator_ref['name']} declares 'pod_secret_refs' but 'kubernetes_secrets' has not been defined." ) secret_entry_ref = payload['kubernetes_secrets'][pod_secret] secret_entry = secret.Secret( # Deploy type: 'env' for environment variable or 'volume' deploy_type=secret_entry_ref['deploy_type'], # The name of the environment variable or the path of the volume deploy_target=secret_entry_ref['deploy_target'], # Name of the Kubernetes Secret secret=secret_entry_ref['secret'], # Key of a secret stored in this Secret object or key in the form of service account file name key=secret_entry_ref['key']) pod_secrets.append(secret_entry) op.secrets = pod_secrets if 'image_pull_policy' in operator_ref: op.image_pull_policy = operator_ref['image_pull_policy'] # define pull secrets image_pull_secrets = [] if 'image_pull_secret_refs' in operator_ref: for image_pull_secret in operator_ref['image_pull_secret_refs']: if not list(find_key_in_dict('kubernetes_secrets', payload)): raise ValueError( f"Pod {operator_ref['name']} declares 'image_pull_secret_refs' but 'kubernetes_secrets' has not been defined." ) secret_entry_ref = payload['kubernetes_secrets'][ image_pull_secret] secret_entry = secret.Secret( # Deploy type: 'env' for environment variable or 'volume' deploy_type=secret_entry_ref['deploy_type'], # The name of the environment variable or the path of the volume deploy_target=secret_entry_ref['deploy_target'], # Name of the Kubernetes Secret secret=secret_entry_ref['secret'], # Key of a secret stored in this Secret object or key in the form of service account file name key=secret_entry_ref['key']) image_pull_secrets.append(secret_entry) op.image_pull_secrets = image_pull_secrets return operator
default_args = {'start_date': YESTERDAY} dag = airflow.DAG('simple_workflow_dag', default_args=default_args, schedule_interval=None) bash_operator_task = bash_operator.BashOperator( task_id='bash_operator_example_task', bash_command='echo "Hello from Airflow Bash Operator"', dag=dag) def python_operator_func(): print("Hello from Airflow Python Operator") python_operator_task = python_operator.PythonOperator( task_id='python_operator_example_task', python_callable=python_operator_func, dag=dag) kubernetes_pod_operator_task = kubernetes_pod_operator.KubernetesPodOperator( task_id='k8s_pod_operator_example_task', name='k8s_pod_example', namespace='default', image='bash', cmds=['echo'], arguments=['"Hello from Airflow Kubernetes Pod Operator"'], dag=dag)