# key='TEST_DB_PASSWORD') # # test_cloud_sql = KubernetesPodOperator( # **default_pod, # name="test-cloud-sql", # task_id="test_cloud_sql", # secrets=[secret_cloud_sql], # cmds=["python3"], # arguments=["test_cloud_sql.py"] # ) # Finish with something simple finish = DummyOperator(name='finish', task_id='finish', dag=dag) # Docs: https://airflow.apache.org/tutorial.html#setting-up-dependencies # These can be written two ways. # Like this: # start >> hello_world >> test_env_vars >> test_gcs_buckets # test_push_xcom >> [test_xcom_vals_file, test_pull] # finish << test_gcs_buckets # finish << test_xcom_vals_file # Or like this: start.set_downstream(hello_world) hello_world.set_downstream(test_env_vars) # test_env_vars.set_downstream(test_gcs_buckets) # finish.set_upstream(test_gcs_buckets) test_push_xcom.set_downstream([test_xcom_vals_file, test_pull]) finish.set_upstream(test_xcom_vals_file)
schedule_interval=timedelta(minutes=10))
start = DummyOperator(task_id='start', dag=dag)
passing = KubernetesPodOperator(namespace='airflow',
image="python:3.6",
cmds=["python", "-c"],
arguments=["print('hello world')"],
labels={"foo": "bar"},
name="passing-test",
task_id="passing-task",
get_logs=True,
dag=dag)
failing = KubernetesPodOperator(namespace='airflow',
image="ubuntu:16.04",
cmds=["python", "-c"],
arguments=["print('hello world')"],
labels={"foo": "bar"},
name="fail",
task_id="failing-task",
get_logs=True,
dag=dag)
end = DummyOperator(task_id='end', dag=dag)
passing.set_upstream(start)
failing.set_upstream(start)
passing.set_downstream(end)
failing.set_downstream(end)
namespace='default',
name='result-run',
task_id="result-task",
image='ubuntu:latest',
cmds=["ls", "/outputs"],
volumes=[volume],
volume_mounts=[volume_mount],
wait_for_downstream=False,
is_delete_operator_pod=False,
get_logs=True,
dag=dag
)
fsc_node = KubernetesPodOperator(
namespace='default',
image="jataware/fsc_model:0.1",
# cmds=["python", "-c"],
arguments=["0", "1", "0.5"],
labels={"model": "fsc"},
image_pull_policy="Always",
volumes=[volume],
volume_mounts=[volume_mount],
name='fsc-run',
task_id='fsc-task',
is_delete_operator_pod=False,
get_logs=True,
dag=dag
)
fsc_node.set_downstream(result_node)
dag = DAG('dag_that_executes_via_KubernetesPodOperator',
default_args=default_args,
schedule_interval=timedelta(minutes=30),
max_active_runs=1,
concurrency=10)
# Generate 2 tasks
tasks = ["task{}".format(i) for i in range(1, 3)]
example_dag_complete_node = DummyOperator(task_id="example_dag_complete",
dag=dag)
org_dags = []
for task in tasks:
bash_command = 'echo HELLO'
org_node = KubernetesPodOperator(namespace='default',
image="localhost:5000/my-python",
cmds=["python", "-c"],
arguments=["print('HELLO')"],
labels={"foo": "bar"},
image_pull_policy="Always",
name=task,
task_id=task,
is_delete_operator_pod=True,
get_logs=True,
dag=dag)
org_node.set_downstream(example_dag_complete_node)
for i in range(0, int(Variable.get('omm_max_input_count'))):
transform_op = KubernetesPodOperator(
namespace='default',
task_id='omm-transform-{}'.format(i),
image='ubuntu:18.10',
in_cluster=True,
get_logs=True,
cmds=['echo'],
arguments=['science_file_{}'.format(INPUT_FILE)],
volume_mounts=[dags_volume_mount, logs_volume_mount],
volumes=[dags_volume, logs_volume],
name='airflow-omm-transform-pod',
dag=dag)
preview_thumbnail_op = KubernetesPodOperator(
namespace='default',
task_id='omm-transform-preview-thumbnail-{}'.format(i),
image='ubuntu:18.10',
in_cluster=True,
get_logs=True,
cmds=['echo'],
arguments=['preview_and_thumbnail_{}'.format(INPUT_FILE)],
volume_mounts=[dags_volume_mount, logs_volume_mount],
volumes=[dags_volume, logs_volume],
name='airflow-omm-preview-thumbnail-pod',
dag=dag)
transform_op.set_upstream(start_op)
preview_thumbnail_op.set_upstream(transform_op)
preview_thumbnail_op.set_downstream(complete_op)
serve_branch = BranchPythonOperator(task_id="serve_or_not",
python_callable=model_exist,
dag=dag)
t3 = KubernetesPodOperator(
namespace="default",
name="{}-restapi".format(model_name.lower()),
image="tensorflow/serving:latest",
env_vars={
'MODEL_NAME': model_name,
'MODEL_BASE_PATH': '/root/runtime/models'
},
task_id="serve_model",
port=8501,
dag=dag,
async=True,
in_cluster=True,
labels={'name': '{}-restapi'.format(model_name.lower())},
volume_mounts=[volume_mount],
volumes=[volume])
t4 = DummyOperator(task_id="update_version_or_not_serve", dag=dag)
t0.set_downstream(cleanup_branch)
cleanup_branch.set_downstream(t1)
cleanup_branch.set_downstream(t2)
t2.set_downstream(serve_branch)
serve_branch.set_downstream(t3)
serve_branch.set_downstream(t4)
dag = DAG(dag_id=ID, default_args=args, schedule_interval=None)
subpath = '{{ subpath }}'
volume_mount = VolumeMount(name='test-volume',
mount_path='/root/runtime',
sub_path=subpath,
read_only=False)
volume_config = {'persistentVolumeClaim': {'claimName': 'test-volume'}}
volume = Volume(name='test-volume', configs=volume_config)
t1 = KubernetesPodOperator(task_id="media_process",
dag=dag,
in_cluster=True,
volume_mounts=[volume_mount],
namespace='default',
name="{}-pod".format(ID.lower()),
volumes=[volume],
image='opencv:latest',
arguments=['python', '/root/runtime/run_job.py'])
t2 = EmailOperator(task_id='send_output',
to='{{ email }}',
subject='Airflow Dag: {} Output File'.format(ID),
dag=dag,
html_content='',
files=[
'/root/airflow/runtime/{}/{}'.format(subpath, file)
for file in {{output}}
])
t1.set_downstream(t2)