def docker_move_subdag(host_top_dir, input_path, output_path): host_path = f"{host_top_dir}/{input_path}" with DAG("docker_backup_db", default_args=default_args, schedule_interval=timedelta(minutes=10)) as dag: locate_file_cmd = """ sleep 10 find {{params.source_location}} -type f -printf "%f\n" | head -1 """ t_view = BashOperator(task_id="view_file", bash_command=locate_file_cmd, xcom_push=True, params={"source_location": host_path}) def is_data_available(*args, **kwargs): ti = kwargs["ti"] data = ti.xcom_pull(key=None, task_ids="view_file") return data is not None t_is_data_available = ShortCircuitOperator( task_id="check_if_data_available", python_callable=is_data_available) t_move = DockerOperator( api_version="auto", docker_url= "tcp://socat:2375", # replace it with swarm/docker endpoint image="centos:latest", network_mode="bridge", volumes=[ f"{host_path}:{input_path}", f"{host_top_dir}/{input_path}:{output_path}", ], command=[ "/bin/bash", "-c", "/bin/sleep 30; " "/bin/mv {{params.source_location}}/{{ ti.xcom_pull('view_file') }} {{params.target_location}};" "/bin/echo '{{params.target_location}}/{{ ti.xcom_pull('view_file') }}';", ], task_id="move_data", xcom_push=True, params={ "source_location": f"{input_path}", "target_location": f"{output_path}" }, ) print_templated_cmd = """ cat {{ ti.xcom_pull('move_data') }} """ t_print = DockerOperator( api_version="auto", docker_url="tcp://socat:2375", image="centos:latest", volumes=[f"{host_top_dir}/{output_path}:{output_path}"], command=print_templated_cmd, task_id="print", ) t_view.set_downstream(t_is_data_available) t_is_data_available.set_downstream(t_move) t_move.set_downstream(t_print)
t1 = BashOperator( task_id='print_date', bash_command='date', dag=dag) t2 = BashOperator( task_id='sleep', bash_command='sleep 5', retries=3, dag=dag) t3 = DockerOperator(api_version='1.19', docker_url='tcp://localhost:2375', #Set your docker URL command='/bin/sleep 30', image='centos:latest', network_mode='bridge', task_id='docker_op_tester', dag=dag) t4 = BashOperator( task_id='print_hello', bash_command='echo "hello world!!!"', dag=dag) t1.set_downstream(t2) t1.set_downstream(t3) t3.set_downstream(t4)
task_id='task_1', bash_command= 'echo "Starting executor Task 1 | Passed Conf : {{ dag_run.conf["json_executor_task"] }}"', dag=dag) executor = DockerOperator( task_id='executor', image='openjdk:8-jre-alpine', api_version='auto', auto_remove=True, volumes=[ '/usr/local/airflow/artifacts:/usr/local/airflow/artifacts', '/var/run/docker.sock:/var/run/docker.sock' ], docker_url="unix://var/run/docker.sock", network_mode="bridge", environment={ 'VPC_EXECUTOR_TASK': '{{ dag_run.conf["json_executor_task"] }}' }, command= 'java -cp /usr/local/airflow/artifacts/jar-with-dependencies.jar <class>', dag=dag) t2 = BashOperator( task_id='task_2', bash_command='echo "Finishing executor Task 2 | Execution Time : {{ ts }}"', dag=dag) executor.set_upstream(t1) executor.set_downstream(t2)
'start_date': datetime.utcnow(), 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False, 'retries': 1, 'retry_delay': timedelta(minutes=5) } dag = DAG('docker_sample', default_args=default_args, schedule_interval=timedelta(minutes=10)) t1 = BashOperator(task_id='print_date', bash_command='date', dag=dag) t2 = BashOperator(task_id='sleep', bash_command='sleep 5', retries=3, dag=dag) t3 = DockerOperator(api_version='1.21', command='/bin/sleep 30', image='busybox:latest', network_mode='bridge', task_id='docker_op_tester', dag=dag) t4 = BashOperator(task_id='print_hello', bash_command='echo "hello world!!!"', dag=dag) t1.set_downstream(t2) t1.set_downstream(t3) t3.set_downstream(t4)
'retry_delay': timedelta(minutes=5) } dag = DAG('docker_sample', default_args=default_args, schedule_interval=timedelta(minutes=10)) t1 = DockerOperator(api_version='1.19', command='/bin/sleep 30', image='centos:latest', network_mode='bridge', task_id='docker_op_tester', dag=dag) t2 = DockerOperator(api_version='1.19', command='/bin/sleep 30', image='centos:latest', network_mode='bridge', task_id='docker_op_tester', dag=dag) t3 = DockerOperator(api_version='1.19', command='/bin/sleep 30', image='centos:latest', network_mode='bridge', task_id='docker_op_tester', dag=dag) t1.set_downstream(t2) t1.set_downstream(t3)