def docker_move_subdag(host_top_dir, input_path, output_path):
host_path = f"{host_top_dir}/{input_path}"
with DAG("docker_backup_db",
default_args=default_args,
schedule_interval=timedelta(minutes=10)) as dag:
locate_file_cmd = """
sleep 10
find {{params.source_location}} -type f -printf "%f\n" | head -1
"""
t_view = BashOperator(task_id="view_file",
bash_command=locate_file_cmd,
xcom_push=True,
params={"source_location": host_path})
def is_data_available(*args, **kwargs):
ti = kwargs["ti"]
data = ti.xcom_pull(key=None, task_ids="view_file")
return data is not None
t_is_data_available = ShortCircuitOperator(
task_id="check_if_data_available",
python_callable=is_data_available)
t_move = DockerOperator(
api_version="auto",
docker_url=
"tcp://socat:2375", # replace it with swarm/docker endpoint
image="centos:latest",
network_mode="bridge",
volumes=[
f"{host_path}:{input_path}",
f"{host_top_dir}/{input_path}:{output_path}",
],
command=[
"/bin/bash",
"-c",
"/bin/sleep 30; "
"/bin/mv {{params.source_location}}/{{ ti.xcom_pull('view_file') }} {{params.target_location}};"
"/bin/echo '{{params.target_location}}/{{ ti.xcom_pull('view_file') }}';",
],
task_id="move_data",
xcom_push=True,
params={
"source_location": f"{input_path}",
"target_location": f"{output_path}"
},
)
print_templated_cmd = """
cat {{ ti.xcom_pull('move_data') }}
"""
t_print = DockerOperator(
api_version="auto",
docker_url="tcp://socat:2375",
image="centos:latest",
volumes=[f"{host_top_dir}/{output_path}:{output_path}"],
command=print_templated_cmd,
task_id="print",
)
t_view.set_downstream(t_is_data_available)
t_is_data_available.set_downstream(t_move)
t_move.set_downstream(t_print)
t1 = BashOperator(
task_id='print_date',
bash_command='date',
dag=dag)
t2 = BashOperator(
task_id='sleep',
bash_command='sleep 5',
retries=3,
dag=dag)
t3 = DockerOperator(api_version='1.19',
docker_url='tcp://localhost:2375', #Set your docker URL
command='/bin/sleep 30',
image='centos:latest',
network_mode='bridge',
task_id='docker_op_tester',
dag=dag)
t4 = BashOperator(
task_id='print_hello',
bash_command='echo "hello world!!!"',
dag=dag)
t1.set_downstream(t2)
t1.set_downstream(t3)
t3.set_downstream(t4)
task_id='task_1',
bash_command=
'echo "Starting executor Task 1 | Passed Conf : {{ dag_run.conf["json_executor_task"] }}"',
dag=dag)
executor = DockerOperator(
task_id='executor',
image='openjdk:8-jre-alpine',
api_version='auto',
auto_remove=True,
volumes=[
'/usr/local/airflow/artifacts:/usr/local/airflow/artifacts',
'/var/run/docker.sock:/var/run/docker.sock'
],
docker_url="unix://var/run/docker.sock",
network_mode="bridge",
environment={
'VPC_EXECUTOR_TASK': '{{ dag_run.conf["json_executor_task"] }}'
},
command=
'java -cp /usr/local/airflow/artifacts/jar-with-dependencies.jar <class>',
dag=dag)
t2 = BashOperator(
task_id='task_2',
bash_command='echo "Finishing executor Task 2 | Execution Time : {{ ts }}"',
dag=dag)
executor.set_upstream(t1)
executor.set_downstream(t2)
'start_date': datetime.utcnow(),
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5)
}
dag = DAG('docker_sample',
default_args=default_args,
schedule_interval=timedelta(minutes=10))
t1 = BashOperator(task_id='print_date', bash_command='date', dag=dag)
t2 = BashOperator(task_id='sleep', bash_command='sleep 5', retries=3, dag=dag)
t3 = DockerOperator(api_version='1.21',
command='/bin/sleep 30',
image='busybox:latest',
network_mode='bridge',
task_id='docker_op_tester',
dag=dag)
t4 = BashOperator(task_id='print_hello',
bash_command='echo "hello world!!!"',
dag=dag)
t1.set_downstream(t2)
t1.set_downstream(t3)
t3.set_downstream(t4)
'retry_delay': timedelta(minutes=5)
}
dag = DAG('docker_sample',
default_args=default_args,
schedule_interval=timedelta(minutes=10))
t1 = DockerOperator(api_version='1.19',
command='/bin/sleep 30',
image='centos:latest',
network_mode='bridge',
task_id='docker_op_tester',
dag=dag)
t2 = DockerOperator(api_version='1.19',
command='/bin/sleep 30',
image='centos:latest',
network_mode='bridge',
task_id='docker_op_tester',
dag=dag)
t3 = DockerOperator(api_version='1.19',
command='/bin/sleep 30',
image='centos:latest',
network_mode='bridge',
task_id='docker_op_tester',
dag=dag)
t1.set_downstream(t2)
t1.set_downstream(t3)
