t0 = DummyOperator(task_id='start')
t1 = DummyOperator(task_id='group_bash_tasks')
t2 = BashOperator(task_id='bash_print_date1',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s && date')
t3 = BashOperator(task_id='bash_print_date2',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s && date')
# Added bash operator task for exercise
t4 = BashOperator(task_id='bash_print_date3',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s && date')
# generate tasks with a loop. task_id must be unique
for task in range(4):
if version.startswith('2'):
tn = PythonOperator(
task_id=f'python_print_date_{task}',
python_callable=
my_custom_function, # make sure you don't include the () of the function
op_kwargs={'task_number': task},
retries=2 #change retries at the task level for exercise
)
else:
tn = PythonOperator(
task_id=f'python_print_date_{task}',
python_callable=
my_custom_function, # make sure you don't include the () of the function
op_kwargs={'task_number': task},
provide_context=True,
)
)
upload_task = DVCUpdateOperator(
dvc_repo=os.environ["REPO"],
files=[
DVCStringUpload(
"data/1.txt",
f"This will be saved into DVC. Current time: {datetime.now()}",
),
],
task_id="update_dvc",
)
# generate tasks with a loop. task_id must be unique
for task in range(5):
if version.startswith("2"):
tn = PythonOperator(
task_id=f"python_custom_task_{task}",
python_callable=custom_python_task_handler, # make sure you don't include the () of the function
op_kwargs={"task_number": task},
)
else:
tn = PythonOperator(
task_id=f"python_custom_task_{task}",
python_callable=custom_python_task_handler, # make sure you don't include the () of the function
op_kwargs={"task_number": task},
provide_context=True,
)
t0 >> tn