"request_memory": "10Gi",
},
volumes=[dataset_volume, outputs_volume],
volume_mounts=[dataset_volume_mount, outputs_volume_mount],
env_vars={
"TRENDS_FOLDERPATH": trends_folderpath,
"MOSAIC_FOLDERPATH": mosaic_folderpath,
"TESTS_FOLDERPATH": tests_folderpath,
},
get_logs=True,
affinity=experiment_affinity,
dag=dag,
)
dag_start = DummyOperator(task_id="Start", dag=dag)
generate_allowed_tiles.set_upstream(dag_start)
generate_analysis_mask.set_upstream(dag_start)
prepare_level2.set_upstream(generate_allowed_tiles)
prepare_level2.set_upstream(generate_analysis_mask)
for task in preprocess_level2_tasks:
task.set_upstream(prepare_level2)
task.set_downstream(prepare_tsa)
for tsa_task, pyramid_task_per_tile in zip(tsa_tasks,
pyramid_tasks_per_tile):
tsa_task.set_upstream(prepare_tsa)
tsa_task.set_downstream(wait_for_trends)
# Start pyramid batch of pyramid tasks for every tile.
tsa_task.set_downstream(pyramid_task_per_tile)
for task in mosaic_tasks:
default_args=default_args,
schedule_interval=timedelta(minutes=10),
tags=['example'])
start = DummyOperator(task_id='run_this_first', dag=dag)
python_task = KubernetesPodOperator(namespace='default',
image="python:3.6",
cmds=["python", "-c"],
arguments=["print('hello world')"],
labels={"foo": "bar"},
name="passing-python",
task_id="passing-task-python",
get_logs=True,
dag=dag)
bash_task = KubernetesPodOperator(
namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["date"],
labels={"foo": "bar"},
name="passing-bash",
# is_delete_operator_pod=False,
task_id="passing-task-bash",
get_logs=True,
dag=dag)
python_task.set_upstream(start)
bash_task.set_upstream(start)
volume_mounts=[volume_mount],
get_logs=True,
dag=dag)
wordcloud_task += 1
wordcloud_task_list.append(create_wordcloud)
#
# Create merge_images tasks
#
merge_images = KubernetesPodOperator(name="merge_images",
namespace=namespace,
image="srnbckr/wordcloud",
cmds=["python", "merge_images.py"],
task_id="merge_images",
resources=compute_resources,
volumes=[volume],
volume_mounts=[volume_mount],
get_logs=True,
dag=dag)
#
# Set dependencies
#
merge_images.set_upstream(wordcloud_task_list)
for task in wordcloud_task_list:
# create_wordcloud tasks should wait for every download_lyrics tasks to finish
task.set_upstream(download_tasks_list)
for task in download_tasks_list:
# Begin with dummy task
task.set_upstream(start)
