"request_memory": "10Gi", }, volumes=[dataset_volume, outputs_volume], volume_mounts=[dataset_volume_mount, outputs_volume_mount], env_vars={ "TRENDS_FOLDERPATH": trends_folderpath, "MOSAIC_FOLDERPATH": mosaic_folderpath, "TESTS_FOLDERPATH": tests_folderpath, }, get_logs=True, affinity=experiment_affinity, dag=dag, ) dag_start = DummyOperator(task_id="Start", dag=dag) generate_allowed_tiles.set_upstream(dag_start) generate_analysis_mask.set_upstream(dag_start) prepare_level2.set_upstream(generate_allowed_tiles) prepare_level2.set_upstream(generate_analysis_mask) for task in preprocess_level2_tasks: task.set_upstream(prepare_level2) task.set_downstream(prepare_tsa) for tsa_task, pyramid_task_per_tile in zip(tsa_tasks, pyramid_tasks_per_tile): tsa_task.set_upstream(prepare_tsa) tsa_task.set_downstream(wait_for_trends) # Start pyramid batch of pyramid tasks for every tile. tsa_task.set_downstream(pyramid_task_per_tile) for task in mosaic_tasks:
default_args=default_args, schedule_interval=timedelta(minutes=10), tags=['example']) start = DummyOperator(task_id='run_this_first', dag=dag) python_task = KubernetesPodOperator(namespace='default', image="python:3.6", cmds=["python", "-c"], arguments=["print('hello world')"], labels={"foo": "bar"}, name="passing-python", task_id="passing-task-python", get_logs=True, dag=dag) bash_task = KubernetesPodOperator( namespace='default', image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["date"], labels={"foo": "bar"}, name="passing-bash", # is_delete_operator_pod=False, task_id="passing-task-bash", get_logs=True, dag=dag) python_task.set_upstream(start) bash_task.set_upstream(start)
volume_mounts=[volume_mount], get_logs=True, dag=dag) wordcloud_task += 1 wordcloud_task_list.append(create_wordcloud) # # Create merge_images tasks # merge_images = KubernetesPodOperator(name="merge_images", namespace=namespace, image="srnbckr/wordcloud", cmds=["python", "merge_images.py"], task_id="merge_images", resources=compute_resources, volumes=[volume], volume_mounts=[volume_mount], get_logs=True, dag=dag) # # Set dependencies # merge_images.set_upstream(wordcloud_task_list) for task in wordcloud_task_list: # create_wordcloud tasks should wait for every download_lyrics tasks to finish task.set_upstream(download_tasks_list) for task in download_tasks_list: # Begin with dummy task task.set_upstream(start)