from airflow.models import DAG from util.settings import default_settings from util.dummy import dummy with DAG(**default_settings()) as dag: dummy('hello-world') >> dummy('first') >> dummy('second')
from airflow.operators.bash import BashOperator from airflow.hooks.base import BaseHook from airflow.models import Variable import pandas as pd from airflow.decorators import dag, task from util.settings import default_settings from airflow.utils.dates import days_ago default_args = { 'owner': 'airflow', } @dag(**default_settings()) def hw_4_dag(): """ # Homework No.4 dag - same functionality as No.3 but realization with @dag and @task features """ @task() def download_titanic_dataset( url='https://web.stanford.edu/class/archive/cs/cs109/cs109.1166/stuff/titanic.csv' ): """ # download_titanic_dataset task This task downloads the sample titanic dataset from predefined URL and returns {'titanic_df' : 'dataframe represented as json data in STR format'} It uses methodology of @task decorator covering all XCOM push / pull mechanics under the hood """ df = pd.read_csv(url) return {'titanic_df_json_str': df.to_json()}