success_finish_operator.set_upstream([cli_download_sra, url_download, download_local_operator, copy_from_biowardrobe]) # # A STEP # error_finish_operator = MySqlOperator( task_id="finish_with_error", mysql_conn_id=biowardrobe_connection_id, sql="""update labdata set libstatus=2000, libstatustxt="{{ ti.xcom_pull(task_ids=['download_sra','download_local', 'download_aria2'], key=None) }}" where uid='{{ ti.xcom_pull(task_ids='branch_download', key='uid') }}'""", trigger_rule=TriggerRule.ONE_FAILED, autocommit=True, dag=dag) error_finish_operator.set_upstream([cli_download_sra, url_download, download_local_operator, copy_from_biowardrobe]) # # BIOWARDROBE DOWNLOAD TRIGGER # dag_t = DAG( dag_id='biowardrobe_download_trigger', default_args={ 'owner': 'airflow', 'start_date': datetime(1970, 1, 1, 1, 1, 1, 1), 'depends_on_past': False, 'email': ['*****@*****.**'], 'email_on_failure': False,
'owner': 'liyi', 'start_date': airflow.utils.dates.days_ago(0), 'retries': 1, 'retry_delay': timedelta(minutes=5) } dag1 = DAG('mysql_create_from', default_args=default_args, description='A simple create_mysql_data DAG', schedule_interval=timedelta(minutes=10)) # print(dag1) # print(type(dag1)) sql1 = ["INSERT INTO airflow_test.test1_from (date1) values (now())"] t1 = MySqlOperator( mysql_conn_id='liyi', sql=sql1, task_id='mysql_create_from', dag=dag1 ) sql2 = ["""insert into airflow_test.test1_in(date1) select * from airflow_test.test1_from ORDER BY date1 DESC LIMIT 1;"""] t2 = MySqlOperator( mysql_conn_id='liyi', sql=sql2, task_id='mysql_insert_into', dag=dag1 ) t2.set_upstream(t1)
success_finish_operator.set_upstream([cli_download_sra, url_download, download_local_operator]) # # A STEP # error_finish_operator = MySqlOperator( task_id="finish_with_error", mysql_conn_id=biowardrobe_connection_id, sql="""update labdata set libstatus=2000, libstatustxt="{{ ti.xcom_pull(task_ids=['download_sra','download_local', 'download_aria2'], key=None) }}" where uid='{{ ti.xcom_pull(task_ids='branch_download', key='uid') }}'""", trigger_rule=TriggerRule.ONE_FAILED, autocommit=True, dag=dag) error_finish_operator.set_upstream([cli_download_sra, url_download, download_local_operator]) # # BIOWARDROBE DOWNLOAD TRIGGER # dag_t = DAG( dag_id='biowardrobe_download_trigger', default_args={ 'owner': 'airflow', 'start_date': days_ago(1), 'depends_on_past': False, 'email': ['*****@*****.**'], 'email_on_failure': False,
task_id='create_spotify_daily_top_50_global', dag=dag, mysql_conn_id="airflow_mysql_db", sql=""" CREATE TABLE IF NOT EXISTS spotify_daily_top_50_global ( artist_name VARCHAR(100), track_id VARCHAR(100), track_name VARCHAR(100), duration_ms INT, explicit BOOL, popularity INT, daily_rank INT, album_id VARCHAR(100), album_name VARCHAR(100), album_type VARCHAR(100), release_date VARCHAR(100), dt VARCHAR(100) ) ; """ ) populate_spotify_daily_top_50_global = BashOperator( task_id='populate_spotify_daily_top_50_global', dag=dag, bash_command='python /usr/local/airflow/dags/Scraper.py' ) create_spotify_daily_top_50_global.set_upstream(check_valid_mysql_connection) populate_spotify_daily_top_50_global.set_upstream(check_valid_mysql_connection)
default_args=default_args, schedule_interval=timedelta(days=1)) # Task 1 t1 = BashOperator(task_id='print_date', bash_command='date', dag=dag) # Task 2 def my_python_function(): now = datetime.now() response = 'This function ran at ' + str(now) return response t2 = PythonOperator(task_id='my_python_task', python_callable=my_python_function, params={'my_param': 'Parameter I passed in'}, dag=dag) # Task 3 t3 = MySqlOperator(task_id='mariadb_task', sql="INSERT INTO test VALUES (1, '이찬호', 'my first dag');", mysql_conn_id='mariadb', autocommit=True, database="djob", dag=dag) # Pipeline Structure t2.set_upstream(t1) t3.set_upstream(t2)