""" This file configures the Apache Airflow DAG to ingest Museum Victoria data. We do this by running `provider_api_scripts.museum_victoria.main` """ # airflow DAG (necessary for Airflow to find this file) from datetime import datetime, timedelta import logging from provider_api_scripts import museum_victoria from util.dag_factory import create_provider_api_workflow logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s: %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) DAG_ID = 'museum_victoria_workflow' START_DATE = datetime(2020, 1, 1) DAGRUN_TIMEOUT = timedelta(hours=24) globals()[DAG_ID] = create_provider_api_workflow(DAG_ID, museum_victoria.main, start_date=START_DATE, schedule_string='@monthly', dated=False, dagrun_timeout=DAGRUN_TIMEOUT)
""" This file configures the Apache Airflow DAG to (re)ingest Flickr data. """ # airflow DAG (necessary for Airflow to find this file) from datetime import datetime import logging from provider_api_scripts import flickr from util.dag_factory import create_provider_api_workflow logging.basicConfig( format='%(asctime)s: [%(levelname)s - DAG Loader] %(message)s', level=logging.DEBUG) logger = logging.getLogger(__name__) DAG_ID = 'flickr_workflow' globals()[DAG_ID] = create_provider_api_workflow(DAG_ID, flickr.main, start_date=datetime( 1970, 1, 1), concurrency=1, schedule_string='@daily', dated=True, day_shift=0)
""" This file configures the Apache Airflow DAG to (re)ingest Flickr data. """ # airflow DAG (necessary for Airflow to find this file) from datetime import datetime, timedelta import logging from provider_api_scripts import walters_art_museum as wam from util.dag_factory import create_provider_api_workflow logging.basicConfig( format='%(asctime)s: [%(levelname)s - DAG Loader] %(message)s', level=logging.DEBUG) logger = logging.getLogger(__name__) DAG_ID = 'walters_workflow' globals()[DAG_ID] = create_provider_api_workflow( DAG_ID, wam.main, start_date=datetime(2020, 9, 27), schedule_string='@monthly', dated=False, dagrun_timeout=timedelta(days=1))
""" This file configures the Apache Airflow DAG to ingest Smithsonian data. We do this by running `provider_api_scripts.smithsonian.main` """ # airflow DAG (necessary for Airflow to find this file) from datetime import datetime, timedelta import logging from provider_api_scripts import smithsonian from util.dag_factory import create_provider_api_workflow logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s: %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) DAG_ID = 'smithsonian_workflow' START_DATE = datetime(2020, 1, 1) DAGRUN_TIMEOUT = timedelta(hours=24) globals()[DAG_ID] = create_provider_api_workflow(DAG_ID, smithsonian.main, start_date=START_DATE, schedule_string='@weekly', dated=False, dagrun_timeout=DAGRUN_TIMEOUT)