"""
This file configures the Apache Airflow DAG to ingest Museum Victoria data.
We do this by running `provider_api_scripts.museum_victoria.main`
"""
# airflow DAG (necessary for Airflow to find this file)
from datetime import datetime, timedelta
import logging
from provider_api_scripts import museum_victoria
from util.dag_factory import create_provider_api_workflow
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s: %(message)s',
level=logging.INFO)
logger = logging.getLogger(__name__)
DAG_ID = 'museum_victoria_workflow'
START_DATE = datetime(2020, 1, 1)
DAGRUN_TIMEOUT = timedelta(hours=24)
globals()[DAG_ID] = create_provider_api_workflow(DAG_ID,
museum_victoria.main,
start_date=START_DATE,
schedule_string='@monthly',
dated=False,
dagrun_timeout=DAGRUN_TIMEOUT)
"""
This file configures the Apache Airflow DAG to (re)ingest Flickr data.
"""
# airflow DAG (necessary for Airflow to find this file)
from datetime import datetime
import logging
from provider_api_scripts import flickr
from util.dag_factory import create_provider_api_workflow
logging.basicConfig(
format='%(asctime)s: [%(levelname)s - DAG Loader] %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
DAG_ID = 'flickr_workflow'
globals()[DAG_ID] = create_provider_api_workflow(DAG_ID,
flickr.main,
start_date=datetime(
1970, 1, 1),
concurrency=1,
schedule_string='@daily',
dated=True,
day_shift=0)
"""
This file configures the Apache Airflow DAG to (re)ingest Flickr data.
"""
# airflow DAG (necessary for Airflow to find this file)
from datetime import datetime, timedelta
import logging
from provider_api_scripts import walters_art_museum as wam
from util.dag_factory import create_provider_api_workflow
logging.basicConfig(
format='%(asctime)s: [%(levelname)s - DAG Loader] %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
DAG_ID = 'walters_workflow'
globals()[DAG_ID] = create_provider_api_workflow(
DAG_ID,
wam.main,
start_date=datetime(2020, 9, 27),
schedule_string='@monthly',
dated=False,
dagrun_timeout=timedelta(days=1))
"""
This file configures the Apache Airflow DAG to ingest Smithsonian data.
We do this by running `provider_api_scripts.smithsonian.main`
"""
# airflow DAG (necessary for Airflow to find this file)
from datetime import datetime, timedelta
import logging
from provider_api_scripts import smithsonian
from util.dag_factory import create_provider_api_workflow
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s: %(message)s',
level=logging.INFO)
logger = logging.getLogger(__name__)
DAG_ID = 'smithsonian_workflow'
START_DATE = datetime(2020, 1, 1)
DAGRUN_TIMEOUT = timedelta(hours=24)
globals()[DAG_ID] = create_provider_api_workflow(DAG_ID,
smithsonian.main,
start_date=START_DATE,
schedule_string='@weekly',
dated=False,
dagrun_timeout=DAGRUN_TIMEOUT)
