requests.post(slack_url, json={"text": message}) # This is very broad but we want to try to log _any_ exception to slack except Exception as e: requests.post(slack_url, json={"text": f"failed to log {type(e)} to slack"}) for dag_directory in dag_directories: dag_id = os.path.basename(dag_directory) globals()[dag_id] = create_dag( dag_directory, tags=["default", "tags"], task_group_defaults={"tooltip": "this is a default tooltip"}, wait_for_defaults={ "retries": 24, "check_existence": True, "timeout": 10 * 60 }, latest_only=False, user_defined_macros={ **user_defined_macros, **macros.data_infra_macros }, user_defined_filters=user_defined_filters, default_args={ "on_failure_callback": log_failure_to_slack, # "on_retry_callback": log_failure_to_slack, }, )
import os import airflow from gusty import create_dag ##################### ## DAG Directories ## ##################### # point to your dags directory dag_parent_dir = os.path.join(os.environ['AIRFLOW_HOME'], "dags") # assumes any subdirectories in the dags directory are Gusty DAGs (with METADATA.yml) (excludes subdirectories like __pycache__) dag_directories = [os.path.join(dag_parent_dir, name) for name in os.listdir(dag_parent_dir) if os.path.isdir(os.path.join(dag_parent_dir, name)) and not name.endswith('__')] #################### ## DAG Generation ## #################### for dag_directory in dag_directories: dag_id = os.path.basename(dag_directory) globals()[dag_id] = create_dag(dag_directory, tags = ['default', 'tags'], task_group_defaults={"tooltip": "default tooltip"}, wait_for_defaults={"retries": 10, "check_existence": True}, latest_only=False)
# TODO: catalogs directory is not a dag, which screws up auto loading folders # assumes any subdirectories in the dags directory are Gusty DAGs (with METADATA.yml) # (excludes subdirectories like __pycache__) # dag_directories = [] # for child in dag_parent_dir.iterdir(): # if child.is_dir() and not str(child).endswith('__'): # dag_directories.append(str(child)) dag_directories = [ dag_parent_dir / "gtfs_downloader", dag_parent_dir / "gtfs_loader", dag_parent_dir / "gtfs_views", ] # DAG Generation ============================================================== for dag_directory in dag_directories: dag_id = os.path.basename(dag_directory) globals()[dag_id] = create_dag( dag_directory, tags=["default", "tags"], task_group_defaults={"tooltip": "this is a default tooltip"}, wait_for_defaults={ "retries": 10, "check_existence": True }, latest_only=False, user_defined_macros=user_defined_macros, user_defined_filters=user_defined_filters, )
import os import airflow from gusty import create_dag ##################### ## DAG Directories ## ##################### # point to your dags directory dag_parent_dir = os.path.join(os.environ['AIRFLOW_HOME'], "dags") # assumes any subdirectories in the dags directory are Gusty DAGs (with METADATA.yml) (excludes subdirectories like __pycache__) dag_directories = [ os.path.join(dag_parent_dir, name) for name in os.listdir(dag_parent_dir) if os.path.isdir(os.path.join(dag_parent_dir, name)) and not name.endswith('__') ] #################### ## DAG Generation ## #################### for dag_directory in dag_directories: dag_id = os.path.basename(dag_directory) globals()[dag_id] = create_dag(dag_directory, ignore_subfolders=True)