requests.post(slack_url, json={"text": message})
# This is very broad but we want to try to log _any_ exception to slack
except Exception as e:
requests.post(slack_url,
json={"text": f"failed to log {type(e)} to slack"})
for dag_directory in dag_directories:
dag_id = os.path.basename(dag_directory)
globals()[dag_id] = create_dag(
dag_directory,
tags=["default", "tags"],
task_group_defaults={"tooltip": "this is a default tooltip"},
wait_for_defaults={
"retries": 24,
"check_existence": True,
"timeout": 10 * 60
},
latest_only=False,
user_defined_macros={
**user_defined_macros,
**macros.data_infra_macros
},
user_defined_filters=user_defined_filters,
default_args={
"on_failure_callback": log_failure_to_slack,
# "on_retry_callback": log_failure_to_slack,
},
)
import os
import airflow
from gusty import create_dag
#####################
## DAG Directories ##
#####################
# point to your dags directory
dag_parent_dir = os.path.join(os.environ['AIRFLOW_HOME'], "dags")
# assumes any subdirectories in the dags directory are Gusty DAGs (with METADATA.yml) (excludes subdirectories like __pycache__)
dag_directories = [os.path.join(dag_parent_dir, name) for name in os.listdir(dag_parent_dir) if os.path.isdir(os.path.join(dag_parent_dir, name)) and not name.endswith('__')]
####################
## DAG Generation ##
####################
for dag_directory in dag_directories:
dag_id = os.path.basename(dag_directory)
globals()[dag_id] = create_dag(dag_directory,
tags = ['default', 'tags'],
task_group_defaults={"tooltip": "default tooltip"},
wait_for_defaults={"retries": 10, "check_existence": True},
latest_only=False)
# TODO: catalogs directory is not a dag, which screws up auto loading folders
# assumes any subdirectories in the dags directory are Gusty DAGs (with METADATA.yml)
# (excludes subdirectories like __pycache__)
# dag_directories = []
# for child in dag_parent_dir.iterdir():
# if child.is_dir() and not str(child).endswith('__'):
# dag_directories.append(str(child))
dag_directories = [
dag_parent_dir / "gtfs_downloader",
dag_parent_dir / "gtfs_loader",
dag_parent_dir / "gtfs_views",
]
# DAG Generation ==============================================================
for dag_directory in dag_directories:
dag_id = os.path.basename(dag_directory)
globals()[dag_id] = create_dag(
dag_directory,
tags=["default", "tags"],
task_group_defaults={"tooltip": "this is a default tooltip"},
wait_for_defaults={
"retries": 10,
"check_existence": True
},
latest_only=False,
user_defined_macros=user_defined_macros,
user_defined_filters=user_defined_filters,
)
import os
import airflow
from gusty import create_dag
#####################
## DAG Directories ##
#####################
# point to your dags directory
dag_parent_dir = os.path.join(os.environ['AIRFLOW_HOME'], "dags")
# assumes any subdirectories in the dags directory are Gusty DAGs (with METADATA.yml) (excludes subdirectories like __pycache__)
dag_directories = [
os.path.join(dag_parent_dir, name) for name in os.listdir(dag_parent_dir)
if os.path.isdir(os.path.join(dag_parent_dir, name))
and not name.endswith('__')
]
####################
## DAG Generation ##
####################
for dag_directory in dag_directories:
dag_id = os.path.basename(dag_directory)
globals()[dag_id] = create_dag(dag_directory, ignore_subfolders=True)