def _validate_task_pools(self,
*,
dagbag: DagBag,
session: Session = NEW_SESSION):
"""
Validates and raise exception if any task in a dag is using a non-existent pool
:meta private:
"""
from airflow.models.pool import Pool
def check_pools(dag):
task_pools = {task.pool for task in dag.tasks}
nonexistent_pools = task_pools - pools
if nonexistent_pools:
return (
f"Dag '{dag.dag_id}' references non-existent pools: {list(sorted(nonexistent_pools))!r}"
)
pools = {p.pool for p in Pool.get_pools(session)}
for dag in dagbag.dags.values():
message = check_pools(dag)
if message:
self.dag_warnings.add(
DagWarning(dag.dag_id, DagWarningType.NONEXISTENT_POOL,
message))
for subdag in dag.subdags:
message = check_pools(subdag)
if message:
self.dag_warnings.add(
DagWarning(subdag.dag_id,
DagWarningType.NONEXISTENT_POOL, message))
def get_pools(self):
return [(p.pool, p.slots, p.description) for p in Pool.get_pools()]
