def _validate_task_pools(self, *, dagbag: DagBag, session: Session = NEW_SESSION): """ Validates and raise exception if any task in a dag is using a non-existent pool :meta private: """ from airflow.models.pool import Pool def check_pools(dag): task_pools = {task.pool for task in dag.tasks} nonexistent_pools = task_pools - pools if nonexistent_pools: return ( f"Dag '{dag.dag_id}' references non-existent pools: {list(sorted(nonexistent_pools))!r}" ) pools = {p.pool for p in Pool.get_pools(session)} for dag in dagbag.dags.values(): message = check_pools(dag) if message: self.dag_warnings.add( DagWarning(dag.dag_id, DagWarningType.NONEXISTENT_POOL, message)) for subdag in dag.subdags: message = check_pools(subdag) if message: self.dag_warnings.add( DagWarning(subdag.dag_id, DagWarningType.NONEXISTENT_POOL, message))
def get_pools(self): return [(p.pool, p.slots, p.description) for p in Pool.get_pools()]