def bag_dag(self, dag, root_dag): """ Adds the DAG into the bag, recurses into sub dags. Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags """ test_cycle(dag) # throws if a task cycle is found dag.resolve_template_files() dag.last_loaded = timezone.utcnow() for task in dag.tasks: settings.policy(task) subdags = dag.subdags try: for subdag in subdags: subdag.full_filepath = dag.full_filepath subdag.parent_dag = dag subdag.is_subdag = True self.bag_dag(dag=subdag, root_dag=root_dag) self.dags[dag.dag_id] = dag self.log.debug('Loaded DAG %s', dag) except AirflowDagCycleException as cycle_exception: # There was an error in bagging the dag. Remove it from the list of dags self.log.exception('Exception bagging dag: %s', dag.dag_id) # Only necessary at the root level since DAG.subdags automatically # performs DFS to search through all subdags if dag == root_dag: for subdag in subdags: if subdag.dag_id in self.dags: del self.dags[subdag.dag_id] raise cycle_exception
def test_policy_function(self): """ Tests that task instances are mutated by the policy function in airflow_local_settings. """ with SettingsContext(SETTINGS_FILE_POLICY, "airflow_local_settings"): from airflow import settings settings.import_local_settings() task_instance = MagicMock() settings.policy(task_instance) assert task_instance.run_as_user == "myself"
def test_import_with_dunder_all(self): """ Tests that if __all__ is specified in airflow_local_settings, only module attributes specified within are imported. """ with SettingsContext(SETTINGS_FILE_POLICY_WITH_DUNDER_ALL, "airflow_local_settings"): from airflow import settings settings.import_local_settings() task_instance = MagicMock() settings.policy(task_instance) assert task_instance.run_as_user == "myself"
def bag_dag(self, dag, parent_dag, root_dag): """ 将加载的dag模块加入到self.dags中 Adds the DAG into the bag, recurses into sub dags. Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags """ # 测试dag中是否有环 dag.test_cycle() # throws if a task cycle is found # 从指定的属性中获取设置的模板文件名,渲染模板并将此属性的值设置为渲染后的模板的内容 dag.resolve_template_files() dag.last_loaded = datetime.now() # 通用预处理 for task in dag.tasks: settings.policy(task) subdags = dag.subdags try: for subdag in subdags: subdag.full_filepath = dag.full_filepath subdag.parent_dag = dag subdag.is_subdag = True self.bag_dag(subdag, parent_dag=dag, root_dag=root_dag) # 将新的dag加入到self.dags中 self.dags[dag.dag_id] = dag self.log.debug('Loaded DAG {dag}'.format(**locals())) except AirflowDagCycleException as cycle_exception: # There was an error in bagging the dag. Remove it from the list of dags self.log.exception( 'Exception bagging dag: {dag.dag_id}'.format(**locals())) # Only necessary at the root level since DAG.subdags automatically # performs DFS to search through all subdags if dag == root_dag: for subdag in subdags: if subdag.dag_id in self.dags: del self.dags[subdag.dag_id] raise cycle_exception