from airflow import models from airflow import configuration from airflow.exceptions import AirflowConfigException from xTool.exceptions import XToolConfigException from xTool.decorators.db import provide_session import traceback import re from xTool.utils.log.logging_mixin import LoggingMixin login_manager = flask_login.LoginManager() login_manager.login_view = 'airflow.login' # Calls login() below login_manager.login_message = None log = LoggingMixin().log class AuthenticationError(Exception): pass class LdapException(Exception): pass def get_ldap_connection(dn=None, password=None): try: cacert = configuration.conf.get("ldap", "cacert") except (AirflowConfigException, XToolConfigException): pass
def list_py_file_paths( directory, followlinks=True, ignore_filename='.ignore', file_ext='.py', safe_mode=False, safe_filters=(b'xTool', b'XTool')): """递归遍历目录,返回匹配规则的文件列表 Traverse a directory and look for Python files. :param directory: the directory to traverse :type directory: unicode :param safe_mode: whether to use a heuristic to determine whether a file contains Airflow DAG definitions :return: a list of paths to Python files in the specified directory :rtype: list[unicode] """ file_paths = [] if directory is None: return [] elif os.path.isfile(directory): return [directory] elif os.path.isdir(directory): patterns_by_dir = {} # 递归遍历目录,包含链接文件 for root, dirs, files in os.walk(directory, followlinks=followlinks): patterns = patterns_by_dir.get(root, []) # 获得需要忽略的文件 ignore_file = os.path.join(root, ignore_filename) if os.path.isfile(ignore_file): with open(ignore_file, 'r') as f: # If we have new patterns create a copy so we don't change # the previous list (which would affect other subdirs) patterns = patterns + \ [p for p in f.read().split('\n') if p] # If we can ignore any subdirs entirely we should - fewer paths # to walk is better. We have to modify the ``dirs`` array in # place for this to affect os.walk dirs[:] = [ d for d in dirs if not any(re.search(p, os.path.join(root, d)) for p in patterns) ] # We want patterns defined in a parent folder's .airflowignore to # apply to subdirs too for d in dirs: patterns_by_dir[os.path.join(root, d)] = patterns for f in files: try: # 获得文件的绝对路径 file_path = os.path.join(root, f) if not os.path.isfile(file_path): continue # 验证文件后缀 mod_name, file_extension = os.path.splitext( os.path.split(file_path)[-1]) if file_extension != file_ext and not zipfile.is_zipfile( file_path): continue # 验证忽略规则 if any([re.findall(p, file_path) for p in patterns]): continue # 使用启发式方式猜测是否是一个DAG文件,DAG文件需要包含DAG 或 airflow # Heuristic that guesses whether a Python file contains an # Airflow DAG definition. might_contain_dag = True if safe_mode and not zipfile.is_zipfile(file_path): with open(file_path, 'rb') as f: content = f.read() might_contain_dag = all( [s in content for s in safe_filters]) if not might_contain_dag: continue file_paths.append(file_path) except Exception: log = LoggingMixin().log log.exception("Error while examining %s", f) return file_paths
def get_connection(cls, conn_id): conn = random.choice(cls.get_connections(conn_id)) if conn.host: log = LoggingMixin().log log.info("Using connection to: %s", conn.host) return conn
def list_py_file_paths(directory, followlinks=True, ignore_filename='.ignore', file_ext='.py', safe_mode=False, safe_filters=(b'xTool', b'XTool')): """递归遍历目录,返回匹配规则的文件列表 Traverse a directory and look for Python files. :param directory: the directory to traverse :type directory: unicode :param safe_mode: whether to use a heuristic to determine whether a file contains Airflow DAG definitions :return: a list of paths to Python files in the specified directory :rtype: list[unicode] """ file_paths = [] if directory is None: return [] elif os.path.isfile(directory): return [directory] elif os.path.isdir(directory): patterns = [] # 递归遍历目录,包含链接文件 for root, dirs, files in os.walk(directory, followlinks=followlinks): # 获得需要忽略的文件 ignore_file = [f for f in files if f == ignore_filename] if ignore_file: f = open(os.path.join(root, ignore_file[0]), 'r') patterns += [p.strip() for p in f.read().split('\n') if p] f.close() for f in files: try: # 获得文件的绝对路径 file_path = os.path.join(root, f) if not os.path.isfile(file_path): continue # 验证文件后缀 mod_name, file_extension = os.path.splitext( os.path.split(file_path)[-1]) if file_extension != file_ext and not zipfile.is_zipfile( file_path): continue # 验证忽略规则 if any([re.findall(p, file_path) for p in patterns]): continue # 使用启发式方式猜测是否是一个DAG文件,DAG文件需要包含DAG 或 airflow # Heuristic that guesses whether a Python file contains an # Airflow DAG definition. might_contain_dag = True if safe_mode and not zipfile.is_zipfile(file_path): with open(file_path, 'rb') as f: content = f.read() might_contain_dag = all( [s in content for s in safe_filters]) if not might_contain_dag: continue file_paths.append(file_path) except Exception: log = LoggingMixin().log log.exception("Error while examining %s", f) return file_paths