from airflow import models
from airflow import configuration
from airflow.exceptions import AirflowConfigException
from xTool.exceptions import XToolConfigException
from xTool.decorators.db import provide_session
import traceback
import re
from xTool.utils.log.logging_mixin import LoggingMixin
login_manager = flask_login.LoginManager()
login_manager.login_view = 'airflow.login' # Calls login() below
login_manager.login_message = None
log = LoggingMixin().log
class AuthenticationError(Exception):
pass
class LdapException(Exception):
pass
def get_ldap_connection(dn=None, password=None):
try:
cacert = configuration.conf.get("ldap", "cacert")
except (AirflowConfigException, XToolConfigException):
pass
def list_py_file_paths(
directory,
followlinks=True,
ignore_filename='.ignore',
file_ext='.py',
safe_mode=False,
safe_filters=(b'xTool', b'XTool')):
"""递归遍历目录，返回匹配规则的文件列表
Traverse a directory and look for Python files.
:param directory: the directory to traverse
:type directory: unicode
:param safe_mode: whether to use a heuristic to determine whether a file
contains Airflow DAG definitions
:return: a list of paths to Python files in the specified directory
:rtype: list[unicode]
"""
file_paths = []
if directory is None:
return []
elif os.path.isfile(directory):
return [directory]
elif os.path.isdir(directory):
patterns_by_dir = {}
# 递归遍历目录，包含链接文件
for root, dirs, files in os.walk(directory, followlinks=followlinks):
patterns = patterns_by_dir.get(root, [])
# 获得需要忽略的文件
ignore_file = os.path.join(root, ignore_filename)
if os.path.isfile(ignore_file):
with open(ignore_file, 'r') as f:
# If we have new patterns create a copy so we don't change
# the previous list (which would affect other subdirs)
patterns = patterns + \
[p for p in f.read().split('\n') if p]
# If we can ignore any subdirs entirely we should - fewer paths
# to walk is better. We have to modify the ``dirs`` array in
# place for this to affect os.walk
dirs[:] = [
d
for d in dirs
if not any(re.search(p, os.path.join(root, d)) for p in patterns)
]
# We want patterns defined in a parent folder's .airflowignore to
# apply to subdirs too
for d in dirs:
patterns_by_dir[os.path.join(root, d)] = patterns
for f in files:
try:
# 获得文件的绝对路径
file_path = os.path.join(root, f)
if not os.path.isfile(file_path):
continue
# 验证文件后缀
mod_name, file_extension = os.path.splitext(
os.path.split(file_path)[-1])
if file_extension != file_ext and not zipfile.is_zipfile(
file_path):
continue
# 验证忽略规则
if any([re.findall(p, file_path) for p in patterns]):
continue
# 使用启发式方式猜测是否是一个DAG文件，DAG文件需要包含DAG 或 airflow
# Heuristic that guesses whether a Python file contains an
# Airflow DAG definition.
might_contain_dag = True
if safe_mode and not zipfile.is_zipfile(file_path):
with open(file_path, 'rb') as f:
content = f.read()
might_contain_dag = all(
[s in content for s in safe_filters])
if not might_contain_dag:
continue
file_paths.append(file_path)
except Exception:
log = LoggingMixin().log
log.exception("Error while examining %s", f)
return file_paths
def get_connection(cls, conn_id):
conn = random.choice(cls.get_connections(conn_id))
if conn.host:
log = LoggingMixin().log
log.info("Using connection to: %s", conn.host)
return conn
def list_py_file_paths(directory,
followlinks=True,
ignore_filename='.ignore',
file_ext='.py',
safe_mode=False,
safe_filters=(b'xTool', b'XTool')):
"""递归遍历目录，返回匹配规则的文件列表
Traverse a directory and look for Python files.
:param directory: the directory to traverse
:type directory: unicode
:param safe_mode: whether to use a heuristic to determine whether a file
contains Airflow DAG definitions
:return: a list of paths to Python files in the specified directory
:rtype: list[unicode]
"""
file_paths = []
if directory is None:
return []
elif os.path.isfile(directory):
return [directory]
elif os.path.isdir(directory):
patterns = []
# 递归遍历目录，包含链接文件
for root, dirs, files in os.walk(directory, followlinks=followlinks):
# 获得需要忽略的文件
ignore_file = [f for f in files if f == ignore_filename]
if ignore_file:
f = open(os.path.join(root, ignore_file[0]), 'r')
patterns += [p.strip() for p in f.read().split('\n') if p]
f.close()
for f in files:
try:
# 获得文件的绝对路径
file_path = os.path.join(root, f)
if not os.path.isfile(file_path):
continue
# 验证文件后缀
mod_name, file_extension = os.path.splitext(
os.path.split(file_path)[-1])
if file_extension != file_ext and not zipfile.is_zipfile(
file_path):
continue
# 验证忽略规则
if any([re.findall(p, file_path) for p in patterns]):
continue
# 使用启发式方式猜测是否是一个DAG文件，DAG文件需要包含DAG 或 airflow
# Heuristic that guesses whether a Python file contains an
# Airflow DAG definition.
might_contain_dag = True
if safe_mode and not zipfile.is_zipfile(file_path):
with open(file_path, 'rb') as f:
content = f.read()
might_contain_dag = all(
[s in content for s in safe_filters])
if not might_contain_dag:
continue
file_paths.append(file_path)
except Exception:
log = LoggingMixin().log
log.exception("Error while examining %s", f)
return file_paths