def send_MIME_email(e_from,
e_to,
mime_msg,
host,
port,
ssl=False,
starttls=False,
user=None,
password=None,
dryrun=False):
"""发送邮件 ."""
log = LoggingMixin().log
if not user or not password:
log.debug(
"No user/password found for SMTP, so logging in with no authentication."
)
if not dryrun:
s = smtplib.SMTP_SSL(host, port) if ssl else smtplib.SMTP(host, port)
if starttls:
s.starttls()
s.login(user, password)
log.info("Sent an alert email to %s", e_to)
s.sendmail(e_from, e_to, mime_msg.as_string())
s.quit()
def send_MIME_email(e_from, e_to, mime_msg, dryrun=False):
"""发送邮件 ."""
log = LoggingMixin().log
SMTP_HOST = configuration.conf.get('smtp', 'SMTP_HOST')
SMTP_PORT = configuration.conf.getint('smtp', 'SMTP_PORT')
SMTP_STARTTLS = configuration.conf.getboolean('smtp', 'SMTP_STARTTLS')
SMTP_SSL = configuration.conf.getboolean('smtp', 'SMTP_SSL')
SMTP_USER = None
SMTP_PASSWORD = None
try:
SMTP_USER = configuration.conf.get('smtp', 'SMTP_USER')
SMTP_PASSWORD = configuration.conf.get('smtp', 'SMTP_PASSWORD')
except XToolConfigException:
log.debug(
"No user/password found for SMTP, so logging in with no authentication."
)
if not dryrun:
s = smtplib.SMTP_SSL(SMTP_HOST,
SMTP_PORT) if SMTP_SSL else smtplib.SMTP(
SMTP_HOST, SMTP_PORT)
if SMTP_STARTTLS:
s.starttls()
if SMTP_USER and SMTP_PASSWORD:
s.login(SMTP_USER, SMTP_PASSWORD)
log.info("Sent an alert email to %s", e_to)
s.sendmail(e_from, e_to, mime_msg.as_string())
s.quit()
def _str(s):
# cloudant-python doesn't support unicode.
if isinstance(s, unicode):
log = LoggingMixin().log
log.debug(
'cloudant-python does not support unicode. Encoding %s as '
'ascii using "ignore".', s)
return s.encode('ascii', 'ignore')
return s
def filter_for_filesize(result, size=None):
"""
Will test the filepath result and test if its size is at least self.filesize
:param result: a list of dicts returned by Snakebite ls
:param size: the file size in MB a file should be at least to trigger True
:return: (bool) depending on the matching criteria
"""
if size:
log = LoggingMixin().log
log.debug('Filtering for file size >= %s in files: %s', size,
map(lambda x: x['path'], result))
size *= settings.MEGABYTE
result = [x for x in result if x['length'] >= size]
log.debug('HdfsSensor.poke: after size filter result is %s',
result)
return result
def filter_for_ignored_ext(result, ignored_ext, ignore_copying):
"""
Will filter if instructed to do so the result to remove matching criteria
:param result: (list) of dicts returned by Snakebite ls
:param ignored_ext: (list) of ignored extensions
:param ignore_copying: (bool) shall we ignore ?
:return: (list) of dicts which were not removed
"""
if ignore_copying:
log = LoggingMixin().log
regex_builder = "^.*\.(%s$)$" % '$|'.join(ignored_ext)
ignored_extentions_regex = re.compile(regex_builder)
log.debug(
'Filtering result for ignored extensions: %s in files %s',
ignored_extentions_regex.pattern,
map(lambda x: x['path'], result))
result = [
x for x in result
if not ignored_extentions_regex.match(x['path'])
]
log.debug('HdfsSensor.poke: after ext filter result is %s', result)
return result
# Crawl through the plugins folder to find AirflowPlugin derivatives
for root, dirs, files in os.walk(plugins_folder, followlinks=True):
for f in files:
try:
# 获得插件目录下的所有文件路径
filepath = os.path.join(root, f)
if not os.path.isfile(filepath):
continue
# 判断文件的后缀是否是.py
mod_name, file_ext = os.path.splitext(
os.path.split(filepath)[-1])
if file_ext != '.py':
continue
log.debug('Importing plugin module %s', filepath)
# 将路径转换为命名空间
# normalize root path as namespace
# e.g.:
# __data__home__user00__application__airflow__plugins_rest_api_plugin
# mod_name 为文件名（去掉后缀）
namespace = '_'.join([re.sub(norm_pattern, '__', root), mod_name])
# 根据文件名加载模块
m = imp.load_source(namespace, filepath)
# 将所加载模块中的 XToolPlugin 子类添加到 plugins 列表中
# 遍历模块的属性值
for obj in list(m.__dict__.values()):
# 判断模块中的类是否为 XToolPlugin 的子类
if (