def send_MIME_email(e_from, e_to, mime_msg, dryrun=False):
log = LoggingMixin().log
SMTP_HOST = configuration.conf.get('smtp', 'SMTP_HOST')
SMTP_PORT = configuration.conf.getint('smtp', 'SMTP_PORT')
SMTP_STARTTLS = configuration.conf.getboolean('smtp', 'SMTP_STARTTLS')
SMTP_SSL = configuration.conf.getboolean('smtp', 'SMTP_SSL')
SMTP_USER = None
SMTP_PASSWORD = None
try:
SMTP_USER = configuration.conf.get('smtp', 'SMTP_USER')
SMTP_PASSWORD = configuration.conf.get('smtp', 'SMTP_PASSWORD')
except AirflowConfigException:
log.debug(
"No user/password found for SMTP, so logging in with no authentication."
)
if not dryrun:
s = smtplib.SMTP_SSL(SMTP_HOST,
SMTP_PORT) if SMTP_SSL else smtplib.SMTP(
SMTP_HOST, SMTP_PORT)
if SMTP_STARTTLS:
s.starttls()
if SMTP_USER and SMTP_PASSWORD:
s.login(SMTP_USER, SMTP_PASSWORD)
log.info("Sent an alert email to %s", e_to)
s.sendmail(e_from, e_to, mime_msg.as_string())
s.quit()
def filter_for_ignored_ext(result, ignored_ext, ignore_copying):
"""
Will filter if instructed to do so the result to remove matching criteria
:param result: list of dicts returned by Snakebite ls
:type result: list[dict]
:param ignored_ext: list of ignored extensions
:type ignored_ext: list
:param ignore_copying: shall we ignore ?
:type ignore_copying: bool
:return: list of dicts which were not removed
:rtype: list[dict]
"""
if ignore_copying:
log = LoggingMixin().log
regex_builder = r"^.*\.(%s$)$" % '$|'.join(ignored_ext)
ignored_extensions_regex = re.compile(regex_builder)
log.debug(
'Filtering result for ignored extensions: %s in files %s',
ignored_extensions_regex.pattern,
map(lambda x: x['path'], result))
result = [
x for x in result
if not ignored_extensions_regex.match(x['path'])
]
log.debug('HdfsSensor.poke: after ext filter result is %s', result)
return result
def send_MIME_email(e_from, e_to, mime_msg, dryrun=False):
log = LoggingMixin().log
SMTP_HOST = configuration.conf.get('smtp', 'SMTP_HOST')
SMTP_PORT = configuration.conf.getint('smtp', 'SMTP_PORT')
SMTP_STARTTLS = configuration.conf.getboolean('smtp', 'SMTP_STARTTLS')
SMTP_SSL = configuration.conf.getboolean('smtp', 'SMTP_SSL')
SMTP_USER = None
SMTP_PASSWORD = None
try:
SMTP_USER = configuration.conf.get('smtp', 'SMTP_USER')
SMTP_PASSWORD = configuration.conf.get('smtp', 'SMTP_PASSWORD')
except AirflowConfigException:
log.debug("No user/password found for SMTP, so logging in with no authentication.")
if not dryrun:
s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT) if SMTP_SSL else smtplib.SMTP(SMTP_HOST, SMTP_PORT)
if SMTP_STARTTLS:
s.starttls()
if SMTP_USER and SMTP_PASSWORD:
s.login(SMTP_USER, SMTP_PASSWORD)
log.info("Sent an alert email to %s", e_to)
s.sendmail(e_from, e_to, mime_msg.as_string())
s.quit()
def send_mime_email(e_from, e_to, mime_msg, dryrun=False):
"""
Send MIME email.
"""
log = LoggingMixin().log
smtp_host = conf.get('smtp', 'SMTP_HOST')
smtp_port = conf.getint('smtp', 'SMTP_PORT')
smtp_starttls = conf.getboolean('smtp', 'SMTP_STARTTLS')
smtp_ssl = conf.getboolean('smtp', 'SMTP_SSL')
smtp_user = None
smtp_password = None
try:
smtp_user = conf.get('smtp', 'SMTP_USER')
smtp_password = conf.get('smtp', 'SMTP_PASSWORD')
except AirflowConfigException:
log.debug("No user/password found for SMTP, so logging in with no authentication.")
if not dryrun:
conn = smtplib.SMTP_SSL(smtp_host, smtp_port) if smtp_ssl else smtplib.SMTP(smtp_host, smtp_port)
if smtp_starttls:
conn.starttls()
if smtp_user and smtp_password:
conn.login(smtp_user, smtp_password)
log.info("Sent an alert email to %s", e_to)
conn.sendmail(e_from, e_to, mime_msg.as_string())
conn.quit()
def _str(s):
# cloudant-python doesn't support unicode.
if isinstance(s, unicode):
log = LoggingMixin().log
log.debug(
'cloudant-python does not support unicode. Encoding %s as ascii using "ignore".',
s)
return s.encode('ascii', 'ignore')
return s
def filter_for_filesize(result: list, min_size: int) -> list:
log = LoggingMixin().log
log.debug(f'Filtering result on size >= {min_size} bytes')
result = [file_ for file_ in result if file_['size'] >= min_size]
log.debug(f"After size filter result is "
f"{list(map(lambda r: r['name'], result))!r}")
return result
def _str(s):
# cloudant-python doesn't support unicode.
if isinstance(s, unicode):
log = LoggingMixin().log
log.debug(
'cloudant-python does not support unicode. Encoding %s as ascii using "ignore".',
s
)
return s.encode('ascii', 'ignore')
return s
def filter_for_ignored_ext(result: list, ignored_ext: list) -> list:
log = LoggingMixin().log
for ext in ignored_ext:
log.debug(f'Filtering result for ignored extension {ext!r}')
result = list(
filter(
lambda file_: not file_['name'].rstrip('/').endswith(ext),
result))
log.debug(f"After ext filter result is "
f"{list(map(lambda r: r['name'], result))!r}")
return result
def filter_for_filesize(result, size=None):
"""
Will test the filepath result and test if its size is at least self.filesize
:param result: a list of dicts returned by Snakebite ls
:param size: the file size in MB a file should be at least to trigger True
:return: (bool) depending on the matching criteria
"""
if size:
log = LoggingMixin().log
log.debug('Filtering for file size >= %s in files: %s', size, map(lambda x: x['path'], result))
size *= settings.MEGABYTE
result = [x for x in result if x['length'] >= size]
log.debug('HdfsSensor.poke: after size filter result is %s', result)
return result
def filter_for_filesize(result, size=None):
"""
Will test the filepath result and test if its size is at least self.filesize
:param result: a list of dicts returned by Snakebite ls
:param size: the file size in MB a file should be at least to trigger True
:return: (bool) depending on the matching criteria
"""
if size:
log = LoggingMixin().log
log.debug('Filtering for file size >= %s in files: %s', size, map(lambda x: x['path'], result))
size *= settings.MEGABYTE
result = [x for x in result if x['length'] >= size]
log.debug('HdfsSensor.poke: after size filter result is %s', result)
return result
def filter_for_ignored_ext(result, ignored_ext, ignore_copying):
"""
Will filter if instructed to do so the result to remove matching criteria
:param result: (list) of dicts returned by Snakebite ls
:param ignored_ext: (list) of ignored extensions
:param ignore_copying: (bool) shall we ignore ?
:return: (list) of dicts which were not removed
"""
if ignore_copying:
log = LoggingMixin().log
regex_builder = "^.*\.(%s$)$" % '$|'.join(ignored_ext)
ignored_extentions_regex = re.compile(regex_builder)
log.debug(
'Filtering result for ignored extensions: %s in files %s',
ignored_extentions_regex.pattern, map(lambda x: x['path'], result)
)
result = [x for x in result if not ignored_extentions_regex.match(x['path'])]
log.debug('HdfsSensor.poke: after ext filter result is %s', result)
return result
def get_dag_fetcher(dagbag, dags_uri):
"""
Factory method that returns an instance of the right
DagFetcher, based on the dags_uri prefix.
Any prefix that does not match keys in the dag_fetchers
dict (or no prefix at all) defaults to FileSystemDagFetcher.
"""
log = LoggingMixin().log
dag_fetchers = dict(hdfs=HDFSDagFetcher,
s3=S3DagFetcher,
gcs=GCSDagFetcher,
git=GitDagFetcher)
uri_schema = dags_uri.split(':')[0]
if uri_schema not in dag_fetchers:
log.debug('Defaulting to FileSystemDagFetcher')
return FileSystemDagFetcher(dagbag, dags_uri)
return dag_fetchers[uri_schema](dagbag, dags_uri)
plugins = []
norm_pattern = re.compile(r'[/|.]')
# Crawl through the plugins folder to find AirflowPlugin derivatives
for root, dirs, files in os.walk(plugins_folder, followlinks=True):
for f in files:
try:
filepath = os.path.join(root, f)
if not os.path.isfile(filepath):
continue
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
if file_ext != '.py':
continue
log.debug('Importing plugin module %s', filepath)
# normalize root path as namespace
namespace = '_'.join([re.sub(norm_pattern, '__', root), mod_name])
m = imp.load_source(namespace, filepath)
for obj in list(m.__dict__.values()):
if (inspect.isclass(obj) and issubclass(obj, AirflowPlugin)
and obj is not AirflowPlugin):
obj.validate()
if obj not in plugins:
plugins.append(obj)
except Exception as e:
log.exception(e)
log.error('Failed to import plugin %s', filepath)
import_errors[filepath] = str(e)
norm_pattern = re.compile(r'[/|.]')
# Crawl through the plugins folder to find AirflowPlugin derivatives
for root, dirs, files in os.walk(plugins_folder, followlinks=True):
for f in files:
try:
filepath = os.path.join(root, f)
if not os.path.isfile(filepath):
continue
mod_name, file_ext = os.path.splitext(
os.path.split(filepath)[-1])
if file_ext != '.py':
continue
log.debug('Importing plugin module %s', filepath)
# normalize root path as namespace
namespace = '_'.join([re.sub(norm_pattern, '__', root), mod_name])
m = imp.load_source(namespace, filepath)
for obj in list(m.__dict__.values()):
if is_valid_plugin(obj, plugins):
plugins.append(obj)
except Exception as e:
log.exception(e)
log.error('Failed to import plugin %s', filepath)
import_errors[filepath] = str(e)
plugins = load_entrypoint_plugins(
pkg_resources.iter_entry_points('airflow.plugins'),
return func(task_instance, *args, session=session, **kwargs)
except Exception as e:
self.add_tagging(task_instance)
self.add_breadcrumbs(task_instance, session=session)
capture_exception(e)
raise
return wrapper
Sentry = DummySentry() # type: DummySentry
try:
# Verify blinker installation
from blinker import signal # noqa: F401 pylint: disable=unused-import
from sentry_sdk.integrations.logging import ignore_logger
from sentry_sdk.integrations.flask import FlaskIntegration
from sentry_sdk import (
push_scope,
configure_scope,
add_breadcrumb,
init,
capture_exception,
)
Sentry = ConfiguredSentry()
except ImportError as e:
log.debug("Could not configure Sentry: %s, using DummySentry instead.", e)
try:
from airflow_local_settings import *
log.info("Loaded airflow_local_settings.")
except:
pass
configure_logging()
configure_vars()
configure_orm()
# TODO: Unify airflow logging setups. Please see AIRFLOW-1457.
logging_config_path = conf.get('core', 'logging_config_path')
try:
from logging_config_path import LOGGING_CONFIG
log.debug("Successfully imported user-defined logging config.")
except Exception as e:
# Import default logging configurations.
log.debug(
"Unable to load custom logging config file: %s. Using default airflow logging config instead",
e
)
from airflow.config_templates.default_airflow_logging import \
DEFAULT_LOGGING_CONFIG as LOGGING_CONFIG
logging.config.dictConfig(LOGGING_CONFIG)
# Const stuff
KILOBYTE = 1024
MEGABYTE = KILOBYTE * KILOBYTE
WEB_COLORS = {'LIGHTBLUE': '#4d9de0',
