def send_MIME_email(e_from, e_to, mime_msg, dryrun=False): log = LoggingMixin().log SMTP_HOST = configuration.conf.get('smtp', 'SMTP_HOST') SMTP_PORT = configuration.conf.getint('smtp', 'SMTP_PORT') SMTP_STARTTLS = configuration.conf.getboolean('smtp', 'SMTP_STARTTLS') SMTP_SSL = configuration.conf.getboolean('smtp', 'SMTP_SSL') SMTP_USER = None SMTP_PASSWORD = None try: SMTP_USER = configuration.conf.get('smtp', 'SMTP_USER') SMTP_PASSWORD = configuration.conf.get('smtp', 'SMTP_PASSWORD') except AirflowConfigException: log.debug( "No user/password found for SMTP, so logging in with no authentication." ) if not dryrun: s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT) if SMTP_SSL else smtplib.SMTP( SMTP_HOST, SMTP_PORT) if SMTP_STARTTLS: s.starttls() if SMTP_USER and SMTP_PASSWORD: s.login(SMTP_USER, SMTP_PASSWORD) log.info("Sent an alert email to %s", e_to) s.sendmail(e_from, e_to, mime_msg.as_string()) s.quit()
def filter_for_ignored_ext(result, ignored_ext, ignore_copying): """ Will filter if instructed to do so the result to remove matching criteria :param result: list of dicts returned by Snakebite ls :type result: list[dict] :param ignored_ext: list of ignored extensions :type ignored_ext: list :param ignore_copying: shall we ignore ? :type ignore_copying: bool :return: list of dicts which were not removed :rtype: list[dict] """ if ignore_copying: log = LoggingMixin().log regex_builder = r"^.*\.(%s$)$" % '$|'.join(ignored_ext) ignored_extensions_regex = re.compile(regex_builder) log.debug( 'Filtering result for ignored extensions: %s in files %s', ignored_extensions_regex.pattern, map(lambda x: x['path'], result)) result = [ x for x in result if not ignored_extensions_regex.match(x['path']) ] log.debug('HdfsSensor.poke: after ext filter result is %s', result) return result
def send_MIME_email(e_from, e_to, mime_msg, dryrun=False): log = LoggingMixin().log SMTP_HOST = configuration.conf.get('smtp', 'SMTP_HOST') SMTP_PORT = configuration.conf.getint('smtp', 'SMTP_PORT') SMTP_STARTTLS = configuration.conf.getboolean('smtp', 'SMTP_STARTTLS') SMTP_SSL = configuration.conf.getboolean('smtp', 'SMTP_SSL') SMTP_USER = None SMTP_PASSWORD = None try: SMTP_USER = configuration.conf.get('smtp', 'SMTP_USER') SMTP_PASSWORD = configuration.conf.get('smtp', 'SMTP_PASSWORD') except AirflowConfigException: log.debug("No user/password found for SMTP, so logging in with no authentication.") if not dryrun: s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT) if SMTP_SSL else smtplib.SMTP(SMTP_HOST, SMTP_PORT) if SMTP_STARTTLS: s.starttls() if SMTP_USER and SMTP_PASSWORD: s.login(SMTP_USER, SMTP_PASSWORD) log.info("Sent an alert email to %s", e_to) s.sendmail(e_from, e_to, mime_msg.as_string()) s.quit()
def send_mime_email(e_from, e_to, mime_msg, dryrun=False): """ Send MIME email. """ log = LoggingMixin().log smtp_host = conf.get('smtp', 'SMTP_HOST') smtp_port = conf.getint('smtp', 'SMTP_PORT') smtp_starttls = conf.getboolean('smtp', 'SMTP_STARTTLS') smtp_ssl = conf.getboolean('smtp', 'SMTP_SSL') smtp_user = None smtp_password = None try: smtp_user = conf.get('smtp', 'SMTP_USER') smtp_password = conf.get('smtp', 'SMTP_PASSWORD') except AirflowConfigException: log.debug("No user/password found for SMTP, so logging in with no authentication.") if not dryrun: conn = smtplib.SMTP_SSL(smtp_host, smtp_port) if smtp_ssl else smtplib.SMTP(smtp_host, smtp_port) if smtp_starttls: conn.starttls() if smtp_user and smtp_password: conn.login(smtp_user, smtp_password) log.info("Sent an alert email to %s", e_to) conn.sendmail(e_from, e_to, mime_msg.as_string()) conn.quit()
def _str(s): # cloudant-python doesn't support unicode. if isinstance(s, unicode): log = LoggingMixin().log log.debug( 'cloudant-python does not support unicode. Encoding %s as ascii using "ignore".', s) return s.encode('ascii', 'ignore') return s
def filter_for_filesize(result: list, min_size: int) -> list: log = LoggingMixin().log log.debug(f'Filtering result on size >= {min_size} bytes') result = [file_ for file_ in result if file_['size'] >= min_size] log.debug(f"After size filter result is " f"{list(map(lambda r: r['name'], result))!r}") return result
def _str(s): # cloudant-python doesn't support unicode. if isinstance(s, unicode): log = LoggingMixin().log log.debug( 'cloudant-python does not support unicode. Encoding %s as ascii using "ignore".', s ) return s.encode('ascii', 'ignore') return s
def filter_for_ignored_ext(result: list, ignored_ext: list) -> list: log = LoggingMixin().log for ext in ignored_ext: log.debug(f'Filtering result for ignored extension {ext!r}') result = list( filter( lambda file_: not file_['name'].rstrip('/').endswith(ext), result)) log.debug(f"After ext filter result is " f"{list(map(lambda r: r['name'], result))!r}") return result
def filter_for_filesize(result, size=None): """ Will test the filepath result and test if its size is at least self.filesize :param result: a list of dicts returned by Snakebite ls :param size: the file size in MB a file should be at least to trigger True :return: (bool) depending on the matching criteria """ if size: log = LoggingMixin().log log.debug('Filtering for file size >= %s in files: %s', size, map(lambda x: x['path'], result)) size *= settings.MEGABYTE result = [x for x in result if x['length'] >= size] log.debug('HdfsSensor.poke: after size filter result is %s', result) return result
def filter_for_filesize(result, size=None): """ Will test the filepath result and test if its size is at least self.filesize :param result: a list of dicts returned by Snakebite ls :param size: the file size in MB a file should be at least to trigger True :return: (bool) depending on the matching criteria """ if size: log = LoggingMixin().log log.debug('Filtering for file size >= %s in files: %s', size, map(lambda x: x['path'], result)) size *= settings.MEGABYTE result = [x for x in result if x['length'] >= size] log.debug('HdfsSensor.poke: after size filter result is %s', result) return result
def filter_for_ignored_ext(result, ignored_ext, ignore_copying): """ Will filter if instructed to do so the result to remove matching criteria :param result: (list) of dicts returned by Snakebite ls :param ignored_ext: (list) of ignored extensions :param ignore_copying: (bool) shall we ignore ? :return: (list) of dicts which were not removed """ if ignore_copying: log = LoggingMixin().log regex_builder = "^.*\.(%s$)$" % '$|'.join(ignored_ext) ignored_extentions_regex = re.compile(regex_builder) log.debug( 'Filtering result for ignored extensions: %s in files %s', ignored_extentions_regex.pattern, map(lambda x: x['path'], result) ) result = [x for x in result if not ignored_extentions_regex.match(x['path'])] log.debug('HdfsSensor.poke: after ext filter result is %s', result) return result
def get_dag_fetcher(dagbag, dags_uri): """ Factory method that returns an instance of the right DagFetcher, based on the dags_uri prefix. Any prefix that does not match keys in the dag_fetchers dict (or no prefix at all) defaults to FileSystemDagFetcher. """ log = LoggingMixin().log dag_fetchers = dict(hdfs=HDFSDagFetcher, s3=S3DagFetcher, gcs=GCSDagFetcher, git=GitDagFetcher) uri_schema = dags_uri.split(':')[0] if uri_schema not in dag_fetchers: log.debug('Defaulting to FileSystemDagFetcher') return FileSystemDagFetcher(dagbag, dags_uri) return dag_fetchers[uri_schema](dagbag, dags_uri)
plugins = [] norm_pattern = re.compile(r'[/|.]') # Crawl through the plugins folder to find AirflowPlugin derivatives for root, dirs, files in os.walk(plugins_folder, followlinks=True): for f in files: try: filepath = os.path.join(root, f) if not os.path.isfile(filepath): continue mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) if file_ext != '.py': continue log.debug('Importing plugin module %s', filepath) # normalize root path as namespace namespace = '_'.join([re.sub(norm_pattern, '__', root), mod_name]) m = imp.load_source(namespace, filepath) for obj in list(m.__dict__.values()): if (inspect.isclass(obj) and issubclass(obj, AirflowPlugin) and obj is not AirflowPlugin): obj.validate() if obj not in plugins: plugins.append(obj) except Exception as e: log.exception(e) log.error('Failed to import plugin %s', filepath) import_errors[filepath] = str(e)
norm_pattern = re.compile(r'[/|.]') # Crawl through the plugins folder to find AirflowPlugin derivatives for root, dirs, files in os.walk(plugins_folder, followlinks=True): for f in files: try: filepath = os.path.join(root, f) if not os.path.isfile(filepath): continue mod_name, file_ext = os.path.splitext( os.path.split(filepath)[-1]) if file_ext != '.py': continue log.debug('Importing plugin module %s', filepath) # normalize root path as namespace namespace = '_'.join([re.sub(norm_pattern, '__', root), mod_name]) m = imp.load_source(namespace, filepath) for obj in list(m.__dict__.values()): if is_valid_plugin(obj, plugins): plugins.append(obj) except Exception as e: log.exception(e) log.error('Failed to import plugin %s', filepath) import_errors[filepath] = str(e) plugins = load_entrypoint_plugins( pkg_resources.iter_entry_points('airflow.plugins'),
return func(task_instance, *args, session=session, **kwargs) except Exception as e: self.add_tagging(task_instance) self.add_breadcrumbs(task_instance, session=session) capture_exception(e) raise return wrapper Sentry = DummySentry() # type: DummySentry try: # Verify blinker installation from blinker import signal # noqa: F401 pylint: disable=unused-import from sentry_sdk.integrations.logging import ignore_logger from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk import ( push_scope, configure_scope, add_breadcrumb, init, capture_exception, ) Sentry = ConfiguredSentry() except ImportError as e: log.debug("Could not configure Sentry: %s, using DummySentry instead.", e)
try: from airflow_local_settings import * log.info("Loaded airflow_local_settings.") except: pass configure_logging() configure_vars() configure_orm() # TODO: Unify airflow logging setups. Please see AIRFLOW-1457. logging_config_path = conf.get('core', 'logging_config_path') try: from logging_config_path import LOGGING_CONFIG log.debug("Successfully imported user-defined logging config.") except Exception as e: # Import default logging configurations. log.debug( "Unable to load custom logging config file: %s. Using default airflow logging config instead", e ) from airflow.config_templates.default_airflow_logging import \ DEFAULT_LOGGING_CONFIG as LOGGING_CONFIG logging.config.dictConfig(LOGGING_CONFIG) # Const stuff KILOBYTE = 1024 MEGABYTE = KILOBYTE * KILOBYTE WEB_COLORS = {'LIGHTBLUE': '#4d9de0',