def test_loading_local_settings_without_logging_config(self): from airflow.logging_config import configure_logging, log with patch.object(log, 'debug') as mock_info: configure_logging() mock_info.assert_called_with( 'Unable to load custom logging, using default config instead' )
def test_1_9_config(self): from airflow.logging_config import configure_logging conf.set('core', 'task_log_reader', 'file.task') try: with self.assertWarnsRegex(DeprecationWarning, r'file.task'): configure_logging() self.assertEqual(conf.get('core', 'task_log_reader'), 'task') finally: conf.remove_option('core', 'task_log_reader', remove_default=False)
def test_loading_invalid_local_settings(self): from airflow.logging_config import configure_logging, log with settings_context(SETTINGS_FILE_INVALID): with patch.object(log, 'warning') as mock_info: # Load config with self.assertRaises(ValueError): configure_logging() mock_info.assert_called_with( 'Unable to load the config, contains a configuration error.' )
def test_loading_valid_local_settings(self): with settings_context(SETTINGS_FILE_VALID): from airflow.logging_config import configure_logging, log with patch.object(log, 'info') as mock_info: configure_logging() mock_info.assert_called_with( 'Successfully imported user-defined logging config from %s', '{}.LOGGING_CONFIG'.format( SETTINGS_DEFAULT_NAME ) )
def test_loading_valid_complex_local_settings(self): # Test what happens when the config is somewhere in a subfolder module_structure = 'etc.airflow.config' dir_structure = module_structure.replace('.', '/') with settings_context(SETTINGS_FILE_VALID, dir_structure): from airflow.logging_config import configure_logging, log with patch.object(log, 'info') as mock_info: configure_logging() mock_info.assert_called_with( 'Successfully imported user-defined logging config from %s', 'etc.airflow.config.{}.LOGGING_CONFIG'.format( SETTINGS_DEFAULT_NAME ) )
def test_when_the_config_key_does_not_exists(self): from airflow import logging_config def side_effect(*args): if args[1] == 'logging_config_class': raise AirflowConfigException else: return "bla_bla_from_test" logging_config.conf.get = mock.Mock(side_effect=side_effect) with patch.object(logging_config.log, 'debug') as mock_debug: logging_config.configure_logging() mock_debug.assert_any_call('Could not find key logging_config_class in config')
def test_loading_remote_logging_with_wasb_handler(self): """Test if logging can be configured successfully for Azure Blob Storage""" import logging from airflow.config_templates import airflow_local_settings from airflow.logging_config import configure_logging from airflow.utils.log.wasb_task_handler import WasbTaskHandler conf.set('core', 'remote_logging', 'True') conf.set('core', 'remote_log_conn_id', 'some_wasb') conf.set('core', 'remote_base_log_folder', 'wasb://some-folder') six.moves.reload_module(airflow_local_settings) configure_logging() logger = logging.getLogger('airflow.task') self.assertIsInstance(logger.handlers[0], WasbTaskHandler)
def initialize(): configure_vars() prepare_classpath() global LOGGING_CLASS_PATH LOGGING_CLASS_PATH = configure_logging() configure_adapters() # The webservers import this file from models.py with the default settings. configure_orm() configure_action_logging() # Ensure we close DB connections at scheduler and gunicon worker terminations atexit.register(dispose_orm)
def start_master(master_yaml_path): configure_logging() server_runner = ai_flow.AIFlowServerRunner(config_file=master_yaml_path) server_runner.start(is_block=False) return server_runner
def create_app(config=None, testing=False, app_name="Airflow"): global app, appbuilder app = Flask(__name__) app.secret_key = conf.get('webserver', 'SECRET_KEY') airflow_home_path = conf.get('core', 'AIRFLOW_HOME') webserver_config_path = airflow_home_path + '/webserver_config.py' app.config.from_pyfile(webserver_config_path, silent=True) app.config['APP_NAME'] = app_name app.config['TESTING'] = testing csrf.init_app(app) db = SQLA(app) from airflow import api api.load_auth() api.api_auth.init_app(app) cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) # noqa from airflow.www_rbac.blueprints import routes app.register_blueprint(routes) configure_logging() with app.app_context(): appbuilder = AppBuilder( app, db.session, security_manager_class=app.config.get('SECURITY_MANAGER_CLASS'), base_template='appbuilder/baselayout.html') def init_views(appbuilder): from airflow.www_rbac import views appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view_no_menu(views.ConfigurationView()) appbuilder.add_view_no_menu(views.VersionView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_link("Configurations", href='/configuration', category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") appbuilder.add_link("Documentation", href='https://airflow.apache.org/', category="Docs", category_icon="fa-cube") appbuilder.add_link("Github", href='https://github.com/apache/incubator-airflow', category="Docs") appbuilder.add_link('Version', href='/version', category='About', category_icon='fa-th') # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. appbuilder.security_cleanup() init_views(appbuilder) from airflow.www_rbac.security import init_roles init_roles(appbuilder) from airflow.www_rbac.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: if six.PY2: reload(e) # noqa else: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': socket.getfqdn(), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app, appbuilder
def configure_action_logging(): """ Any additional configuration (register callback) for airflow.utils.action_loggers module :return: None """ pass try: from airflow_local_settings import * log.info("Loaded airflow_local_settings.") except: pass configure_logging() configure_vars() configure_adapters() # The webservers import this file from models.py with the default settings. configure_orm() configure_action_logging() # Ensure we close DB connections at scheduler and gunicon worker terminations atexit.register(dispose_orm) # Const stuff KILOBYTE = 1024 MEGABYTE = KILOBYTE * KILOBYTE WEB_COLORS = {'LIGHTBLUE': '#4d9de0', 'LIGHTORANGE': '#FF9933'}
def create_app(config=None, testing=False): """Create a new instance of Airflow WWW app""" flask_app = Flask(__name__) flask_app.secret_key = conf.get('webserver', 'SECRET_KEY') flask_app.config['PERMANENT_SESSION_LIFETIME'] = timedelta( minutes=settings.get_session_lifetime_config()) flask_app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True) flask_app.config['APP_NAME'] = conf.get(section="webserver", key="instance_name", fallback="Airflow") flask_app.config['TESTING'] = testing flask_app.config['SQLALCHEMY_DATABASE_URI'] = conf.get( 'core', 'SQL_ALCHEMY_CONN') flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False flask_app.config['SESSION_COOKIE_HTTPONLY'] = True flask_app.config['SESSION_COOKIE_SECURE'] = conf.getboolean( 'webserver', 'COOKIE_SECURE') cookie_samesite_config = conf.get('webserver', 'COOKIE_SAMESITE') if cookie_samesite_config == "": warnings.warn( "Old deprecated value found for `cookie_samesite` option in `[webserver]` section. " "Using `Lax` instead. Change the value to `Lax` in airflow.cfg to remove this warning.", DeprecationWarning, ) cookie_samesite_config = "Lax" flask_app.config['SESSION_COOKIE_SAMESITE'] = cookie_samesite_config if config: flask_app.config.from_mapping(config) if 'SQLALCHEMY_ENGINE_OPTIONS' not in flask_app.config: flask_app.config[ 'SQLALCHEMY_ENGINE_OPTIONS'] = settings.prepare_engine_args() # Configure the JSON encoder used by `|tojson` filter from Flask flask_app.json_encoder = AirflowJsonEncoder csrf.init_app(flask_app) init_wsgi_middleware(flask_app) db = SQLA() db.session = settings.Session db.init_app(flask_app) init_dagbag(flask_app) init_api_experimental_auth(flask_app) Cache(app=flask_app, config={ 'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp' }) init_flash_views(flask_app) configure_logging() configure_manifest_files(flask_app) with flask_app.app_context(): init_appbuilder(flask_app) init_appbuilder_views(flask_app) init_appbuilder_links(flask_app) init_plugins(flask_app) init_connection_form() init_error_handlers(flask_app) init_api_connexion(flask_app) init_api_experimental(flask_app) sync_appbuilder_roles(flask_app) init_jinja_globals(flask_app) init_xframe_protection(flask_app) init_permanent_session(flask_app) return flask_app
def test_loading_no_local_settings(self): with settings_context(SETTINGS_FILE_EMPTY): from airflow.logging_config import configure_logging with self.assertRaises(ImportError): configure_logging()
def create_app(config=None, session=None, testing=False, app_name="Airflow"): global app, appbuilder app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = conf.get('webserver', 'SECRET_KEY') airflow_home_path = conf.get('core', 'AIRFLOW_HOME') webserver_config_path = airflow_home_path + '/webserver_config.py' app.config.from_pyfile(webserver_config_path, silent=True) app.config['APP_NAME'] = app_name app.config['TESTING'] = testing csrf.init_app(app) db = SQLA(app) from airflow import api api.load_auth() api.api_auth.init_app(app) # flake8: noqa: F841 cache = Cache(app=app, config={ 'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp' }) from airflow.www.blueprints import routes app.register_blueprint(routes) configure_logging() configure_manifest_files(app) with app.app_context(): from airflow.www.security import AirflowSecurityManager security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \ AirflowSecurityManager if not issubclass(security_manager_class, AirflowSecurityManager): raise Exception( """Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager, not FAB's security manager.""") appbuilder = AppBuilder(app, db.session if not session else session, security_manager_class=security_manager_class, base_template='appbuilder/baselayout.html') def init_views(appbuilder): from airflow.www import views appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view_no_menu(views.ConfigurationView()) appbuilder.add_view_no_menu(views.VersionView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_link("Configurations", href='/configuration', category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") appbuilder.add_link("Documentation", href='https://airflow.apache.org/', category="Docs", category_icon="fa-cube") appbuilder.add_link("Github", href='https://github.com/apache/airflow', category="Docs") appbuilder.add_link('Version', href='/version', category='About', category_icon='fa-th') def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import ( flask_appbuilder_views, flask_appbuilder_menu_links) for v in flask_appbuilder_views: log.debug("Adding view %s", v["name"]) appbuilder.add_view(v["view"], v["name"], category=v["category"]) for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]): log.debug("Adding menu link %s", ml["name"]) appbuilder.add_link(ml["name"], href=ml["href"], category=ml["category"], category_icon=ml["category_icon"]) integrate_plugins() # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. def init_plugin_blueprints(app): from airflow.plugins_manager import flask_blueprints for bp in flask_blueprints: log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name) app.register_blueprint(bp["blueprint"]) init_views(appbuilder) init_plugin_blueprints(app) security_manager = appbuilder.sm security_manager.sync_roles() from airflow.www.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: if six.PY2: reload(e) # noqa else: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': socket.getfqdn(), 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app, appbuilder
def create_app(config=None, session=None, testing=False, app_name="Airflow"): global app, appbuilder app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies=conf.get("webserver", "PROXY_FIX_NUM_PROXIES", fallback=None), x_for=conf.getint("webserver", "PROXY_FIX_X_FOR", fallback=1), x_proto=conf.getint("webserver", "PROXY_FIX_X_PROTO", fallback=1), x_host=conf.getint("webserver", "PROXY_FIX_X_HOST", fallback=1), x_port=conf.getint("webserver", "PROXY_FIX_X_PORT", fallback=1), x_prefix=conf.getint("webserver", "PROXY_FIX_X_PREFIX", fallback=1)) app.secret_key = conf.get('webserver', 'SECRET_KEY') session_lifetime_days = conf.getint('webserver', 'SESSION_LIFETIME_DAYS', fallback=30) app.config['PERMANENT_SESSION_LIFETIME'] = timedelta( days=session_lifetime_days) app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True) app.config['APP_NAME'] = app_name app.config['TESTING'] = testing app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['SESSION_COOKIE_HTTPONLY'] = True app.config['SESSION_COOKIE_SECURE'] = conf.getboolean( 'webserver', 'COOKIE_SECURE') app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE') if config: app.config.from_mapping(config) # Configure the JSON encoder used by `|tojson` filter from Flask app.json_encoder = AirflowJsonEncoder csrf.init_app(app) db = SQLA(app) from airflow import api api.load_auth() api.API_AUTH.api_auth.init_app(app) Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) from airflow.www.blueprints import routes app.register_blueprint(routes) configure_logging() configure_manifest_files(app) with app.app_context(): from airflow.www.security import AirflowSecurityManager security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \ AirflowSecurityManager if not issubclass(security_manager_class, AirflowSecurityManager): raise Exception( """Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager, not FAB's security manager.""") appbuilder = AppBuilder(app, db.session if not session else session, security_manager_class=security_manager_class, base_template='appbuilder/baselayout.html', update_perms=conf.getboolean( 'webserver', 'UPDATE_FAB_PERMS')) def init_views(appbuilder): from airflow.www import views # Remove the session from scoped_session registry to avoid # reusing a session with a disconnected connection appbuilder.session.remove() appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_view(views.ConfigurationView, "Configurations", category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") if "dev" in version.version: airflow_doc_site = "https://airflow.readthedocs.io/en/latest" else: airflow_doc_site = 'https://airflow.apache.org/docs/{}'.format( version.version) appbuilder.add_link("Website", href='https://airflow.apache.org', category="Docs", category_icon="fa-globe") appbuilder.add_link("Documentation", href=airflow_doc_site, category="Docs", category_icon="fa-cube") appbuilder.add_link("GitHub", href='https://github.com/apache/airflow', category="Docs") appbuilder.add_view(views.VersionView, 'Version', category='About', category_icon='fa-th') def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import ( flask_appbuilder_views, flask_appbuilder_menu_links) for v in flask_appbuilder_views: log.debug("Adding view %s", v["name"]) appbuilder.add_view(v["view"], v["name"], category=v["category"]) for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]): log.debug("Adding menu link %s", ml["name"]) appbuilder.add_link(ml["name"], href=ml["href"], category=ml["category"], category_icon=ml["category_icon"]) integrate_plugins() # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. def init_plugin_blueprints(app): from airflow.plugins_manager import flask_blueprints for bp in flask_blueprints: log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name) app.register_blueprint(bp["blueprint"]) init_views(appbuilder) init_plugin_blueprints(app) if conf.getboolean('webserver', 'UPDATE_FAB_PERMS'): security_manager = appbuilder.sm security_manager.sync_roles() from airflow.www.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): # pylint: disable=unused-variable globals = { 'hostname': socket.getfqdn() if conf.getboolean( 'webserver', 'EXPOSE_HOSTNAME', fallback=True) else 'redact', 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), 'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000) } if 'analytics_tool' in conf.getsection('webserver'): globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID') }) return globals @app.before_request def before_request(): _force_log_out_after = conf.getint('webserver', 'FORCE_LOG_OUT_AFTER', fallback=0) if _force_log_out_after > 0: flask.session.permanent = True app.permanent_session_lifetime = datetime.timedelta( minutes=_force_log_out_after) flask.session.modified = True flask.g.user = flask_login.current_user @app.after_request def apply_caching(response): _x_frame_enabled = conf.getboolean('webserver', 'X_FRAME_ENABLED', fallback=True) if not _x_frame_enabled: response.headers["X-Frame-Options"] = "DENY" return response @app.teardown_appcontext def shutdown_session(exception=None): # pylint: disable=unused-variable settings.Session.remove() @app.before_request def make_session_permanent(): flask_session.permanent = True return app, appbuilder
def test_1_9_config(self): from airflow.logging_config import configure_logging with conf_vars({('logging', 'task_log_reader'): 'file.task'}): with self.assertWarnsRegex(DeprecationWarning, r'file.task'): configure_logging() self.assertEqual(conf.get('logging', 'task_log_reader'), 'task')
def tearDown(self) -> None: for handler_ref in logging._handlerList[:]: logging._removeHandlerRef(handler_ref) importlib.reload(airflow_local_settings) configure_logging()
def create_app(config=None, testing=False): app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies=conf.get("webserver", "PROXY_FIX_NUM_PROXIES", fallback=None), x_for=conf.get("webserver", "PROXY_FIX_X_FOR", fallback=1), x_proto=conf.get("webserver", "PROXY_FIX_X_PROTO", fallback=1), x_host=conf.get("webserver", "PROXY_FIX_X_HOST", fallback=1), x_port=conf.get("webserver", "PROXY_FIX_X_PORT", fallback=1), x_prefix=conf.get("webserver", "PROXY_FIX_X_PREFIX", fallback=1)) app.secret_key = conf.get('webserver', 'SECRET_KEY') app.config['LOGIN_DISABLED'] = not conf.getboolean('webserver', 'AUTHENTICATE') app.config['SESSION_COOKIE_HTTPONLY'] = True app.config['SESSION_COOKIE_SECURE'] = conf.getboolean( 'webserver', 'COOKIE_SECURE') app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE') if config: app.config.from_mapping(config) csrf.init_app(app) app.config['TESTING'] = testing airflow.load_login() airflow.login.LOGIN_MANAGER.init_app(app) from airflow import api api.load_auth() api.API_AUTH.api_auth.init_app(app) # flake8: noqa: F841 cache = Cache(app=app, config={ 'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp' }) app.register_blueprint(routes) configure_logging() with app.app_context(): from airflow.www import views admin = Admin( app, name='Airflow', static_url_path='/admin', index_view=views.HomeView(endpoint='', url='/admin', name="DAGs"), template_mode='bootstrap3', ) av = admin.add_view vs = views av(vs.Airflow(name='DAGs', category='DAGs')) if not conf.getboolean('core', 'secure_mode'): av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling")) av( vs.ChartModelView(models.Chart, Session, name="Charts", category="Data Profiling")) av( vs.KnownEventView(models.KnownEvent, Session, name="Known Events", category="Data Profiling")) av( vs.SlaMissModelView(models.SlaMiss, Session, name="SLA Misses", category="Browse")) av( vs.TaskInstanceModelView(models.TaskInstance, Session, name="Task Instances", category="Browse")) av(vs.LogModelView(models.Log, Session, name="Logs", category="Browse")) av( vs.JobModelView(jobs.BaseJob, Session, name="Jobs", category="Browse")) av( vs.PoolModelView(models.Pool, Session, name="Pools", category="Admin")) av(vs.ConfigurationView(name='Configuration', category="Admin")) av( vs.UserModelView(models.User, Session, name="Users", category="Admin")) av( vs.ConnectionModelView(Connection, Session, name="Connections", category="Admin")) av( vs.VariableView(models.Variable, Session, name="Variables", category="Admin")) av(vs.XComView(models.XCom, Session, name="XComs", category="Admin")) if "dev" in version.version: airflow_doc_site = "https://airflow.readthedocs.io/en/latest" else: airflow_doc_site = 'https://airflow.apache.org/docs/{}'.format( version.version) admin.add_link( base.MenuLink(category='Docs', name='Documentation', url=airflow_doc_site)) admin.add_link( base.MenuLink(category='Docs', name='GitHub', url='https://github.com/apache/airflow')) av(vs.VersionView(name='Version', category="About")) av( vs.DagRunModelView(models.DagRun, Session, name="DAG Runs", category="Browse")) av(vs.DagModelView(models.DagModel, Session, name=None)) # Hack to not add this view to the menu admin._menu = admin._menu[:-1] def integrate_plugins(): """Integrate plugins to the context""" log = LoggingMixin().log from airflow.plugins_manager import (admin_views, flask_blueprints, menu_links) for v in admin_views: log.debug('Adding view %s', v.name) admin.add_view(v) for bp in flask_blueprints: log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name) app.register_blueprint(bp["blueprint"]) for ml in sorted(menu_links, key=lambda x: x.name): log.debug('Adding menu link %s', ml.name) admin.add_link(ml) integrate_plugins() import airflow.www.api.experimental.endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: six.moves.reload_module(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': get_hostname(), 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app
def create_app(config=None, testing=False, app_name="Airflow"): """Create a new instance of Airflow WWW app""" flask_app = Flask(__name__) flask_app.secret_key = conf.get('webserver', 'SECRET_KEY') session_lifetime_days = conf.getint('webserver', 'SESSION_LIFETIME_DAYS', fallback=30) flask_app.config['PERMANENT_SESSION_LIFETIME'] = timedelta( days=session_lifetime_days) flask_app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True) flask_app.config['APP_NAME'] = app_name flask_app.config['TESTING'] = testing flask_app.config['SQLALCHEMY_DATABASE_URI'] = conf.get( 'core', 'SQL_ALCHEMY_CONN') flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False flask_app.config['SESSION_COOKIE_HTTPONLY'] = True flask_app.config['SESSION_COOKIE_SECURE'] = conf.getboolean( 'webserver', 'COOKIE_SECURE') flask_app.config['SESSION_COOKIE_SAMESITE'] = conf.get( 'webserver', 'COOKIE_SAMESITE') if config: flask_app.config.from_mapping(config) # Configure the JSON encoder used by `|tojson` filter from Flask flask_app.json_encoder = AirflowJsonEncoder csrf.init_app(flask_app) init_wsgi_middleware(flask_app) db = SQLA() db.session = settings.Session db.init_app(flask_app) init_dagbag(flask_app) init_api_experimental_auth(flask_app) Cache(app=flask_app, config={ 'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp' }) init_flash_views(flask_app) configure_logging() configure_manifest_files(flask_app) with flask_app.app_context(): init_appbuilder(flask_app) init_appbuilder_views(flask_app) init_appbuilder_links(flask_app) init_plugins(flask_app) init_error_handlers(flask_app) init_api_connexion(flask_app) init_api_experimental(flask_app) sync_appbuilder_roles(flask_app) init_jinja_globals(flask_app) init_logout_timeout(flask_app) init_xframe_protection(flask_app) init_permanent_session(flask_app) return flask_app
def create_app(config=None, session=None, testing=False, app_name="Airflow"): cluster_id = os.getenv('CLUSTER_ID', "") cdn_url = 'https://d3chibfrmv0usk.cloudfront.net/airflow/v1.10.9' airflow_webserver_proxy_uri = "airflow-webserver-{0}".format(cluster_id) airflow_webserver_proxy_admin_uri = "/{0}/home".format(airflow_webserver_proxy_uri) global app, appbuilder app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix( app.wsgi_app, x_for=conf.getint("webserver", "PROXY_FIX_X_FOR", fallback=1), x_proto=conf.getint("webserver", "PROXY_FIX_X_PROTO", fallback=1), x_host=conf.getint("webserver", "PROXY_FIX_X_HOST", fallback=1), x_port=conf.getint("webserver", "PROXY_FIX_X_PORT", fallback=1), x_prefix=conf.getint("webserver", "PROXY_FIX_X_PREFIX", fallback=1) ) app.secret_key = conf.get('webserver', 'SECRET_KEY') session_lifetime_days = conf.getint('webserver', 'SESSION_LIFETIME_DAYS', fallback=30) app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=session_lifetime_days) app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True) app.config['APP_NAME'] = app_name app.config['TESTING'] = testing app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['SESSION_COOKIE_HTTPONLY'] = True app.config['SESSION_COOKIE_SECURE'] = conf.getboolean('webserver', 'COOKIE_SECURE') app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE') if config: app.config.from_mapping(config) # Configure the JSON encoder used by `|tojson` filter from Flask app.json_encoder = AirflowJsonEncoder csrf.init_app(app) app.config["SESSION_PROTECTION"] = False app.config["SESSION_COOKIE_NAME"] = "webserver_{}_session".format(cluster_id) db = SQLA(app) def can_see_profile(user): for role in user.roles: for perms in role.permissions: if perms.permission.name == "can_userinfo": return True return False app.jinja_env.tests['can_see_profile'] = can_see_profile BaseView.get_redirect = proxified_get_redirect AppBuilder.get_url_for_index = proxified_get_url_for_index AppBuilder.get_url_for_login = proxified_get_url_for_login from airflow import api api.load_auth() api.API_AUTH.api_auth.init_app(app) Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) from airflow.www.blueprints import routes app.register_blueprint(routes) configure_logging() configure_manifest_files(app) with app.app_context(): from airflow.www.qubole_security import QuboleSecurityManager security_manager_class = QuboleSecurityManager # if not issubclass(security_manager_class, AirflowSecurityManager): # raise Exception( # """Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager, # not FAB's security manager.""") from airflow.utils.qubole_utils import override_route_base override_route_base("appbuilder") appbuilder = AppBuilder( app, db.session if not session else session, security_manager_class=security_manager_class, base_template='airflow/master.html', update_perms=conf.getboolean('webserver', 'UPDATE_FAB_PERMS')) def init_views(appbuilder): override_route_base("airflow") from airflow.www import views # Remove the session from scoped_session registry to avoid # reusing a session with a disconnected connection appbuilder.session.remove() appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_view(views.ConfigurationView, "Configurations", category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") if "dev" in version.version: airflow_doc_site = "https://airflow.readthedocs.io/en/latest" else: airflow_doc_site = 'https://airflow.apache.org/docs/1.10.9' appbuilder.add_link("Website", href='https://airflow.apache.org', category="Docs", category_icon="fa-globe") appbuilder.add_link("Documentation", href=airflow_doc_site, category="Docs", category_icon="fa-cube") appbuilder.add_link("GitHub", href='https://github.com/apache/airflow', category="Docs") appbuilder.add_link("Qubole Documentation", href='http://docs.qubole.com/en/latest/user-guide/airflow/index.html', category="Docs", category_icon="fa-cube") appbuilder.add_link("Blog", href='https://www.qubole.com/tech-category/airflow/', category="Docs", category_icon="fa-cube") appbuilder.add_link("Video Tutorial", href='https://www.youtube.com/watch?v=4_Ifm4PNRyg&list=PL5_c35Deekdm6N1OBHdQm7JZECTdm7zl-', category="Docs", category_icon="fa-cube") appbuilder.add_view(views.VersionView, 'Version', category='About', category_icon='fa-th') def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import ( flask_appbuilder_views, flask_appbuilder_menu_links ) for v in flask_appbuilder_views: log.debug("Adding view %s", v["name"]) appbuilder.add_view(v["view"], v["name"], category=v["category"]) for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]): log.debug("Adding menu link %s", ml["name"]) appbuilder.add_link(ml["name"], href=ml["href"], category=ml["category"], category_icon=ml["category_icon"]) integrate_plugins() # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. def init_plugin_blueprints(app): from airflow.plugins_manager import flask_blueprints for bp in flask_blueprints: log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name) app.register_blueprint(bp["blueprint"]) init_views(appbuilder) init_plugin_blueprints(app) if conf.getboolean('webserver', 'UPDATE_FAB_PERMS'): security_manager = appbuilder.sm security_manager.sync_roles() from airflow.www.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: import importlib importlib.reload(e) airflow_webserver_proxy_uri = os.environ.get('AIRFLOW_WEBSERVER_PROXY_URI', "") app.register_blueprint(e.api_experimental, url_prefix='/{}/api/experimental'.format(airflow_webserver_proxy_uri)) @app.context_processor def jinja_globals(): # pylint: disable=unused-variable globals = { 'hostname': socket.getfqdn() if conf.getboolean( 'webserver', 'EXPOSE_HOSTNAME', fallback=True) else 'redact', 'navbar_color': conf.get( 'webserver', 'NAVBAR_COLOR'), 'cluster_id': cluster_id, 'log_fetch_delay_sec': conf.getint( 'webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint( 'webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint( 'webserver', 'log_animation_speed', fallback=1000) } if 'analytics_tool' in conf.getsection('webserver'): globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID') }) return globals @app.before_request def before_request(): _force_log_out_after = conf.getint('webserver', 'FORCE_LOG_OUT_AFTER', fallback=0) if _force_log_out_after > 0: flask.session.permanent = True app.permanent_session_lifetime = datetime.timedelta(minutes=_force_log_out_after) flask.session.modified = True flask.g.user = flask_login.current_user @app.after_request def apply_caching(response): _x_frame_enabled = conf.getboolean('webserver', 'X_FRAME_ENABLED', fallback=True) if not _x_frame_enabled: response.headers["X-Frame-Options"] = "DENY" return response @app.teardown_appcontext def shutdown_session(exception=None): # pylint: disable=unused-variable settings.Session.remove() @app.before_request def make_session_permanent(): flask_session.permanent = True @app.before_request def clear_cookies(): from flask import session user_id = session.get('user_id', None) if user_id: from airflow.settings import Session from flask_appbuilder.security.sqla.models import User user = Session.query(User).filter_by(id = user_id).first() if user is None: session.clear() return app, appbuilder
def create_app(config=None, testing=False): app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix(app.wsgi_app, x_for=conf.getint("webserver", "PROXY_FIX_X_FOR", fallback=1), x_proto=conf.getint("webserver", "PROXY_FIX_X_PROTO", fallback=1), x_host=conf.getint("webserver", "PROXY_FIX_X_HOST", fallback=1), x_port=conf.getint("webserver", "PROXY_FIX_X_PORT", fallback=1), x_prefix=conf.getint("webserver", "PROXY_FIX_X_PREFIX", fallback=1)) app.config['PERMANENT_SESSION_LIFETIME'] = datetime.timedelta( minutes=settings.get_session_lifetime_config()) app.config['LOGIN_DISABLED'] = not conf.getboolean('webserver', 'AUTHENTICATE') app.secret_key = conf.get('webserver', 'SECRET_KEY') app.config['SESSION_COOKIE_HTTPONLY'] = True app.config['SESSION_COOKIE_SECURE'] = conf.getboolean( 'webserver', 'COOKIE_SECURE') app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE') if config: app.config.from_mapping(config) if 'SQLALCHEMY_ENGINE_OPTIONS' not in app.config: app.config['SQLALCHEMY_ENGINE_OPTIONS'] = settings.prepare_engine_args( ) csrf.init_app(app) app.config['TESTING'] = testing airflow.load_login() airflow.login.LOGIN_MANAGER.init_app(app) from airflow import api api.load_auth() api.API_AUTH.api_auth.init_app(app) # flake8: noqa: F841 cache = Cache(app=app, config={ 'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp' }) app.register_blueprint(routes) configure_logging() with app.app_context(): from airflow.www import views admin = Admin( app, name='Airflow', static_url_path='/admin', index_view=views.HomeView(endpoint='', url='/admin', name="DAGs"), template_mode='bootstrap3', ) av = admin.add_view vs = views av(vs.Airflow(name='DAGs', category='DAGs')) if not conf.getboolean('core', 'secure_mode'): av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling")) av( vs.ChartModelView(models.Chart, Session, name="Charts", category="Data Profiling")) av( vs.KnownEventView(models.KnownEvent, Session, name="Known Events", category="Data Profiling")) av( vs.SlaMissModelView(models.SlaMiss, Session, name="SLA Misses", category="Browse")) av( vs.TaskInstanceModelView(models.TaskInstance, Session, name="Task Instances", category="Browse")) av(vs.LogModelView(models.Log, Session, name="Logs", category="Browse")) av( vs.JobModelView(jobs.BaseJob, Session, name="Jobs", category="Browse")) av( vs.PoolModelView(models.Pool, Session, name="Pools", category="Admin")) av(vs.ConfigurationView(name='Configuration', category="Admin")) av( vs.UserModelView(models.User, Session, name="Users", category="Admin")) av( vs.ConnectionModelView(Connection, Session, name="Connections", category="Admin")) av( vs.VariableView(models.Variable, Session, name="Variables", category="Admin")) av(vs.XComView(models.XCom, Session, name="XComs", category="Admin")) if "dev" in version.version: airflow_doc_site = "https://airflow.readthedocs.io/en/latest" else: airflow_doc_site = 'https://airflow.apache.org/docs/{}'.format( version.version) admin.add_link( base.MenuLink(name="Website", url='https://airflow.apache.org', category="Docs")) admin.add_link( base.MenuLink(category='Docs', name='Documentation', url=airflow_doc_site)) admin.add_link( base.MenuLink(category='Docs', name='GitHub', url='https://github.com/apache/airflow')) av(vs.VersionView(name='Version', category="About")) av( vs.DagRunModelView(models.DagRun, Session, name="DAG Runs", category="Browse")) av(vs.DagModelView(models.DagModel, Session, name=None)) # Hack to not add this view to the menu admin._menu = admin._menu[:-1] def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import (admin_views, flask_blueprints, menu_links) for v in admin_views: log.debug('Adding view %s', v.name) admin.add_view(v) for bp in flask_blueprints: log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name) app.register_blueprint(bp["blueprint"]) for ml in sorted(menu_links, key=lambda x: x.name): log.debug('Adding menu link %s', ml.name) admin.add_link(ml) integrate_plugins() import airflow.www.api.experimental.endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: six.moves.reload_module(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': get_hostname() if conf.getboolean('webserver', 'EXPOSE_HOSTNAME', fallback=True) else 'redact', 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), 'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2), 'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30), 'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000), 'state_color_mapping': STATE_COLORS } @app.before_request def before_request(): _force_log_out_after = conf.getint('webserver', 'FORCE_LOG_OUT_AFTER', fallback=0) if _force_log_out_after > 0: flask.session.permanent = True app.permanent_session_lifetime = datetime.timedelta( minutes=_force_log_out_after) flask.session.modified = True flask.g.user = flask_login.current_user @app.after_request def apply_caching(response): _x_frame_enabled = conf.getboolean('webserver', 'X_FRAME_ENABLED', fallback=True) if not _x_frame_enabled: response.headers["X-Frame-Options"] = "DENY" return response @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app
def configure_action_logging(): """ Any additional configuration (register callback) for airflow.utils.action_loggers module :return: None """ pass try: from airflow_local_settings import * # noqa F403 F401 log.info("Loaded airflow_local_settings.") except Exception: pass logging_class_path = configure_logging() configure_vars() configure_adapters() # The webservers import this file from models.py with the default settings. configure_orm() configure_action_logging() # Ensure we close DB connections at scheduler and gunicon worker terminations atexit.register(dispose_orm) # Const stuff KILOBYTE = 1024 MEGABYTE = KILOBYTE * KILOBYTE WEB_COLORS = {'LIGHTBLUE': '#4d9de0', 'LIGHTORANGE': '#FF9933'}
def create_app(config=None, testing=False): app = Flask(__name__) app.secret_key = configuration.get('webserver', 'SECRET_KEY') app.config['LOGIN_DISABLED'] = not configuration.getboolean( 'webserver', 'AUTHENTICATE') csrf.init_app(app) app.config['TESTING'] = testing airflow.load_login() airflow.login.login_manager.init_app(app) from airflow import api api.load_auth() api.api_auth.init_app(app) cache = Cache(app=app, config={ 'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp' }) app.register_blueprint(routes) configure_logging() with app.app_context(): from airflow.www import views admin = Admin( app, name='Airflow', static_url_path='/admin', index_view=views.HomeView(endpoint='', url='/admin', name="DAGs"), template_mode='bootstrap3', ) av = admin.add_view vs = views av(vs.Airflow(name='DAGs', category='DAGs')) if not conf.getboolean('core', 'secure_mode'): av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling")) av( vs.ChartModelView(models.Chart, Session, name="Charts", category="Data Profiling")) av( vs.KnownEventView(models.KnownEvent, Session, name="Known Events", category="Data Profiling")) av( vs.SlaMissModelView(models.SlaMiss, Session, name="SLA Misses", category="Browse")) av( vs.TaskInstanceModelView(models.TaskInstance, Session, name="Task Instances", category="Browse")) av(vs.LogModelView(models.Log, Session, name="Logs", category="Browse")) av( vs.JobModelView(jobs.BaseJob, Session, name="Jobs", category="Browse")) av( vs.PoolModelView(models.Pool, Session, name="Pools", category="Admin")) av(vs.ConfigurationView(name='Configuration', category="Admin")) av( vs.UserModelView(models.User, Session, name="Users", category="Admin")) av( vs.ConnectionModelView(models.Connection, Session, name="Connections", category="Admin")) av( vs.VariableView(models.Variable, Session, name="Variables", category="Admin")) av(vs.XComView(models.XCom, Session, name="XComs", category="Admin")) admin.add_link( base.MenuLink(category='Docs', name='Documentation', url='https://airflow.incubator.apache.org/')) admin.add_link( base.MenuLink(category='Docs', name='Github', url='https://github.com/apache/incubator-airflow')) av(vs.VersionView(name='Version', category="About")) av( vs.DagRunModelView(models.DagRun, Session, name="DAG Runs", category="Browse")) av(vs.DagModelView(models.DagModel, Session, name=None)) # Hack to not add this view to the menu admin._menu = admin._menu[:-1] def integrate_plugins(): """Integrate plugins to the context""" log = LoggingMixin().log from airflow.plugins_manager import (admin_views, flask_blueprints, menu_links) for v in admin_views: log.debug('Adding view %s', v.name) admin.add_view(v) for bp in flask_blueprints: log.debug('Adding blueprint %s', bp.name) app.register_blueprint(bp) for ml in sorted(menu_links, key=lambda x: x.name): log.debug('Adding menu link %s', ml.name) admin.add_link(ml) integrate_plugins() import airflow.www.api.experimental.endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: if six.PY2: reload(e) else: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': socket.getfqdn(), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app
def create_app(config=None, testing=False, app_name="Airflow"): global app, appbuilder app = Flask(__name__) app.secret_key = conf.get('webserver', 'SECRET_KEY') airflow_home_path = conf.get('core', 'AIRFLOW_HOME') webserver_config_path = airflow_home_path + '/webserver_config.py' app.config.from_pyfile(webserver_config_path, silent=True) app.config['APP_NAME'] = app_name app.config['TESTING'] = testing csrf.init_app(app) db = SQLA(app) from airflow import api api.load_auth() api.api_auth.init_app(app) cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) # noqa from airflow.www_rbac.blueprints import routes app.register_blueprint(routes) configure_logging() with app.app_context(): appbuilder = AppBuilder( app, db.session, security_manager_class=app.config.get('SECURITY_MANAGER_CLASS'), base_template='appbuilder/baselayout.html') def init_views(appbuilder): from airflow.www_rbac import views appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view_no_menu(views.ConfigurationView()) appbuilder.add_view_no_menu(views.VersionView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_link("Configurations", href='/configuration', category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") appbuilder.add_link("Documentation", href='https://airflow.apache.org/', category="Docs", category_icon="fa-cube") appbuilder.add_link("Github", href='https://github.com/apache/incubator-airflow', category="Docs") appbuilder.add_link('Version', href='/version', category='About', category_icon='fa-th') # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. appbuilder.security_cleanup() init_views(appbuilder) from airflow.www_rbac.security import init_roles init_roles(appbuilder) from airflow.www_rbac.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: if six.PY2: reload(e) # noqa else: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': socket.getfqdn(), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app, appbuilder
def configure_action_logging(): """ Any additional configuration (register callback) for airflow.utils.action_loggers module :return: None """ pass try: from airflow_local_settings import * # noqa F403 F401 log.info("Loaded airflow_local_settings.") except Exception: pass configure_logging() configure_vars() configure_adapters() # The webservers import this file from models.py with the default settings. configure_orm() configure_action_logging() # Ensure we close DB connections at scheduler and gunicon worker terminations atexit.register(dispose_orm) # Const stuff KILOBYTE = 1024 MEGABYTE = KILOBYTE * KILOBYTE WEB_COLORS = {'LIGHTBLUE': '#4d9de0', 'LIGHTORANGE': '#FF9933'}
def tearDown(self) -> None: importlib.reload(airflow_local_settings) configure_logging()
def test_loading_local_settings_without_logging_config(self): from airflow.logging_config import configure_logging, log with patch.object(log, 'debug') as mock_info: configure_logging() mock_info.assert_called_with( 'Unable to load custom logging, using default config instead')
def create_app(config=None, session=None, testing=False, app_name="Airflow"): global app, appbuilder app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies=None, x_for=1, x_proto=1, x_host=1, x_port=1, x_prefix=1) app.secret_key = conf.get('webserver', 'SECRET_KEY') app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True) app.config['APP_NAME'] = app_name app.config['TESTING'] = testing app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['SESSION_COOKIE_HTTPONLY'] = True app.config['SESSION_COOKIE_SECURE'] = conf.getboolean( 'webserver', 'COOKIE_SECURE') app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE') if config: app.config.from_mapping(config) # Configure the JSON encoder used by `|tojson` filter from Flask app.json_encoder = AirflowJsonEncoder csrf.init_app(app) db = SQLA(app) from airflow import api api.load_auth() api.API_AUTH.api_auth.init_app(app) Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) from airflow.www.blueprints import routes app.register_blueprint(routes) configure_logging() configure_manifest_files(app) with app.app_context(): from airflow.www.security import AirflowSecurityManager security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \ AirflowSecurityManager if not issubclass(security_manager_class, AirflowSecurityManager): raise Exception( """Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager, not FAB's security manager.""") appbuilder = AppBuilder(app, db.session if not session else session, security_manager_class=security_manager_class, base_template='appbuilder/baselayout.html') def init_views(appbuilder): from airflow.www import views # Remove the session from scoped_session registry to avoid # reusing a session with a disconnected connection appbuilder.session.remove() appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view_no_menu(views.ConfigurationView()) appbuilder.add_view_no_menu(views.VersionView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_link("Configurations", href='/configuration', category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") appbuilder.add_link("Documentation", href='https://airflow.apache.org/', category="Docs", category_icon="fa-cube") appbuilder.add_link("GitHub", href='https://github.com/apache/airflow', category="Docs") appbuilder.add_link('Version', href='/version', category='About', category_icon='fa-th') def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import ( flask_appbuilder_views, flask_appbuilder_menu_links) for v in flask_appbuilder_views: log.debug("Adding view %s", v["name"]) appbuilder.add_view(v["view"], v["name"], category=v["category"]) for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]): log.debug("Adding menu link %s", ml["name"]) appbuilder.add_link(ml["name"], href=ml["href"], category=ml["category"], category_icon=ml["category_icon"]) integrate_plugins() # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. def init_plugin_blueprints(app): from airflow.plugins_manager import flask_blueprints for bp in flask_blueprints: log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name) app.register_blueprint(bp["blueprint"]) init_views(appbuilder) init_plugin_blueprints(app) security_manager = appbuilder.sm security_manager.sync_roles() from airflow.www.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): # pylint: disable=unused-variable globals = { 'hostname': socket.getfqdn(), 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), } if 'analytics_tool' in conf.getsection('webserver'): globals.update({ 'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'), 'analytics_id': conf.get('webserver', 'ANALYTICS_ID') }) return globals @app.teardown_appcontext def shutdown_session(exception=None): # pylint: disable=unused-variable settings.Session.remove() return app, appbuilder
def configure_action_logging(): """ Any additional configuration (register callback) for airflow.utils.action_loggers module :rtype: None """ pass try: from airflow_local_settings import * # noqa F403 F401 log.info("Loaded airflow_local_settings.") except Exception: pass logging_class_path = configure_logging() configure_vars() configure_adapters() # The webservers import this file from models.py with the default settings. configure_orm() configure_action_logging() # Ensure we close DB connections at scheduler and gunicon worker terminations atexit.register(dispose_orm) # Const stuff KILOBYTE = 1024 MEGABYTE = KILOBYTE * KILOBYTE WEB_COLORS = {'LIGHTBLUE': '#4d9de0', 'LIGHTORANGE': '#FF9933'}
def test_loading_no_local_settings(self): with settings_context(SETTINGS_FILE_EMPTY): from airflow.logging_config import configure_logging with self.assertRaises(ImportError): configure_logging()
def create_app(config=None, session=None, testing=False, app_name="Airflow"): global app, appbuilder app = Flask(__name__) if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'): app.wsgi_app = ProxyFix(app.wsgi_app) app.secret_key = conf.get('webserver', 'SECRET_KEY') airflow_home_path = conf.get('core', 'AIRFLOW_HOME') webserver_config_path = airflow_home_path + '/webserver_config.py' app.config.from_pyfile(webserver_config_path, silent=True) app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['APP_NAME'] = app_name app.config['TESTING'] = testing csrf.init_app(app) db = SQLA(app) from airflow import api api.load_auth() api.api_auth.init_app(app) # flake8: noqa: F841 cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) from airflow.www_rbac.blueprints import routes app.register_blueprint(routes) configure_logging() configure_manifest_files(app) with app.app_context(): from airflow.www_rbac.security import AirflowSecurityManager security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \ AirflowSecurityManager if not issubclass(security_manager_class, AirflowSecurityManager): raise Exception( """Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager, not FAB's security manager.""") appbuilder = AppBuilder( app, db.session if not session else session, security_manager_class=security_manager_class, base_template='appbuilder/baselayout.html') def init_views(appbuilder): from airflow.www_rbac import views appbuilder.add_view_no_menu(views.Airflow()) appbuilder.add_view_no_menu(views.DagModelView()) appbuilder.add_view_no_menu(views.ConfigurationView()) appbuilder.add_view_no_menu(views.VersionView()) appbuilder.add_view(views.DagRunModelView, "DAG Runs", category="Browse", category_icon="fa-globe") appbuilder.add_view(views.JobModelView, "Jobs", category="Browse") appbuilder.add_view(views.LogModelView, "Logs", category="Browse") appbuilder.add_view(views.SlaMissModelView, "SLA Misses", category="Browse") appbuilder.add_view(views.TaskInstanceModelView, "Task Instances", category="Browse") appbuilder.add_link("Configurations", href='/configuration', category="Admin", category_icon="fa-user") appbuilder.add_view(views.ConnectionModelView, "Connections", category="Admin") appbuilder.add_view(views.PoolModelView, "Pools", category="Admin") appbuilder.add_view(views.VariableModelView, "Variables", category="Admin") appbuilder.add_view(views.XComModelView, "XComs", category="Admin") appbuilder.add_link("Documentation", href='https://airflow.apache.org/', category="Docs", category_icon="fa-cube") appbuilder.add_link("Github", href='https://github.com/apache/incubator-airflow', category="Docs") appbuilder.add_link('Version', href='/version', category='About', category_icon='fa-th') def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import ( flask_appbuilder_views, flask_appbuilder_menu_links) for v in flask_appbuilder_views: log.debug("Adding view %s", v["name"]) appbuilder.add_view(v["view"], v["name"], category=v["category"]) for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]): log.debug("Adding menu link %s", ml["name"]) appbuilder.add_link(ml["name"], href=ml["href"], category=ml["category"], category_icon=ml["category_icon"]) integrate_plugins() # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. init_views(appbuilder) security_manager = appbuilder.sm security_manager.sync_roles() from airflow.www_rbac.api.experimental import endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: if six.PY2: reload(e) # noqa else: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': socket.getfqdn(), 'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app, appbuilder
def start_master(master_yaml_path): configure_logging() master = ai_flow.AIFlowMaster(config_file=master_yaml_path) master.start(is_block=False) return master
def create_app(config=None, testing=False): log = LoggingMixin().log app = Flask(__name__) app.wsgi_app = ProxyFix(app.wsgi_app) if configuration.conf.get('webserver', 'SECRET_KEY') == "temporary_key": log.info("SECRET_KEY for Flask App is not specified. Using a random one.") app.secret_key = os.urandom(16) else: app.secret_key = configuration.conf.get('webserver', 'SECRET_KEY') app.config['LOGIN_DISABLED'] = not configuration.conf.getboolean( 'webserver', 'AUTHENTICATE') csrf.init_app(app) app.config['TESTING'] = testing airflow.load_login() airflow.login.login_manager.init_app(app) from airflow import api api.load_auth() api.api_auth.init_app(app) cache = Cache( app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'}) app.register_blueprint(routes) configure_logging() with app.app_context(): from airflow.www import views admin = Admin( app, name='Airflow', static_url_path='/admin', index_view=views.HomeView(endpoint='', url='/admin', name="DAGs"), template_mode='bootstrap3', ) av = admin.add_view vs = views av(vs.Airflow(name='DAGs', category='DAGs')) if not conf.getboolean('core', 'secure_mode'): av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling")) av(vs.ChartModelView( models.Chart, Session, name="Charts", category="Data Profiling")) av(vs.KnownEventView( models.KnownEvent, Session, name="Known Events", category="Data Profiling")) av(vs.SlaMissModelView( models.SlaMiss, Session, name="SLA Misses", category="Browse")) av(vs.TaskInstanceModelView(models.TaskInstance, Session, name="Task Instances", category="Browse")) av(vs.LogModelView( models.Log, Session, name="Logs", category="Browse")) av(vs.JobModelView( jobs.BaseJob, Session, name="Jobs", category="Browse")) av(vs.PoolModelView( models.Pool, Session, name="Pools", category="Admin")) av(vs.ConfigurationView( name='Configuration', category="Admin")) av(vs.UserModelView( models.User, Session, name="Users", category="Admin")) av(vs.ConnectionModelView( models.Connection, Session, name="Connections", category="Admin")) av(vs.VariableView( models.Variable, Session, name="Variables", category="Admin")) av(vs.XComView( models.XCom, Session, name="XComs", category="Admin")) admin.add_link(base.MenuLink( category='Docs', name='Documentation', url='https://airflow.incubator.apache.org/')) admin.add_link( base.MenuLink(category='Docs', name='Github', url='https://github.com/apache/incubator-airflow')) av(vs.VersionView(name='Version', category="About")) av(vs.DagRunModelView( models.DagRun, Session, name="DAG Runs", category="Browse")) av(vs.DagModelView(models.DagModel, Session, name=None)) # Hack to not add this view to the menu admin._menu = admin._menu[:-1] def integrate_plugins(): """Integrate plugins to the context""" from airflow.plugins_manager import ( admin_views, flask_blueprints, menu_links) for v in admin_views: log.debug('Adding view %s', v.name) admin.add_view(v) for bp in flask_blueprints: log.debug('Adding blueprint %s', bp.name) app.register_blueprint(bp) for ml in sorted(menu_links, key=lambda x: x.name): log.debug('Adding menu link %s', ml.name) admin.add_link(ml) integrate_plugins() import airflow.www.api.experimental.endpoints as e # required for testing purposes otherwise the module retains # a link to the default_auth if app.config['TESTING']: if six.PY2: reload(e) else: import importlib importlib.reload(e) app.register_blueprint(e.api_experimental, url_prefix='/api/experimental') @app.context_processor def jinja_globals(): return { 'hostname': get_hostname(), 'navbar_color': configuration.get('webserver', 'NAVBAR_COLOR'), } @app.teardown_appcontext def shutdown_session(exception=None): settings.Session.remove() return app
def test_logging(self): configure_logging()