def create_app(config=None, testing=False):
app = Flask(__name__)
app.secret_key = configuration.get('webserver', 'SECRET_KEY')
app.config['LOGIN_DISABLED'] = not configuration.getboolean(
'webserver', 'AUTHENTICATE')
csrf.init_app(app)
app.config['TESTING'] = testing
sebflow.load_login()
sebflow.login.login_manager.init_app(app)
from sebflow import api
api.load_auth()
api.api_auth.init_app(app)
cache = Cache(app=app,
config={
'CACHE_TYPE': 'filesystem',
'CACHE_DIR': '/tmp'
})
app.register_blueprint(routes)
configure_logging()
return app
def load_auth():
auth_backend = 'sebflow.api.auth.backend.default'
try:
auth_backend = conf.get("api", "auth_backend")
except conf.SebflowConfigException:
pass
try:
global api_auth
api_auth = import_module(auth_backend)
except ImportError as err:
log.critical(
"Cannot import %s for API authentication due to: %s",
auth_backend, err
)
raise SebflowException(err)
def get_fernet():
"""
Deferred load of Fernet key.
This function could fail either because Cryptography is not installed
or because the Fernet key is invalid.
:return: Fernet object
:raises: SebflowException if there's a problem trying to load Fernet
"""
try:
from cryptography.fernet import Fernet
except:
raise SebflowException('Failed to import Fernet, it may not be installed')
try:
return Fernet(configuration.get('core', 'FERNET_KEY').encode('utf-8'))
except ValueError as ve:
raise SebflowException("Could not create Fernet object: {}".format(ve))
def load_login():
log = LoggingMixin().log
auth_backend = 'airflow.default_login'
try:
if conf.getboolean('webserver', 'AUTHENTICATE'):
auth_backend = conf.get('webserver', 'auth_backend')
except conf.SebflowConfigException:
if conf.getboolean('webserver', 'AUTHENTICATE'):
log.warning(
"auth_backend not found in webserver config reverting to "
"*deprecated* behavior of importing airflow_login")
auth_backend = "airflow_login"
try:
global login
login = import_module(auth_backend)
except ImportError as err:
log.critical(
"Cannot import authentication module %s. "
"Please correct your authentication backend or disable authentication: %s",
auth_backend, err)
if conf.getboolean('webserver', 'AUTHENTICATE'):
raise SebflowException("Failed to import authentication backend")
def prepare_classpath():
config_path = os.path.join(conf.get('core', 'sebflow_home'), 'config')
config_path = os.path.expanduser(config_path)
if config_path not in sys.path:
sys.path.append(config_path)
import logging
import os
import sys
from logging.config import dictConfig
import sebflow.configuration as conf
log = logging.getLogger(__name__)
LOG_FORMAT = conf.get('core', 'log_format')
BASE_LOG_FOLDER = conf.get('core', 'BASE_LOG_FOLDER')
FILENAME_TEMPLATE = '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log'
PROCESSOR_LOG_FOLDER = conf.get('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY')
PROCESSOR_FILENAME_TEMPLATE = '{{ filename }}.log'
LOG_LEVEL = conf.get('core', 'LOGGING_LEVEL').upper()
DEFAULT_LOGGING_CONFIG = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'sebflow': {
'format': LOG_FORMAT,
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'sebflow',
'stream': 'ext://sys.stdout'
},
},
def configure_vars():
global SEBFLOW_HOME
global SQL_ALCHEMY_CONN
SEBFLOW_HOME = os.path.expanduser(conf.get('core', 'SEBFLOW_HOME'))
SQL_ALCHEMY_CONN = conf.get('core', 'SQL_ALCHEMY_CONN')
import logging
import os
import pendulum
import sebflow.configuration as conf
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from logging_config import configure_logging
log = logging.getLogger(__name__)
TIMEZONE = pendulum.timezone('UTC')
try:
tz = conf.get('core', 'default_timezone')
if tz == 'system':
TIMEZONE = pendulum.local_timezone()
else:
TIMEZONE = pendulum.timezone(tz)
except:
pass
CONFIG_DIR = os.path.join(os.path.dirname(__file__), 'config_files')
HEADER = '''
_____ __________ ________ ____ _ __
/ ___// ____/ __ )/ ____/ / / __ \ | / /
\__ \/ __/ / __ / /_ / / / / / / | /| / /
___/ / /___/ /_/ / __/ / /___/ /_/ /| |/ |/ /
/____/_____/_____/_/ /_____/\____/ |__/|__/
)
t9 = PythonOperator(
task_id='task1_1311',
dag=dag,
python_callable=sleep1,
)
tsql = MsSqlOperator(task_id='test_mssql',
dag=dag,
sql='select * from sebflow.dbo.task',
mssql_conn_id='densql')
t10 = SlackAPIPostOperator(task_id='test_slack',
dag=dag,
channel='denver_notifications',
token=conf.get('core', 'slack_token'))
t2.set_upstream(t1)
t3.set_upstream(t1)
t4.set_upstream(t2)
t5.set_upstream(t2)
t6.set_upstream(t4)
t7.set_upstream(t3)
t8.set_upstream(t7)
t9.set_upstream([t1, t3, t5])
tsql.set_upstream(t1)
t10.set_upstream(tsql)
if __name__ == '__main__':
dag.tree_view()
dag.run()