from flask import Flask
from werkzeug.contrib.fixers import ProxyFix
from website.content import pages, blogs
from website.page import page
from website.blog import blog
from website.error import error_page
from markdown import markdown
site = Flask(__name__, template_folder='content')
site.wsgi_app = ProxyFix(site.wsgi_app)
@site.context_processor
def inject_navlinks():
return dict(navlinks=pages)
@page.context_processor
def inject_blogs():
return dict(bloglinks=blogs)
@site.template_filter('markdown')
def markdown_filter(text):
return markdown(text, extensions=['extra'])
site.register_blueprint(page)
site.register_blueprint(blog, url_prefix='/blog')
site.register_error_handler(404, error_page)
def __init__(self, app, forwarded_proto):
self.app = ProxyFix(app)
self.forwarded_proto = forwarded_proto
import os
from flask import Flask, redirect, url_for
from flask_api import status
from flask_dance.contrib.github import github, make_github_blueprint
from werkzeug.contrib.fixers import ProxyFix
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
app.secret_key = os.environ.get("FLASK_SECRET_KEY", "youllnevercrackme")
app.config["GITHUB_OAUTH_CLIENT_ID"] = os.environ.get("GITHUB_OAUTH_CLIENT_ID")
app.config["GITHUB_OAUTH_CLIENT_SECRET"] = os.environ.get(
"GITHUB_OAUTH_CLIENT_SECRET")
app.register_blueprint(blueprint=make_github_blueprint(scope='public_repo'),
url_prefix="/login")
host_username = os.environ.get("GITHUB_USERNAME", 'mykhaly')
repo_name = os.environ.get("REPOSITORY_NAME", "self-replicating-repo")
def get_link_to_repo(username, repo):
return f"https://github.com/{username}/{repo}"
@app.route("/")
def index():
if not github.authorized:
return redirect(url_for("github.login"))
def init_app(cls, app):
ProductionConfig.init_app(app)
# Handle proxy server headers
from werkzeug.contrib.fixers import ProxyFix
app.wsgi_app = ProxyFix(app.wsgi_app)
return app_iter
def _sr_callback(self, start_response):
def callback(status, headers, exc_info=None):
# Call upstream start_response
start_response(status, headers, exc_info)
return callback
app = Flask(__name__, static_url_path=config.APPLICATION_ROOT + '/static')
bp = Blueprint(config.APPLICATION_ROOT, __name__)
from werkzeug.contrib.fixers import ProxyFix
app.wsgi_app = WSGIRawBody(ProxyFix(app.wsgi_app))
app.debug = config.DEBUG
app.secret_key = config.FLASK_SESSION_SECRET_KEY
app.root_path = os.path.abspath(os.path.dirname(__file__))
app.config.APPLICATION_ROOT = config.APPLICATION_ROOT
app.config.PREFERRED_URL_SCHEME = config.PREFERRED_URL_SCHEME
app.config.AUTH_COOKIE_NAME = config.AUTH_COOKIE_NAME
app.config.AUTH_REDIRECT = config.AUTH_REDIRECT
app.config.AUTH_SECRET = config.AUTH_SECRET
# Always log
import logging
file_handler = logging.FileHandler(config.LOGFILE)
file_handler.setLevel(logging.DEBUG)
def create_app(config=None, session=None, testing=False, app_name="Airflow"):
global app, appbuilder
app = Flask(__name__)
if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'):
app.wsgi_app = ProxyFix(app.wsgi_app)
app.secret_key = conf.get('webserver', 'SECRET_KEY')
app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True)
app.config['APP_NAME'] = app_name
app.config['TESTING'] = testing
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SESSION_COOKIE_HTTPONLY'] = True
app.config['SESSION_COOKIE_SECURE'] = conf.getboolean('webserver', 'COOKIE_SECURE')
app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE')
if config:
app.config.from_mapping(config)
# Configure the JSON encoder used by `|tojson` filter from Flask
app.json_encoder = AirflowJsonEncoder
csrf.init_app(app)
db = SQLA(app)
from airflow import api
api.load_auth()
api.API_AUTH.api_auth.init_app(app)
# flake8: noqa: F841
cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
from airflow.www.blueprints import routes
app.register_blueprint(routes)
configure_logging()
configure_manifest_files(app)
with app.app_context():
from airflow.www.security import AirflowSecurityManager
security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \
AirflowSecurityManager
if not issubclass(security_manager_class, AirflowSecurityManager):
raise Exception(
"""Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager,
not FAB's security manager.""")
appbuilder = AppBuilder(
app,
db.session if not session else session,
security_manager_class=security_manager_class,
base_template='appbuilder/baselayout.html')
def init_views(appbuilder):
from airflow.www import views
appbuilder.add_view_no_menu(views.Airflow())
appbuilder.add_view_no_menu(views.DagModelView())
appbuilder.add_view_no_menu(views.ConfigurationView())
appbuilder.add_view_no_menu(views.VersionView())
appbuilder.add_view(views.DagRunModelView,
"DAG Runs",
category="Browse",
category_icon="fa-globe")
appbuilder.add_view(views.JobModelView,
"Jobs",
category="Browse")
appbuilder.add_view(views.LogModelView,
"Logs",
category="Browse")
appbuilder.add_view(views.SlaMissModelView,
"SLA Misses",
category="Browse")
appbuilder.add_view(views.TaskInstanceModelView,
"Task Instances",
category="Browse")
appbuilder.add_link("Configurations",
href='/configuration',
category="Admin",
category_icon="fa-user")
appbuilder.add_view(views.ConnectionModelView,
"Connections",
category="Admin")
appbuilder.add_view(views.PoolModelView,
"Pools",
category="Admin")
appbuilder.add_view(views.VariableModelView,
"Variables",
category="Admin")
appbuilder.add_view(views.XComModelView,
"XComs",
category="Admin")
appbuilder.add_link("Documentation",
href='https://airflow.apache.org/',
category="Docs",
category_icon="fa-cube")
appbuilder.add_link("GitHub",
href='https://github.com/apache/airflow',
category="Docs")
appbuilder.add_link('Version',
href='/version',
category='About',
category_icon='fa-th')
def integrate_plugins():
"""Integrate plugins to the context"""
from airflow.plugins_manager import (
flask_appbuilder_views, flask_appbuilder_menu_links
)
for v in flask_appbuilder_views:
log.debug("Adding view %s", v["name"])
appbuilder.add_view(v["view"],
v["name"],
category=v["category"])
for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]):
log.debug("Adding menu link %s", ml["name"])
appbuilder.add_link(ml["name"],
href=ml["href"],
category=ml["category"],
category_icon=ml["category_icon"])
integrate_plugins()
# Garbage collect old permissions/views after they have been modified.
# Otherwise, when the name of a view or menu is changed, the framework
# will add the new Views and Menus names to the backend, but will not
# delete the old ones.
def init_plugin_blueprints(app):
from airflow.plugins_manager import flask_blueprints
for bp in flask_blueprints:
log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name)
app.register_blueprint(bp["blueprint"])
init_views(appbuilder)
init_plugin_blueprints(app)
security_manager = appbuilder.sm
security_manager.sync_roles()
from airflow.www.api.experimental import endpoints as e
# required for testing purposes otherwise the module retains
# a link to the default_auth
if app.config['TESTING']:
import importlib
importlib.reload(e)
app.register_blueprint(e.api_experimental, url_prefix='/api/experimental')
@app.context_processor
def jinja_globals():
return {
'hostname': socket.getfqdn(),
'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'),
}
@app.teardown_appcontext
def shutdown_session(exception=None):
settings.Session.remove()
return app, appbuilder
def create_app(config='Testing'):
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
# Configure the flask app
app.config.from_object("croplands_api.config." + config)
# initialize all of the extensions
jwt.init_app(app)
celery.init_app(app)
db.init_app(app)
limiter.init_app(app)
cache.init_app(app)
compress.init_app(app)
mail.init_app(app)
api.init_app(app, flask_sqlalchemy_db=db)
# initialize google earth engine
ee.Initialize(ServiceAccountCredentials._from_parsed_json_keyfile(
app.config['GOOGLE_SERVICE_ACCOUNT'],
scopes=app.config['GOOGLE_SERVICE_ACCOUNT_SCOPES']))
# import and register all of the blueprints
from croplands_api.views.public import public
from croplands_api.views.auth import auth
from croplands_api.views.gee import gee
from croplands_api.views.aws import aws
from croplands_api.views.upload import upload
from croplands_api.views.stats import stats_blueprint
from croplands_api.views.data import data_blueprint
app.register_blueprint(public)
app.register_blueprint(gee)
app.register_blueprint(aws)
app.register_blueprint(auth)
app.register_blueprint(upload)
app.register_blueprint(stats_blueprint)
app.register_blueprint(data_blueprint)
from croplands_api.views.api import init_api
init_api(app)
# import and init error handlers
from croplands_api.views.errors import init_error_handlers
init_error_handlers(app)
# cors headers and cache
app.after_request(add_cors_headers)
from croplands_api.auth import load_user, is_anonymous
app.before_request(load_user)
from croplands_api.utils.log import log
app.after_request(log)
@limiter.request_filter
def registered():
"""
Removes limit if user is registered and using a token.
:return:
"""
return not is_anonymous()
if 'POSTMARK_API_KEY' in app.config:
email_handler = PostMarkHandler(api_key=app.config['POSTMARK_API_KEY'])
email_handler.setLevel(logging.ERROR)
app.logger.addHandler(email_handler)
import croplands_api.tasks.high_res_imagery
import croplands_api.tasks.classifications
import croplands_api.tasks.reference_data_coverage
import croplands_api.tasks.records
return app
def init(conf=None, verbose=0, logfile=None, gunicorn=True, unittest=False, debug=False):
"""Initialize the whole application.
:param conf: Configuration file to use
:type conf: str
:param verbose: Set the verbosity level
:type verbose: int
:param logfile: Store the logs in the given file
:type logfile: str
:param gunicorn: Enable gunicorn engine instead of flask's default
:type gunicorn: bool
:param unittest: Are we running tests (used for test only)
:type unittest: bool
:param debug: Enable debug mode
:type debug: bool
:returns: A :class:`burpui.server.BUIServer` object
"""
from flask_login import LoginManager
from flask_bower import Bower
from .utils import basic_login_from_request, ReverseProxied
from .server import BUIServer as BurpUI
from .routes import view
from .api import api, apibp
logger = logging.getLogger('burp-ui')
# The debug argument used to be a boolean so we keep supporting this format
if isinstance(verbose, bool):
if verbose:
verbose = logging.DEBUG
else:
verbose = logging.CRITICAL
else:
levels = [
logging.CRITICAL,
logging.ERROR,
logging.WARNING,
logging.INFO,
logging.DEBUG
]
if verbose >= len(levels):
verbose = len(levels) - 1
if not verbose:
verbose = 0
verbose = levels[verbose]
if logfile:
from logging.handlers import RotatingFileHandler
handler = RotatingFileHandler(
logfile,
maxBytes=1024 * 1024 * 100,
backupCount=5
)
else:
from logging import StreamHandler
handler = StreamHandler()
if verbose > logging.DEBUG:
LOG_FORMAT = (
'[%(asctime)s] %(levelname)s in '
'%(module)s.%(funcName)s: %(message)s'
)
else:
LOG_FORMAT = (
'-' * 80 + '\n' +
'%(levelname)s in %(module)s.%(funcName)s ' +
'[%(pathname)s:%(lineno)d]:\n' +
'%(message)s\n' +
'-' * 80
)
handler.setLevel(verbose)
handler.setFormatter(Formatter(LOG_FORMAT))
logger.setLevel(verbose)
logger.addHandler(handler)
logger.debug(
'conf: {}\n'.format(conf) +
'verbose: {}\n'.format(logging.getLevelName(verbose)) +
'logfile: {}\n'.format(logfile) +
'gunicorn: {}\n'.format(gunicorn) +
'debug: {}\n'.format(debug) +
'unittest: {}'.format(unittest)
)
if not unittest:
from ._compat import patch_json
patch_json()
if gunicorn:
from gevent import monkey
monkey.patch_all()
# We initialize the core
app = BurpUI()
app.enable_logger()
app.gunicorn = gunicorn
app.config['CFG'] = None
# FIXME: strange behavior when bundling errors
# app.config['BUNDLE_ERRORS'] = True
app.config['REMEMBER_COOKIE_HTTPONLY'] = True
app.jinja_env.globals.update(
isinstance=isinstance,
list=list,
version_id='{}-{}'.format(__version__, __release__)
)
if debug and not gunicorn: # pragma: no cover
app.config['DEBUG'] = True and not unittest
app.config['TESTING'] = True and not unittest
# Still need to test conf file here because the init function can be called
# by gunicorn directly
app.config['CFG'] = lookup_config(conf)
logger.info('Using configuration: {}'.format(app.config['CFG']))
app.setup(app.config['CFG'])
if debug:
app.config.setdefault('TEMPLATES_AUTO_RELOAD', True)
# manage application secret key
if app.secret_key and (app.secret_key.lower() == 'none' or
(app.secret_key.lower() == 'random' and gunicorn)):
logger.warning('Your setup is not secure! Please consider setting a'
' secret key in your configuration file')
app.secret_key = 'Burp-UI'
if not app.secret_key or app.secret_key.lower() == 'random':
from base64 import b64encode
app.secret_key = b64encode(os.urandom(256))
app.wsgi_app = ReverseProxied(app.wsgi_app, app)
# Manage gunicorn special tricks & improvements
if gunicorn: # pragma: no cover
logger.info('Using gunicorn')
from werkzeug.contrib.fixers import ProxyFix
if app.storage and app.storage.lower() == 'redis':
if app.redis:
part = app.redis.split(':')
host = part[0]
try:
port = int(part[1])
except:
port = 6379
else:
host = 'localhost'
port = 6379
logger.debug('Using redis {}:{}'.format(host, port))
try:
from redis import Redis
from flask_session import Session
red = Redis(host=host, port=port)
app.config['SESSION_TYPE'] = 'redis'
app.config['SESSION_REDIS'] = red
app.config['SESSION_USE_SIGNER'] = app.secret_key is not None
app.config['SESSION_PERMANENT'] = False
ses = Session()
ses.init_app(app)
except Exception as e:
logger.warning('Unable to initialize redis: {}'.format(str(e)))
pass
api.cache.init_app(
app,
config={
'CACHE_TYPE': 'redis',
'CACHE_REDIS_HOST': host,
'CACHE_REDIS_PORT': port,
'CACHE_REDIS_DB': 1
}
)
# clear cache at startup in case we removed or added servers
with app.app_context():
api.cache.clear()
else:
api.cache.init_app(app)
app.wsgi_app = ProxyFix(app.wsgi_app)
else:
api.cache.init_app(app)
# We initialize the API
api.version = __version__
api.release = __release__
api.__url__ = __url__
api.__doc__ = __doc__
api.load_all()
app.register_blueprint(apibp)
# Then we load our routes
view.__url__ = __url__
view.__doc__ = __doc__
app.register_blueprint(view)
# And the login_manager
app.login_manager = LoginManager()
app.login_manager.login_view = 'view.login'
app.login_manager.login_message_category = 'info'
app.login_manager.session_protection = 'strong'
app.login_manager.init_app(app)
app.config.setdefault(
'BOWER_COMPONENTS_ROOT',
os.path.join('static', 'vendor')
)
app.config.setdefault('BOWER_REPLACE_URL_FOR', True)
bower = Bower()
bower.init_app(app)
@app.before_request
def setup_request():
if app.scookie:
from flask import request
criteria = [
request.is_secure,
request.headers.get('X-Forwarded-Proto', 'http') == 'https'
]
app.config['SESSION_COOKIE_SECURE'] = \
app.config['REMEMBER_COOKIE_SECURE'] = any(criteria)
@app.login_manager.user_loader
def load_user(userid):
"""User loader callback"""
if app.auth != 'none':
return app.uhandler.user(userid)
return None
@app.login_manager.request_loader
def load_user_from_request(request):
"""User loader from request callback"""
if app.auth != 'none':
return basic_login_from_request(request, app)
return app
def create_app():
"""Create Flask app."""
config = load_config()
app = Flask(__name__)
app.config.from_object(config)
if not hasattr(app, 'production'):
app.production = not app.debug and not app.testing
# Proxy fix
app.wsgi_app = ProxyFix(app.wsgi_app)
# CSRF protect
CsrfProtect(app)
if app.debug or app.testing:
DebugToolbarExtension(app)
# Serve static files
app.wsgi_app = SharedDataMiddleware(
app.wsgi_app, {
'/pages':
os.path.join(app.config.get('PROJECT_PATH'),
'application/pages')
})
else:
# Log errors to stderr in production mode
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.ERROR)
# Enable Sentry
if app.config.get('SENTRY_DSN'):
from .utils.sentry import sentry
sentry.init_app(app,
dsn=app.config.get('SENTRY_DSN'),
logging=True,
level=logging.ERROR)
# Serve static files
app.wsgi_app = SharedDataMiddleware(
app.wsgi_app, {
'/static':
os.path.join(app.config.get('PROJECT_PATH'), 'output/static'),
'/pkg':
os.path.join(app.config.get('PROJECT_PATH'), 'output/pkg'),
'/pages':
os.path.join(app.config.get('PROJECT_PATH'), 'output/pages')
})
# Register components
register_db(app)
register_routes(app)
register_admin(app)
register_security(app)
register_babel(app)
register_api(app)
register_jinja(app)
register_error_handle(app)
register_hooks(app)
register_socketio(app)
register_redis(app)
register_context_processor(app)
register_before_first_request(app)
return app
def make_app(debug=False, **app_options):
global db, app, admin
app = Flask(__name__)
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
logg.debug("creating flask app %s", __name__)
try:
import arguments.config
app.config.from_object(arguments.config)
except ImportError:
pass
if app_options:
app.config.update(app_options)
app.config["RESTFUL_JSON"] = {'ensure_ascii': False}
app.config["SECRET_KEY"] = "dev"
app.config["DEBUG"] = debug
logg.debug("app config is:\n%s", pformat(dict(app.config)))
if debug:
app.debug = True
from werkzeug.debug import DebuggedApplication
app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
app.jinja_env.add_extension('arguments.helper.templating.PyJadeExtension')
# initialize extensions
# flask-sqlalchemy
db = SQLAlchemy(app)
import arguments.database.datamodel
# flask-admin
admin = Admin(app, name="Arguments", template_mode="bootstrap3")
# markdown via flask-misaka
# TODO: markdown options should be configurable
markdown_opts = dict(
autolink=True,
fenced_code=True,
no_intra_emphasis=True,
strikethrough=True,
tables=True,
safelink=True,
escape=True,
smartypants=True
)
Misaka(app, **markdown_opts)
# user management provided by flask_login
login_manager = LoginManager(app)
login_manager.login_view = 'ekklesia.login'
# XXX: for testing: just use first user from the DB as anon user
# login_manager.anonymous_user = lambda: User.query.first()
from arguments.database.datamodel import User
@login_manager.user_loader
def load_user(user_id):
return User.query.get(int(user_id))
# i18n via flask-babelex
babel = Babel(app)
@babel.localeselector
def get_locale():
return session["locale"]
# OAuth2 using flask-dance
init_oauth_ext(app)
# ajax lib flask-sijax
path = os.path.join('.', os.path.dirname(__file__), 'static/js/sijax/')
app.config['SIJAX_STATIC_PATH'] = path
app.config['SIJAX_JSON_URI'] = '/static/js/sijax/json2.js'
flask_sijax.Sijax(app)
@app.before_request
def set_locale():
locale = session.get("locale")
if locale:
logg.debug("locale from session: %s", locale)
else:
locale = request.accept_languages.best_match(['de', 'en', 'fr'])
logg.debug("locale from request: %s", locale)
session["locale"] = locale
g.locale = locale
import arguments.views
import arguments.views.admin
# import arguments_rest.api
# needed when running behind a reverse proxy.
app.wsgi_app = ProxyFix(app.wsgi_app)
return app
# coding:utf-8
import logging.config
logging.config.fileConfig('log.conf')
from werkzeug.serving import run_simple
from werkzeug.wsgi import DispatcherMiddleware
from werkzeug.contrib.fixers import ProxyFix
import ca.frontend as frontend
import ca.backend as backend
frontend_app = frontend.create_app()
backend_app = backend.create_app()
application = ProxyFix(
DispatcherMiddleware(frontend_app, {
'/backend': backend_app,
}))
if __name__ == "__main__":
run_simple("0.0.0.0", 7200, application)
from werkzeug.contrib.fixers import ProxyFix
bp = Blueprint('bp', __name__, template_folder='templates', static_folder='static')
app = Flask(__name__, instance_relative_config = True)
# Load default config
app.config.from_object('config.default')
# Load instance configuration if exists
app.config.from_pyfile('config.py', silent = True)
# Load configuration file specified in BRAVO_CONFIG_FILE environment variable if exists
app.config.from_envvar('BRAVO_CONFIG_FILE', silent = True)
#app.config.from_object('flask_config.BravoFreeze5GRCh38Config')
if app.config['PROXY']: app.wsgi_app = ProxyFix(app.wsgi_app)
if 'GVS_URL_PREFIX' in os.environ: app.config['URL_PREFIX'] = os.environ['GVS_URL_PREFIX']
if 'BRAVO_ADMIN_MODE' in os.environ: app.config['ADMIN'] = True if os.environ['BRAVO_ADMIN_MODE'].lower() == 'true' else False
mail_on_500(app, app.config['ADMINS'])
app.config['COMPRESS_LEVEL'] = 2 # Since we don't cache, faster=better
Compress(app)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 5 # 5 second browser cache timeout
app.config['TEMPLATES_AUTO_RELOAD'] = True
BASE_COVERAGE = []
BASE_COVERAGE.extend({'bp-min-length':0, 'path':path} for path in glob.glob(app.config['BASE_COVERAGE_DIRECTORY'] + 'full/*.json.gz'))
BASE_COVERAGE.extend({'bp-min-length':300, 'binned':True, 'path':path} for path in glob.glob(app.config['BASE_COVERAGE_DIRECTORY'] + 'bin_25e-2/*.json.gz'))
BASE_COVERAGE.extend({'bp-min-length':1000,'binned':True, 'path':path} for path in glob.glob(app.config['BASE_COVERAGE_DIRECTORY'] + 'bin_50e-2/*.json.gz'))
BASE_COVERAGE.extend({'bp-min-length':3000,'binned':True, 'path':path} for path in glob.glob(app.config['BASE_COVERAGE_DIRECTORY'] + 'bin_75e-2/*.json.gz'))
MAX_REGION_LENGTH = int(350e3) # Longer than TTN (305kb), short enough to perform okay.
def create_app(config=None, testing=False):
log = LoggingMixin().log
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
if configuration.conf.get('webserver', 'SECRET_KEY') == "temporary_key":
log.info(
"SECRET_KEY for Flask App is not specified. Using a random one.")
app.secret_key = os.urandom(16)
else:
app.secret_key = configuration.conf.get('webserver', 'SECRET_KEY')
app.config['LOGIN_DISABLED'] = not configuration.conf.getboolean(
'webserver', 'AUTHENTICATE')
csrf.init_app(app)
app.config['TESTING'] = testing
airflow.load_login()
airflow.login.login_manager.init_app(app)
from airflow import api
api.load_auth()
api.api_auth.init_app(app)
cache = Cache(app=app,
config={
'CACHE_TYPE': 'filesystem',
'CACHE_DIR': '/tmp'
})
app.register_blueprint(routes)
configure_logging()
with app.app_context():
from airflow.www import views
admin = Admin(
app,
name='Airflow',
static_url_path='/admin',
index_view=views.HomeView(endpoint='', url='/admin', name="DAGs"),
template_mode='bootstrap3',
)
av = admin.add_view
vs = views
av(vs.Airflow(name='DAGs', category='DAGs'))
if not conf.getboolean('core', 'secure_mode'):
av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling"))
av(
vs.ChartModelView(models.Chart,
Session,
name="Charts",
category="Data Profiling"))
av(
vs.KnownEventView(models.KnownEvent,
Session,
name="Known Events",
category="Data Profiling"))
av(
vs.SlaMissModelView(models.SlaMiss,
Session,
name="SLA Misses",
category="Browse"))
av(
vs.TaskInstanceModelView(models.TaskInstance,
Session,
name="Task Instances",
category="Browse"))
av(vs.LogModelView(models.Log, Session, name="Logs",
category="Browse"))
av(
vs.JobModelView(jobs.BaseJob,
Session,
name="Jobs",
category="Browse"))
av(
vs.PoolModelView(models.Pool,
Session,
name="Pools",
category="Admin"))
av(vs.ConfigurationView(name='Configuration', category="Admin"))
av(
vs.UserModelView(models.User,
Session,
name="Users",
category="Admin"))
av(
vs.ConnectionModelView(models.Connection,
Session,
name="Connections",
category="Admin"))
av(
vs.VariableView(models.Variable,
Session,
name="Variables",
category="Admin"))
av(vs.XComView(models.XCom, Session, name="XComs", category="Admin"))
admin.add_link(
base.MenuLink(category='Docs',
name='Documentation',
url='https://airflow.incubator.apache.org/'))
admin.add_link(
base.MenuLink(category='Docs',
name='Github',
url='https://github.com/apache/incubator-airflow'))
av(vs.VersionView(name='Version', category="About"))
av(
vs.DagRunModelView(models.DagRun,
Session,
name="DAG Runs",
category="Browse"))
av(vs.DagModelView(models.DagModel, Session, name=None))
# Hack to not add this view to the menu
admin._menu = admin._menu[:-1]
def integrate_plugins():
"""Integrate plugins to the context"""
from airflow.plugins_manager import (admin_views, flask_blueprints,
menu_links)
for v in admin_views:
log.debug('Adding view %s', v.name)
admin.add_view(v)
for bp in flask_blueprints:
log.debug('Adding blueprint %s', bp.name)
app.register_blueprint(bp)
for ml in sorted(menu_links, key=lambda x: x.name):
log.debug('Adding menu link %s', ml.name)
admin.add_link(ml)
integrate_plugins()
import airflow.www.api.experimental.endpoints as e
# required for testing purposes otherwise the module retains
# a link to the default_auth
if app.config['TESTING']:
if six.PY2:
reload(e)
else:
import importlib
importlib.reload(e)
app.register_blueprint(e.api_experimental,
url_prefix='/api/experimental')
@app.context_processor
def jinja_globals():
return {
'hostname': get_hostname(),
'navbar_color': configuration.get('webserver', 'NAVBAR_COLOR'),
}
@app.teardown_appcontext
def shutdown_session(exception=None):
settings.Session.remove()
return app
hovermode='closest')
}
else:
return {
'data': [
go.Scatter(
x=dff[dff['name'] == i]['date'],
y=dff[dff['name'] == i][priceType],
text=dff[dff['name'] == i]['name'],
# mode='markers', #scatter/ no--line
name=i) for i in dff.name.unique()
],
'layout':
go.Layout(
title=industry + ' Stock Price',
xaxis={'title': 'date'},
yaxis={'title': priceType + ' price'},
# margin={'l': 40, 'b': 40, 't': 10, 'r': 10},
hovermode='closest')
}
server = app.server
if __name__ == '__main__':
from werkzeug.contrib.fixers import ProxyFix
server.wsgi_app = ProxyFix(server.wsgi_app)
app.run_server(debug=True, host='0.0.0.0')
thread_sleep_time_max=settings.TASK_TEMPLATE_THREAD_SLEEP_TIME_MAX)
sync_actions_thread = SyncActionsHandlerThread(
thread_sleep_time_min=settings.SYNC_ACTIONS_THREAD_SLEEP_TIME_MIN,
thread_sleep_time_max=settings.SYNC_ACTIONS_THREAD_SLEEP_TIME_MAX,
run_once=False)
datasaur_sync_manager = DatasaurSyncManager(
thread_sleep_time_min=settings.DATASAUR_SYNC_THREAD_SLEEP_TIME_MIN,
thread_sleep_time_max=settings.DATASAUR_SYNC_THREAD_SLEEP_TIME_MAX)
datasaur_sync_manager.start_sync_loop()
print("Startup in", time.time() - start_time)
# actions_thread = ActionFlowTriggerQueueThread(thread_sleep_time_min=5,
# thread_sleep_time_max=6,
# run_once=False)
# Debug
if __name__ == '__main__':
settings.NAME_EQUALS_MAIN = True # can adjust this for local deferral testing if needed?
limiter.enabled = False
# os.environ['test'] = "test_os_environ"
app.run(host='0.0.0.0', port=8082, debug=True)
# CAUTION . app.run() is BLOCKING
# code below app.run will not execute!!!
else:
print("settings.NAME_EQUALS_MAIN", settings.NAME_EQUALS_MAIN)
app.debug == False
app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies=2)
def create_app():
_app = Flask(__name__)
# used for encrypting cookies for handling sessions
_app.config['SECRET_KEY'] = 'abc492ee-9739-11e6-a174-07f6b92d4a4b'
message_queue_type = environ.env.config.get(ConfigKeys.TYPE,
domain=ConfigKeys.COORDINATOR,
default=None)
if message_queue_type is None and not (len(environ.env.config) == 0
or environ.env.config.get(
ConfigKeys.TESTING)):
raise RuntimeError('no message queue type specified')
queue_host = environ.env.config.get(ConfigKeys.HOST,
domain=ConfigKeys.COORDINATOR,
default='')
message_channel = ''
message_queue = None
if message_queue_type == 'redis':
message_db = environ.env.config.get(ConfigKeys.DB,
domain=ConfigKeys.COORDINATOR,
default=0)
message_env = environ.env.config.get(ConfigKeys.ENVIRONMENT,
default='test')
message_channel = 'dino_{}_{}'.format(message_env, message_db)
message_queue = 'redis://{}'.format(queue_host)
elif message_queue_type == 'amqp':
message_channel = 'dino_%s' % environ.env.config.get(
ConfigKeys.ENVIRONMENT, default='test')
message_queue = ';'.join([
'amqp://%s:%s@%s:%s%s' % (
environ.env.config.get(ConfigKeys.USER,
domain=ConfigKeys.COORDINATOR,
default=''),
environ.env.config.get(ConfigKeys.PASSWORD,
domain=ConfigKeys.COORDINATOR,
default=''),
host,
environ.env.config.get(ConfigKeys.PORT,
domain=ConfigKeys.COORDINATOR,
default=''),
environ.env.config.get(ConfigKeys.VHOST,
domain=ConfigKeys.COORDINATOR,
default=''),
) for host in queue_host.split(';')
])
elif not environ.env.config.get(ConfigKeys.TESTING, False):
raise RuntimeError(
'unknown message queue type {} specified: {}'.format(
message_queue_type, environ.env.config.params))
logger.info('message_queue: %s' % message_queue)
cors = environ.env.config.get(ConfigKeys.CORS_ORIGINS,
default='*').split(',')
if cors == ['*']:
cors = cors[0]
_socketio = SocketIO(_app,
logger=socket_logger,
engineio_logger=os.environ.get('DINO_DEBUG',
'0') == '1',
async_mode='eventlet',
message_queue=message_queue,
channel=message_channel,
cors_allowed_origins=cors)
# preferably "emit" should be set during env creation, but the socketio object is not created until after env is
environ.env.out_of_scope_emit = _socketio.emit
_app.wsgi_app = ProxyFix(_app.wsgi_app)
return _app, _socketio
def create_app(oidc_blueprint=None):
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_port=1, x_prefix=1)
app.secret_key = "8210f566-4981-11ea-92d1-f079596e599b"
app.config.from_json('config.json')
settings = Settings(app.config)
cred = Credentials(settings.db_url)
infra = Infrastructures(settings.db_url)
toscaTemplates = utils.loadToscaTemplates(settings.toscaDir)
toscaInfo = utils.extractToscaInfo(settings.toscaDir, settings.toscaParamsDir, toscaTemplates)
app.jinja_env.filters['tojson_pretty'] = utils.to_pretty_json
app.logger.debug("TOSCA INFO: " + json.dumps(toscaInfo))
loglevel = app.config.get("LOG_LEVEL") if app.config.get("LOG_LEVEL") else "INFO"
numeric_level = getattr(logging, loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level)
oidc_base_url = app.config['OIDC_BASE_URL']
oidc_token_url = oidc_base_url + '/token'
oidc_refresh_url = oidc_base_url + '/token'
oidc_authorization_url = oidc_base_url + '/authorize'
if not oidc_blueprint:
oidc_blueprint = OAuth2ConsumerBlueprint(
"oidc", __name__,
client_id=app.config['OIDC_CLIENT_ID'],
client_secret=app.config['OIDC_CLIENT_SECRET'],
scope=app.config['OIDC_SCOPES'],
base_url=oidc_base_url,
token_url=oidc_token_url,
auto_refresh_url=oidc_refresh_url,
authorization_url=oidc_authorization_url,
redirect_to='home'
)
app.register_blueprint(oidc_blueprint, url_prefix="/login")
@app.before_request
def before_request_checks():
if 'external_links' not in session:
session['external_links'] = settings.external_links
g.analytics_tag = settings.analytics_tag
g.settings = settings
def authorized_with_valid_token(f):
@wraps(f)
def decorated_function(*args, **kwargs):
try:
if not oidc_blueprint.session.authorized or 'username' not in session:
return redirect(url_for('login'))
if oidc_blueprint.session.token['expires_in'] < 20:
app.logger.debug("Force refresh token")
oidc_blueprint.session.get('/userinfo')
except (InvalidTokenError, TokenExpiredError):
flash("Token expired.", 'warning')
return redirect(url_for('login'))
return f(*args, **kwargs)
return decorated_function
@app.route('/settings')
@authorized_with_valid_token
def show_settings():
return render_template('settings.html', oidc_url=settings.oidcUrl, im_url=settings.imUrl)
@app.route('/login')
def login():
session.clear()
return render_template('home.html', oidc_name=settings.oidcName)
@app.route('/')
def home():
if not oidc_blueprint.session.authorized:
return redirect(url_for('login'))
try:
account_info = oidc_blueprint.session.get(urlparse(settings.oidcUrl)[2] + "/userinfo")
except (InvalidTokenError, TokenExpiredError):
flash("Token expired.", 'warning')
return redirect(url_for('login'))
if account_info.ok:
account_info_json = account_info.json()
session["vos"] = None
if 'eduperson_entitlement' in account_info_json:
session["vos"] = utils.getUserVOs(account_info_json['eduperson_entitlement'])
if settings.oidcGroups:
user_groups = []
if 'groups' in account_info_json:
user_groups = account_info_json['groups']
elif 'eduperson_entitlement' in account_info_json:
user_groups = account_info_json['eduperson_entitlement']
if not set(settings.oidcGroups).issubset(user_groups):
app.logger.debug("No match on group membership. User group membership: " + json.dumps(user_groups))
message = Markup('You need to be a member of the following groups: {0}. <br>'
' Please, visit <a href="{1}">{1}</a> and apply for the requested '
'membership.'.format(json.dumps(settings.oidcGroups), settings.oidcUrl))
raise Forbidden(description=message)
session['userid'] = account_info_json['sub']
if 'name' in account_info_json:
session['username'] = account_info_json['name']
else:
session['username'] = ""
if 'given_name' in account_info_json:
session['username'] = account_info_json['given_name']
if 'family_name' in account_info_json:
session['username'] += " " + account_info_json['family_name']
if session['username'] == "":
session['username'] = account_info_json['sub']
if 'email' in account_info_json:
session['gravatar'] = utils.avatar(account_info_json['email'], 26)
else:
session['gravatar'] = utils.avatar(account_info_json['sub'], 26)
return render_template('portfolio.html', templates=toscaInfo)
else:
flash("Error getting User info: \n" + account_info.text, 'error')
return render_template('home.html', oidc_name=settings.oidcName)
@app.route('/vminfo/<infid>/<vmid>')
@authorized_with_valid_token
def showvminfo(infid=None, vmid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data, "Accept": "application/json"}
url = "%s/infrastructures/%s/vms/%s" % (settings.imUrl, infid, vmid)
response = requests.get(url, headers=headers)
vminfo = {}
state = ""
nets = ""
deployment = ""
if not response.ok:
flash("Error retrieving VM info: \n" + response.text, 'error')
else:
app.logger.debug("VM Info: %s" % response.text)
vminfo = utils.format_json_radl(response.json()["radl"])
if "cpu.arch" in vminfo:
del vminfo["cpu.arch"]
if "state" in vminfo:
state = vminfo["state"]
del vminfo["state"]
if "provider.type" in vminfo:
deployment = vminfo["provider.type"]
del vminfo["provider.type"]
if "provider.host" in vminfo:
if "provider.port" in vminfo:
deployment += ": %s:%s" % (vminfo["provider.host"], vminfo["provider.port"])
del vminfo["provider.port"]
else:
deployment += ": " + vminfo["provider.host"]
del vminfo["provider.host"]
cont = 0
while "net_interface.%s.ip" % cont in vminfo:
if cont > 0:
nets += Markup('<br/>')
nets += Markup('<i class="fa fa-network-wired"></i>')
nets += " %s: %s" % (cont, vminfo["net_interface.%s.ip" % cont])
del vminfo["net_interface.%s.ip" % cont]
cont += 1
cont = 0
while "net_interface.%s.connection" % cont in vminfo:
del vminfo["net_interface.%s.connection" % cont]
cont += 1
for elem in vminfo:
if elem.endswith("size") and isinstance(vminfo[elem], int):
vminfo[elem] = "%d GB" % (vminfo[elem] / 1073741824)
return render_template('vminfo.html', infid=infid, vmid=vmid, vminfo=vminfo,
state=state, nets=nets, deployment=deployment)
@app.route('/managevm/<op>/<infid>/<vmid>')
@authorized_with_valid_token
def managevm(op=None, infid=None, vmid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data, "Accept": "application/json"}
op = op.lower()
if op in ["stop", "start", "reboot"]:
url = "%s/infrastructures/%s/vms/%s/%s" % (settings.imUrl, infid, vmid, op)
response = requests.put(url, headers=headers)
elif op == "terminate":
url = "%s/infrastructures/%s/vms/%s" % (settings.imUrl, infid, vmid)
response = requests.delete(url, headers=headers)
else:
flash("Error: invalid operation: %s." % op, 'error')
return redirect(url_for('showinfrastructures'))
if response.ok:
flash("Operation '%s' successfully made on VM ID: %s" % (op, vmid), 'info')
else:
flash("Error making %s op on VM %s: \n%s" % (op, vmid, response.text), 'error')
if op == "terminate":
return redirect(url_for('showinfrastructures'))
else:
return redirect(url_for('showvminfo', infid=infid, vmid=vmid))
@app.route('/infrastructures')
@authorized_with_valid_token
def showinfrastructures():
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data, "Accept": "application/json"}
url = "%s/infrastructures" % settings.imUrl
response = requests.get(url, headers=headers)
infrastructures = {}
if not response.ok:
flash("Error retrieving infrastructure list: \n" + response.text, 'error')
else:
app.logger.debug("Infrastructures: %s" % response.text)
state_res = response.json()
if "uri-list" in state_res:
inf_id_list = [elem["uri"] for elem in state_res["uri-list"]]
else:
inf_id_list = []
for inf_id in inf_id_list:
url = "%s/state" % inf_id
response = requests.get(url, headers=headers)
if not response.ok:
flash("Error retrieving infrastructure %s state: \n%s" % (inf_id, response.text), 'warning')
else:
inf_state = response.json()
infrastructures[os.path.basename(inf_id)] = inf_state['state']
try:
infra_name = infra.get_infra(os.path.basename(inf_id))["name"]
except Exception:
infra_name = ""
infrastructures[os.path.basename(inf_id)]['name'] = infra_name
return render_template('infrastructures.html', infrastructures=infrastructures)
@app.route('/reconfigure/<infid>')
@authorized_with_valid_token
def infreconfigure(infid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data}
url = "%s/infrastructures/%s/reconfigure" % (settings.imUrl, infid)
response = requests.put(url, headers=headers)
if response.ok:
flash("Infrastructure successfuly reconfigured.", "info")
else:
flash("Error reconfiguring Infrastructure: \n" + response.text, "error")
return redirect(url_for('showinfrastructures'))
@app.route('/template/<infid>')
@authorized_with_valid_token
def template(infid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data}
url = "%s/infrastructures/%s/tosca" % (settings.imUrl, infid)
response = requests.get(url, headers=headers)
if not response.ok:
flash("Error getting template: \n" + response.text, "error")
template = ""
else:
template = response.text
return render_template('deptemplate.html', template=template)
@app.route('/log/<infid>')
@authorized_with_valid_token
def inflog(infid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data}
url = "%s/infrastructures/%s/contmsg" % (settings.imUrl, infid)
response = requests.get(url, headers=headers, verify=False)
if not response.ok:
log = "Not found"
else:
log = response.text
return render_template('inflog.html', log=log)
@app.route('/vmlog/<infid>/<vmid>')
@authorized_with_valid_token
def vmlog(infid=None, vmid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data}
url = "%s/infrastructures/%s/vms/%s/contmsg" % (settings.imUrl, infid, vmid)
response = requests.get(url, headers=headers, verify=False)
if not response.ok:
log = "Not found"
else:
log = response.text
return render_template('inflog.html', log=log, vmid=vmid)
@app.route('/outputs/<infid>')
@authorized_with_valid_token
def infoutputs(infid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data}
url = "%s/infrastructures/%s/outputs" % (settings.imUrl, infid)
response = requests.get(url, headers=headers, verify=False)
if not response.ok:
outputs = {}
else:
outputs = response.json()["outputs"]
for elem in outputs:
if isinstance(outputs[elem], str) and (outputs[elem].startswith('http://') or
outputs[elem].startswith('https://')):
outputs[elem] = Markup("<a href='%s' target='_blank'>%s</a>" % (outputs[elem], outputs[elem]))
return render_template('outputs.html', infid=infid, outputs=outputs)
@app.route('/delete/<infid>/<force>')
@authorized_with_valid_token
def infdel(infid=None, force=0):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data}
url = "%s/infrastructures/%s?async=1" % (settings.imUrl, infid)
if force:
url += "&force=1"
response = requests.delete(url, headers=headers)
if not response.ok:
flash("Error deleting infrastructure: " + response.text, "error")
else:
flash("Infrastructure '%s' successfuly deleted." % infid, "info")
try:
infra.delete_infra(infid)
except Exception as ex:
flash("Error deleting infrastructure name: %s" + str(ex), "warning")
return redirect(url_for('showinfrastructures'))
@app.route('/configure')
@authorized_with_valid_token
def configure():
selected_tosca = request.args['selected_tosca']
app.logger.debug("Template: " + json.dumps(toscaInfo[selected_tosca]))
vos = utils.getStaticVOs()
vos.extend(appdb.get_vo_list())
vos = list(set(vos))
if "vos" in session and session["vos"]:
vos = [vo for vo in vos if vo in session["vos"]]
return render_template('createdep.html',
template=toscaInfo[selected_tosca],
selectedTemplate=selected_tosca,
vos=vos)
@app.route('/sites/<vo>')
def getsites(vo=None):
res = ""
appdb_sites = appdb.get_sites(vo)
for site_name, site in appdb_sites.items():
if site["state"]:
site["state"] = " (WARNING: %s state!)" % site["state"]
res += '<option name="selectedSite" value=%s>%s%s</option>' % (site_name, site_name, site["state"])
for site_name, _ in utils.getStaticSites(vo).items():
# avoid site duplication
if site_name not in appdb_sites:
res += '<option name="selectedSite" value=%s>%s</option>' % (site_name, site_name)
return res
@app.route('/images/<site>/<vo>')
@authorized_with_valid_token
def getimages(site=None, vo=None):
res = ""
if vo == "local":
access_token = oidc_blueprint.session.token['access_token']
for image_name, image_id in utils.get_site_images(site, vo, access_token, cred, session["userid"]):
res += '<option name="selectedSiteImage" value=%s>%s</option>' % (image_id, image_name)
else:
for image in appdb.get_images(site, vo):
res += '<option name="selectedImage" value=%s>%s</option>' % (image, image)
return res
@app.route('/usage/<site>/<vo>')
@authorized_with_valid_token
def getusage(site=None, vo=None):
try:
access_token = oidc_blueprint.session.token['access_token']
quotas_dict = utils.get_site_usage(site, vo, access_token, cred, session["userid"])
return json.dumps(quotas_dict)
except Exception as ex:
return "Error loading site quotas: %s!" % str(ex), 400
def add_image_to_template(template, image):
# Add the image to all compute nodes
for node in list(template['topology_template']['node_templates'].values()):
if node["type"] == "tosca.nodes.indigo.Compute":
node["capabilities"]["os"]["properties"]["image"] = image
app.logger.debug(yaml.dump(template, default_flow_style=False))
return template
def add_auth_to_template(template, auth_data):
# Add the auth_data ElasticCluster node
for node in list(template['topology_template']['node_templates'].values()):
if node["type"] == "tosca.nodes.ec3.ElasticCluster":
if "properties" not in node:
node["properties"] = {}
node["properties"]["im_auth"] = auth_data
app.logger.debug(yaml.dump(template, default_flow_style=False))
return template
def set_inputs_to_template(template, inputs):
# Add the image to all compute nodes
for name, value in template['topology_template']['inputs'].items():
if name in inputs:
if value["type"] == "integer":
value["default"] = int(inputs[name])
elif value["type"] == "float":
value["default"] = float(inputs[name])
elif value["type"] == "boolean":
if inputs[name].lower() in ['yes', 'true', '1']:
value["default"] = True
else:
value["default"] = False
# Special case for ports, convert a comma separated list of ints
# to a PortSpec map
elif value["type"] == "map" and name == "ports":
ports = inputs[name].split(",")
ports_value = {}
for port in ports:
# Should we also open UDP?
ports_value["port_%s" % port] = {"protocol": "tcp", "source": int(port)}
value["default"] = ports_value
else:
value["default"] = inputs[name]
app.logger.debug(yaml.dump(template, default_flow_style=False))
return template
@app.route('/submit', methods=['POST'])
@authorized_with_valid_token
def createdep():
form_data = request.form.to_dict()
vo = form_data['extra_opts.selectedVO']
site = form_data['extra_opts.selectedSite']
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"], vo, site)
app.logger.debug("Form data: " + json.dumps(request.form.to_dict()))
with io.open(settings.toscaDir + request.args.get('template')) as stream:
template = yaml.full_load(stream)
if form_data['extra_opts.selectedImage'] != "":
image = "appdb://%s/%s?%s" % (form_data['extra_opts.selectedSite'],
form_data['extra_opts.selectedImage'],
form_data['extra_opts.selectedVO'])
elif form_data['extra_opts.selectedSiteImage'] != "":
site_url = utils.get_ost_image_url(form_data['extra_opts.selectedSite'])
image = "ost://%s/%s" % (site_url, form_data['extra_opts.selectedSiteImage'])
else:
flash("No correct image selected.", "error")
return redirect(url_for('showinfrastructures'))
template = add_image_to_template(template, image)
template = add_auth_to_template(template, auth_data)
inputs = {k: v for (k, v) in form_data.items() if not k.startswith("extra_opts.")}
app.logger.debug("Parameters: " + json.dumps(inputs))
template = set_inputs_to_template(template, inputs)
payload = yaml.dump(template, default_flow_style=False, sort_keys=False)
headers = {"Authorization": auth_data, "Content-Type": "text/yaml"}
url = "%s/infrastructures?async=1" % settings.imUrl
response = requests.post(url, headers=headers, data=payload)
if not response.ok:
flash("Error creating infrastrucrure: \n" + response.text, "error")
else:
try:
inf_id = os.path.basename(response.text)
infra.write_infra(inf_id, {"name": form_data['infra_name']})
except Exception as ex:
flash("Error storing Infrastructure name: %s" % str(ex), "warning")
return redirect(url_for('showinfrastructures'))
@app.route('/manage_creds')
@authorized_with_valid_token
def manage_creds():
sites = {}
try:
sites = utils.getCachedSiteList()
except Exception as e:
flash("Error retrieving sites list: \n" + str(e), 'warning')
return render_template('service_creds.html', sites=sites)
@app.route('/write_creds', methods=['GET', 'POST'])
@authorized_with_valid_token
def write_creds():
serviceid = request.args.get('service_id', "")
servicename = request.args.get('service_name', "")
app.logger.debug("service_id={}".format(serviceid))
if request.method == 'GET':
res = {}
projects = {}
try:
res = cred.get_cred(servicename, session["userid"])
projects = utils.getCachedProjectIDs(serviceid)
app.logger.debug("projects={}".format(projects))
if session["vos"]:
filter_projects = {}
for vo, project in projects.items():
if vo in session["vos"]:
filter_projects[vo] = project
projects = filter_projects
except Exception as ex:
flash("Error reading credentials %s!" % ex, 'error')
return render_template('modal_creds.html', service_creds=res, service_id=serviceid,
service_name=servicename, projects=projects)
else:
app.logger.debug("Form data: " + json.dumps(request.form.to_dict()))
creds = request.form.to_dict()
try:
cred.write_creds(servicename, session["userid"], creds)
flash("Credentials successfully written!", 'info')
except Exception as ex:
flash("Error writing credentials %s!" % ex, 'error')
return redirect(url_for('manage_creds'))
@app.route('/delete_creds')
@authorized_with_valid_token
def delete_creds():
serviceid = request.args.get('service_id', "")
try:
cred.delete_cred(serviceid, session["userid"])
flash("Credentials successfully deleted!", 'info')
except Exception as ex:
flash("Error deleting credentials %s!" % ex, 'error')
return redirect(url_for('manage_creds'))
@app.route('/addresourcesform/<infid>')
@authorized_with_valid_token
def addresourcesform(infid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data, "Accept": "text/plain"}
url = "%s/infrastructures/%s/radl" % (settings.imUrl, infid)
response = requests.get(url, headers=headers)
if response.ok:
systems = []
try:
radl = radl_parse.parse_radl(response.text)
systems = radl.systems
except Exception as ex:
flash("Error parsing RADL: \n%s" % str(ex), 'error')
return render_template('addresource.html', infid=infid, systems=systems)
else:
flash("Error getting RADL: \n%s" % (response.text), 'error')
return redirect(url_for('showinfrastructures'))
@app.route('/addresources/<infid>', methods=['POST'])
@authorized_with_valid_token
def addresources(infid=None):
access_token = oidc_blueprint.session.token['access_token']
auth_data = utils.getUserAuthData(access_token, cred, session["userid"])
headers = {"Authorization": auth_data, "Accept": "text/plain"}
form_data = request.form.to_dict()
url = "%s/infrastructures/%s/radl" % (settings.imUrl, infid)
response = requests.get(url, headers=headers)
if response.ok:
radl = None
try:
radl = radl_parse.parse_radl(response.text)
radl.deploys = []
for system in radl.systems:
if "%s_num" % system.name in form_data:
vm_num = int(form_data["%s_num" % system.name])
if vm_num > 0:
radl.deploys.append(deploy(system.name, vm_num))
except Exception as ex:
flash("Error parsing RADL: \n%s\n%s" % (str(ex), response.text), 'error')
if radl:
headers = {"Authorization": auth_data, "Accept": "application/json"}
url = "%s/infrastructures/%s" % (settings.imUrl, infid)
response = requests.post(url, headers=headers, data=str(radl))
if response.ok:
num = len(response.json()["uri-list"])
flash("%d nodes added successfully" % num, 'info')
else:
flash("Error adding nodesL: \n%s" % (response.text), 'error')
return redirect(url_for('showinfrastructures'))
else:
flash("Error getting RADL: \n%s" % (response.text), 'error')
return redirect(url_for('showinfrastructures'))
@app.route('/logout')
def logout():
session.clear()
oidc_blueprint.session.get("/logout")
return redirect(url_for('login'))
@app.errorhandler(403)
def forbidden(error):
return render_template('error_pages/403.html', message=error.description)
@app.errorhandler(404)
def page_not_found(error):
app.logger.error('Page not found: %s', (request.path))
return render_template('error_pages/404.html'), 404
@app.errorhandler(500)
def internal_server_error(error):
app.logger.error('Server Error: %s', (error))
return render_template('error_pages/500.html', support_email=app.config.get('SUPPORT_EMAIL')), 500
return app
socket.send('quit')
print "done"
@app.route('/')
def index():
nonce = ''.join(random.sample(
string.lowercase+string.digits, 16
))
r = Response(render_template("otm.jinja",
nonce=nonce
))
r.headers['Content-Security-Policy'] = ';'.join((
"default-src 'none'",
"style-src 'nonce-%s'" % nonce,
"script-src 'nonce-%s'" % nonce,
"connect-src %s://%s/ws" % (
"wss" if request.is_secure else "ws",
request.host,
),
))
r.headers['X-Frame-Options'] = 'DENY'
return r
if __name__ == "__main__":
from gevent import pywsgi
from geventwebsocket.handler import WebSocketHandler
from werkzeug.contrib.fixers import ProxyFix
app = ProxyFix(app)
server = pywsgi.WSGIServer(('127.0.0.1', 8080), app, handler_class=WebSocketHandler)
server.serve_forever()
how-to-run-flask-applications-with-nginx-using-gunicorn/
"""
import argparse
from werkzeug.serving import run_simple
from werkzeug.wsgi import DispatcherMiddleware
from werkzeug.contrib.fixers import ProxyFix
from quokka import create_app, create_api
from quokka.utils.paas import activate
application = DispatcherMiddleware(create_app(), {
'/api': create_api()
})
application.wsgi_app = ProxyFix(application.app.wsgi_app)
application = app = activate(application)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Run Quokka App for WSGI")
parser.add_argument('-p', '--port', help='App Port')
parser.add_argument('-i', '--host', help='App Host')
parser.add_argument('-r', '--reloader', action='store_true',
help='Turn reloader on')
parser.add_argument('-d', '--debug', action='store_true',
help='Turn debug on')
args = parser.parse_args()
run_simple(
args.host or '0.0.0.0',
int(args.port) if args.port else 5000,
from flask import Flask
from flask_admin import Admin
from flask_mail import Mail
from flask_migrate import Migrate
from flask_restless import APIManager
from flask_security import Security, SQLAlchemyUserDatastore
from flask_sqlalchemy import SQLAlchemy
from werkzeug.contrib.fixers import ProxyFix # use remote address header
from . import config
from .meta import fullname
# main objects
app = Flask(__name__)
app.config.from_object(config)
app.wsgi_app = ProxyFix(app.wsgi_app) # remote address header fix
db = SQLAlchemy(app)
mail = Mail(app)
migrate = Migrate(app, db)
# additional imports below avoid circular import issues w/flask objects.
from .logcfg import log # noqa: E402
from . import models, forms, views, admin, utils, errors # noqa: F401,E402
# admin setup
adm = Admin(
app,
name=fullname,
template_mode='bootstrap3',
index_view=admin.HomeView(),
category_icon_classes=admin.category_icon_classes,
def __init__(self, import_name, key, num_proxies=None):
Flask.__init__(self, import_name)
if num_proxies is not None:
self.wsgi_app = ProxyFix(self.wsgi_app, num_proxies=num_proxies)
self.config['KEY'] = key
self.hooks = {}
@self.errorhandler(400)
@self.errorhandler(403)
@self.errorhandler(404)
@self.errorhandler(500)
def handle_error(e):
if isinstance(e, HTTPException):
msg = e.description
status = e.code
else:
msg = 'Internal server error'
status = 500
return msg, status
@self.before_request
def validate_ip():
if not self.debug:
# Python 2.x
if hasattr(str, 'decode'):
ip = ip_address(request.remote_addr.decode('utf8'))
# Python 3.x
else:
ip = ip_address(request.remote_addr)
for block in get(
'https://api.github.com/meta').json()['hooks']:
if ip in ip_network(block):
break
else:
raise Forbidden('Requests must originate from GitHub')
@self.before_request
def validate_hmac():
if not self.debug:
key = self.config['KEY']
signature = request.headers.get('X-Hub-Signature')
if not signature:
raise BadRequest('Missing HMAC signature')
payload = request.get_data()
digest = new(key, payload, sha1).hexdigest()
if ('sha1=%s' % digest) != signature:
raise BadRequest('Wrong HMAC signature')
@self.route('/hooks', methods=['POST'])
def hook():
event = request.headers.get('X-GitHub-Event')
if not event:
raise BadRequest('No hook given')
guid = request.headers.get('X-GitHub-Delivery')
if not guid:
raise BadRequest('No event GUID')
data = request.get_json()
if not data:
raise BadRequest('No payload data')
if event in self.hooks:
return self.hooks[event](data, guid)
else:
return 'Hook not used'
def create_app(config='CTFd.config.Config'):
app = CTFdFlask(__name__)
with app.app_context():
app.config.from_object(config)
theme_loader = ThemeLoader(os.path.join(app.root_path, 'themes'),
followlinks=True)
app.jinja_loader = theme_loader
from CTFd.models import db, Teams, Solves, Challenges, Fails, Flags, Tags, Files, Tracking
url = create_database()
# This allows any changes to the SQLALCHEMY_DATABASE_URI to get pushed back in
# This is mostly so we can force MySQL's charset
app.config['SQLALCHEMY_DATABASE_URI'] = str(url)
# Register database
db.init_app(app)
# Register Flask-Migrate
migrations.init_app(app, db)
# Alembic sqlite support is lacking so we should just create_all anyway
if url.drivername.startswith('sqlite'):
db.create_all()
stamp()
else:
# This creates tables instead of db.create_all()
# Allows migrations to happen properly
upgrade()
from CTFd.models import ma
ma.init_app(app)
app.db = db
app.VERSION = __version__
from CTFd.cache import cache
cache.init_app(app)
app.cache = cache
# If you have multiple workers you must have a shared cache
socketio.init_app(app,
async_mode=app.config.get('SOCKETIO_ASYNC_MODE'),
message_queue=app.config.get('CACHE_REDIS_URL'))
if app.config.get('REVERSE_PROXY'):
app.wsgi_app = ProxyFix(app.wsgi_app)
version = utils.get_config('ctf_version')
# Upgrading from an older version of CTFd
if version and (StrictVersion(version) < StrictVersion(__version__)):
if confirm_upgrade():
run_upgrade()
else:
exit()
if not version:
utils.set_config('ctf_version', __version__)
if not utils.get_config('ctf_theme'):
utils.set_config('ctf_theme', 'core')
update_check(force=True)
init_request_processors(app)
init_template_filters(app)
init_template_globals(app)
# Importing here allows tests to use sensible names (e.g. api instead of api_bp)
from CTFd.views import views
from CTFd.teams import teams
from CTFd.users import users
from CTFd.challenges import challenges
from CTFd.scoreboard import scoreboard
from CTFd.auth import auth
from CTFd.admin import admin
from CTFd.api import api
from CTFd.events import events
from CTFd.errors import page_not_found, forbidden, general_error, gateway_error
app.register_blueprint(views)
app.register_blueprint(teams)
app.register_blueprint(users)
app.register_blueprint(challenges)
app.register_blueprint(scoreboard)
app.register_blueprint(auth)
app.register_blueprint(api)
app.register_blueprint(events)
app.register_blueprint(admin)
app.register_error_handler(404, page_not_found)
app.register_error_handler(403, forbidden)
app.register_error_handler(500, general_error)
app.register_error_handler(502, gateway_error)
init_logs(app)
init_plugins(app)
return app
'SECURITY_CAPTCHABLE': True,
'SECURITY_CAPTCHA_FUNC': rand_captcha,
'SECURITY_POST_LOGIN_VIEW': 'http://design.hnu.edu.cn/pinwall',
'SECURITY_POST_LOGOUT_VIEW': 'http://design.hnu.edu.cn/pinwall',
'SECURITY_POST_REGISTER_VIEW': 'http://design.hnu.edu.cn/pinwall',
'SECURITY_POST_CONFIRM_VIEW': 'http://design.hnu.edu.cn/pinwall',
'SECURITY_POST_RESET_VIEW': 'http://design.hnu.edu.cn/pinwall',
'SECURITY_POST_CHANGE_VIEW': 'http://design.hnu.edu.cn/pinwall',
}
api_setting_override = {
'SECURITY_LOGINABLE': False,
'SECURITY_LOGOUTABLE': False,
'SECURITY_CONFIRMABLE': False,
'SECURITY_REGISTERABLE': False,
'SECURITY_RECOVERABLE': False,
'SECURITY_TRACKABLE': False,
'SECURITY_CHANGEABLE': False,
}
application = ProxyFix(
DispatcherMiddleware(
frontend.create_app(settings_override=frontend_setting_override), {
'/api': api.create_app(register_security_blueprint=False),
'/weixin': weixin.create_app()
}))
if __name__ == "__main__":
# run_simple('0.0.0.0', 80, application, use_reloader=True, use_debugger=True)
run_simple('0.0.0.0', 80, application)
def create_app() -> Flask:
app = Flask(__name__, static_folder='assets')
app.config.from_pyfile('config.py')
app.session_interface = RedisSessionInterface()
app.wsgi_app = ProxyFix(app.wsgi_app)
db.init_app(app)
init_celery(app, celery)
if app.config['SENTRY_DSN']:
sentry = Sentry(app, dsn=app.config['SENTRY_DSN'])
setup_templating(app)
register_blueprints(app)
@app.before_first_request
def setup_logging():
if not app.debug:
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.WARNING)
@app.before_first_request
def setup_db():
db.engine.pool._use_threadlocal = True
@app.before_request
def get_user_data():
try:
g.user = {"id": session['steamid'], "nick": session['nick']}
except KeyError:
g.user = None
@app.before_request
def set_default_page_number():
try:
g.page = int(request.args.get('p', 1))
except ValueError:
g.page = 1
@app.before_request
def generate_csrf_token():
if '_csrf_token' not in session:
session['_csrf_token'] = binascii.b2a_hex(os.urandom(16)).decode()
g.token = session['_csrf_token']
@app.context_processor
def classes():
"""Convenience function to return a list of TF2 classes."""
return {
"classes": [
'scout', 'soldier', 'pyro', 'demoman', 'heavyweapons',
'engineer', 'medic', 'sniper', 'spy'
]
}
@app.errorhandler(404)
def page_not_found(e):
"""404 Page not found"""
return redirect('/')
@app.errorhandler(500)
def server_error(e):
"""500 Internal server error"""
return render_template('errors/500.html'), 500
return app
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /myprefix;
}
:param app: the WSGI application
'''
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
if script_name:
environ['SCRIPT_NAME'] = script_name
path_info = environ['PATH_INFO']
if path_info.startswith(script_name):
environ['PATH_INFO'] = path_info[len(script_name):]
scheme = environ.get('HTTP_X_SCHEME', '')
if scheme:
environ['wsgi.url_scheme'] = scheme
return self.app(environ, start_response)
application = app
# patch app to handle non root url-s behind proxy & wsgi
app.wsgi_app = ReverseProxyPathFix(ProxyFix(application.wsgi_app))
if __name__ == "__main__":
run()
def main():
parser = optparse.OptionParser()
parser.add_option('-l',
'--location',
dest='location',
action='store',
type="string",
help='path prefix to store the database in',
default="dtg_")
parser.add_option('-i',
'--server-ip',
dest='server_ip',
action='store',
type="string",
help='ip/hostname to run the server on',
default="127.0.0.1")
parser.add_option('-p',
'--server-port',
dest='server_port',
action='store',
type="int",
help='port to run the server on',
default=5005)
parser.add_option('-e',
'--email',
dest='email',
action='store',
type="string",
help='e-mail address of the admin',
default=None)
parser.add_option(
'-P',
'--path',
dest='path',
action='store',
type="string",
help=
'Path of DTG below the HTTP root folder (e.g. PATH in http://SERVERIP:SERVERPORT/PATH/)',
default=None)
parser.add_option(
'-R',
'--proxy',
dest='proxy',
action='store_true',
help='Proxy mode (use when running behind an HTTP proxy)',
default=False)
parser.add_option('-D',
'--debug',
dest='debug',
action='store_true',
help='Debug mode',
default=False)
parser.add_option('-M',
'--allow-migrations',
dest='migrate',
action='store_true',
help='Allow DB migrations. Only use it with backups :-)',
default=False)
parser.add_option('--add-user',
dest='adduser',
action='store',
type="string",
help="Username to add, password will be asked for",
default="")
parser.add_option('--del-user',
dest='deluser',
action='store',
type="string",
help="Username to delete",
default="")
parser.add_option('--change-pwd',
dest='changepwd',
action='store',
type="string",
help="Username to change password of",
default="")
options, args = parser.parse_args()
if args:
parser.error("don't know what to do with additional arguments")
# HACK! :)
sys.dtg_db_path = lambda x: "sqlite:///" + os.path.abspath(options.location
+ x + ".db")
sys.dtg_do_upgrade = options.migrate
sys.dtg_debug = options.debug
from dtg.webapp import app, add_user, del_user, change_pwd
if options.debug:
app.secret_key = "insecure"
if options.adduser or options.changepwd:
password, password2 = getpass(), getpass("Password, again: ")
if password != password2:
print "Passwords do not match"
return
if options.adduser:
print add_user(options.adduser, password)
return
if options.deluser:
print del_user(options.deluser)
return
if options.changepwd:
print change_pwd(options.changepwd, password)
return
#app.wsgi_app = GzipMiddleware(app.wsgi_app)
if options.email is not None:
kwargs = dict(error_email=options.email,
from_address=options.email,
smtp_server="localhost")
else:
kwargs = {}
app.wsgi_app = ErrorMiddleware(app.wsgi_app, **kwargs)
if options.path:
if not options.path.startswith("/"):
options.path = "/" + options.path
print "Mounting DTG under", options.path
app.wsgi_app = DispatcherMiddleware(
lambda e, s: [s("404 Not found", []), "Not found"][1:],
mounts={options.path: app.wsgi_app})
if options.proxy:
app.wsgi_app = ProxyFix(app.wsgi_app)
app.run(host=options.server_ip,
port=options.server_port,
threaded=True,
use_reloader=options.debug,
passthrough_errors=True)
from config import DefaultConfig
logger.debug("Loading default config.")
app.config.from_object(DefaultConfig())
app.teardown_request(database.close_db_filter)
# Load the override config via the provider.
config_provider.update_app_config(app.config)
# Update any configuration found in the override environment variable.
environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, "{}"))
app.config.update(environ_config)
# Fix remote address handling for Flask.
if app.config.get("PROXY_COUNT", 1):
app.wsgi_app = ProxyFix(app.wsgi_app,
num_proxies=app.config.get("PROXY_COUNT", 1))
# Ensure the V3 upgrade key is specified correctly. If not, simply fail.
# TODO: Remove for V3.1.
if not is_testing and not is_building and app.config.get(
"SETUP_COMPLETE", False):
v3_upgrade_mode = app.config.get("V3_UPGRADE_MODE")
if v3_upgrade_mode is None:
raise Exception(
"Configuration flag `V3_UPGRADE_MODE` must be set. Please check the upgrade docs"
)
if (v3_upgrade_mode != "background" and v3_upgrade_mode != "complete"
and v3_upgrade_mode != "production-transition"
and v3_upgrade_mode != "post-oci-rollout"
and v3_upgrade_mode != "post-oci-roll-back-compat"):
def create_app(config={}):
"""
Configure and create the Flask app via factory function.
Args:
config (optional): dict of app.config settings to override
"""
app = Flask(__name__, static_url_path="/")
app.wsgi_app = ProxyFix(app.wsgi_app)
# Load default Flask settings
app.config.from_pyfile('default_settings.py')
# Override defaults with settings file passed in environment variable
app.config.from_envvar('APP_SETTINGS_FILE', silent=True)
# If any config settings specified, set them
for k, v in config.items():
app.config[k] = v
# Add any new runtime settings to DB
with app.app_context():
api.config.merge_new_settings()
# Register blueprints
app.register_blueprint(v1_blueprint, url_prefix="/api/v1")
# Report all validation errors (RequestParser-specific setting)
app.config['BUNDLE_ERRORS'] = True
# Register error handlers
@app.errorhandler(PicoException)
def handle_pico_exception(e):
"""Handle exceptions."""
response = jsonify(e.to_dict())
response.status_code = e.status_code
return response
if not app.debug:
@app.errorhandler(Exception)
def handle_generic_exception(e):
# @TODO log picoexceptions also?
get_origin_logger(e).error(traceback.format_exc())
response = jsonify({
'message':
"An internal error occurred. " +
"Please contact an administrator."
})
response.status_code = 500
return response
# Configure logging
with app.app_context():
api.logger.setup_logs({"verbose": 2})
# Register a post-request function
@app.after_request
def after_request(response):
response.headers.add('Access-Control-Allow-Methods',
'GET, POST, PUT, PATCH, DELETE, OPTIONS')
response.headers.add('Access-Control-Allow-Credentials', 'true')
response.headers.add('Access-Control-Allow-Headers', 'Content-Type, *')
response.headers.add('Cache-Control', 'no-cache')
response.headers.add('Cache-Control', 'no-store')
with app.app_context():
if app.debug:
response.headers.add('Access-Control-Allow-Origin', '*')
if api.user.is_logged_in():
# Flask 1.0+ bug loads config SESSION_COOKIE_DOMAIN
# correctly as None but later converts it to bool false.
domain = app.config['SESSION_COOKIE_DOMAIN']
if not domain:
domain = None
# Set the CSRF token cookie
if 'token' not in session:
csrf_token = api.common.token()
session['token'] = csrf_token
response.set_cookie('token', session['token'], domain=domain)
return response
return app
def make_app(build_dir: str,
models: Dict[str, DemoModel],
demo_db: Optional[DemoDatabase] = None,
cache_size: int = 128) -> Flask:
if not os.path.exists(build_dir):
logger.error("app directory %s does not exist, aborting", build_dir)
sys.exit(-1)
app = Flask(__name__) # pylint: disable=invalid-name
start_time = datetime.now(pytz.utc)
start_time_str = start_time.strftime("%Y-%m-%d %H:%M:%S %Z")
app.predictors = {}
app.max_request_lengths = {
} # requests longer than these will be rejected to prevent OOME
app.wsgi_app = ProxyFix(
app.wsgi_app) # sets the requester IP with the X-Forwarded-For header
for name, demo_model in models.items():
if demo_model is not None:
logger.info(f"loading {name} model")
predictor = demo_model.predictor()
app.predictors[name] = predictor
app.max_request_lengths[name] = demo_model.max_request_length
# Disable caching for HTML documents and API responses so that clients
# always talk to the source (this server).
@app.after_request
def set_cache_headers(resp: Response) -> Response:
if resp.mimetype == "text/html" or resp.mimetype == "application/json":
return with_no_cache_headers(resp)
else:
return resp
@app.errorhandler(ServerError)
def handle_invalid_usage(error: ServerError) -> Response: # pylint: disable=unused-variable
response = jsonify(error.to_dict())
response.status_code = error.status_code
return response
@lru_cache(maxsize=cache_size)
def _caching_prediction(model: Predictor, data: str) -> JsonDict:
"""
Just a wrapper around ``model.predict_json`` that allows us to use a cache decorator.
"""
return model.predict_json(json.loads(data))
@app.route('/')
def index() -> Response: # pylint: disable=unused-variable
return send_file(os.path.join(build_dir, 'index.html'))
@app.route('/permadata', methods=['POST', 'OPTIONS'])
def permadata() -> Response: # pylint: disable=unused-variable
"""
If the user requests a permalink, the front end will POST here with the payload
{ slug: slug }
which we convert to an integer id and use to retrieve saved results from the database.
"""
# This is just CORS boilerplate.
if request.method == "OPTIONS":
return Response(response="", status=200)
# If we don't have a database configured, there are no permalinks.
if demo_db is None:
raise ServerError('Permalinks are not enabled', 400)
# Convert the provided slug to an integer id.
slug = request.get_json()["slug"]
perma_id = slug_to_int(slug)
if perma_id is None:
# Malformed slug
raise ServerError("Unrecognized permalink: {}".format(slug), 400)
# Fetch the results from the database.
try:
permadata = demo_db.get_result(perma_id)
except psycopg2.Error:
logger.exception(
"Unable to get results from database: perma_id %s", perma_id)
raise ServerError('Database trouble', 500)
if permadata is None:
# No data found, invalid id?
raise ServerError("Unrecognized permalink: {}".format(slug), 400)
return jsonify({
"modelName": permadata.model_name,
"requestData": permadata.request_data,
"responseData": permadata.response_data
})
@app.route('/predict/<model_name>', methods=['POST', 'OPTIONS'])
def predict(model_name: str) -> Response: # pylint: disable=unused-variable
"""make a prediction using the specified model and return the results"""
if request.method == "OPTIONS":
return Response(response="", status=200)
# Do log if no argument is specified
record_to_database = request.args.get("record",
"true").lower() != "false"
# Do use the cache if no argument is specified
use_cache = request.args.get("cache", "true").lower() != "false"
lowered_model_name = model_name.lower()
model = app.predictors.get(lowered_model_name)
if model is None:
raise ServerError("unknown model: {}".format(model_name),
status_code=400)
max_request_length = app.max_request_lengths[lowered_model_name]
data = request.get_json()
serialized_request = json.dumps(data)
if len(serialized_request) > max_request_length:
raise ServerError(
f"Max request length exceeded for model {model_name}! " +
f"Max: {max_request_length} Actual: {len(serialized_request)}")
logger.info("request: %s",
json.dumps({
"model": model_name,
"inputs": data
}))
log_blob = {
"model": model_name,
"inputs": data,
"cached": False,
"outputs": {}
}
# Record the number of cache hits before we hit the cache so we can tell whether we hit or not.
# In theory this could result in false positives.
pre_hits = _caching_prediction.cache_info().hits # pylint: disable=no-value-for-parameter
if record_to_database and demo_db is not None:
try:
perma_id = None
perma_id = demo_db.insert_request(
headers=dict(request.headers),
requester=request.remote_addr,
model_name=model_name,
inputs=data)
except Exception: # pylint: disable=broad-except
# TODO(joelgrus): catch more specific errors
logger.exception("Unable to add request to database",
exc_info=True)
if use_cache and cache_size > 0:
# lru_cache insists that all function arguments be hashable,
# so unfortunately we have to stringify the data.
prediction = _caching_prediction(model, json.dumps(data))
else:
# if cache_size is 0, skip caching altogether
prediction = model.predict_json(data)
post_hits = _caching_prediction.cache_info().hits # pylint: disable=no-value-for-parameter
if record_to_database and demo_db is not None and perma_id is not None:
try:
demo_db.update_response(perma_id=perma_id, outputs=prediction)
slug = int_to_slug(perma_id)
prediction["slug"] = slug
log_blob["slug"] = slug
except Exception: # pylint: disable=broad-except
# TODO(joelgrus): catch more specific errors
logger.exception("Unable to add response to database",
exc_info=True)
if use_cache and post_hits > pre_hits:
# Cache hit, so insert an artifical pause
log_blob["cached"] = True
time.sleep(0.25)
# The model predictions are extremely verbose, so we only log the most human-readable
# parts of them.
if "comprehension" in model_name:
if 'best_span_str' in prediction:
answer = prediction['best_span_str']
else:
answer = prediction['answer']
log_blob["outputs"]["answer"] = answer
elif model_name == "coreference-resolution":
log_blob["outputs"]["clusters"] = prediction["clusters"]
log_blob["outputs"]["document"] = prediction["document"]
elif model_name == "textual-entailment":
log_blob["outputs"]["label_probs"] = prediction["label_probs"]
elif model_name == "sentiment-analysis":
log_blob["outputs"]["probs"] = prediction["probs"]
elif model_name == "named-entity-recognition":
log_blob["outputs"]["tags"] = prediction["tags"]
elif model_name == "semantic-role-labeling":
verbs = []
for verb in prediction["verbs"]:
# Don't want to log boring verbs with no semantic parses.
good_tags = [tag for tag in verb["tags"] if tag != "0"]
if len(good_tags) > 1:
verbs.append({
"verb": verb["verb"],
"description": verb["description"]
})
log_blob["outputs"]["verbs"] = verbs
elif model_name == "constituency-parsing":
log_blob["outputs"]["trees"] = prediction["trees"]
elif model_name == "wikitables-parser":
log_blob['outputs']['logical_form'] = prediction['logical_form']
log_blob['outputs']['answer'] = prediction['answer']
elif model_name == "quarel-parser-zero":
log_blob['outputs']['logical_form'] = prediction['logical_form']
log_blob['outputs']['answer'] = prediction['answer']
log_blob['outputs']['score'] = prediction['score']
elif model_name == "nlvr-parser":
log_blob['outputs']['logical_form'] = prediction['logical_form'][0]
log_blob['outputs']['answer'] = prediction['denotations'][0][0]
elif model_name == "atis-parser":
log_blob['outputs']['predicted_sql_query'] = prediction[
'predicted_sql_query']
# TODO(brendanr): Add event2mind log_blob here?
logger.info("prediction: %s", json.dumps(log_blob))
return jsonify(prediction)
@app.route('/models')
def list_models() -> Response: # pylint: disable=unused-variable
"""list the available models"""
return jsonify({"models": list(app.predictors.keys())})
@app.route('/info')
def info() -> Response: # pylint: disable=unused-variable
"""List metadata about the running webserver"""
uptime = str(datetime.now(pytz.utc) - start_time)
git_version = os.environ.get('ALLENNLP_DEMO_SOURCE_COMMIT') or ""
return jsonify({
"start_time":
start_time_str,
"uptime":
uptime,
"git_version":
git_version,
"peak_memory_mb":
peak_memory_mb(),
"githubUrl":
"http://github.com/allenai/allennlp-demo/commit/" + git_version
})
@app.route('/health')
def health() -> Response: # pylint: disable=unused-variable
return "healthy"
# As an SPA, we need to return index.html for /model-name and /model-name/permalink
def return_page(permalink: str = None) -> Response: # pylint: disable=unused-argument, unused-variable
"""return the page"""
return send_file(os.path.join(build_dir, 'index.html'))
for model_name in models:
logger.info(f"setting up default routes for {model_name}")
app.add_url_rule(f"/{model_name}", view_func=return_page)
app.add_url_rule(f"/{model_name}/<permalink>", view_func=return_page)
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def static_proxy(path: str) -> Response: # pylint: disable=unused-variable
if os.path.isfile(os.path.join(build_dir, path)):
return send_from_directory(build_dir, path)
else:
# Send the index.html page back to the client as a catch-all, since
# we're an SPA and JavaScript acts to handle routes the server
# doesn't.
return send_file(os.path.join(build_dir, 'index.html'))
@app.route('/static/js/<path:path>')
def static_js_proxy(path: str) -> Response: # pylint: disable=unused-variable
return send_from_directory(os.path.join(build_dir, 'static/js'), path)
@app.route('/static/css/<path:path>')
def static_css_proxy(path: str) -> Response: # pylint: disable=unused-variable
return send_from_directory(os.path.join(build_dir, 'static/css'), path)
@app.route('/static/media/<path:path>')
def static_media_proxy(path: str) -> Response: # pylint: disable=unused-variable
return send_from_directory(os.path.join(build_dir, 'static/media'),
path)
return app
def create_app(environment_name=None):
"""Create and return a Flask application. Reads a config file path from the
OK_SERVER_CONFIG environment variable. If it is not set, it imports and reads
from the config module (using environment_name). This is so we can default to a development
environment locally, but the app will fail in production if there is no
config file rather than dangerously defaulting to a development environment.
"""
app = Flask(__name__)
env_module_path = os.getenv('OK_SERVER_CONFIG',
'server.settings.{}'.format(environment_name))
if '/' in env_module_path:
env_module_path = env_module_path.replace('/', '.')
env_module_path = env_module_path.replace('.py', '')
app.config.from_object(env_module_path + '.Config')
# Set REMOTE_ADDR for proxies
num_proxies = app.config.get('NUM_PROXIES', 0)
if num_proxies:
app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies=num_proxies)
# Sentry Error Reporting
sentry_dsn = os.getenv('SENTRY_DSN')
if not app.debug and sentry_dsn:
sentry.init_app(app, dsn=sentry_dsn)
@app.errorhandler(500)
def internal_server_error(error):
return render_template('errors/500.html',
event_id=g.sentry_event_id,
public_dsn=sentry.client.get_public_dsn('https')
), 500
# Azure Application Insights request and error tracking
appinsights.init_app(app)
@app.errorhandler(404)
def not_found_error(error):
if request.path.startswith("/api"):
return api.handle_error(error)
return render_template('errors/404.html'), 404
@app.route("/healthz")
def health_check():
return 'OK'
@app.after_request
def flush_appinsights_telemetry(response):
appinsights.flush()
return response
@app.context_processor
def inject_root_url():
return {'index_url': app.config['APPLICATION_ROOT']}
# initialize the cache
cache.init_app(app)
# initialize redis task queues
RQ(app)
# Protect All Routes from csrf
csrf.init_app(app)
# initialize the debug tool bar
debug_toolbar.init_app(app)
# initialize SQLAlchemy
db.init_app(app)
# Flask-Login manager
login_manager.init_app(app)
# initalize cloud storage
storage.init_app(app)
# Set up logging
logging.init_app(app)
# Import and register the different asset bundles
assets_env.init_app(app)
assets_loader = PythonAssetsLoader(assets)
for name, bundle in assets_loader.load_bundles().items():
assets_env.register(name, bundle)
# custom URL handling
converters.init_app(app)
# custom Jinja rendering
app.jinja_env.globals.update({
'utils': utils,
'debug': app.debug,
'instantclick': app.config.get('INSTANTCLICK', True),
'CSRFForm': CSRFForm
})
app.jinja_env.filters.update({
'markdown': utils.convert_markdown,
'pluralize': utils.pluralize,
})
# register our blueprints
# OAuth should not need CSRF protection
csrf.exempt(auth)
app.register_blueprint(auth)
csrf.exempt(oauth)
app.register_blueprint(oauth)
app.register_blueprint(files)
app.register_blueprint(student)
app.register_blueprint(admin, url_prefix='/admin')
app.register_blueprint(about, url_prefix='/about')
# Redis Queue dashboard
csrf.exempt(queue)
app.register_blueprint(queue, url_prefix='/rq')
# API does not need CSRF protection
csrf.exempt(api_endpoints)
app.register_blueprint(api_endpoints, url_prefix=API_PREFIX)
return app
