def superproxy(config, listen, pool_size, stop_timeout, dozer):
import sys
import signal
import gevent
from gevent.pywsgi import WSGIServer
from gevent.pool import Pool
from .superproxy import WSGISuperProxy
from .proxylist import ProxyList
conf = config.get('superproxy', {})
fetcher = config.get('proxyfetcher')
checker = config.get('proxychecker')
proxylist = ProxyList(fetcher=fetcher, checker=checker, **conf.pop('proxylist', {}))
listen = listen or conf.pop('listen', '0.0.0.0:8088')
pool_size = pool_size or conf.pop('pool_size', 500)
stop_timeout = stop_timeout or conf.pop('stop_timeout', 5)
dozer = conf.pop('dozer', False)
iface, port = listen.split(':')
app = WSGISuperProxy(proxylist, **conf)
if dozer:
from dozer import Dozer
app = Dozer(app)
server = WSGIServer((iface, int(port)), app, spawn=Pool(pool_size))
server.stop_timeout = stop_timeout
logger = logging.getLogger('proxytools.superproxy')
def stop(*args):
if server.closed:
try:
logger.error('Server stopping - multiple exit signals received - aborting.')
finally:
sys.exit('Multiple exit signals received - aborting.')
logger.info('Server stopping %s', args)
server.stop()
[gevent.signal(sig, stop) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT)]
logger.info('Server started')
server.serve_forever()
logger.debug('Server stopped')
def run_command(rule_filename, debug=False, interactive_debugger=False,
debug_headers=False, profile=False, memory_profile=False,
garbage_collect=False):
"""Actually runs the command from the parsed arguments"""
settings = ProxySettings.parse_file(rule_filename)
app = ReloadingApp(rule_filename, settings)
if profile:
try:
from repoze.profile.profiler import AccumulatingProfileMiddleware
except ImportError:
print('Error: you must manually install repoze.profiler to use --profile')
sys.exit(1)
app = AccumulatingProfileMiddleware(
app,
log_filename='/tmp/deliverance-proxy-profiling.log',
discard_first_request=True,
flush_at_shutdown=True,
path='/.deliverance/profile')
if memory_profile:
try:
from dozer import Dozer
except ImportError:
print('Error: you must manually install Dozer to use --memory-profile')
sys.exit(1)
app = Dozer(app)
if interactive_debugger:
from weberror.evalexception import EvalException
app = EvalException(app, debug=True)
else:
from weberror.errormiddleware import ErrorMiddleware
app = ErrorMiddleware(app, debug=debug)
if debug_headers:
from wsgifilter.proxyapp import DebugHeaders
app = DebugHeaders(app, show_body=debug_headers > 1)
if garbage_collect:
from deliverance.garbagecollect import GarbageCollectingMiddleware
app = GarbageCollectingMiddleware(app)
print('To see logging, visit %s/.deliverance/login' % settings.base_url)
print(' after login go to %s/?deliv_log' % settings.base_url)
if profile:
print('To see profiling information visit %s/.deliverance/profile' % settings.base_url)
serve(app, host=settings.host, port=settings.port)
def inner_run(self, *args, **options):
# Flag the server as active
from devserver import settings
import devserver
settings.DEVSERVER_ACTIVE = True
settings.DEBUG = True
from django.conf import settings
from django.utils import translation
shutdown_message = options.get('shutdown_message', '')
use_werkzeug = options.get('use_werkzeug', False)
quit_command = (sys.platform
== 'win32') and 'CTRL-BREAK' or 'CONTROL-C'
wsgi_app = options.get('wsgi_app', None)
if use_werkzeug:
try:
from werkzeug import run_simple, DebuggedApplication
except ImportError as e:
self.stderr.write(
"WARNING: Unable to initialize werkzeug: %s\n" % e)
use_werkzeug = False
else:
from django.views import debug
debug.technical_500_response = null_technical_500_response
self.stdout.write("Validating models...\n\n")
self.check(display_num_errors=True)
self.check_migrations()
self.stdout.write(
("Django version %(version)s, using settings %(settings)r\n"
"Running django-devserver %(devserver_version)s\n"
"%(server_model)s %(server_type)s server is running at http://%(addr)s:%(port)s/\n"
"Quit the server with %(quit_command)s.\n") % {
"server_type": use_werkzeug and 'werkzeug' or 'Django',
"server_model": options['use_forked'] and 'Forked'
or 'Threaded',
"version": self.get_version(),
"devserver_version": devserver.get_version(),
"settings": settings.SETTINGS_MODULE,
"addr": self._raw_ipv6 and '[%s]' % self.addr or self.addr,
"port": self.port,
"quit_command": quit_command,
})
# django.core.management.base forces the locale to en-us. We should
# set it up correctly for the first request (particularly important
# in the "--noreload" case).
translation.activate(settings.LANGUAGE_CODE)
app = self.get_handler(*args, **options)
if wsgi_app:
self.stdout.write("Using WSGI application %r\n" % wsgi_app)
if os.path.exists(os.path.abspath(wsgi_app)):
# load from file
app = imp.load_source('wsgi_app',
os.path.abspath(wsgi_app)).application
else:
try:
app = __import__(wsgi_app, {}, {},
['application']).application
except (ImportError, AttributeError):
raise
if options['use_forked']:
mixin = SocketServer.ForkingMixIn
else:
mixin = SocketServer.ThreadingMixIn
middleware = getattr(settings, 'DEVSERVER_WSGI_MIDDLEWARE', [])
for middleware in middleware:
module, class_name = middleware.rsplit('.', 1)
app = getattr(__import__(module, {}, {}, [class_name]),
class_name)(app)
if options['use_dozer']:
from dozer import Dozer
app = Dozer(app)
try:
if use_werkzeug:
run_simple(self.addr,
int(self.port),
DebuggedApplication(app, True),
use_reloader=False,
use_debugger=True)
else:
run(self.addr, int(self.port), app, mixin, ipv6=self.use_ipv6)
except wsgi_server_exc_cls as e:
# Use helpful error messages instead of ugly tracebacks.
ERRORS = {
errno.EACCES: "You don't have permission to access that port.",
errno.EADDRINUSE: "That port is already in use.",
errno.EADDRNOTAVAIL: "That IP address can't be assigned-to.",
}
if not isinstance(e, socket.error): # Django < 1.6
ERRORS[13] = ERRORS.pop(errno.EACCES)
ERRORS[98] = ERRORS.pop(errno.EADDRINUSE)
ERRORS[99] = ERRORS.pop(errno.EADDRNOTAVAIL)
try:
if not isinstance(e, socket.error): # Django < 1.6
error_text = ERRORS[e.args[0].args[0]]
else:
error_text = ERRORS[e.errno]
except (AttributeError, KeyError):
error_text = str(e)
sys.stderr.write(self.style.ERROR("Error: %s" % error_text) + '\n')
# Need to use an OS exit because sys.exit doesn't work in a thread
os._exit(1)
except KeyboardInterrupt:
if shutdown_message:
self.stdout.write("%s\n" % shutdown_message)
sys.exit(0)
# -*- coding: utf-8 -*- from __future__ import unicode_literals from dozer import Dozer from .wsgi import application application = Dozer(application)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option(
'--werkzeug',
action='store_true',
dest='use_werkzeug',
default=False,
help='Tells Django to use the Werkzeug interactive debugger.'),
make_option(
'--forked',
action='store_true',
dest='use_forked',
default=False,
help='Use forking instead of threading for multiple web requests.'
),
make_option('--dozer',
action='store_true',
dest='use_dozer',
default=False,
help='Enable the Dozer memory debugging middleware.'),
make_option(
'--wsgi-app',
dest='wsgi_app',
default=None,
help='Load the specified WSGI app as the server endpoint.'),
)
if any(map(lambda app: app in settings.INSTALLED_APPS, STATICFILES_APPS)):
option_list += make_option(
'--nostatic',
dest='use_static_files',
action='store_false',
default=True,
help=
'Tells Django to NOT automatically serve static files at STATIC_URL.'
),
help = "Starts a lightweight Web server for development which outputs additional debug information."
args = '[optional port number, or ipaddr:port]'
# Validation is called explicitly each time the server is reloaded.
requires_model_validation = False
def run_from_argv(self, argv):
parser = self.create_parser(argv[0], argv[1])
default_args = getattr(settings, 'DEVSERVER_ARGS', None)
if default_args:
options, args = parser.parse_args(default_args)
else:
options = None
options, args = parser.parse_args(argv[2:], options)
handle_default_options(options)
self.execute(*args, **options.__dict__)
def handle(self, addrport='', *args, **options):
if args:
raise CommandError('Usage is runserver %s' % self.args)
if not addrport:
addr = getattr(settings, 'DEVSERVER_DEFAULT_ADDR', '127.0.0.1')
port = getattr(settings, 'DEVSERVER_DEFAULT_PORT', '8000')
addrport = '%s:%s' % (addr, port)
return super(Command, self).handle(addrport=addrport, *args, **options)
def get_handler(self, *args, **options):
if int(options['verbosity']) < 1:
handler = WSGIHandler()
else:
handler = DevServerHandler()
# AdminMediaHandler is removed in Django 1.5
# Add it only when it avialable.
try:
from django.core.servers.basehttp import AdminMediaHandler
except ImportError:
pass
else:
handler = AdminMediaHandler(handler, options['admin_media_path'])
if 'django.contrib.staticfiles' in settings.INSTALLED_APPS and options[
'use_static_files']:
from django.contrib.staticfiles.handlers import StaticFilesHandler
handler = StaticFilesHandler(handler)
return handler
def inner_run(self, *args, **options):
# Flag the server as active
from devserver import settings
import devserver
settings.DEVSERVER_ACTIVE = True
settings.DEBUG = True
from django.conf import settings
from django.utils import translation
shutdown_message = options.get('shutdown_message', '')
use_werkzeug = options.get('use_werkzeug', False)
quit_command = (sys.platform
== 'win32') and 'CTRL-BREAK' or 'CONTROL-C'
wsgi_app = options.get('wsgi_app', None)
if use_werkzeug:
try:
from werkzeug import run_simple, DebuggedApplication
except ImportError, e:
self.stderr.write(
"WARNING: Unable to initialize werkzeug: %s\n" % e)
use_werkzeug = False
else:
from django.views import debug
debug.technical_500_response = null_technical_500_response
self.stdout.write("Validating models...\n\n")
self.validate(display_num_errors=True)
self.stdout.write(
("Django version %(version)s, using settings %(settings)r\n"
"Running django-devserver %(devserver_version)s\n"
"%(server_model)s %(server_type)s server is running at http://%(addr)s:%(port)s/\n"
"Quit the server with %(quit_command)s.\n") % {
"server_type": use_werkzeug and 'werkzeug' or 'Django',
"server_model": options['use_forked'] and 'Forked'
or 'Threaded',
"version": self.get_version(),
"devserver_version": devserver.get_version(),
"settings": settings.SETTINGS_MODULE,
"addr": self._raw_ipv6 and '[%s]' % self.addr or self.addr,
"port": self.port,
"quit_command": quit_command,
})
# django.core.management.base forces the locale to en-us. We should
# set it up correctly for the first request (particularly important
# in the "--noreload" case).
translation.activate(settings.LANGUAGE_CODE)
app = self.get_handler(*args, **options)
if wsgi_app:
self.stdout.write("Using WSGI application %r\n" % wsgi_app)
if os.path.exists(os.path.abspath(wsgi_app)):
# load from file
app = imp.load_source('wsgi_app',
os.path.abspath(wsgi_app)).application
else:
try:
app = __import__(wsgi_app, {}, {},
['application']).application
except (ImportError, AttributeError):
raise
if options['use_forked']:
mixin = SocketServer.ForkingMixIn
else:
mixin = SocketServer.ThreadingMixIn
middleware = getattr(settings, 'DEVSERVER_WSGI_MIDDLEWARE', [])
for middleware in middleware:
module, class_name = middleware.rsplit('.', 1)
app = getattr(__import__(module, {}, {}, [class_name]),
class_name)(app)
if options['use_dozer']:
from dozer import Dozer
app = Dozer(app)
try:
if use_werkzeug:
run_simple(self.addr,
int(self.port),
DebuggedApplication(app, True),
use_reloader=False,
use_debugger=True)
else:
run(self.addr,
int(self.port),
app,
mixin,
ipv6=options['use_ipv6'])
except WSGIServerException, e:
# Use helpful error messages instead of ugly tracebacks.
ERRORS = {
13: "You don't have permission to access that port.",
98: "That port is already in use.",
99: "That IP address can't be assigned-to.",
}
try:
error_text = ERRORS[e.args[0].args[0]]
except (AttributeError, KeyError):
error_text = str(e)
sys.stderr.write(self.style.ERROR("Error: %s" % error_text) + '\n')
# Need to use an OS exit because sys.exit doesn't work in a thread
os._exit(1)
def make_app_for_config_and_path(config, base):
if config.get_option('trunk_dir') and not config.get_option('user_dirs'):
print "--trunk-dir is only valid with --user-dirs"
sys.exit(1)
if config.get_option('reload'):
if Reloader.is_installed():
Reloader.install()
else:
return Reloader.restart_with_reloader()
if config.get_option('user_dirs'):
if not config.get_option('trunk_dir'):
print "You didn't specify a directory for the trunk directories."
sys.exit(1)
app = UserBranchesFromTransportRoot(base, config)
else:
app = BranchesFromTransportRoot(base, config)
setup_logging(config)
if config.get_option('profile'):
from loggerhead.middleware.profile import LSProfMiddleware
app = LSProfMiddleware(app)
if config.get_option('memory_profile'):
from dozer import Dozer
app = Dozer(app)
if not config.get_option('user_prefix'):
prefix = '/'
else:
prefix = config.get_option('user_prefix')
if not prefix.startswith('/'):
prefix = '/' + prefix
try:
from paste.deploy.config import PrefixMiddleware
except ImportError:
cant_proxy_correctly_message = (
'Unsupported configuration: PasteDeploy not available, but '
'loggerhead appears to be behind a proxy.')
def check_not_proxied(app):
def wrapped(environ, start_response):
if 'HTTP_X_FORWARDED_SERVER' in environ:
exc = HTTPInternalServerError()
exc.explanation = cant_proxy_correctly_message
raise exc
return app(environ, start_response)
return wrapped
app = check_not_proxied(app)
else:
app = PrefixMiddleware(app, prefix=prefix)
app = HTTPExceptionHandler(app)
app = ErrorHandlerApp(app)
app = TransLogger(app, logger=logging.getLogger('loggerhead'))
return app
def configure(self,
cors=False,
sqlalchemy=False,
marshmallow=False,
redis=False,
celery=False,
gevent=False):
self.config['TESTING'] = (os.environ.get('FLASK_TESTING', False)
or sys.argv[0].endswith('pytest')
or self.config.get('TESTING', False))
self.config.resolve_lazy_values()
if self.testing:
for key, value in self.config.items():
if key.startswith('TESTING_'):
self.config[key[8:]] = value
register_test_helpers(self)
if self.config.get('LOGGING'):
# Turn off werkzeug default handlers not to duplicate logs
self.logger.info('Configuring logging')
print(logging.getLogger('werkzeug').handlers)
logging.getLogger('werkzeug').handlers = []
logging.config.dictConfig(self.config['LOGGING'])
self.logger.info('Logger configured')
if self.config.get('FLASK_SHELL_CONTEXT'):
register_shell_context(self, *self.config['FLASK_SHELL_CONTEXT'])
if self.config.get('DOZER'):
from dozer import Dozer
self.wsgi_app = Dozer(self.wsgi_app)
if self.config.get('DOZER_PROFILER'):
from dozer import Profiler
self.wsgi_app = Profiler(self.wsgi_app)
if cors:
# options parsed from config
# see https://github.com/corydolphin/flask-cors/blob/master/flask_cors/core.py
from flask_cors import CORS
CORS(self)
if sqlalchemy:
from .sqla import SQLAlchemy
db = SQLAlchemy(self)
try:
from flask_migrate import Migrate
except ImportError:
pass
else:
Migrate(self, db, compare_type=True)
if marshmallow:
from flask_marshmallow import Marshmallow
Marshmallow(self)
if redis:
from .redis import create_redis
create_redis(self)
if celery:
from .celery import create_celery
create_celery(self, task_views=True)
if gevent:
from flask_gevent import Gevent
Gevent(self)
def inner_run():
# Flag the server as active
from devserver import settings
import devserver
settings.DEVSERVER_ACTIVE = True
settings.DEBUG = True
from django.conf import settings
from django.utils import translation
print "Validating models..."
self.validate(display_num_errors=True)
print "\nDjango version %s, using settings %r" % (
django.get_version(), settings.SETTINGS_MODULE)
print "Running django-devserver %s" % (devserver.get_version(), )
if use_werkzeug:
server_type = 'werkzeug'
else:
server_type = 'django'
print "%s %s server is running at http://%s:%s/" % (
options['use_forked'] and 'Forked'
or 'Threaded', server_type, addr, port)
print "Quit the server with %s." % quit_command
# django.core.management.base forces the locale to en-us. We should
# set it up correctly for the first request (particularly important
# in the "--noreload" case).
translation.activate(settings.LANGUAGE_CODE)
if int(options['verbosity']) < 1:
app = WSGIHandler()
else:
app = DevServerHandler()
if wsgi_app:
try:
app = __import__(wsgi_app, {}, {},
['application']).application
except (ImportError, AttributeError):
raise
if options['use_forked']:
mixin = SocketServer.ForkingMixIn
else:
mixin = SocketServer.ThreadingMixIn
middleware = getattr(settings, 'DEVSERVER_WSGI_MIDDLEWARE', [])
for middleware in middleware:
module, class_name = middleware.rsplit('.', 1)
app = getattr(__import__(module, {}, {}, [class_name]),
class_name)(app)
if 'django.contrib.staticfiles' in settings.INSTALLED_APPS and use_static_files:
from django.contrib.staticfiles.handlers import StaticFilesHandler
app = StaticFilesHandler(app)
else:
app = AdminMediaHandler(app, admin_media_path)
if options['use_dozer']:
from dozer import Dozer
app = Dozer(app)
try:
if use_werkzeug:
run_simple(addr,
int(port),
DebuggedApplication(app, True),
use_reloader=False,
use_debugger=True)
else:
run(addr, int(port), app, mixin)
except WSGIServerException, e:
# Use helpful error messages instead of ugly tracebacks.
ERRORS = {
13: "You don't have permission to access that port.",
98: "That port is already in use.",
99: "That IP address can't be assigned-to.",
}
try:
error_text = ERRORS[e.args[0].args[0]]
except (AttributeError, KeyError):
error_text = str(e)
sys.stderr.write(
self.style.ERROR("Error: %s" % error_text) + '\n')
# Need to use an OS exit because sys.exit doesn't work in a thread
os._exit(1)
def create_app():
# get current configuration, or default to 'production' for safety
config_name = os.environ.get('FLASK_ENV') or 'production'
# load configuration files from 'instance' folder
instance_dir = (Path(__file__).parent / "instance").absolute()
app = Flask(__name__, instance_relative_config=True, instance_path=str(instance_dir))
app.config.from_object(app_config[config_name])
app.config.from_pyfile('secrets.py')
app.config.from_pyfile('mail.py')
app.config.from_pyfile('rollbar.py')
# app.config.from_pyfile('scout.py')
app.config.from_pyfile('local.py')
# create a long-lived Redis connection
app.config['REDIS_SESSION'] = redis.Redis.from_url(url=app.config['CACHE_REDIS_URL'])
# create long-lived Mongo connection for Flask-Sessionstore
app.config['SESSION_MONGODB'] = MongoClient(host=app.config['SESSION_MONGO_URL'])
# we have two proxies -- we're behind both waitress and nginx
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=2)
if app.config.get('PROFILE_MEMORY', False):
app.wsgi_app = Dozer(app.wsgi_app)
db.init_app(app)
migrate = Migrate(app, db)
bootstrap = Bootstrap(app)
mail = Mail(app)
bleach = Bleach(app)
md = Markdown(app, extensions=[makeExtension(configs={'entities': 'named'})])
rb = Rollbar(app)
qr = QRcode(app)
bbl = Babel(app)
session_store = Session(app)
cache.init_app(app)
# add endpoint profiler and rate limiter in production mode
# also add handler to direct Waitress logging output to the console
if config_name == 'production':
profiler = Profiler(app)
# set up Flask-Limiter
limiter.init_app(app)
# add debug toolbar if in debug mode
if config_name == 'development':
toolbar = DebugToolbarExtension(app)
# api_toolbar = DebugAPIExtension(app)
# panels = list(app.config['DEBUG_TB_PANELS'])
# panels.append('flask_debug_api.BrowseAPIPanel')
# panels.append('flask_debugtoolbar_lineprofilerpanel.panels.LineProfilerPanel')
# app.config['DEBUG_TB_PANELS'] = panels
# set up CSS and javascript assets
env = Environment(app)
if not app.debug:
from logging import INFO, Formatter, basicConfig
from logging.handlers import RotatingFileHandler
basicConfig(level=INFO)
file_handler = RotatingFileHandler(app.config['LOG_FILE'], 'a', 1 * 1024 * 1024, 10)
file_handler.setFormatter(Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
app.logger.setLevel(INFO)
file_handler.setLevel(INFO)
app.logger.addHandler(file_handler)
app.logger.info('MPS Project Manager starting')
# use Werkzeug built-in profiler if profile-to-disk is enabled
if app.config.get('PROFILE_TO_DISK', False):
app.config['PROFILE'] = True
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=app.config.get('PROFILE_DIRECTORY'))
app.logger.info('Profiling to disk enabled')
# configure behaviour for uploaded files
asset_folder = Path(app.config.get('ASSETS_FOLDER'))
uploaded_subfolder = Path(app.config.get('ASSETS_UPLOADED_SUBFOLDER'))
submitted_subfolder = Path(app.config.get('ASSETS_SUBMITTED_SUBFOLDER'))
abs_uploaded_path = asset_folder / uploaded_subfolder
abs_uploaded_path.mkdir(parents=True, exist_ok=True)
abs_submissions_path = asset_folder / submitted_subfolder
abs_submissions_path.mkdir(parents=True, exist_ok=True)
app.config['UPLOADED_SOLUTIONS_DEST'] = abs_uploaded_path
app.config['UPLOADED_BATCHUSERLIST_DEST'] = abs_uploaded_path
app.config['UPLOADED_SUBMISSIONS_DEST'] = abs_submissions_path
configure_uploads(app, [solution_files, batch_user_files, submitted_files])
# configure Flask-Security, which needs access to the database models for User and Role
from app import models
user_datastore = SQLAlchemyUserDatastore(db, models.User, models.Role)
# patch Flask-Security's login form to include some descriptive text on the email field
security = Security(app, user_datastore, login_form=PatchedLoginForm, mail_util_cls=PatchedMailUtil)
if config_name == 'production':
# set up more stringent limits for login view and forgot-password view
# add to a particular view function.
login = app.view_functions['security.login']
forgot = app.view_functions['security.forgot_password']
limiter.limit("50/day;5/minute")(login)
limiter.limit("50/day;5/minute")(forgot)
# set up celery and store in extensions dictionary
celery = make_celery(app)
app.extensions['celery'] = celery
# register celery tasks
# there doesn't seem a good way of doing this using factory functions! Here I compromise by passing the
# celery application instance to a collection of register_*() functions, which use an @celery decorator
# to register callables. Then we write the callable into the app, in the 'tasks' dictionary
app.tasks = {}
tasks.register_send_log_email(celery, mail)
tasks.register_utility_tasks(celery)
tasks.register_prune_email(celery)
tasks.register_backup_tasks(celery)
tasks.register_rollover_tasks(celery)
tasks.register_issue_confirm_tasks(celery)
tasks.register_golive_tasks(celery)
tasks.register_close_selection_tasks(celery)
tasks.register_user_launch_tasks(celery)
tasks.register_popularity_tasks(celery)
tasks.register_matching_tasks(celery)
tasks.register_matching_email_tasks(celery)
tasks.register_availability_tasks(celery)
tasks.register_scheduling_tasks(celery)
tasks.register_maintenance_tasks(celery)
tasks.register_assessment_tasks(celery)
tasks.register_assessor_tasks(celery)
tasks.register_email_notification_tasks(celery)
tasks.register_precompute_tasks(celery)
tasks.register_push_feedback_tasks(celery)
tasks.register_system_tasks(celery)
tasks.register_batch_create_tasks(celery)
tasks.register_selecting_tasks(celery)
tasks.register_session_tasks(celery)
tasks.register_marking_tasks(celery)
tasks.register_services_tasks(celery)
tasks.register_test_tasks(celery)
@security.login_context_processor
def login_context_processor():
# build list of system messages to consider displaying on login screen
messages = []
for message in MessageOfTheDay.query.filter_by(show_login=True).all():
if message.project_classes.first() is None:
messages.append(message)
return dict(messages=messages)
@app.before_request
def before_request_handler():
if current_user.is_authenticated:
if request.endpoint is not None and 'ajax' not in request.endpoint:
try:
Notification.query.filter_by(remove_on_pageload=True).delete()
db.session.commit()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception", exc_info=e)
@app.template_filter('dealingwithdollars')
def dealingwithdollars(latex_string):
if latex_string is None:
return r'<div class="alert alert-danger">An empty string was supplied. ' \
r'Please check your project description.</div>'
splat = list(latex_string) # Splits string into list of characters
dollar_inds = [i for i in range(0, len(splat)) if splat[i] == "$"] # Finds indices of all dollar signs
display_inds = [] # Less pythonic than list comprehension, but now inline_inds can exclude double dollar signs
for elem in dollar_inds:
if elem != len(splat) - 1:
if splat[elem + 1] == r"$":
display_inds.append(elem)
display_inds.append(elem + 1)
inline_inds = [elem for elem in dollar_inds if splat[elem - 1] != "\\" and elem not in display_inds] # \$ is allowed in LaTeX, $ is not.
just_dollar = [elem for elem in dollar_inds if elem not in inline_inds and elem not in display_inds]
if len(inline_inds) % 2 != 0: # Checks for lonely dollar signs
latex_string = r'<div class="alert alert-danger">Failed to match LaTeX dollar delimiters. ' \
r'Please check the markup in your project description.</div>' + latex_string
else: # Only converts inline math delimiters, as latex2markdown seems to convert display math delimiters
for i in range(0, len(inline_inds)):
if i % 2 == 0:
splat[inline_inds[i]] = r"\\("
else:
splat[inline_inds[i]] = r"\\)"
for elem in just_dollar:
splat.pop(elem - 1)
latex_string = ''.join(splat)
l2m_obj = latex2markdown.LaTeX2Markdown(latex_string)
mathjax_string = l2m_obj.to_markdown()
return mathjax_string
@app.template_filter('urlencode')
def urlencode_filter(s):
if s is None:
return None
s = s.encode('utf8')
s = parse.quote_plus(s)
return bleach.clean(s)
def _get_previous_login():
if not has_request_context():
return None
if session.get('previous_login', None) is not None:
real_id = session['previous_login']
real_user = db.session.query(User).filter_by(id=real_id).first()
else:
real_user = None
return real_user
def _get_live_platform():
if not has_request_context():
return None
return current_app.config.get('EMAIL_IS_LIVE', False)
# collect all global context functions together in an attempt to avoid function call overhead
@app.context_processor
def global_context():
if not has_request_context():
return {}
return {'get_previous_login': _get_previous_login,
'website_revision': site_revision,
'website_copyright_dates': site_copyright_dates,
'build_version': git_tag,
'home_dashboard_url': home_dashboard_url(),
'get_base_context': get_global_context_data,
'get_live_platform': _get_live_platform}
@app.errorhandler(404)
def not_found_error(error):
return render_template('errors/404.html'), 404
@app.errorhandler(429)
def rate_limit_error(error):
return render_template('errors/429.html'), 429
@app.errorhandler(500)
def internal_error(error):
db.session.rollback()
return render_template('errors/500.html'), 500
if not app.debug:
@app.after_request
def after_request(response):
timeout = app.config['DATABASE_QUERY_TIMEOUT']
for query in get_debug_queries():
if query.duration >= timeout:
app.logger.warning("SLOW QUERY: %s\nParameters: %s\nDuration: %fs\nContext: %s\n" % (
query.statement, query.parameters, query.duration, query.context))
return response
@user_logged_in.connect_via(app)
def login_callback(self, user):
# DS 3 Feb 2019 - why did we want to clear notifications?
# disabled this for now
# # clear notifications for the user who has just logged in
# Notification.query.filter_by(user_id=user.id).delete()
# force precompute of expensive views
celery = current_app.extensions['celery']
precompute_at_login(user, celery, autocommit=False)
user.last_active = datetime.now()
db.session.commit()
from flask import Request
class CustomRequest(Request):
@property
def rollbar_person(self):
db.session.rollback()
if current_user is None:
return None
# 'id' is required, 'username' and 'email' are indexed but optional.
# all values are strings.
return {'id': str(current_user.id),
'username': str(current_user.username),
'email': str(current_user.email)}
app.request_class = CustomRequest
# IMPORT BLUEPRINTS
from .home import home as home_blueprint
app.register_blueprint(home_blueprint, url_prefix='/')
from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth')
from .admin import admin as admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin')
from .faculty import faculty as faculty_blueprint
app.register_blueprint(faculty_blueprint, url_prefix='/faculty')
from .convenor import convenor as convenor_blueprint
app.register_blueprint(convenor_blueprint, url_prefix='/convenor')
from .student import student as student_blueprint
app.register_blueprint(student_blueprint, url_prefix='/student')
from .office import office as office_blueprint
app.register_blueprint(office_blueprint, url_prefix='/office')
from .reports import reports as reports_blueprint
app.register_blueprint(reports_blueprint, url_prefix='/reports')
from .user_approver import user_approver as user_approver_blueprint
app.register_blueprint(user_approver_blueprint, url_prefix='/user_approver')
from .project_approver import project_approver as project_approver_blueprint
app.register_blueprint(project_approver_blueprint, url_prefix='/project_approver')
from .loadbalancer import alb as alb_blueprint
app.register_blueprint(alb_blueprint, url_prefix='/alb')
from .manage_users import manage_users as manage_users_blueprint
app.register_blueprint(manage_users_blueprint, url_prefix='/manage_users')
from .documents import documents as documents_blueprint
app.register_blueprint(documents_blueprint, url_prefix='/documents')
from .services import services as services_blueprint
app.register_blueprint(services_blueprint, url_prefix='/services')
from .projecthub import projecthub as projecthub_blueprint
app.register_blueprint(projecthub_blueprint, url_prefix='/projecthub')
return app, celery
def superproxy(config, listen, pool_size, stop_timeout, dozer):
# TODO: clean this chaotic configuration, obviously we should not configure
# from strings/dicts inside classes, it will make things more obvious
import sys
import signal
import gevent
from gevent.pywsgi import WSGIServer
from gevent.pool import Pool
conf = config.get('superproxy', {})
superproxy_cls = import_string(conf.pop('cls', 'proxytools.superproxy.WSGISuperProxy'))
if conf.get('verify', None) is False:
# Assuming you know what you're doing
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fetcher = config.get('proxyfetcher')
checker = config.get('proxychecker')
proxylist_cls = import_string(conf.get('proxylist', {})
.pop('cls', 'proxytools.proxylist.ProxyList'))
proxylist = proxylist_cls(fetcher=fetcher, checker=checker, **conf.pop('proxylist', {}))
listen = listen or conf.pop('listen', '0.0.0.0:8088')
pool_size = pool_size or conf.pop('pool_size', 500)
stop_timeout = stop_timeout or conf.pop('stop_timeout', 5)
dozer = conf.pop('dozer', False)
http_debug = conf.pop('http_debug', False)
app = superproxy_cls(proxylist, **conf)
if dozer:
from dozer import Dozer
app = Dozer(app)
if http_debug:
import http.client
http.client.HTTPConnection.debuglevel = 10 if http_debug is True else http_debug
iface, port = listen.split(':')
server = WSGIServer((iface, int(port)), app, spawn=Pool(pool_size))
server.stop_timeout = stop_timeout
logger = logging.getLogger('proxytools.superproxy')
def stop(*args):
if server.closed:
try:
logger.error('Server stopping - multiple exit signals received - aborting.')
finally:
sys.exit('Multiple exit signals received - aborting.')
logger.info('Server stopping %s', args)
server.stop()
[gevent.signal(sig, stop) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT)]
logger.info('Server started')
server.serve_forever()
logger.debug('Server stopped')
