def check_uwsgi(worker_callback=None, atexit=None):
# type: (Optional[Callable], Optional[Callable]) -> None
"""Check whetever uwsgi is running and what needs to be done.
:param worker_callback: Callback function to call in uWSGI worker processes.
"""
try:
import uwsgi # type: ignore[import]
except ImportError:
return
if not uwsgi.opt.get("enable-threads"):
raise uWSGIConfigError("enable-threads option must be set to true")
# If uwsgi has more than one process, it is running in prefork operational mode: uwsgi is going to fork multiple
# sub-processes.
# If lazy-app is enabled, then the app is loaded in each subprocess independently. This is fine.
# If it's not enabled, then the app will be loaded in the master process, and uwsgi will `fork()` abruptly,
# bypassing Python sanity checks. We need to handle this case properly.
# The proper way to handle that is to allow to register a callback function to run in the subprocess at their
# startup, and warn the caller that this is the master process and that (probably) nothing should be done.
if uwsgi.numproc > 1 and not uwsgi.opt.get(
"lazy-apps") and uwsgi.worker_id() == 0:
if not uwsgi.opt.get("master"):
# Having multiple workers without the master process is not supported:
# the postfork hooks are not available, so there's no way to start a different profiler in each
# worker
raise uWSGIConfigError(
"master option must be enabled when multiple processes are used"
)
# Register the function to be called in child process at startup
if worker_callback is not None:
try:
import uwsgidecorators # type: ignore[import]
except ImportError:
raise uWSGIConfigError(
"Running under uwsgi but uwsgidecorators cannot be imported"
)
uwsgidecorators.postfork(worker_callback)
if atexit is not None:
original_atexit = getattr(uwsgi, "atexit", None)
def _atexit():
try:
atexit()
except Exception:
pass
if original_atexit is not None:
original_atexit()
uwsgi.atexit = _atexit
raise uWSGIMasterProcess()
def init(uwsgi_borker):
UwsgiPostFork._uwsgi_broker = uwsgi_borker
UwsgiPostFork._configure_op = getattr(uwsgi_borker.restful_comp(),
ComponentConstants.CONFIGURE_CALLBACK_FUNC_NAME, None)
UwsgiPostFork._postfork_op = getattr(uwsgi_borker.restful_comp(),
ComponentConstants.POST_FORK_CALLBACK_FUNC_NAME, None)
UwsgiPostFork._verbose = uwsgi_borker.w_logger.isEnabledFor(logging.DEBUG)
uwsgidecorators.postfork(UwsgiPostFork.do_post_fork)
def create_app(
test_config: typing.Optional[typing.Mapping[str, typing.Any]] = None,
) -> Flask:
"""Flask app factory."""
app = Flask(__name__)
if test_config is None:
from src.common.setup import config
app.config.from_object(config.Config)
else:
# load the test config if passed in
app.config.from_object(test_config)
from src.common.setup import log, sentry
log.setup(app)
sentry.connect(app)
@app.errorhandler(Exception)
def handle_bad_request(e: Exception) -> typing.Any:
"""Catch-all exception handler."""
# Create an id for the exception to make searching the logs
# easier.
e_id = id(e)
setattr(e, "log_id", e_id)
app.logger.exception(f"Unhandled exception ({e_id}): {e}")
# All werkzeug raised exceptions (404 etc) have response code
# assigned to them.
code = getattr(e, "code", 500)
return {
"error": {
"message":
str(e) if code < 500 else f"{code}: Unexpected error",
"id": e_id,
}
}, code
def setup_worker() -> None:
# Connect infra
from src.common.setup import db, storage
db.connect(app)
storage.connect(app)
api.register_rest_routes(app)
api.register_graphql_resolvers(app)
if os.environ.get("FLASK_ENV") == "development":
setup_worker()
else:
import uwsgidecorators
uwsgidecorators.postfork(setup_worker)
return app
def create_app(
test_config: typing.Optional[typing.Mapping[str, typing.Any]] = None,
) -> Flask:
"""Flask app factory.
"""
app = Flask(__name__)
if test_config is None:
from . import config
app.config.from_object(config.Config)
else:
# load the test config if passed in
app.config.from_object(test_config)
from . import log, sentry
log.setup(app)
sentry.connect(app)
@app.errorhandler(Exception)
def handle_bad_request(e: Exception) -> typing.Any:
"""Catch-all exception handler.
"""
# Create an id for the exception to make searching the logs
# easier.
e_id = id(e)
setattr(e, 'log_id', e_id)
app.logger.exception(f'Unhandled exception ({e_id}): {e}')
# All werkzeug raised exceptions (404 etc) have response code
# assigned to them.
code = getattr(e, 'code', 500)
return {'error': {'message': str(e), 'id': e_id}}, code
def setup_worker() -> None:
# Connect infra
from . import db, storage
db.connect(app)
storage.connect(app)
# Register routes
from . import posts, routes
app.register_blueprint(routes.bp)
app.register_blueprint(posts.routes.bp)
try:
# If running behind uwsgi, init db as post fork op to avoid workers
# sharing the connection pool.
import uwsgidecorators
except ImportError:
setup_worker()
else:
uwsgidecorators.postfork(setup_worker)
return app
def run_helper(self):
self._conf, args = self._parse_config()
if args.prepare_storage:
# Logging should be the first plugin (outermost wrapper)
self._configure_logging(self._conf)
qvarn.log.set_context('prepare-storage')
self._connect_to_storage(self._conf)
self._prepare_storage(self._conf)
else:
# Logging should be the first plugin (outermost wrapper)
self._configure_logging(self._conf)
qvarn.log.set_context('setup')
self._install_logging_plugin()
# Error catching should also be as high as possible to catch all
self._app.install(qvarn.ErrorTransformPlugin())
self._setup_auth(self._conf)
self._app.install(qvarn.StringToUnicodePlugin())
# Import is here to not fail tests and is only used on uWSGI
import uwsgidecorators
uwsgidecorators.postfork(self._uwsgi_postfork_setup)
def run_helper(self, specdir=None, argv=None, uwsgi_postfork_setup=True):
self._conf, args = get_configuration(argv)
if args.prepare_storage:
specdir = specdir or self._conf.get('main', 'specdir')
# Logging should be the first plugin (outermost wrapper)
self._configure_logging(self._conf)
qvarn.log.set_context('prepare-storage')
self._connect_to_storage(self._conf, specdir, prepare_storage=True)
else:
# Logging should be the first plugin (outermost wrapper)
self._configure_logging(self._conf)
qvarn.log.set_context('setup')
self._install_logging_plugin(self._conf)
# Error catching should also be as high as possible to catch all
self._app.install(qvarn.ErrorTransformPlugin())
self._setup_auth_token_endpoint(self._conf)
self._setup_auth(self._conf)
if uwsgi_postfork_setup:
# Import is here to not fail tests and is only used on uWSGI
import uwsgidecorators
uwsgidecorators.postfork(self._uwsgi_postfork_setup)
def run_helper(self, specdir): # pragma: no cover
self._conf, args = self._parse_config()
if args.prepare_storage:
# Logging should be the first plugin (outermost wrapper)
self._configure_logging(self._conf)
qvarn.log.set_context('prepare-storage')
self._connect_to_storage(self._conf)
specs_and_texts = self._load_specs_from_files(specdir)
self._store_resource_types(specs_and_texts)
specs = self._load_specs_from_db()
self._add_resource_types_from_specs(s for s in specs)
self._prepare_storage(self._conf)
else:
# Logging should be the first plugin (outermost wrapper)
self._configure_logging(self._conf)
qvarn.log.set_context('setup')
self._install_logging_plugin()
# Error catching should also be as high as possible to catch all
self._app.install(qvarn.ErrorTransformPlugin())
self._setup_auth(self._conf)
# Import is here to not fail tests and is only used on uWSGI
import uwsgidecorators
uwsgidecorators.postfork(self._uwsgi_postfork_setup)
ttl=config.getint("security", "token_ttl_secs"),
token_recreate_before=config.getint("security", "token_recreate_before_secs"),
id_field="user_uid",
add_cookie=True,
cookie_secret=cookie_sign_secret,
auth_redirect_rule=(lambda: True if bottle.request.params.get("client_type") == "browser" else False),
auth_redirect_to="/cabinet",
soft_authentication_keyword="is_authenticated",
on_jwt_exception=on_jwt_auth_failed,
on_auth_redirect=on_auth_redirect
)
application.install(provider_plugin)
application.jwt_plugin = provider_plugin
application.cookie_secret = cookie_sign_secret
from reqlog.dbschema import * # noqa
from reqlog.api import * # noqa
from reqlog.pages import * # noqa
try:
import uwsgidecorators
uwsgidecorators.postfork(setup_app)
except ImportError:
logger.info("NOT RUNNING ON UWSGI")
import session.views # NOQA
import gevent
from session.regions import run_region_reloader
gevent.spawn(run_region_reloader)
try:
from uwsgidecorators import postfork
except ImportError:
import gevent.monkey
gevent.monkey.patch_all()
init()
else:
init = postfork(init)
from .app import application
if __name__ == '__main__':
import settings
import gevent_profiler
import signal
from gevent.pywsgi import WSGIServer
gevent_profiler.attach_on_signal(signum=signal.SIGUSR1, duration=30)
gevent_profiler.set_stats_output('stats.txt')
gevent_profiler.set_summary_output('summary.txt')
gevent_profiler.set_trace_output('trace.txt')
logger.info('listening %s:%d', settings.SESSION['host'],
# application = HelloWorldApplication(application)
# from http://projects.unbit.it/uwsgi/wiki/TipsAndTricks
# AUTHOR: Simone Federici
try:
# uwsgi module is only available when running from uwsgi
import uwsgi
except ImportError:
# we're probably running from django's built-in server
pass
else:
from uwsgidecorators import postfork, thread, timer
from django.utils import autoreload
# autodiscover SSH handlers
import okupy.accounts.ssh # noqa
from okupy.common.ssh import ssh_main
import Crypto.Random
postfork(thread(ssh_main))
@postfork
def reset_rng():
Crypto.Random.atfork()
@timer(5)
def change_code_gracefull_reload(sig):
if autoreload.code_changed():
uwsgi.reload()
def init_worker(self):
self.queue = Queue(self.queue_size)
from uwsgidecorators import postfork, thread
self._post = postfork(thread(self._post))
try:
import uwsgi # noqa
except ImportError:
if os.getenv('CELERY_CONTEXT'):
log.info('Celery context')
from celery.signals import worker_process_init
worker_process_init.connect(mongo_connect)
else:
log.debug('Not in uwsgi/celery context')
mongo_connect()
else:
log.info('Uwsgi context')
from uwsgidecorators import postfork
mongo_connect = postfork(mongo_connect)
def main(global_config, **settings):
"""This function returns a Pyramid WSGI application."""
import mist.api.auth.middleware
settings = {}
configurator = Configurator(root_factory=Root, settings=settings)
# Add custom adapter to the JSON renderer to avoid serialization errors
json_renderer = JSON()
def string_adapter(obj, request):
bottle_application.catchall = False
config = init_config()
logging_level = logging.DEBUG
logger.setLevel(logging_level)
logger.debug("config contents: {}".format(config))
bottle_application.install(SessionIdPlugin())
repository = RedisRepository(logger, config)
shortener = Shortener(logger, config, repository)
controller_list = [
controllers.ErrorController(),
controllers.ShortenUrlController(logger, config, shortener),
controllers.RedirectController(logger, config, repository),
]
for controller in controller_list:
for route in controller.get_routes():
logger.debug("initializing route: {}".format(route))
bottle_application.route(**route)
try:
import uwsgidecorators
uwsgidecorators.postfork(setup_application)
except ImportError:
logger.info("NOT RUNNING ON UWSGI")
def gpfork(fn):
postfork(gevent.Greenlet(fn).start)
