def post_fork(server, worker):
"""
This is necessary to patch
See: https://github.com/jneight/django-db-geventpool#patch-psycopg2
"""
patch_psycopg()
worker.log.info("django-db-geventpool: make psycopg2 green")
def patch(self):
"""Override `patch` so that we can patch some extra libraries.
Gevent patches base libraries that perform I/O for us already (e.g. `os` and `sockets). We need to patch any
additional libraries that perform I/O outside of the base libraries in order to make them cooperative.
See https://docs.gunicorn.org/en/stable/design.html#async-workers and
https://docs.gunicorn.org/en/stable/design.html#async-workers.
For a longer discussion of why this is important, see description in #3850.
"""
# pylint: disable=import-outside-toplevel
super().patch()
# patch grpc
from grpc.experimental import gevent as grpc_gevent
grpc_gevent.init_gevent()
self.log.info("patched grpc")
# patch psycopg2
from psycogreen.gevent import patch_psycopg
patch_psycopg()
self.log.info("patched psycopg2")
def post_fork(server, worker):
try:
from psycogreen.gevent import patch_psycopg
patch_psycopg()
worker.log.info("Made Psycopg2 Green")
except ImportError:
worker.log.info("Psycopg2 not patched")
def post_fork(server, worker):
"""
Called just after a worker has been forked.
Enables async processing in Psycopg2 if GUNICORN_ENABLE_ASYNC_PSYCOPG2 is set.
"""
patch_psycopg()
worker.log.info("Enabled async Psycopg2")
def greenify():
"""Patch threading and psycopg2 modules for green threads."""
from gevent.monkey import patch_all, saved
if ('threading' in sys.modules) and ('threading' not in saved):
raise Exception('threading module loaded before patching!')
patch_all()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
def post_fork(server, worker):
"""
Called just after a worker has been forked.
Enables async processing in Psycopg2 if GUNICORN_ENABLE_ASYNC_PSYCOPG2 is set.
"""
if worker_class == 'gevent' and _enable_async_psycopg2:
patch_psycopg()
worker.log.info('Enabled async Psycopg2')
def greenify():
"""Patch threading and psycopg2 modules for green threads."""
from gevent.monkey import patch_all, saved
if ('threading' in sys.modules) and ('threading' not in saved):
raise Exception('threading module loaded before patching!')
patch_all()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
def patch_psycopg():
"""
Patch postgresql to be friendly with gevent
"""
try:
from psycogreen.gevent import patch_psycopg
logger.info('Patching psycopg for gevent')
patch_psycopg()
except ImportError:
pass
def postfork(server, worker):
# Make sure any lazily initialized SQLAlchemy state, notably the
# `returns_unicode_strings` flag on the engine's `Dialect`, is initialized
# before we start accepting requests. Otherwise, some requests could start
# making queries before the initialization was complete, and those queries
# would observe the incorrect, uninitialized values for certain
# engine/dialect state variables, leading to errors.
angular_momentum.db.get_engine().connect().close()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
def configure_workers(sender=None, conf=None, **kwargs):
def patch_psycopg():
"""
Patch postgresql to be friendly with gevent
"""
try:
from psycogreen.gevent import patch_psycopg
logger.info('Patching psycopg for gevent')
patch_psycopg()
except ImportError:
pass
patch_psycopg()
def create_app(testing=False):
if ENABLE_SUBSCRIPTIONS:
from gevent.monkey import patch_all
patch_all()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
app = Flask('voyage')
app.wsgi_app = ProxyFix(app.wsgi_app)
app.secret_key = os.environ.get('FLASK_SECRET', 's3cr3t')
app.config.update({
'SQLALCHEMY_DATABASE_URI':
os.environ['SQLALCHEMY_DATABASE_URI'],
'SQLALCHEMY_TRACK_MODIFICATIONS':
False,
})
from voyage.schema import schema
app.add_url_rule('/',
view_func=login_required(
GraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True,
allow_subscriptions=ENABLE_SUBSCRIPTIONS,
)))
@app.route('/graphiql')
@login_required
def graphiql_view():
from voyage.graphiql import render_graphiql
return make_response(
render_graphiql(enable_subscriptions=ENABLE_SUBSCRIPTIONS))
if not testing:
if os.environ.get('LOCAL_AUTH', False):
# Just set up local auth
from voyage.auth import local_auth
local_auth(app)
else:
from voyage.auth import google_auth
google_auth(app)
from voyage.extensions import configure_extensions
configure_extensions(app)
return app
def _patch_gevent_if_required(args, gevent_commands):
if len(args) <= 1:
return
for gevent_command in gevent_commands:
should_patch = args[1] == gevent_command.command
if gevent_command.contains:
should_patch = should_patch and gevent_command.contains in ' '.join(
args)
if should_patch:
monkey.patch_all(subprocess=True)
patch_psycopg()
if gevent_command.http_adapter_pool_size:
# requests must be imported after `patch_all()` is called
import requests
requests.adapters.DEFAULT_POOLSIZE = gevent_command.http_adapter_pool_size
break
def main():
args = parser.parse_args()
if not args.debug:
patch_all()
patch_psycopg()
from word_way.app import create_app
wsgi_app = create_app(args.config)
if args.debug:
wsgi_app.run(host=args.host, port=args.port, debug=True)
else:
http_server = WSGIServer((args.host, args.port), wsgi_app)
try:
http_server.serve_forever()
except KeyboardInterrupt:
raise SystemExit
def handle(self, addrport='', *args, **options):
if not addrport:
self.addr = ''
self.port = DEFAULT_PORT
else:
m = match(naiveip_re, addrport)
if m is None:
raise CommandError('"%s" is not a valid port number '
'or address:port pair.' % addrport)
self.addr, _, _, _, self.port = m.groups()
environ['DJANGO_SOCKETIO_PORT'] = str(self.port)
if options.get('use_psyco'):
try:
from psycogreen.gevent import patch_psycopg
except ImportError:
raise CommandError(
'Could not patch psycopg. '
'Is psycogreen installed?')
patch_psycopg()
if options.get('use_reloader'):
start_new_thread(reload_watcher, ())
try:
bind = (self.addr, int(self.port))
print 'SocketIOServer running on %s:%s\n\n' % bind
handler = self.get_handler(*args, **options)
server = SocketIOServer(
bind, handler, resource='socket.io', policy_server=True)
server.serve_forever()
except KeyboardInterrupt:
socks = []
for key, sock in six.iteritems(server.sockets):
socks.append(sock)
for sock in socks:
sock.kill(detach=True)
server.stop()
if RELOAD:
print 'Reloading...\n\n'
restart_with_reloader()
def handle(self, addrport='', *args, **options):
if not addrport:
self.addr = ''
self.port = 8000
else:
m = match(naiveip_re, addrport)
if m is None:
raise CommandError('"%s" is not a valid port number '
'or address:port pair.' % addrport)
self.addr, _, _, _, self.port = m.groups()
environ['DJANGO_SOCKETIO_PORT'] = str(self.port)
if options.get('use_psyco'):
try:
from psycogreen.gevent import patch_psycopg
except ImportError:
raise CommandError('Could not patch psycopg. '
'Is psycogreen installed?')
patch_psycopg()
if options.get('use_reloader'):
start_new_thread(reload_watcher, ())
try:
bind = (self.addr, int(self.port))
print 'SocketIOServer running on %s:%s\n\n' % bind
handler = self.get_handler(*args, **options)
server = SocketIOServer(bind,
handler,
resource='socket.io',
policy_server=True)
server.serve_forever()
except KeyboardInterrupt:
for key, sock in six.iteritems(server.sockets):
sock.kill(detach=True)
server.stop()
if RELOAD:
print 'Reloading...\n\n'
restart_with_reloader()
def post_fork(server, worker):
"""
Called just after a worker has been forked.
Enables async processing in Psycopg2 if GUNICORN_ENABLE_ASYNC_PSYCOPG2 is set.
"""
if worker_class == 'gevent':
if _enable_async_psycopg2:
patch_psycopg()
worker.log.info('Enabled async Psycopg2')
# Temporary workaround for https://github.com/django/asgiref/issues/144.
# Essentially reverts part of
# https://github.com/django/django/commit/a415ce70bef6d91036b00dd2c8544aed7aeeaaed.
#
# TODO: Remove once there is a better fix for https://github.com/django/asgiref/issues/144.
if _patch_asgiref:
import asgiref.local
import threading
asgiref.local.Local = lambda **kwargs: threading.local()
worker.log.info('Patched asgiref.local.Local')
def greenify():
"""Patch threading and psycopg2 modules for green threads."""
# don't greenify twice.
if _GREEN:
return
_GREEN[True] = True
from gevent.monkey import patch_all, saved
if ('threading' in sys.modules) and ('threading' not in saved):
raise Exception('threading module loaded before patching!')
patch_all()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
try:
# Use psycopg2 by default
import psycopg2
del psycopg2
except ImportError:
# Fallback to psycopg2cffi if required (eg: pypy)
from psycopg2cffi import compat
compat.register()
def greenify():
"""Patch threading and psycopg2 modules for green threads."""
# don't greenify twice.
if _GREEN:
return
_GREEN[True] = True
from gevent.monkey import patch_all, saved
if ('threading' in sys.modules) and ('threading' not in saved):
raise Exception('threading module loaded before patching!')
patch_all()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
try:
# Use psycopg2 by default
import psycopg2
del psycopg2
except ImportError:
# Fallback to psycopg2cffi if required (eg: pypy)
from psycopg2cffi import compat
compat.register()
#!/usr/bin/env python
import os
import sys
from gevent import monkey#; monkey.patch_all()
monkey.patch_os()
monkey.patch_time()
monkey.patch_thread( _threading_local=False)
monkey.patch_sys()
monkey.patch_socket()
monkey.patch_select()
monkey.patch_ssl()
monkey.patch_subprocess()
from psycogreen.gevent import patch_psycopg; patch_psycopg();
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geventpooltest.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
def post_fork(server, worker):
os.environ["GEVENT_RESOLVER"] = "ares"
from psycogreen.gevent import patch_psycopg
patch_psycopg()
def post_fork(server, worker):
patch_psycopg()
worker.log.info("Made Psycopg2 Green")
server.log.info("Worker spawned (pid: %s)", worker.pid)
def post_fork(server, worker):
from psycogreen.gevent import patch_psycopg
patch_psycopg()
def post_fork(server, worker):
patch_psycopg()
worker.log.info("Made Psycopg2 Green")
def post_fork(server, worker):
from psycogreen.gevent import patch_psycopg
patch_psycopg()
def post_fork(server, worker):
patch_psycopg()
def __init__(self, connect_timeout=30, **connect_kwargs):
patch_psycopg()
self.conns = []
self.connect_timeout = connect_timeout
self.connect_kwargs = connect_kwargs
def do_post_fork(server, worker):
monkey.patch_all()
patch_psycopg()
# you should see this text in your gunicorn logs if it was successful
worker.log.info("Made Psycopg2 Green")
def post_fork(_, __):
patch_psycopg()
def _post_fork_handler(server, worker):
patch_psycopg()
worker.log.info("Made Psycopg2 Green")
def patch():
# Patch all the things that gevent knows about.
patch_all()
# Patch psycopg2 to also be gevent friendly.
patch_psycopg()
def pre_fork(server, worker):
patch_psycopg()
worker.log.info('Patched psycopg with gevent support')
For more information on this file, see
https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
"""
from website import settings
from api.base import settings as api_settings
if not settings.DEBUG_MODE:
from gevent import monkey
monkey.patch_all()
# PATCH: avoid deadlock on getaddrinfo, this patch is necessary while waiting for
# the final gevent 1.1 release (https://github.com/gevent/gevent/issues/349)
# unicode('foo').encode('idna') # noqa
from psycogreen.gevent import patch_psycopg # noqa
patch_psycopg()
import os # noqa
from django.core.wsgi import get_wsgi_application # noqa
from website.app import init_app # noqa
if os.environ.get('API_REMOTE_DEBUG', None):
import pydevd
remote_parts = os.environ.get('API_REMOTE_DEBUG').split(':')
pydevd.settrace(remote_parts[0], port=int(remote_parts[1]), suspend=False, stdoutToServer=True, stderrToServer=True, trace_only_current_thread=False)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.base.settings')
#### WARNING: Here be monkeys ###############
import six
# applying this later can lead to inconsistencies and threading issues
# but compressor doesn't like it
# ('module' object has no attribute 'poll' which has to do with
# gevent-patching subprocess)
import sys
GEVENT_COMMANDS = (
GeventCommand('run_gunicorn'),
GeventCommand('run_sql'),
GeventCommand('run_blob_migration'),
GeventCommand('check_blob_logs'),
GeventCommand('preindex_everything'),
GeventCommand('migrate_multi'),
GeventCommand('prime_views'),
GeventCommand('ptop_preindex'),
GeventCommand('sync_prepare_couchdb_multi'),
GeventCommand('sync_couch_views'),
GeventCommand('populate_form_date_modified'),
GeventCommand('migrate_domain_from_couch_to_sql', http_adapter_pool_size=32),
GeventCommand('migrate_multiple_domains_from_couch_to_sql', http_adapter_pool_size=32),
)
if len(sys.argv) > 1 and _should_patch_gevent(sys.argv, GEVENT_COMMANDS):
from gevent.monkey import patch_all; patch_all(subprocess=True)
from psycogreen.gevent import patch_psycopg; patch_psycopg()
init_hq_python_path()
run_patches()
set_default_settings_path(sys.argv)
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
def post_fork(server, worker):
# patch psycopg2 for gevent compatibility
patch_psycopg()
# Monkey-patching should happen as early as possible ...
# pylint: disable=multiple-statements,wrong-import-position,wrong-import-order
# fmt: off
from psycogreen.gevent import patch_psycopg
patch_psycopg() # noqa: E402,E702
# fmt: on
import logging
from celery import Celery
from celery.signals import after_setup_logger
from django.conf import settings
celery_app = Celery("dataworkspace")
celery_app.config_from_object("django.conf:settings", namespace="CELERY")
celery_app.autodiscover_tasks()
# this *shouldn't* need to be applied directly but
# automatically configured via ^^ config_from_object
celery_app.conf.task_routes = settings.CELERY_ROUTES
@after_setup_logger.connect
def setup_loggers(*args, **kwargs):
logging.config.dictConfig(settings.LOGGING)
def post_fork(server, worker):
from psycogreen.gevent import patch_psycopg
patch_psycopg()
worker.log.info("Made psycopg Green")
def post_fork(server, worker):
from psycogreen.gevent import patch_psycopg
patch_psycopg()
worker.log.info('Made psycopg2 green :)')
from sio.monitor import que_monitor_loop
spawn(que_monitor_loop, server, worker)
import sqlalchemy as sql
from sqlalchemy import or_
import bottle_sqlalchemy as sqlalchemy
from sqlalchemy import create_engine, Column, Sequence
from sqlalchemy.ext.declarative import declarative_base
import psycopg2, random, urllib, os
from bin.utils import isInt, gen_string
from dataobjects import Debug
from datetime import datetime
from gevent import monkey
monkey.patch_all()
from psycogreen.gevent import patch_psycopg
patch_psycopg()
Base = declarative_base()
class Postgres():
def __init__(self, cfg, app):
self._cfg = cfg
self._db_host = self._cfg.get('Postgres', 'host')
self._db_port = self._cfg.get('Postgres', 'port')
self._db_database = self._cfg.get('Postgres', 'db')
self._db_user = self._cfg.get('Postgres', 'user')
self._db_pass = self._cfg.get('Postgres', 'pass')
self.engine = create_engine('postgresql+psycopg2://%s:%s@%s:%s/%s' % (
self._db_user,
self._db_pass,
self._db_host,
def post_fork(server, worker):
patch_psycopg()
worker.log.info("Made Psycopg2 Green")
server.log.info("Worker spawned (pid: %s)", worker.pid)
def smtp():
"Run smtp smarthost for transactional service."
from gevent import monkey
# Do not patch threads, it may lead to Django DatabaseWrapper being
# shared between threads.
# See: https://code.djangoproject.com/ticket/17998#comment:6
monkey.patch_all(thread=False)
import os
import sys
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "munch.settings")
from psycogreen.gevent import patch_psycopg
patch_psycopg()
sys.path.append(
os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
import django
django.setup()
import time
from django.conf import settings
from gevent import sleep
from gevent.pool import Pool
from slimta.system import drop_privileges
from munch.core.utils import monkey_patch_slimta_exception
from munch.apps.transactional.queue import queue
from munch.apps.transactional.edge import (
TransactionalSmtpEdge, ProxyProtocolTransactionalSmtpEdge)
from munch.apps.transactional.edge import EdgeValidators
monkey_patch_slimta_exception()
tls_settings = settings.TRANSACTIONAL.get('SMTP_SMARTHOST_TLS')
if tls_settings is not None:
if not isinstance(tls_settings, dict):
raise Exception(
('Setting SMTP_SMARTHOST_TLS should be a dict with '
'"keyfile" and "certfile" keys'))
for i in ('keyfile', 'certfile'):
if i not in tls_settings:
raise Exception(
'{} is required if you plan to offer STARTTLS support')
else:
path = tls_settings[i]
if not os.access(path, os.R_OK):
raise Exception(
'{} is not readable or inexistant.'.format(path))
ssl_context = SSLContext(PROTOCOL_SSLv23)
ssl_context.load_cert_chain(tls_settings.get('certfile'),
keyfile=tls_settings.get('keyfile'))
else:
ssl_context = None
pool = Pool(settings.TRANSACTIONAL.get('EDGE_MAX_CONN', 200))
edge_class = TransactionalSmtpEdge
if settings.TRANSACTIONAL.get('PROXYPROTO_ENABLED', False):
edge_class = ProxyProtocolTransactionalSmtpEdge
edge = edge_class(
(settings.TRANSACTIONAL.get('SMTP_BIND_HOST'),
settings.TRANSACTIONAL.get('SMTP_BIND_PORT')),
queue,
data_timeout=settings.TRANSACTIONAL.get('EDGE_TIMEOUTS',
{}).get('data_timeout'),
command_timeout=settings.TRANSACTIONAL.get('EDGE_TIMEOUTS',
{}).get('command_timeout'),
pool=pool,
hostname=settings.TRANSACTIONAL.get('EDGE_EHLO_AS', None),
validator_class=EdgeValidators,
context=ssl_context,
auth=True,
)
log.info('Listening on {}:{}'.format(
settings.TRANSACTIONAL.get('SMTP_BIND_HOST'),
settings.TRANSACTIONAL.get('SMTP_BIND_PORT')))
edge.start()
if settings.TRANSACTIONAL.get('DROP_PRIVILEGES_USER') is not None:
# ??? Really needed? See python-slimta/examples/slimta-mail.py...
sleep(0.1)
# If this command is run with root user (to be allowed to
# open reserved ports like 25), we should then "switch" to a
# normal user for security.
drop_privileges(settings.TRANSACTIONAL.get('DROP_PRIVILEGES_USER'),
settings.TRANSACTIONAL.get('DROP_PRIVILEGES_GROUP'))
log.info('Dropping privileges to {}:{}'.format(
settings.TRANSACTIONAL.get('DROP_PRIVILEGES_USER'),
settings.TRANSACTIONAL.get('DROP_PRIVILEGES_GROUP')))
try:
edge.get()
except KeyboardInterrupt:
try:
stop_timeout = settings.TRANSACTIONAL.get('SMTP_STOP_TIMEOUT', 5)
edge.server.close()
ts = time.time()
log.info('Stop accepting connections.')
if stop_timeout > 0:
log.info('Waiting {} seconds before timing out '
'current connections...'.format(stop_timeout))
while time.time() - ts <= stop_timeout:
sleep(1.0)
log.info('Edge stopped...')
edge.server.stop(timeout=1)
except KeyboardInterrupt:
log.info('Forcing shutdown...')
edge.server.stop(timeout=1)
def post_fork(server, worker):
"""
https://github.com/jneight/django-db-geventpool
"""
patch_psycopg()
worker.log.info("Made Psycopg2 Green")
