def make_server(config, listener, app):
"""Make a Gevent server for WSGI apps."""
max_concurrency = int(config.get("max_concurrency", 0)) or None
stop_timeout = int(config.get("stop_timeout", 0))
handler = config.get("handler", None)
pool = Pool(size=max_concurrency)
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs = {}
if gevent.version_info[:2] >= (1, 1): # error_log is new in 1.1
kwargs["error_log"] = LoggingLogAdapter(logger, level=logging.ERROR)
if handler:
kwargs["handler_class"] = _load_factory(handler, default_name=None)
# pylint: disable=star-args
server = WSGIServer(
listener,
application=app,
spawn=pool,
log=log,
**kwargs
)
server.stop_timeout = stop_timeout
return server
def make_server(server_config, listener, app):
"""Make a gevent server for WSGI apps."""
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"handler": config.Optional(config.String, default=None),
"max_concurrency": config.Integer,
"stop_timeout": config.Optional(config.Integer, default=0),
},
)
pool = Pool(size=cfg.max_concurrency)
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs = {}
if gevent.version_info[:2] >= (1, 1): # error_log is new in 1.1
kwargs["error_log"] = LoggingLogAdapter(logger, level=logging.ERROR)
if cfg.handler:
kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None)
server = WSGIServer(listener, application=app, spawn=pool, log=log, **kwargs)
server.stop_timeout = cfg.stop_timeout
runtime_monitor.start(server_config, app, pool)
return server
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
"""Make a gevent server for WSGI apps."""
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"handler": config.Optional(config.String, default=None),
"max_concurrency": config.Integer,
"stop_timeout": config.Optional(config.Integer, default=0),
},
)
pool = Pool(size=cfg.max_concurrency)
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs: Dict[str, Any] = {}
if cfg.handler:
kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None)
server = WSGIServer(
listener,
application=app,
spawn=pool,
log=log,
error_log=LoggingLogAdapter(logger, level=logging.ERROR),
**kwargs,
)
server.stop_timeout = cfg.stop_timeout
runtime_monitor.start(server_config, app, pool)
return server
def superproxy(config, listen, pool_size, stop_timeout, dozer):
import sys
import signal
import gevent
from gevent.pywsgi import WSGIServer
from gevent.pool import Pool
from .superproxy import WSGISuperProxy
from .proxylist import ProxyList
conf = config.get('superproxy', {})
fetcher = config.get('proxyfetcher')
checker = config.get('proxychecker')
proxylist = ProxyList(fetcher=fetcher, checker=checker, **conf.pop('proxylist', {}))
listen = listen or conf.pop('listen', '0.0.0.0:8088')
pool_size = pool_size or conf.pop('pool_size', 500)
stop_timeout = stop_timeout or conf.pop('stop_timeout', 5)
dozer = conf.pop('dozer', False)
iface, port = listen.split(':')
app = WSGISuperProxy(proxylist, **conf)
if dozer:
from dozer import Dozer
app = Dozer(app)
server = WSGIServer((iface, int(port)), app, spawn=Pool(pool_size))
server.stop_timeout = stop_timeout
logger = logging.getLogger('proxytools.superproxy')
def stop(*args):
if server.closed:
try:
logger.error('Server stopping - multiple exit signals received - aborting.')
finally:
sys.exit('Multiple exit signals received - aborting.')
logger.info('Server stopping %s', args)
server.stop()
[gevent.signal(sig, stop) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT)]
logger.info('Server started')
server.serve_forever()
logger.debug('Server stopped')
def make_server(server_config: Dict[str, str], listener: socket.socket,
app: Any) -> StreamServer:
"""Make a gevent server for WSGI apps."""
# pylint: disable=maybe-no-member
cfg = config.parse_config(
server_config,
{
"handler":
config.Optional(config.String, default=None),
"max_concurrency":
config.Optional(config.Integer),
"stop_timeout":
config.Optional(config.TimespanWithLegacyFallback,
default=datetime.timedelta(seconds=10)),
},
)
if cfg.max_concurrency is not None:
raise Exception(
"The max_concurrency setting is not allowed for WSGI servers. See https://git.io/Jeywc."
)
pool = Pool()
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs: Dict[str, Any] = {}
if cfg.handler:
kwargs["handler_class"] = _load_factory(cfg.handler, default_name=None)
server = WSGIServer(
listener,
application=app,
spawn=pool,
log=log,
error_log=LoggingLogAdapter(logger, level=logging.ERROR),
**kwargs,
)
server.stop_timeout = cfg.stop_timeout.total_seconds()
runtime_monitor.start(server_config, app, pool)
return server
def make_server(config, listener, app):
"""Make a Gevent server for WSGI apps."""
max_concurrency = int(config.get("max_concurrency", 0)) or None
stop_timeout = int(config.get("stop_timeout", 0))
handler = config.get("handler", None)
pool = Pool(size=max_concurrency)
log = LoggingLogAdapter(logger, level=logging.DEBUG)
kwargs = {}
if gevent.version_info[:2] >= (1, 1): # error_log is new in 1.1
kwargs["error_log"] = LoggingLogAdapter(logger, level=logging.ERROR)
if handler:
kwargs["handler_class"] = _load_factory(handler, default_name=None)
server = WSGIServer(listener,
application=app,
spawn=pool,
log=log,
**kwargs)
server.stop_timeout = stop_timeout
return server
def superproxy(config, listen, pool_size, stop_timeout, dozer):
# TODO: clean this chaotic configuration, obviously we should not configure
# from strings/dicts inside classes, it will make things more obvious
import sys
import signal
import gevent
from gevent.pywsgi import WSGIServer
from gevent.pool import Pool
conf = config.get('superproxy', {})
superproxy_cls = import_string(conf.pop('cls', 'proxytools.superproxy.WSGISuperProxy'))
if conf.get('verify', None) is False:
# Assuming you know what you're doing
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fetcher = config.get('proxyfetcher')
checker = config.get('proxychecker')
proxylist_cls = import_string(conf.get('proxylist', {})
.pop('cls', 'proxytools.proxylist.ProxyList'))
proxylist = proxylist_cls(fetcher=fetcher, checker=checker, **conf.pop('proxylist', {}))
listen = listen or conf.pop('listen', '0.0.0.0:8088')
pool_size = pool_size or conf.pop('pool_size', 500)
stop_timeout = stop_timeout or conf.pop('stop_timeout', 5)
dozer = conf.pop('dozer', False)
http_debug = conf.pop('http_debug', False)
app = superproxy_cls(proxylist, **conf)
if dozer:
from dozer import Dozer
app = Dozer(app)
if http_debug:
import http.client
http.client.HTTPConnection.debuglevel = 10 if http_debug is True else http_debug
iface, port = listen.split(':')
server = WSGIServer((iface, int(port)), app, spawn=Pool(pool_size))
server.stop_timeout = stop_timeout
logger = logging.getLogger('proxytools.superproxy')
def stop(*args):
if server.closed:
try:
logger.error('Server stopping - multiple exit signals received - aborting.')
finally:
sys.exit('Multiple exit signals received - aborting.')
logger.info('Server stopping %s', args)
server.stop()
[gevent.signal(sig, stop) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT)]
logger.info('Server started')
server.serve_forever()
logger.debug('Server stopped')
