class NoGoStatusCodes(TestCase):
def __init__(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.resource = HendrixWSGIResource(reactor,
self.tp,
self.wsgi_thing)
self.nameSpace = TestNameSpace()
self.nameSpace.async_thing_complete = Queue()
return super(NoGoStatusCodes, self).__init__(*args, **kwargs)
def setUp(self, *args, **kwargs):
self.addCleanup(self.tp.stop)
super(NoGoStatusCodes, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('404 NOT FOUND', [('Content-type','text/plain')])
@crosstown_traffic.follow_response(
no_go_status_codes=self.no_go_status_codes,
same_thread=True
)
def long_thing_on_same_thread():
self.nameSpace.async_task_was_run = True
logger.info("No bad status codes; went ahead with async thing.")
return "Nothing."
def test_bad_status_codes_cause_no_go_in_wsgi_response(self):
self.no_go_status_codes = [404, '6xx']
request = DummyRequest('r1')
request.isSecure = lambda: False
request.content = "llamas"
finished = request.notifyFinish()
self.resource.render(request)
# This must wait until the WSGI response is closed.
finished.addCallback(
lambda _: self.assertFalse(
self.nameSpace.async_task_was_run
)
)
def test_bad_status_codes_cause_no_go_flag(self):
through_to_you = crosstown_traffic.follow_response(no_go_status_codes=[418])
through_to_you.status_code = 418
through_to_you.check_status_code_against_no_go_list()
self.assertTrue(through_to_you.no_go)
def test_no_bad_status_codes_are_cool(self):
through_to_you = crosstown_traffic.follow_response(no_go_status_codes=[418])
through_to_you.status_code = 404
through_to_you.check_status_code_against_no_go_list()
self.assertFalse(through_to_you.no_go)
def __init__(self, core):
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
application = get_flask_application(core)
wsgi_resource = WSGIResource(reactor, thread_pool, application)
Site.__init__(self, wsgi_resource)
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet(
[fernet.Fernet(key) for key in config["encryption_keys"]]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
class SeleniumDownloadHandler(HTTP11DownloadHandler):
def __init__(self, settings):
super(SeleniumDownloadHandler, self).__init__(settings)
self._enable_driver = settings.getbool('WEB_DRIVER_ENABLED')
self._driver_name = settings.get('WEB_DRIVER_NAME')
self._driver_path = settings.get('WEB_DRIVER_PATH')
selenium_concurrent_request = settings.get(
'WEB_DRIVER_CONCURRENT_REQUESTS', 16)
self._thread_pool = ThreadPool(minthreads=selenium_concurrent_request,
maxthreads=selenium_concurrent_request)
self._thread_pool.start()
self._driver_timeout = settings.get('WEB_DRIVER_TIMEOUT', 300)
def download_request(self, request, spider):
if self._enable_driver and request.meta.get('selenium_needed', False):
agent = SeleniumAsyncAgent(contextFactory=self._contextFactory,
pool=self._thread_pool,
driverName=self._driver_name,
driverPath=self._driver_path,
connectTimeout=self._driver_timeout)
return agent.download_request(request)
else:
return super(SeleniumDownloadHandler,
self).download_request(request, spider)
def close(self):
super(SeleniumDownloadHandler, self).close()
self._thread_pool.stop()
class AddressResolver(object):
pool = None
def __init__(self, minthreads=1, maxthreads=4):
self.pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads)
# unclosed ThreadPool leads to reactor hangs at shutdown
# this is a problem in many situation, so better enforce pool stop here
reactor.addSystemEventTrigger(
"before", "shutdown", self.pool.stop
)
self.pool.start()
def get_host_by_name(self, address):
d = defer.Deferred()
def func():
try:
reactor.callFromThread(
d.callback, socket.gethostbyname(address)
)
except Exception as e:
reactor.callFromThread(d.errback, e)
self.pool.callInThread(func)
return d
def close(self):
self.pool.stop()
class AddressResolver(object):
pool = None
def __init__(self, minthreads=1, maxthreads=4):
self.pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads)
# unclosed ThreadPool leads to reactor hangs at shutdown
# this is a problem in many situation, so better enforce pool stop here
reactor.addSystemEventTrigger("before", "shutdown", self.pool.stop)
self.pool.start()
def get_host_by_name(self, address):
d = defer.Deferred()
def func():
try:
reactor.callFromThread(d.callback,
socket.gethostbyname(address))
except Exception as e:
reactor.callFromThread(d.errback, e)
self.pool.callInThread(func)
return d
def close(self):
self.pool.stop()
class ProviderServer(object):
def __init__(self, reactor, port):
self.reactor = reactor
self.reactor.addSystemEventTrigger('before', 'shutdown',
self.before_shutdown)
self.reactor.suggestThreadPoolSize(config.REACTOR_THREAD_POOL_MAX)
self.app = create_app()
self.thread_pool = ThreadPool(maxthreads=config.FLASK_THREAD_POOL_MAX)
self.thread_pool.start()
wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app)
root_resource = RootResource(wsgi_resource)
root_resource.putChild("metrics", MetricsResource())
site = Site(root_resource)
self.bind = self.reactor.listenTCP(port, site)
log.info('Provider is listening on {} ...'.format(port))
def run(self):
self.reactor.run()
def stop_services(self):
log.info("Shutting down provider...")
self.app.cleanup()
self.thread_pool.stop()
return maybeDeferred(self.bind.stopListening).addCallbacks(
callback=lambda _: log.info("Port listening was stopped"),
errback=lambda failure: log.error(
"Error while stopping port listening: {}".format(failure)))
@inlineCallbacks
def before_shutdown(self):
self.app.stop()
yield self.stop_services()
def __init__(self, core):
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger("after", "shutdown", thread_pool.stop)
application = get_flask_application(core)
wsgi_resource = WSGIResource(reactor, thread_pool, application)
Site.__init__(self, wsgi_resource)
def makeService(self, options):
app = Application("Mercurial SSH Server") #, uid, gid)
services = IServiceCollection(app)
service = options.subOptions.getService()
service.setServiceParent(services)
wsgi_app = WSGIApplication()
threadpool = ThreadPool(config.web.min_threads, config.web.max_threads)
threadpool.start()
reactor.addSystemEventTrigger('after', 'shutdown', threadpool.stop)
root = wsgi.WSGIResource(reactor, threadpool, wsgi_app)
factory = server.Site(root)
if isfile(config.web.certificate):
from OpenSSL import SSL
# Run SSL Server
class SSLContext(object):
def getContext(self):
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_privatekey_file(config.private_key)
ctx.use_certificate_file(config.web.certificate)
return ctx
config_service = internet.SSLServer(config.web.port, factory,
SSLContext())
else:
config_service = internet.TCPServer(config.web.port, factory)
config_service.setServiceParent(app)
clean_changes_task = task.LoopingCall(self.__clean_old_changes_from_db)
clean_changes_task.start(5*60, now=True) # Every 5 minutes
reactor.addSystemEventTrigger('after', 'shutdown',
clean_changes_task.stop)
return services
class TwistedApplicationServer(ApplicationServer):
def __init__(self, app, port, host, debug):
super(TwistedApplicationServer,
self).__init__(app, port, host, debug)
self.reactor = reactor
self.thread_pool = ThreadPool(5, 40)
self.resource = WSGIResource(self.reactor, self.thread_pool,
self.app)
self.reactor.addSystemEventTrigger('after', 'shutdown',
self.thread_pool.stop)
def run(self):
print("Begin Listening")
self.thread_pool.start()
self.reactor.listenTCP(self.port,
Site(self.resource),
interface=self.host)
self.reactor.run()
def shutdown(self):
print("Received call to shutdown")
print("Reactor should have stopped")
self.reactor.callFromThread(self.reactor.stop)
print("Reactor stop listening")
self.reactor.callFromThread(self.reactor.stopListening)
print("Reactor Shutdown")
class FakeReactor(object):
"""
A fake reactor implementation which just supports enough reactor APIs for
L{ThreadedResolver}.
"""
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def callFromThread(self, f, *args, **kwargs):
self._threadCalls.put((f, args, kwargs))
def _runThreadCalls(self):
f, args, kwargs = self._threadCalls.get()
f(*args, **kwargs)
def _stop(self):
self._threadpool.stop()
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet([
fernet.Fernet(key) for key in config["encryption_keys"]
]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
class FakeReactor(object):
"""
A fake reactor implementation which just supports enough reactor APIs for
L{ThreadedResolver}.
"""
implements(IReactorTime, IReactorThreads)
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def callFromThread(self, f, *args, **kwargs):
self._threadCalls.put((f, args, kwargs))
def _runThreadCalls(self):
f, args, kwargs = self._threadCalls.get()
f(*args, **kwargs)
def _stop(self):
self._threadpool.stop()
def test_contemporaneous_requests(self):
'''
We're going to create two request-response cycles here:
Cycle 1 will begin.
Cycle 2 will begin.
Cycle 2 will return.
Cycle 1 will return.
This way, we can prove that the crosstown_traffic created
by cycle 1 is not resolved by the return of cycle 2.
'''
tp = ThreadPool(maxthreads=20)
tp.start()
self.addCleanup(tp.stop)
log.debug("\n\nStarting the two stream stuff.")
request1 = DummyRequest('r1')
request1.isSecure = lambda: False
request1.content = "Nothing really here."
request1.headers['llamas'] = 'dingo'
nameSpace.test_case = self
hr = HendrixWSGIResource(reactor, tp, wsgi_application)
d1 = deferToThreadPool(reactor, tp, hr.render, request1)
request2 = DummyRequest('r2')
request2.isSecure = lambda: False
request2.content = "Nothing really here."
request2.headers['llamas'] = 'dingo'
d2 = deferToThreadPool(reactor, tp, hr.render, request2)
def woah_stop(failure):
nameSpace.async_task_was_done.put_nowait(False)
nameSpace.second_cycle_complete.put_nowait(False)
nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)
d1.addErrback(woah_stop)
d2.addErrback(woah_stop)
combo_deferred = gatherResults([d1, d2])
def wait_for_queue_resolution():
nameSpace.async_task_was_done.get(True, 3)
combo_deferred.addCallback(
lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution)
)
combo_deferred.addCallback(
lambda _: self.assertTrue(nameSpace.async_task_was_run)
)
return combo_deferred
def runtwisted(config=None):
"""
Run the Twisted server.
"""
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
app = api.makeapp(config=config)
wsgi_app = WSGIResource(reactor, threadpool, app)
class OptimaResource(Resource):
isLeaf = True
def __init__(self, wsgi):
self._wsgi = wsgi
def render(self, request):
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
# If we have a full path for the client directory, use that directory.
if os.path.isabs(config.CLIENT_DIR):
clientDirTarget = config.CLIENT_DIR
# Otherwise (we have a relative path), use it (correcting so it is with
# respect to the sciris repo directory).
else:
clientDirTarget = '%s%s%s' % (os.pardir, os.sep, config.CLIENT_DIR)
base_resource = File('%s%sdist%s' % (clientDirTarget, os.sep, os.sep))
base_resource.putChild(
'dev', File('%s%ssrc%s' % (clientDirTarget, os.sep, os.sep)))
base_resource.putChild('api', OptimaResource(wsgi_app))
site = Site(base_resource)
try:
port = str(sys.argv[1])
except IndexError:
port = "8091"
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + port)
endpoint.listen(site)
reactor.run()
def _test_StreamingServer():
expected_id = 'test'
headers = {
#'TimeSeekRange.dlna.org': 'npt=30.000-',
'TimeSeekRange.dlna.org': 'npt=20.000-50.000',
#'Range': 'bytes=5-',
'Connection': 'close',
}
duration = '00:01:00'
content_length = 64 * 1024
interface = '192.168.0.103'
class StreamingServerStub(ByteSeekMixin, TimeSeekMixin, StreamingServer):
def get_content(self, id, environ):
eq_(expected_id, id)
#raise ValueError(repr(environ))
for k in headers:
eq_(headers[k], environ['HTTP_' + k.upper()])
return ContentStub(content_length,
'video/mpeg',
'DLNA.ORG_PN=MPEG_PS_NTSC;DLNA.ORG_OP=11')
from twisted.web import client
class HTTPPageGetter(client.HTTPPageGetter):
handleStatus_206 = lambda self: self.handleStatus_200()
class HTTPClientFactory(client.HTTPClientFactory):
protocol = HTTPPageGetter
def getPageFactory(url, *args, **kwargs):
scheme, host, port, path = client._parse(url)
factory = HTTPClientFactory(url, *args, **kwargs)
reactor.connectTCP(host, port, factory)
return factory
app = StreamingServerStub('test')
mapper = Mapper()
mapper.connect(':id', controller='mt', action='get')
app = RoutesMiddleware(app, mapper)
tpool = ThreadPool()
tpool.start()
resource = upnp.WSGIResource(reactor, tpool, app)
port = reactor.listenTCP(0, server.Site(resource), interface=interface)
port_num = port.socket.getsockname()[1]
url = 'http://%s:%i/%s?duration=%s' % (interface, port_num, expected_id, duration)
factory = getPageFactory(url, headers=headers)
def check_result(contents):
raise ValueError('expected: %d, actual: %d' % (content_length, len(contents)))
#raise ValueError(repr(factory.response_headers))
eq_(content_length, len(contents))
factory.deferred.addCallback(check_result)
reactor.callLater(5, reactor.stop)
reactor.run()
def StartStorageService(config, block_store):
try:
http_port = config['StorageService']['HttpPort']
http_host = config['StorageService']['Host']
worker_threads = config['StorageService'].get('WorkerThreads', 8)
reactor_threads = config['StorageService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on port %s', http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
block = Resource()
block.putChild(
b'get',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlockApp(block_store))))
block.putChild(
b'gets',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlocksApp(block_store))))
block.putChild(
b'store',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(StoreBlocksApp(block_store))))
block.putChild(
b'list',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(ListBlocksApp(block_store))))
block.putChild(
b'check',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(CheckBlocksApp(block_store))))
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(block_store))))
root.putChild(b'block', block)
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def test_contemporaneous_requests(self):
'''
We're going to create two request-response cycles here:
Cycle 1 will begin.
Cycle 2 will begin.
Cycle 2 will return.
Cycle 1 will return.
This way, we can prove that the crosstown_traffic created
by cycle 1 is not resolved by the return of cycle 2.
'''
tp = ThreadPool(maxthreads=20)
tp.start()
self.addCleanup(tp.stop)
log.debug("\n\nStarting the two stream stuff.")
request1 = DummyRequest([b'r1'])
request1.isSecure = lambda: False
request1.content = "Nothing really here."
request1.requestHeaders.addRawHeader('llamas', 'dingo')
request1.client = IPv4Address("TCP", b"50.0.50.0", 5000)
nameSpace.test_case = self
hr = HendrixWSGIResource(reactor, tp, wsgi_application)
d1 = deferToThreadPool(reactor, tp, hr.render, request1)
request2 = DummyRequest([b'r2'])
request2.isSecure = lambda: False
request2.content = b"Nothing really here."
request2.requestHeaders.addRawHeader('llamas', 'dingo')
request2.client = IPv4Address("TCP", b"100.0.50.0", 5000)
d2 = deferToThreadPool(reactor, tp, hr.render, request2)
def woah_stop(failure):
nameSpace.async_task_was_done.put_nowait(False)
nameSpace.second_cycle_complete.put_nowait(False)
nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)
d1.addErrback(woah_stop)
d2.addErrback(woah_stop)
combo_deferred = gatherResults([d1, d2])
def wait_for_queue_resolution():
nameSpace.async_task_was_done.get(True, 3)
combo_deferred.addCallback(lambda _: deferToThreadPool(
reactor, tp, wait_for_queue_resolution))
combo_deferred.addCallback(
lambda _: self.assertTrue(nameSpace.async_task_was_run))
return combo_deferred
def start(self):
"""
Starts the logger pool and sets up the required shutdown event
trigger which will call :meth:`stop` on exit.
"""
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
ThreadPool.start(self)
logger.debug(
"Started logger thread pool (min=%s, max=%s)", self.min, self.max)
def run(self, handler):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler))
reactor.listenTCP(self.port, factory, interface=self.host)
class URLTranServer(object):
def __init__(self, app):
self.pool = ThreadPool()
self.pool.start()
self.resource = _WSGIResource(reactor, self.pool, app)
self.site = _Site(self.resource)
def register(self, port):
serve_register_tcp(self.site, port)
class VMMasterServer(object):
def __init__(self, reactor, port):
self.reactor = reactor
self.app = create_app()
self.thread_pool = ThreadPool(maxthreads=config.THREAD_POOL_MAX)
self.thread_pool.start()
wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app)
root_resource = RootResource(wsgi_resource)
root_resource.putChild("proxy", ProxyResource(self.app))
site = Site(root_resource)
site.protocol = HTTPChannelWithClient
self.bind = self.reactor.listenTCP(port, site)
log.info('Server is listening on %s ...' % port)
def run(self):
self.reactor.addSystemEventTrigger('before', 'shutdown',
self.before_shutdown)
self.reactor.run()
del self
def __del__(self):
log.info("Shutting down server...")
d = self.bind.stopListening()
_block_on(d, 20)
self.app.cleanup()
self.thread_pool.stop()
log.info("Server gracefully shut down")
def wait_for_end_active_sessions(self):
active_sessions = self.app.sessions.active()
def wait_for():
while active_sessions:
log.info("Waiting for {} sessions to complete: {}".format(
len(active_sessions),
[(i.id, i.status) for i in active_sessions]))
for session in active_sessions:
if session.is_done:
log.debug("Session {} is done".format(session.id))
active_sessions.remove(session)
time.sleep(1)
log.info("Wait for end %s active session[s]:"
" %s" % (len(active_sessions), active_sessions))
return deferToThread(wait_for).addCallbacks(
callback=lambda _: log.info(
"All active sessions has been completed"),
errback=lambda failure: log.error(
"Error while waiting for active_sessions: {}".format(failure)))
@inlineCallbacks
def before_shutdown(self):
self.app.running = False
yield self.wait_for_end_active_sessions()
def twisted(app, address, **options):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, app))
reactor.listenTCP(address[1], factory, interface=address[0])
reactor.run()
def twisted(app, address, **options):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, app))
reactor.listenTCP(address[1], factory, interface=address[0])
reactor.run()
def run(self, handler):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler))
reactor.listenTCP(self.port, factory, interface=self.host)
reactor.run()
class NoGoStatusCodes(TestCase):
def __init__(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.resource = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
self.nameSpace = TestNameSpace()
self.nameSpace.async_thing_complete = Queue()
return super(NoGoStatusCodes, self).__init__(*args, **kwargs)
def setUp(self, *args, **kwargs):
self.addCleanup(self.tp.stop)
super(NoGoStatusCodes, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('404 NOT FOUND', [('Content-type', 'text/plain')])
@crosstown_traffic(no_go_status_codes=self.no_go_status_codes,
same_thread=True)
def long_thing_on_same_thread():
self.nameSpace.async_task_was_run = True
log.debug("No bad status codes; went ahead with async thing.")
return [b"Nothing."]
def test_bad_status_codes_cause_no_go_in_wsgi_response(self):
self.no_go_status_codes = [404, '6xx']
request = DummyRequest([b'r1'])
request.isSecure = lambda: False
request.content = "llamas"
request.client = IPv4Address("TCP", b"50.0.50.0", 5000)
finished = request.notifyFinish()
self.resource.render(request)
# This must wait until the WSGI response is closed.
finished.addCallback(
lambda _: self.assertFalse(self.nameSpace.async_task_was_run))
return finished
def test_bad_status_codes_cause_no_go_flag(self):
through_to_you = crosstown_traffic(no_go_status_codes=[418])
through_to_you.status_code = 418
through_to_you.check_status_code_against_no_go_list()
self.assertTrue(through_to_you.no_go)
def test_no_bad_status_codes_are_cool(self):
through_to_you = crosstown_traffic(no_go_status_codes=[418])
through_to_you.status_code = 404
through_to_you.check_status_code_against_no_go_list()
self.assertFalse(through_to_you.no_go)
def test_threadLocality(self):
"""
An 'Record' repr()'d in two separate threads at the same time should
look the same (i.e. the repr state tracking for '...' should be
thread-local).
"""
pool = ThreadPool(2, 2)
pool.start()
self.addCleanup(pool.stop)
class StickyRepr(object):
"""
This has a __repr__ which will block until a separate thread
notifies it that it should return. We use this to create a race
condition.
"""
waited = False
def __init__(self):
self.set = threading.Event()
self.wait = threading.Event()
def __repr__(self):
if not self.waited:
self.set.set()
self.wait.wait()
return 'sticky'
r = StickyRepr()
mr = MyRecord(something=1, somethingElse=r)
d = deferToThreadPool(reactor, pool, repr, mr)
def otherRepr():
# First we wait for the first thread doing a repr() to enter its
# __repr__()...
r.set.wait()
# OK, now it's blocked. Let's make sure that subsequent calls to
# this repr() won't block.
r.waited = True
# Do it! This is a concurrent repr().
result = repr(mr)
# Now we're done, wake up the other repr and let it complete.
r.wait.set()
return result
d2 = deferToThreadPool(reactor, pool, otherRepr)
def done(xxx_todo_changeme):
(thread1repr, thread2repr) = xxx_todo_changeme
knownGood = 'MyRecord(something=1, somethingElse=sticky)'
# self.assertEquals(thread1repr, thread2repr)
self.assertEqual(thread1repr, knownGood)
self.assertEqual(thread2repr, knownGood)
return gatherResults([d, d2]).addCallback(done)
class ThreadPoolService(Service):
def __init__(self):
self.threadpool = ThreadPool()
def startService(self):
self.threadpool.start()
def stopService(self):
self.threadpool.stop()
def get_twisted_pool():
global pool
try:
return pool
except:
pool = ThreadPool(minthreads=min_t, maxthreads=max_t, name='core')
pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
return pool
def StartService(config):
# load the eservice database
try:
dbfile_name = config['Service']['EnclaveServiceDatabaseFile']
except KeyError:
logger.error(
'missing required configuration for enclave service database')
sys.exit(-1)
try:
eservice_db.load_database(dbfile_name)
except Exception as e:
logger.error('error loading eservice database from file %s', str(e))
sys.exit(-1)
try:
http_port = config['Service']['HttpPort']
http_host = config['Service']['Host']
worker_threads = config['Service'].get('WorkerThreads', 8)
reactor_threads = config['Service'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on host %s, port %s', http_host, http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
flask_app = toxaway.views.register(config)
flask_site = WSGIResource(reactor, reactor.getThreadPool(), flask_app)
root = Resource()
root.putChild(b'shutdown', ShutdownResource())
root.putChild(b'flask', flask_site)
files = config.get('StaticContent', {})
logger.info('files=%s', files)
for key, val in files.items():
logger.info('map <%s> to <%s>', key, val)
root.putChild(key.encode(), File(val.encode()))
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
class SameOrDifferentThread(TestCase):
def setUp(self, *args, **kwargs):
self.tp = ThreadPool()
self.tp.start()
self.addCleanup(self.tp.stop)
super(SameOrDifferentThread, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('200 OK', [('Content-type', 'text/plain')])
nameSpace.this_thread = threading.current_thread()
@crosstown_traffic(
same_thread=self.use_same_thread
)
def long_thing_on_same_thread():
nameSpace.thread_that_is_supposed_to_be_the_same = threading.current_thread()
log.debug("Finished async thing on same thread.")
return [b"Nothing."]
def assert_that_threads_are_the_same(self):
self.assertEqual(
nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_the_same
)
def assert_that_threads_are_different(self):
self.assertNotEqual(nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_different)
def request_same_or_different_thread_thread(self):
hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
request1 = DummyRequest([b'r1'])
request1.isSecure = lambda: False
request1.content = b"llamas"
request1.client = IPv4Address("TCP", b"50.0.50.0", 5000)
d = deferToThreadPool(reactor, self.tp, hr.render, request1)
d.addCallback(lambda _: request1.notifyFinish())
return d
def test_that_threads_are_the_same(self):
self.use_same_thread = True
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_the_same)
return pytest_twisted.blockon(d)
def test_that_threads_are_different(self):
self.use_same_thread = False
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_different)
return pytest_twisted.blockon(d)
def StartEnclaveService(config, enclave):
try:
http_port = config['EnclaveService']['HttpPort']
http_host = config['EnclaveService']['Host']
storage_url = config['StorageService']['URL']
worker_threads = config['EnclaveService'].get('WorkerThreads', 8)
reactor_threads = config['EnclaveService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('enclave service started on %s:%s', http_host, http_port)
logger.info('verifying_key: %s', enclave.verifying_key)
logger.info('encryption_key: %s', enclave.encryption_key)
logger.info('enclave_id: %s', enclave.enclave_id)
logger.info('storage service: %s', storage_url)
thread_pool = ThreadPool(minthreads=1, maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(enclave, storage_url))))
root.putChild(
b'initialize',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InitializeApp(enclave))))
root.putChild(
b'invoke',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InvokeApp(enclave))))
root.putChild(
b'verify',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(VerifyApp(enclave))))
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def twisted_adapter(host, port):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
ittyResource = wsgi.WSGIResource(reactor, thread_pool, handle_request)
site = server.Site(ittyResource)
reactor.listenTCP(port, site)
reactor.run()
def twisted_adapter(host, port):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
ittyResource = wsgi.WSGIResource(reactor, thread_pool, handle_request)
site = server.Site(ittyResource)
reactor.listenTCP(port, site)
reactor.run()
class Main(object):
__metaclass__ = Singleton
def __init__(self):
log.debug("[D] %s %s " % (__file__, __name__), "Starting db_threadpool")
self.db_threadpool = ThreadPool(0, config.advanced.db_thread_pool_size)
self.db_threadpool.start()
log.debug("[D] %s %s " % (__file__, __name__), "Starting scheduler_threadpool")
self.scheduler_threadpool = ThreadPool(0, config.advanced.scheduler_thread_pool_size)
self.scheduler_threadpool.start()
self.transactor = Transactor(self.db_threadpool)
self.transactor.retries = 0
def __init__(self, pool=None, minthreads=1, maxthreads=4, **kwargs):
"""Creates a twisted aware Session
Notes
~~~~~
* If you provide both `pool` and `max_workers`, the latter is
ignored and provided threadpool is used as is.
"""
requestsSession.__init__(self, **kwargs)
if pool is None:
pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads)
self.pool = pool
pool.start()
def run(port):
# Create and start a thread pool,
wsgiThreadPool = ThreadPool()
wsgiThreadPool.start()
service_list = []
b = WebRoot()
global_cfg = getGlobalCFG()
global_cfg["local_web_port"] = port
def loadmodule(ob, packagename):
'''
载入扩展模块
'''
if hasattr(ob, "get_module_config"):
config_dict = ob.get_module_config()
web_define = config_dict.get("webmgt")
if web_define != None:
for (url_str,web_cls) in web_define.items():
print("loaded web: {0} ".format(url_str))
web_obj = web_cls()
web_obj.global_cfg = global_cfg
b.mount(url_str, web_obj)
service_define = config_dict.get("service")
if service_define != None:
for (key, service_cls) in service_define.items():
print("loaded service: {0} ".format(str(service_cls)))
service_obj = service_cls()
service_obj.global_cfg = global_cfg
service_list.append(service_obj)
''' 扫描模块存放目录， 加载扩展模块 '''
dir_list = os.listdir(MODULES_PATH)
from dlutil import util_classutil as clsutil
for directory in dir_list:
path_tmp = "{0}/{1}".format(MODULES_PATH, directory)
module_pack = MODULES_PATH.split('/')[-1] + "." + directory
if os.path.isdir(path_tmp) and path_tmp[0:1] != ".":
pack_name = clsutil.find_class_from_path(path_tmp,
loadmodule,
module_pack)
wsgiAppAsResource = WSGIResource(reactor, wsgiThreadPool, b)
site = server.Site(wsgiAppAsResource)
reactor.listenTCP(port, site)
reactor.run()
def twisted_adapter(host, port, application):
from twisted.application import service, strports
from twisted.web import server, http, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
resource = wsgi.WSGIResource(reactor, thread_pool, application)
site = server.Site(resource)
reactor.listenTCP(port, site)
reactor.run()
def twisted_adapter(host, port, application):
from twisted.application import service, strports
from twisted.web import server, http, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
resource = wsgi.WSGIResource(reactor, thread_pool, application)
site = server.Site(resource)
reactor.listenTCP(port, site)
reactor.run()
class TestDonationAPI(TestCase):
"""
Tests for L{bdm.resource.DonationAPI}.
"""
def setUp(self):
self.store = Store()
self.threadPool = ThreadPool()
self.threadPool.start()
self.api = DonationAPI(self.store, 'nothing', self.threadPool)
def tearDown(self):
self.threadPool.stop()
def test_serverStatsSuccess(self):
"""
L{serverStats} returns the expected dictionary results when passed a
valid [IP, PORT].
"""
def _cb(result):
expected = [{
'server_name': 'Test Server',
'map': 'testmap',
'player_count': 8,
'max_players': 16,
'online': True,
'location': 'ZA'
}]
self.assertEqual(expected, result)
servers = [['1.1.1.1', 27015, "ZA"]]
return self.api.serverStats(servers,
querier=MockServerQuerier).addCallback(_cb)
def test_serverStatsOffline(self):
"""
No exception is raised if the server is inaccesable, and the online
status is set to C{False}
"""
def _cb(result):
expected = [{
'server_name': '1.1.1.2',
'online': False,
'location': 'ZA'
}]
self.assertEqual(expected, result)
servers = [['1.1.1.2', 27015, "ZA"]]
return self.api.serverStats(servers,
querier=MockServerQuerier).addCallback(_cb)
class SameOrDifferentThread(TestCase):
def setUp(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.addCleanup(self.tp.stop)
super(SameOrDifferentThread, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('200 OK', [('Content-type','text/plain')])
nameSpace.this_thread = threading.current_thread()
@crosstown_traffic.follow_response(same_thread=self.use_same_thread)
def long_thing_on_same_thread():
nameSpace.thread_that_is_supposed_to_be_the_same = threading.current_thread()
logger.info("Finished async thing on same thread.")
return "Nothing."
def assert_that_threads_are_the_same(self):
self.assertEqual(nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_the_same
)
def assert_that_threads_are_different(self):
self.assertNotEqual(nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_different)
def request_same_or_different_thread_thread(self):
hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
request1 = DummyRequest('r1')
request1.isSecure = lambda: False
request1.content = "llamas"
d = deferToThreadPool(reactor, self.tp, hr.render, request1)
return d
def test_that_threads_are_the_same(self):
self.use_same_thread = True
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_the_same)
return d
def test_that_threads_are_different(self):
self.use_same_thread = False
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_different)
return d
def test_threadLocality(self):
"""
An 'Record' repr()'d in two separate threads at the same time should
look the same (i.e. the repr state tracking for '...' should be
thread-local).
"""
pool = ThreadPool(2, 2)
pool.start()
self.addCleanup(pool.stop)
class StickyRepr(object):
"""
This has a __repr__ which will block until a separate thread
notifies it that it should return. We use this to create a race
condition.
"""
waited = False
def __init__(self):
self.set = threading.Event()
self.wait = threading.Event()
def __repr__(self):
if not self.waited:
self.set.set()
self.wait.wait()
return 'sticky'
r = StickyRepr()
mr = MyRecord(something=1, somethingElse=r)
d = deferToThreadPool(reactor, pool, repr, mr)
def otherRepr():
# First we wait for the first thread doing a repr() to enter its
# __repr__()...
r.set.wait()
# OK, now it's blocked. Let's make sure that subsequent calls to
# this repr() won't block.
r.waited = True
# Do it! This is a concurrent repr().
result = repr(mr)
# Now we're done, wake up the other repr and let it complete.
r.wait.set()
return result
d2 = deferToThreadPool(reactor, pool, otherRepr)
def done(xxx_todo_changeme):
(thread1repr, thread2repr) = xxx_todo_changeme
knownGood = 'MyRecord(something=1, somethingElse=sticky)'
# self.assertEquals(thread1repr, thread2repr)
self.assertEqual(thread1repr, knownGood)
self.assertEqual(thread2repr, knownGood)
return gatherResults([d, d2]).addCallback(done)
def main(config_path):
cfg = ConfigParser()
cfg.read(config_path)
# Start Twisted logging to console.
log.startLogging(stderr)
# Read database configuration options.
db_url = cfg.get('database', 'url')
# Read website configuration options.
http_debug = cfg.getboolean('http', 'debug', fallback=False)
http_host = cfg.get('http', 'host', fallback='localhost')
http_port = cfg.getint('http', 'port', fallback=5000)
http_pool = cfg.getint('http', 'pool_size', fallback=4)
# Default to much saner database query defaults and always
# commit and/or flush statements explicitly.
# factory = sessionmaker(autocommit=False, autoflush=False)
# Prepare database connection with table reflection.
engine = create_engine(db_url)
session = scoped_session(sessionmaker(autocommit=False, autoflush=False))
db = SQLSoup(engine, session=session)
# Extract manager options, sans the pool_size we handle here.
# pool_size = int(manager_opts.pop('pool_size', 2))
pool_size = 2
# Set the correct thread pool size for the manager.
reactor.suggestThreadPoolSize(pool_size)
# Prepare the website that will get exposed to the users.
site = make_site(db, debug=http_debug)
# Prepare WSGI site with a separate thread pool.
pool = ThreadPool(http_pool, http_pool, 'http')
site = Site(WSGIResource(reactor, pool, site))
pool.start()
# Bind the website to it's address.
reactor.listenTCP(http_port, site, interface=http_host)
# Run the Twisted reactor until the user terminates us.
reactor.run()
# Kill the HTTP ThreadPool.
pool.stop()
class FakeReactor:
"""
A fake reactor implementation which just supports enough reactor APIs for
L{ThreadedResolver}.
"""
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def callFromThread(self, callable: Callable[..., Any], *args, **kwargs):
self._threadCalls.put((callable, args, kwargs))
def _runThreadCalls(self):
f, args, kwargs = self._threadCalls.get()
f(*args, **kwargs)
def _stop(self):
self._threadpool.stop()
def getDelayedCalls(self):
# IReactorTime.getDelayedCalls
pass
def seconds(self):
# IReactorTime.seconds
pass
def callInThread(self, callable: Callable[..., Any], *args, **kwargs):
# IReactorInThreads.callInThread
pass
def suggestThreadPoolSize(self, size):
# IReactorThreads.suggestThreadPoolSize
pass
def run_by_pool():
urls = [LINK_URL % n for n in range(1, PAGE_NUM + 1)]
print (urls)
# 5*20 最大100线程在运行
error_log("start：" + str(time.time()))
pool = ThreadPool(minthreads=1, maxthreads=5)
for url in urls:
pool.callInThread(start, url, save_path=IMAGE_SAVE_BASEPATH)
pool.start()
while True:
# 每20s判断一次线程池状态，没有线程正在运行则停止下载进程
time.sleep(20)
if len(pool.working) == 0:
pool.stop()
error_log("end：" + str(time.time()))
break
def add_site(config_func, ini_path="app.ini", relative_to="."):
# Create and start a thread pool,
pool = ThreadPool()
pool.start()
# ensuring that it will be stopped when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
# init site
app = loadapp("config:" + ini_path, relative_to=relative_to)
resource = WSGIResource(reactor, pool, app)
site = Site(resource)
port = config_func("port", default=8080, _type=int)
# just listen
try:
reactor.listenTCP(port, site)
except Exception, e:
reactor.stop()
class VMMasterServer(object):
def __init__(self, reactor, port):
self.reactor = reactor
self.app = create_app()
self.thread_pool = ThreadPool(maxthreads=config.THREAD_POOL_MAX)
self.thread_pool.start()
wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app)
root_resource = RootResource(wsgi_resource)
root_resource.putChild("proxy", ProxyResource(self.app))
site = Site(root_resource)
site.protocol = HTTPChannelWithClient
self.bind = self.reactor.listenTCP(port, site)
log.info('Server is listening on %s ...' % port)
def run(self):
self.reactor.addSystemEventTrigger('before', 'shutdown',
self.before_shutdown)
self.reactor.run()
del self
def __del__(self):
d = self.bind.stopListening()
_block_on(d, 20)
self.app.cleanup()
self.thread_pool.stop()
def wait_for_end_active_sessions(self):
active_sessions = self.app.sessions.active()
def wait_for():
while active_sessions:
for session in active_sessions:
if session.status in ('failed', 'succeed'):
active_sessions.remove(session)
time.sleep(1)
log.info("Wait for end %s active session[s]:"
" %s" % (len(active_sessions), active_sessions))
return deferToThread(wait_for, self).addBoth(
lambda i: log.info("All active sessions has been completed"))
@inlineCallbacks
def before_shutdown(self):
self.app.running = False
yield self.wait_for_end_active_sessions()
def main():
# create thread pool used to serve WSGI requests
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
# create a Twisted Web WSGI resource for our Flask server
wsgi_resource = WSGIResource(reactor, thread_pool, app)
root_resource = WSGIRootResource(wsgi_resource, {})
factory = Site(root_resource)
# mumudvb status thread
statusThread = task.LoopingCall(mumudvbThread,"MuMuDVB thread")
statusThread.start(10, False)
reactor.listenTCP(port, factory)
reactor.run()
def run_app_in_twisted():
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
wsgi_app = WSGIResource(reactor, threadpool, app)
class ServerResource(Resource):
isLeaf = True
def __init__(self, wsgi):
Resource.__init__(self)
self._wsgi = wsgi
def render(self, request):
"""
Adds headers to disable caching of api calls
"""
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
# web-client files served from here
base_resource = File('../client/dist')
# api requests must go through /api
base_resource.putChild('api', ServerResource(wsgi_app))
# downloadable files go here
base_resource.putChild('file', File(config.SAVE_FOLDER))
site = Site(base_resource)
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + str(config.PORT))
endpoint.listen(site)
reactor.run()
class ThreadedRunner(SimpleRunner):
"""Run tests using a threadpool.
Uses TwistedPython's thread pool"""
def __init__(self, result_class):
from twisted.python.threadpool import ThreadPool
SimpleRunner.__init__(self, _threadclass(result_class))
self._pool = ThreadPool()
self._pool.start()
def run(self, fixture):
assert not self._done
self._pool.dispatch(None, fixture, self._result)
def result(self):
self._pool.stop()
return SimpleRunner.result(self)
class TestDonationAPI(TestCase):
"""
Tests for L{bdm.resource.DonationAPI}.
"""
def setUp(self):
self.store = Store()
self.threadPool = ThreadPool()
self.threadPool.start()
self.api = DonationAPI(self.store, 'nothing', self.threadPool)
def tearDown(self):
self.threadPool.stop()
def test_serverStatsSuccess(self):
"""
L{serverStats} returns the expected dictionary results when passed a
valid [IP, PORT].
"""
def _cb(result):
expected = [{'server_name': 'Test Server',
'map': 'testmap',
'player_count': 8,
'max_players': 16,
'online': True,
'location': 'ZA'}]
self.assertEqual(expected, result)
servers = [['1.1.1.1', 27015, "ZA"]]
return self.api.serverStats(servers, querier=MockServerQuerier).addCallback(_cb)
def test_serverStatsOffline(self):
"""
No exception is raised if the server is inaccesable, and the online
status is set to C{False}
"""
def _cb(result):
expected = [{'server_name': '1.1.1.2', 'online': False, 'location':'ZA'}]
self.assertEqual(expected, result)
servers = [['1.1.1.2', 27015, "ZA"]]
return self.api.serverStats(servers, querier=MockServerQuerier).addCallback(_cb)
def __init__(self, pool=None, minthreads=1, maxthreads=4, **kwargs):
"""Creates a twisted aware Session
Notes
~~~~~
* If you provide both `pool` and `max_workers`, the latter is
ignored and provided threadpool is used as is.
"""
requestsSession.__init__(self, **kwargs)
self.ownPool = False
if pool is None:
self.ownPool = True
pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads)
# unclosed ThreadPool leads to reactor hangs at shutdown
# this is a problem in many situation, so better enforce pool stop here
reactor.addSystemEventTrigger("before", "shutdown", lambda:pool.stop())
self.pool = pool
if self.ownPool:
pool.start()
def test_integration(self):
"""
``auto_threaded`` works with ``twisted.python.threads.ThreadPool``.
"""
from twisted.internet import reactor
threadpool = ThreadPool(minthreads=1, name=self.id())
threadpool.start()
self.addCleanup(threadpool.stop)
spy = Spy()
async_spy = AsyncSpy(
reactor=reactor, threadpool=threadpool, provider=spy
)
a = [object()]
b = [object()]
c = [object()]
result = async_spy.method(a, b, c)
result.addCallback(self.assertEqual, spy.method(a, b, c))
return result
def create(transport, path, config):
personality = transport.worker.personality
personality.WEB_SERVICE_CHECKERS['wsgi'](personality, config)
reactor = transport.worker.components_shared['reactor']
if 'module' not in config:
raise ApplicationError(u'crossbar.error.invalid_configuration', 'missing WSGI app module')
if 'object' not in config:
raise ApplicationError(u'crossbar.error.invalid_configuration', 'missing WSGI app object')
# import WSGI app module and object
mod_name = config['module']
try:
mod = importlib.import_module(mod_name)
except ImportError as e:
raise ApplicationError(u'crossbar.error.invalid_configuration', 'WSGI app module "{}" import failed: {} - Python search path was {}'.format(mod_name, e, sys.path))
obj_name = config['object']
if obj_name not in mod.__dict__:
raise ApplicationError(u'crossbar.error.invalid_configuration', 'WSGI app object "{}" not in module "{}"'.format(obj_name, mod_name))
else:
app = getattr(mod, obj_name)
# Create a thread-pool for running the WSGI requests in
pool = ThreadPool(maxthreads=config.get('maxthreads', 20),
minthreads=config.get('minthreads', 0),
name='crossbar_wsgi_threadpool')
reactor.addSystemEventTrigger('before', 'shutdown', pool.stop)
pool.start()
# Create a Twisted Web WSGI resource from the user's WSGI application object
try:
resource = WSGIResource(reactor, pool, app)
if path == '/':
resource = WSGIRootResource(resource, {})
except Exception as e:
raise ApplicationError(u'crossbar.error.invalid_configuration', 'could not instantiate WSGI resource: {}'.format(e))
else:
return RouterWebServiceWsgi(transport, path, config, resource)
def main():
""" Punto de entrada de la aplicacion. """
# Habilitamos la interfaz Web solo si hay conexion a la red local.
if CONFIG["NC_ADDRESS"] is not None:
# Creamos un thread pool para la aplicacion.
threadPool = ThreadPool(maxthreads=CONFIG["SERVER_MAX_THREADS"])
threadPool.start()
reactor.addSystemEventTrigger("before", "shutdown", threadPool.stop)
app.mutex = threading.Lock()
# Creamos los diferentes recursos que servira el servidor de Twisted.
wsgiResource = WSGIResource(reactor, threadPool, app)
staticResource = File(STATIC_FILES_PATH)
tmpCSVResource = File("/tmp")
webSocketFactory = WebSocketServerFactory(
"ws://{0}:{1}".format(CONFIG["SERVER_INTERFACE"], CONFIG["SERVER_PORT"])
)
webSocketFactory.protocol = LibraServerProtocol
webSocketResource = WebSocketResource(webSocketFactory)
rootResource = WSGIRootResource(
wsgiResource, {"static": staticResource, "ws": webSocketResource, "csv": tmpCSVResource}
)
site = Site(rootResource)
reactor.listenTCP(CONFIG["SERVER_PORT"], site)
# Mostramos si hay conexion a la red local.
if not CONFIG["DEVELOPMENT"]:
if CONFIG["NC_ADDRESS"] is not None:
lcd.showMessage("{0}".format(CONFIG["NC_ADDRESS"]), "{0}".format(CONFIG["SERVER_PORT"]))
else:
lcd.showMessage("Desconectado")
reactor.run()
def _create_resource(self, path_config, nested=True):
"""
Creates child resource to be added to the parent.
:param path_config: Configuration for the new child resource.
:type path_config: dict
:returns: Resource -- the new child resource
"""
# WAMP-WebSocket resource
#
if path_config['type'] == 'websocket':
ws_factory = WampWebSocketServerFactory(self._router_session_factory, self.config.extra.cbdir, path_config, self._templates)
# FIXME: Site.start/stopFactory should start/stop factories wrapped as Resources
ws_factory.startFactory()
return WebSocketResource(ws_factory)
# Static file hierarchy resource
#
elif path_config['type'] == 'static':
static_options = path_config.get('options', {})
if 'directory' in path_config:
static_dir = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['directory']))
elif 'package' in path_config:
if 'resource' not in path_config:
raise ApplicationError(u"crossbar.error.invalid_configuration", "missing resource")
try:
mod = importlib.import_module(path_config['package'])
except ImportError as e:
emsg = "Could not import resource {} from package {}: {}".format(path_config['resource'], path_config['package'], e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
try:
static_dir = os.path.abspath(pkg_resources.resource_filename(path_config['package'], path_config['resource']))
except Exception as e:
emsg = "Could not import resource {} from package {}: {}".format(path_config['resource'], path_config['package'], e)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
else:
raise ApplicationError(u"crossbar.error.invalid_configuration", "missing web spec")
static_dir = static_dir.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770
# create resource for file system hierarchy
#
if static_options.get('enable_directory_listing', False):
static_resource_class = StaticResource
else:
static_resource_class = StaticResourceNoListing
cache_timeout = static_options.get('cache_timeout', DEFAULT_CACHE_TIMEOUT)
static_resource = static_resource_class(static_dir, cache_timeout=cache_timeout)
# set extra MIME types
#
static_resource.contentTypes.update(EXTRA_MIME_TYPES)
if 'mime_types' in static_options:
static_resource.contentTypes.update(static_options['mime_types'])
patchFileContentTypes(static_resource)
# render 404 page on any concrete path not found
#
static_resource.childNotFound = Resource404(self._templates, static_dir)
return static_resource
# WSGI resource
#
elif path_config['type'] == 'wsgi':
if not _HAS_WSGI:
raise ApplicationError(u"crossbar.error.invalid_configuration", "WSGI unsupported")
if 'module' not in path_config:
raise ApplicationError(u"crossbar.error.invalid_configuration", "missing WSGI app module")
if 'object' not in path_config:
raise ApplicationError(u"crossbar.error.invalid_configuration", "missing WSGI app object")
# import WSGI app module and object
mod_name = path_config['module']
try:
mod = importlib.import_module(mod_name)
except ImportError as e:
raise ApplicationError(u"crossbar.error.invalid_configuration", "WSGI app module '{}' import failed: {} - Python search path was {}".format(mod_name, e, sys.path))
else:
obj_name = path_config['object']
if obj_name not in mod.__dict__:
raise ApplicationError(u"crossbar.error.invalid_configuration", "WSGI app object '{}' not in module '{}'".format(obj_name, mod_name))
else:
app = getattr(mod, obj_name)
# Create a threadpool for running the WSGI requests in
pool = ThreadPool(maxthreads=path_config.get("maxthreads", 20),
minthreads=path_config.get("minthreads", 0),
name="crossbar_wsgi_threadpool")
self._reactor.addSystemEventTrigger('before', 'shutdown', pool.stop)
pool.start()
# Create a Twisted Web WSGI resource from the user's WSGI application object
try:
wsgi_resource = WSGIResource(self._reactor, pool, app)
if not nested:
wsgi_resource = WSGIRootResource(wsgi_resource, {})
except Exception as e:
raise ApplicationError(u"crossbar.error.invalid_configuration", "could not instantiate WSGI resource: {}".format(e))
else:
return wsgi_resource
# Redirecting resource
#
elif path_config['type'] == 'redirect':
redirect_url = path_config['url'].encode('ascii', 'ignore')
return RedirectResource(redirect_url)
# JSON value resource
#
elif path_config['type'] == 'json':
value = path_config['value']
return JsonResource(value)
# CGI script resource
#
elif path_config['type'] == 'cgi':
cgi_processor = path_config['processor']
cgi_directory = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['directory']))
cgi_directory = cgi_directory.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770
return CgiDirectory(cgi_directory, cgi_processor, Resource404(self._templates, cgi_directory))
# WAMP-Longpoll transport resource
#
elif path_config['type'] == 'longpoll':
path_options = path_config.get('options', {})
lp_resource = WampLongPollResource(self._router_session_factory,
timeout=path_options.get('request_timeout', 10),
killAfter=path_options.get('session_timeout', 30),
queueLimitBytes=path_options.get('queue_limit_bytes', 128 * 1024),
queueLimitMessages=path_options.get('queue_limit_messages', 100),
debug=path_options.get('debug', False),
debug_transport_id=path_options.get('debug_transport_id', None)
)
lp_resource._templates = self._templates
return lp_resource
# Publisher resource (part of REST-bridge)
#
elif path_config['type'] == 'publisher':
# create a vanilla session: the publisher will use this to inject events
#
publisher_session_config = ComponentConfig(realm=path_config['realm'], extra=None)
publisher_session = ApplicationSession(publisher_session_config)
# add the publisher session to the router
#
self._router_session_factory.add(publisher_session, authrole=path_config.get('role', 'anonymous'))
# now create the publisher Twisted Web resource
#
return PublisherResource(path_config.get('options', {}), publisher_session)
# Webhook resource (part of REST-bridge)
#
elif path_config['type'] == 'webhook':
# create a vanilla session: the webhook will use this to inject events
#
webhook_session_config = ComponentConfig(realm=path_config['realm'], extra=None)
webhook_session = ApplicationSession(webhook_session_config)
# add the webhook session to the router
#
self._router_session_factory.add(webhook_session, authrole=path_config.get('role', 'anonymous'))
# now create the webhook Twisted Web resource
#
return WebhookResource(path_config.get('options', {}), webhook_session)
# Caller resource (part of REST-bridge)
#
elif path_config['type'] == 'caller':
# create a vanilla session: the caller will use this to inject calls
#
caller_session_config = ComponentConfig(realm=path_config['realm'], extra=None)
caller_session = ApplicationSession(caller_session_config)
# add the calling session to the router
#
self._router_session_factory.add(caller_session, authrole=path_config.get('role', 'anonymous'))
# now create the caller Twisted Web resource
#
return CallerResource(path_config.get('options', {}), caller_session)
# File Upload resource
#
elif path_config['type'] == 'upload':
upload_directory = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['directory']))
upload_directory = upload_directory.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770
if not os.path.isdir(upload_directory):
emsg = "configured upload directory '{}' in file upload resource isn't a directory".format(upload_directory)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
if 'temp_directory' in path_config:
temp_directory = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['temp_directory']))
temp_directory = temp_directory.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770
else:
temp_directory = os.path.abspath(tempfile.gettempdir())
temp_directory = os.path.join(temp_directory, 'crossbar-uploads')
if not os.path.exists(temp_directory):
os.makedirs(temp_directory)
if not os.path.isdir(temp_directory):
emsg = "configured temp directory '{}' in file upload resource isn't a directory".format(temp_directory)
self.log.error(emsg)
raise ApplicationError(u"crossbar.error.invalid_configuration", emsg)
# file upload progress and finish events are published via this session
#
upload_session_config = ComponentConfig(realm=path_config['realm'], extra=None)
upload_session = ApplicationSession(upload_session_config)
self._router_session_factory.add(upload_session, authrole=path_config.get('role', 'anonymous'))
self.log.info("File upload resource started. Uploads to {upl} using temp folder {tmp}.", upl=upload_directory, tmp=temp_directory)
return FileUploadResource(upload_directory, temp_directory, path_config['form_fields'], upload_session, path_config.get('options', {}))
# Generic Twisted Web resource
#
elif path_config['type'] == 'resource':
try:
klassname = path_config['classname']
self.log.debug("Starting class '{}'".format(klassname))
c = klassname.split('.')
module_name, klass_name = '.'.join(c[:-1]), c[-1]
module = importlib.import_module(module_name)
make = getattr(module, klass_name)
return make(path_config.get('extra', {}))
except Exception as e:
emsg = "Failed to import class '{}' - {}".format(klassname, e)
self.log.error(emsg)
self.log.error("PYTHONPATH: {pythonpath}", pythonpath=sys.path)
raise ApplicationError(u"crossbar.error.class_import_failed", emsg)
# Schema Docs resource
#
elif path_config['type'] == 'schemadoc':
realm = path_config['realm']
if realm not in self.realm_to_id:
raise ApplicationError(u"crossbar.error.no_such_object", "No realm with URI '{}' configured".format(realm))
realm_id = self.realm_to_id[realm]
realm_schemas = self.realms[realm_id].session._schemas
return SchemaDocResource(self._templates, realm, realm_schemas)
# Nested subpath resource
#
elif path_config['type'] == 'path':
nested_paths = path_config.get('paths', {})
if '/' in nested_paths:
nested_resource = self._create_resource(nested_paths['/'])
else:
nested_resource = Resource404(self._templates, b'')
# nest subpaths under the current entry
#
self._add_paths(nested_resource, nested_paths)
return nested_resource
else:
raise ApplicationError(u"crossbar.error.invalid_configuration",
"invalid Web path type '{}' in {} config".format(path_config['type'],
'nested' if nested else 'root'))
def start(self):
reactor.addSystemEventTrigger('during', 'shutdown', self.stop)
ThreadPool.start(self)
