class NoGoStatusCodes(TestCase): def __init__(self, *args, **kwargs): self.tp = ThreadPool(maxthreads=20) self.tp.start() self.resource = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing) self.nameSpace = TestNameSpace() self.nameSpace.async_thing_complete = Queue() return super(NoGoStatusCodes, self).__init__(*args, **kwargs) def setUp(self, *args, **kwargs): self.addCleanup(self.tp.stop) super(NoGoStatusCodes, self).setUp(*args, **kwargs) def wsgi_thing(self, environ, start_response): start_response('404 NOT FOUND', [('Content-type','text/plain')]) @crosstown_traffic.follow_response( no_go_status_codes=self.no_go_status_codes, same_thread=True ) def long_thing_on_same_thread(): self.nameSpace.async_task_was_run = True logger.info("No bad status codes; went ahead with async thing.") return "Nothing." def test_bad_status_codes_cause_no_go_in_wsgi_response(self): self.no_go_status_codes = [404, '6xx'] request = DummyRequest('r1') request.isSecure = lambda: False request.content = "llamas" finished = request.notifyFinish() self.resource.render(request) # This must wait until the WSGI response is closed. finished.addCallback( lambda _: self.assertFalse( self.nameSpace.async_task_was_run ) ) def test_bad_status_codes_cause_no_go_flag(self): through_to_you = crosstown_traffic.follow_response(no_go_status_codes=[418]) through_to_you.status_code = 418 through_to_you.check_status_code_against_no_go_list() self.assertTrue(through_to_you.no_go) def test_no_bad_status_codes_are_cool(self): through_to_you = crosstown_traffic.follow_response(no_go_status_codes=[418]) through_to_you.status_code = 404 through_to_you.check_status_code_against_no_go_list() self.assertFalse(through_to_you.no_go)
def __init__(self, core): thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) application = get_flask_application(core) wsgi_resource = WSGIResource(reactor, thread_pool, application) Site.__init__(self, wsgi_resource)
def from_config(cls, reactor, logger, queue, config_path): with open(config_path) as f: config = yaml.safe_load(f) # TODO: bump this once alchimia properly handles pinning thread_pool = ThreadPool(minthreads=1, maxthreads=1) thread_pool.start() reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop) return cls( logger, DownloadDatabase(reactor, thread_pool, config["db"]["uri"]), FilePath(config["storage"]["filesystem"]), fernet.MultiFernet( [fernet.Fernet(key) for key in config["encryption_keys"]]), VBMSClient( reactor, connect_vbms_path=config["connect_vbms"]["path"], bundle_path=config["connect_vbms"]["bundle_path"], endpoint_url=config["vbms"]["endpoint_url"], keyfile=config["vbms"]["keyfile"], samlfile=config["vbms"]["samlfile"], key=config["vbms"].get("key"), keypass=config["vbms"]["keypass"], ca_cert=config["vbms"].get("ca_cert"), client_cert=config["vbms"].get("client_cert"), ), queue, config["env"], )
class SeleniumDownloadHandler(HTTP11DownloadHandler): def __init__(self, settings): super(SeleniumDownloadHandler, self).__init__(settings) self._enable_driver = settings.getbool('WEB_DRIVER_ENABLED') self._driver_name = settings.get('WEB_DRIVER_NAME') self._driver_path = settings.get('WEB_DRIVER_PATH') selenium_concurrent_request = settings.get( 'WEB_DRIVER_CONCURRENT_REQUESTS', 16) self._thread_pool = ThreadPool(minthreads=selenium_concurrent_request, maxthreads=selenium_concurrent_request) self._thread_pool.start() self._driver_timeout = settings.get('WEB_DRIVER_TIMEOUT', 300) def download_request(self, request, spider): if self._enable_driver and request.meta.get('selenium_needed', False): agent = SeleniumAsyncAgent(contextFactory=self._contextFactory, pool=self._thread_pool, driverName=self._driver_name, driverPath=self._driver_path, connectTimeout=self._driver_timeout) return agent.download_request(request) else: return super(SeleniumDownloadHandler, self).download_request(request, spider) def close(self): super(SeleniumDownloadHandler, self).close() self._thread_pool.stop()
class AddressResolver(object): pool = None def __init__(self, minthreads=1, maxthreads=4): self.pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads) # unclosed ThreadPool leads to reactor hangs at shutdown # this is a problem in many situation, so better enforce pool stop here reactor.addSystemEventTrigger( "before", "shutdown", self.pool.stop ) self.pool.start() def get_host_by_name(self, address): d = defer.Deferred() def func(): try: reactor.callFromThread( d.callback, socket.gethostbyname(address) ) except Exception as e: reactor.callFromThread(d.errback, e) self.pool.callInThread(func) return d def close(self): self.pool.stop()
class AddressResolver(object): pool = None def __init__(self, minthreads=1, maxthreads=4): self.pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads) # unclosed ThreadPool leads to reactor hangs at shutdown # this is a problem in many situation, so better enforce pool stop here reactor.addSystemEventTrigger("before", "shutdown", self.pool.stop) self.pool.start() def get_host_by_name(self, address): d = defer.Deferred() def func(): try: reactor.callFromThread(d.callback, socket.gethostbyname(address)) except Exception as e: reactor.callFromThread(d.errback, e) self.pool.callInThread(func) return d def close(self): self.pool.stop()
class ProviderServer(object): def __init__(self, reactor, port): self.reactor = reactor self.reactor.addSystemEventTrigger('before', 'shutdown', self.before_shutdown) self.reactor.suggestThreadPoolSize(config.REACTOR_THREAD_POOL_MAX) self.app = create_app() self.thread_pool = ThreadPool(maxthreads=config.FLASK_THREAD_POOL_MAX) self.thread_pool.start() wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app) root_resource = RootResource(wsgi_resource) root_resource.putChild("metrics", MetricsResource()) site = Site(root_resource) self.bind = self.reactor.listenTCP(port, site) log.info('Provider is listening on {} ...'.format(port)) def run(self): self.reactor.run() def stop_services(self): log.info("Shutting down provider...") self.app.cleanup() self.thread_pool.stop() return maybeDeferred(self.bind.stopListening).addCallbacks( callback=lambda _: log.info("Port listening was stopped"), errback=lambda failure: log.error( "Error while stopping port listening: {}".format(failure))) @inlineCallbacks def before_shutdown(self): self.app.stop() yield self.stop_services()
def __init__(self, core): thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger("after", "shutdown", thread_pool.stop) application = get_flask_application(core) wsgi_resource = WSGIResource(reactor, thread_pool, application) Site.__init__(self, wsgi_resource)
def makeService(self, options): app = Application("Mercurial SSH Server") #, uid, gid) services = IServiceCollection(app) service = options.subOptions.getService() service.setServiceParent(services) wsgi_app = WSGIApplication() threadpool = ThreadPool(config.web.min_threads, config.web.max_threads) threadpool.start() reactor.addSystemEventTrigger('after', 'shutdown', threadpool.stop) root = wsgi.WSGIResource(reactor, threadpool, wsgi_app) factory = server.Site(root) if isfile(config.web.certificate): from OpenSSL import SSL # Run SSL Server class SSLContext(object): def getContext(self): ctx = SSL.Context(SSL.SSLv23_METHOD) ctx.use_privatekey_file(config.private_key) ctx.use_certificate_file(config.web.certificate) return ctx config_service = internet.SSLServer(config.web.port, factory, SSLContext()) else: config_service = internet.TCPServer(config.web.port, factory) config_service.setServiceParent(app) clean_changes_task = task.LoopingCall(self.__clean_old_changes_from_db) clean_changes_task.start(5*60, now=True) # Every 5 minutes reactor.addSystemEventTrigger('after', 'shutdown', clean_changes_task.stop) return services
class TwistedApplicationServer(ApplicationServer): def __init__(self, app, port, host, debug): super(TwistedApplicationServer, self).__init__(app, port, host, debug) self.reactor = reactor self.thread_pool = ThreadPool(5, 40) self.resource = WSGIResource(self.reactor, self.thread_pool, self.app) self.reactor.addSystemEventTrigger('after', 'shutdown', self.thread_pool.stop) def run(self): print("Begin Listening") self.thread_pool.start() self.reactor.listenTCP(self.port, Site(self.resource), interface=self.host) self.reactor.run() def shutdown(self): print("Received call to shutdown") print("Reactor should have stopped") self.reactor.callFromThread(self.reactor.stop) print("Reactor stop listening") self.reactor.callFromThread(self.reactor.stopListening) print("Reactor Shutdown")
class FakeReactor(object): """ A fake reactor implementation which just supports enough reactor APIs for L{ThreadedResolver}. """ def __init__(self): self._clock = Clock() self.callLater = self._clock.callLater self._threadpool = ThreadPool() self._threadpool.start() self.getThreadPool = lambda: self._threadpool self._threadCalls = Queue() def callFromThread(self, f, *args, **kwargs): self._threadCalls.put((f, args, kwargs)) def _runThreadCalls(self): f, args, kwargs = self._threadCalls.get() f(*args, **kwargs) def _stop(self): self._threadpool.stop()
def from_config(cls, reactor, logger, queue, config_path): with open(config_path) as f: config = yaml.safe_load(f) # TODO: bump this once alchimia properly handles pinning thread_pool = ThreadPool(minthreads=1, maxthreads=1) thread_pool.start() reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop) return cls( logger, DownloadDatabase(reactor, thread_pool, config["db"]["uri"]), FilePath(config["storage"]["filesystem"]), fernet.MultiFernet([ fernet.Fernet(key) for key in config["encryption_keys"] ]), VBMSClient( reactor, connect_vbms_path=config["connect_vbms"]["path"], bundle_path=config["connect_vbms"]["bundle_path"], endpoint_url=config["vbms"]["endpoint_url"], keyfile=config["vbms"]["keyfile"], samlfile=config["vbms"]["samlfile"], key=config["vbms"].get("key"), keypass=config["vbms"]["keypass"], ca_cert=config["vbms"].get("ca_cert"), client_cert=config["vbms"].get("client_cert"), ), queue, config["env"], )
class FakeReactor(object): """ A fake reactor implementation which just supports enough reactor APIs for L{ThreadedResolver}. """ implements(IReactorTime, IReactorThreads) def __init__(self): self._clock = Clock() self.callLater = self._clock.callLater self._threadpool = ThreadPool() self._threadpool.start() self.getThreadPool = lambda: self._threadpool self._threadCalls = Queue() def callFromThread(self, f, *args, **kwargs): self._threadCalls.put((f, args, kwargs)) def _runThreadCalls(self): f, args, kwargs = self._threadCalls.get() f(*args, **kwargs) def _stop(self): self._threadpool.stop()
def test_contemporaneous_requests(self): ''' We're going to create two request-response cycles here: Cycle 1 will begin. Cycle 2 will begin. Cycle 2 will return. Cycle 1 will return. This way, we can prove that the crosstown_traffic created by cycle 1 is not resolved by the return of cycle 2. ''' tp = ThreadPool(maxthreads=20) tp.start() self.addCleanup(tp.stop) log.debug("\n\nStarting the two stream stuff.") request1 = DummyRequest('r1') request1.isSecure = lambda: False request1.content = "Nothing really here." request1.headers['llamas'] = 'dingo' nameSpace.test_case = self hr = HendrixWSGIResource(reactor, tp, wsgi_application) d1 = deferToThreadPool(reactor, tp, hr.render, request1) request2 = DummyRequest('r2') request2.isSecure = lambda: False request2.content = "Nothing really here." request2.headers['llamas'] = 'dingo' d2 = deferToThreadPool(reactor, tp, hr.render, request2) def woah_stop(failure): nameSpace.async_task_was_done.put_nowait(False) nameSpace.second_cycle_complete.put_nowait(False) nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False) d1.addErrback(woah_stop) d2.addErrback(woah_stop) combo_deferred = gatherResults([d1, d2]) def wait_for_queue_resolution(): nameSpace.async_task_was_done.get(True, 3) combo_deferred.addCallback( lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution) ) combo_deferred.addCallback( lambda _: self.assertTrue(nameSpace.async_task_was_run) ) return combo_deferred
def runtwisted(config=None): """ Run the Twisted server. """ globalLogBeginner.beginLoggingTo( [FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")]) threadpool = ThreadPool(maxthreads=30) app = api.makeapp(config=config) wsgi_app = WSGIResource(reactor, threadpool, app) class OptimaResource(Resource): isLeaf = True def __init__(self, wsgi): self._wsgi = wsgi def render(self, request): request.prepath = [] request.postpath = ['api'] + request.postpath[:] r = self._wsgi.render(request) request.responseHeaders.setRawHeaders( b'Cache-Control', [b'no-cache', b'no-store', b'must-revalidate']) request.responseHeaders.setRawHeaders(b'expires', [b'0']) return r # If we have a full path for the client directory, use that directory. if os.path.isabs(config.CLIENT_DIR): clientDirTarget = config.CLIENT_DIR # Otherwise (we have a relative path), use it (correcting so it is with # respect to the sciris repo directory). else: clientDirTarget = '%s%s%s' % (os.pardir, os.sep, config.CLIENT_DIR) base_resource = File('%s%sdist%s' % (clientDirTarget, os.sep, os.sep)) base_resource.putChild( 'dev', File('%s%ssrc%s' % (clientDirTarget, os.sep, os.sep))) base_resource.putChild('api', OptimaResource(wsgi_app)) site = Site(base_resource) try: port = str(sys.argv[1]) except IndexError: port = "8091" # Start the threadpool now, shut it down when we're closing threadpool.start() reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop) endpoint = serverFromString(reactor, "tcp:port=" + port) endpoint.listen(site) reactor.run()
def _test_StreamingServer(): expected_id = 'test' headers = { #'TimeSeekRange.dlna.org': 'npt=30.000-', 'TimeSeekRange.dlna.org': 'npt=20.000-50.000', #'Range': 'bytes=5-', 'Connection': 'close', } duration = '00:01:00' content_length = 64 * 1024 interface = '192.168.0.103' class StreamingServerStub(ByteSeekMixin, TimeSeekMixin, StreamingServer): def get_content(self, id, environ): eq_(expected_id, id) #raise ValueError(repr(environ)) for k in headers: eq_(headers[k], environ['HTTP_' + k.upper()]) return ContentStub(content_length, 'video/mpeg', 'DLNA.ORG_PN=MPEG_PS_NTSC;DLNA.ORG_OP=11') from twisted.web import client class HTTPPageGetter(client.HTTPPageGetter): handleStatus_206 = lambda self: self.handleStatus_200() class HTTPClientFactory(client.HTTPClientFactory): protocol = HTTPPageGetter def getPageFactory(url, *args, **kwargs): scheme, host, port, path = client._parse(url) factory = HTTPClientFactory(url, *args, **kwargs) reactor.connectTCP(host, port, factory) return factory app = StreamingServerStub('test') mapper = Mapper() mapper.connect(':id', controller='mt', action='get') app = RoutesMiddleware(app, mapper) tpool = ThreadPool() tpool.start() resource = upnp.WSGIResource(reactor, tpool, app) port = reactor.listenTCP(0, server.Site(resource), interface=interface) port_num = port.socket.getsockname()[1] url = 'http://%s:%i/%s?duration=%s' % (interface, port_num, expected_id, duration) factory = getPageFactory(url, headers=headers) def check_result(contents): raise ValueError('expected: %d, actual: %d' % (content_length, len(contents))) #raise ValueError(repr(factory.response_headers)) eq_(content_length, len(contents)) factory.deferred.addCallback(check_result) reactor.callLater(5, reactor.stop) reactor.run()
def StartStorageService(config, block_store): try: http_port = config['StorageService']['HttpPort'] http_host = config['StorageService']['Host'] worker_threads = config['StorageService'].get('WorkerThreads', 8) reactor_threads = config['StorageService'].get('ReactorThreads', 8) except KeyError as ke: logger.error('missing configuration for %s', str(ke)) sys.exit(-1) logger.info('service started on port %s', http_port) thread_pool = ThreadPool(maxthreads=worker_threads) thread_pool.start() reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop) block = Resource() block.putChild( b'get', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(GetBlockApp(block_store)))) block.putChild( b'gets', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(GetBlocksApp(block_store)))) block.putChild( b'store', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(StoreBlocksApp(block_store)))) block.putChild( b'list', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(ListBlocksApp(block_store)))) block.putChild( b'check', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(CheckBlocksApp(block_store)))) root = Resource() root.putChild( b'info', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(InfoApp(block_store)))) root.putChild(b'block', block) site = Site(root, timeout=60) site.displayTracebacks = True reactor.suggestThreadPoolSize(reactor_threads) signal.signal(signal.SIGQUIT, __shutdown__) signal.signal(signal.SIGTERM, __shutdown__) endpoint = TCP4ServerEndpoint(reactor, http_port, backlog=32, interface=http_host) endpoint.listen(site)
def test_contemporaneous_requests(self): ''' We're going to create two request-response cycles here: Cycle 1 will begin. Cycle 2 will begin. Cycle 2 will return. Cycle 1 will return. This way, we can prove that the crosstown_traffic created by cycle 1 is not resolved by the return of cycle 2. ''' tp = ThreadPool(maxthreads=20) tp.start() self.addCleanup(tp.stop) log.debug("\n\nStarting the two stream stuff.") request1 = DummyRequest([b'r1']) request1.isSecure = lambda: False request1.content = "Nothing really here." request1.requestHeaders.addRawHeader('llamas', 'dingo') request1.client = IPv4Address("TCP", b"50.0.50.0", 5000) nameSpace.test_case = self hr = HendrixWSGIResource(reactor, tp, wsgi_application) d1 = deferToThreadPool(reactor, tp, hr.render, request1) request2 = DummyRequest([b'r2']) request2.isSecure = lambda: False request2.content = b"Nothing really here." request2.requestHeaders.addRawHeader('llamas', 'dingo') request2.client = IPv4Address("TCP", b"100.0.50.0", 5000) d2 = deferToThreadPool(reactor, tp, hr.render, request2) def woah_stop(failure): nameSpace.async_task_was_done.put_nowait(False) nameSpace.second_cycle_complete.put_nowait(False) nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False) d1.addErrback(woah_stop) d2.addErrback(woah_stop) combo_deferred = gatherResults([d1, d2]) def wait_for_queue_resolution(): nameSpace.async_task_was_done.get(True, 3) combo_deferred.addCallback(lambda _: deferToThreadPool( reactor, tp, wait_for_queue_resolution)) combo_deferred.addCallback( lambda _: self.assertTrue(nameSpace.async_task_was_run)) return combo_deferred
def start(self): """ Starts the logger pool and sets up the required shutdown event trigger which will call :meth:`stop` on exit. """ reactor.addSystemEventTrigger("before", "shutdown", self.stop) ThreadPool.start(self) logger.debug( "Started logger thread pool (min=%s, max=%s)", self.min, self.max)
def run(self, handler): from twisted.web import server, wsgi from twisted.python.threadpool import ThreadPool from twisted.internet import reactor thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) reactor.listenTCP(self.port, factory, interface=self.host)
class URLTranServer(object): def __init__(self, app): self.pool = ThreadPool() self.pool.start() self.resource = _WSGIResource(reactor, self.pool, app) self.site = _Site(self.resource) def register(self, port): serve_register_tcp(self.site, port)
class VMMasterServer(object): def __init__(self, reactor, port): self.reactor = reactor self.app = create_app() self.thread_pool = ThreadPool(maxthreads=config.THREAD_POOL_MAX) self.thread_pool.start() wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app) root_resource = RootResource(wsgi_resource) root_resource.putChild("proxy", ProxyResource(self.app)) site = Site(root_resource) site.protocol = HTTPChannelWithClient self.bind = self.reactor.listenTCP(port, site) log.info('Server is listening on %s ...' % port) def run(self): self.reactor.addSystemEventTrigger('before', 'shutdown', self.before_shutdown) self.reactor.run() del self def __del__(self): log.info("Shutting down server...") d = self.bind.stopListening() _block_on(d, 20) self.app.cleanup() self.thread_pool.stop() log.info("Server gracefully shut down") def wait_for_end_active_sessions(self): active_sessions = self.app.sessions.active() def wait_for(): while active_sessions: log.info("Waiting for {} sessions to complete: {}".format( len(active_sessions), [(i.id, i.status) for i in active_sessions])) for session in active_sessions: if session.is_done: log.debug("Session {} is done".format(session.id)) active_sessions.remove(session) time.sleep(1) log.info("Wait for end %s active session[s]:" " %s" % (len(active_sessions), active_sessions)) return deferToThread(wait_for).addCallbacks( callback=lambda _: log.info( "All active sessions has been completed"), errback=lambda failure: log.error( "Error while waiting for active_sessions: {}".format(failure))) @inlineCallbacks def before_shutdown(self): self.app.running = False yield self.wait_for_end_active_sessions()
def twisted(app, address, **options): from twisted.web import server, wsgi from twisted.python.threadpool import ThreadPool from twisted.internet import reactor thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, app)) reactor.listenTCP(address[1], factory, interface=address[0]) reactor.run()
def run(self, handler): from twisted.web import server, wsgi from twisted.python.threadpool import ThreadPool from twisted.internet import reactor thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) reactor.listenTCP(self.port, factory, interface=self.host) reactor.run()
class NoGoStatusCodes(TestCase): def __init__(self, *args, **kwargs): self.tp = ThreadPool(maxthreads=20) self.tp.start() self.resource = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing) self.nameSpace = TestNameSpace() self.nameSpace.async_thing_complete = Queue() return super(NoGoStatusCodes, self).__init__(*args, **kwargs) def setUp(self, *args, **kwargs): self.addCleanup(self.tp.stop) super(NoGoStatusCodes, self).setUp(*args, **kwargs) def wsgi_thing(self, environ, start_response): start_response('404 NOT FOUND', [('Content-type', 'text/plain')]) @crosstown_traffic(no_go_status_codes=self.no_go_status_codes, same_thread=True) def long_thing_on_same_thread(): self.nameSpace.async_task_was_run = True log.debug("No bad status codes; went ahead with async thing.") return [b"Nothing."] def test_bad_status_codes_cause_no_go_in_wsgi_response(self): self.no_go_status_codes = [404, '6xx'] request = DummyRequest([b'r1']) request.isSecure = lambda: False request.content = "llamas" request.client = IPv4Address("TCP", b"50.0.50.0", 5000) finished = request.notifyFinish() self.resource.render(request) # This must wait until the WSGI response is closed. finished.addCallback( lambda _: self.assertFalse(self.nameSpace.async_task_was_run)) return finished def test_bad_status_codes_cause_no_go_flag(self): through_to_you = crosstown_traffic(no_go_status_codes=[418]) through_to_you.status_code = 418 through_to_you.check_status_code_against_no_go_list() self.assertTrue(through_to_you.no_go) def test_no_bad_status_codes_are_cool(self): through_to_you = crosstown_traffic(no_go_status_codes=[418]) through_to_you.status_code = 404 through_to_you.check_status_code_against_no_go_list() self.assertFalse(through_to_you.no_go)
def test_threadLocality(self): """ An 'Record' repr()'d in two separate threads at the same time should look the same (i.e. the repr state tracking for '...' should be thread-local). """ pool = ThreadPool(2, 2) pool.start() self.addCleanup(pool.stop) class StickyRepr(object): """ This has a __repr__ which will block until a separate thread notifies it that it should return. We use this to create a race condition. """ waited = False def __init__(self): self.set = threading.Event() self.wait = threading.Event() def __repr__(self): if not self.waited: self.set.set() self.wait.wait() return 'sticky' r = StickyRepr() mr = MyRecord(something=1, somethingElse=r) d = deferToThreadPool(reactor, pool, repr, mr) def otherRepr(): # First we wait for the first thread doing a repr() to enter its # __repr__()... r.set.wait() # OK, now it's blocked. Let's make sure that subsequent calls to # this repr() won't block. r.waited = True # Do it! This is a concurrent repr(). result = repr(mr) # Now we're done, wake up the other repr and let it complete. r.wait.set() return result d2 = deferToThreadPool(reactor, pool, otherRepr) def done(xxx_todo_changeme): (thread1repr, thread2repr) = xxx_todo_changeme knownGood = 'MyRecord(something=1, somethingElse=sticky)' # self.assertEquals(thread1repr, thread2repr) self.assertEqual(thread1repr, knownGood) self.assertEqual(thread2repr, knownGood) return gatherResults([d, d2]).addCallback(done)
class ThreadPoolService(Service): def __init__(self): self.threadpool = ThreadPool() def startService(self): self.threadpool.start() def stopService(self): self.threadpool.stop()
def get_twisted_pool(): global pool try: return pool except: pool = ThreadPool(minthreads=min_t, maxthreads=max_t, name='core') pool.start() reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) return pool
def StartService(config): # load the eservice database try: dbfile_name = config['Service']['EnclaveServiceDatabaseFile'] except KeyError: logger.error( 'missing required configuration for enclave service database') sys.exit(-1) try: eservice_db.load_database(dbfile_name) except Exception as e: logger.error('error loading eservice database from file %s', str(e)) sys.exit(-1) try: http_port = config['Service']['HttpPort'] http_host = config['Service']['Host'] worker_threads = config['Service'].get('WorkerThreads', 8) reactor_threads = config['Service'].get('ReactorThreads', 8) except KeyError as ke: logger.error('missing configuration for %s', str(ke)) sys.exit(-1) logger.info('service started on host %s, port %s', http_host, http_port) thread_pool = ThreadPool(maxthreads=worker_threads) thread_pool.start() reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop) flask_app = toxaway.views.register(config) flask_site = WSGIResource(reactor, reactor.getThreadPool(), flask_app) root = Resource() root.putChild(b'shutdown', ShutdownResource()) root.putChild(b'flask', flask_site) files = config.get('StaticContent', {}) logger.info('files=%s', files) for key, val in files.items(): logger.info('map <%s> to <%s>', key, val) root.putChild(key.encode(), File(val.encode())) site = Site(root, timeout=60) site.displayTracebacks = True reactor.suggestThreadPoolSize(reactor_threads) endpoint = TCP4ServerEndpoint(reactor, http_port, backlog=32, interface=http_host) endpoint.listen(site)
class SameOrDifferentThread(TestCase): def setUp(self, *args, **kwargs): self.tp = ThreadPool() self.tp.start() self.addCleanup(self.tp.stop) super(SameOrDifferentThread, self).setUp(*args, **kwargs) def wsgi_thing(self, environ, start_response): start_response('200 OK', [('Content-type', 'text/plain')]) nameSpace.this_thread = threading.current_thread() @crosstown_traffic( same_thread=self.use_same_thread ) def long_thing_on_same_thread(): nameSpace.thread_that_is_supposed_to_be_the_same = threading.current_thread() log.debug("Finished async thing on same thread.") return [b"Nothing."] def assert_that_threads_are_the_same(self): self.assertEqual( nameSpace.this_thread, nameSpace.thread_that_is_supposed_to_be_the_same ) def assert_that_threads_are_different(self): self.assertNotEqual(nameSpace.this_thread, nameSpace.thread_that_is_supposed_to_be_different) def request_same_or_different_thread_thread(self): hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing) request1 = DummyRequest([b'r1']) request1.isSecure = lambda: False request1.content = b"llamas" request1.client = IPv4Address("TCP", b"50.0.50.0", 5000) d = deferToThreadPool(reactor, self.tp, hr.render, request1) d.addCallback(lambda _: request1.notifyFinish()) return d def test_that_threads_are_the_same(self): self.use_same_thread = True d = self.request_same_or_different_thread_thread() d.addCallback(lambda _: self.assert_that_threads_are_the_same) return pytest_twisted.blockon(d) def test_that_threads_are_different(self): self.use_same_thread = False d = self.request_same_or_different_thread_thread() d.addCallback(lambda _: self.assert_that_threads_are_different) return pytest_twisted.blockon(d)
def StartEnclaveService(config, enclave): try: http_port = config['EnclaveService']['HttpPort'] http_host = config['EnclaveService']['Host'] storage_url = config['StorageService']['URL'] worker_threads = config['EnclaveService'].get('WorkerThreads', 8) reactor_threads = config['EnclaveService'].get('ReactorThreads', 8) except KeyError as ke: logger.error('missing configuration for %s', str(ke)) sys.exit(-1) logger.info('enclave service started on %s:%s', http_host, http_port) logger.info('verifying_key: %s', enclave.verifying_key) logger.info('encryption_key: %s', enclave.encryption_key) logger.info('enclave_id: %s', enclave.enclave_id) logger.info('storage service: %s', storage_url) thread_pool = ThreadPool(minthreads=1, maxthreads=worker_threads) thread_pool.start() reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop) root = Resource() root.putChild( b'info', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(InfoApp(enclave, storage_url)))) root.putChild( b'initialize', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(InitializeApp(enclave)))) root.putChild( b'invoke', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(InvokeApp(enclave)))) root.putChild( b'verify', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(VerifyApp(enclave)))) site = Site(root, timeout=60) site.displayTracebacks = True reactor.suggestThreadPoolSize(reactor_threads) signal.signal(signal.SIGQUIT, __shutdown__) signal.signal(signal.SIGTERM, __shutdown__) endpoint = TCP4ServerEndpoint(reactor, http_port, backlog=32, interface=http_host) endpoint.listen(site)
def twisted_adapter(host, port): from twisted.web import server, wsgi from twisted.python.threadpool import ThreadPool from twisted.internet import reactor thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) ittyResource = wsgi.WSGIResource(reactor, thread_pool, handle_request) site = server.Site(ittyResource) reactor.listenTCP(port, site) reactor.run()
class Main(object): __metaclass__ = Singleton def __init__(self): log.debug("[D] %s %s " % (__file__, __name__), "Starting db_threadpool") self.db_threadpool = ThreadPool(0, config.advanced.db_thread_pool_size) self.db_threadpool.start() log.debug("[D] %s %s " % (__file__, __name__), "Starting scheduler_threadpool") self.scheduler_threadpool = ThreadPool(0, config.advanced.scheduler_thread_pool_size) self.scheduler_threadpool.start() self.transactor = Transactor(self.db_threadpool) self.transactor.retries = 0
def __init__(self, pool=None, minthreads=1, maxthreads=4, **kwargs): """Creates a twisted aware Session Notes ~~~~~ * If you provide both `pool` and `max_workers`, the latter is ignored and provided threadpool is used as is. """ requestsSession.__init__(self, **kwargs) if pool is None: pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads) self.pool = pool pool.start()
def run(port): # Create and start a thread pool, wsgiThreadPool = ThreadPool() wsgiThreadPool.start() service_list = [] b = WebRoot() global_cfg = getGlobalCFG() global_cfg["local_web_port"] = port def loadmodule(ob, packagename): ''' 载入扩展模块 ''' if hasattr(ob, "get_module_config"): config_dict = ob.get_module_config() web_define = config_dict.get("webmgt") if web_define != None: for (url_str,web_cls) in web_define.items(): print("loaded web: {0} ".format(url_str)) web_obj = web_cls() web_obj.global_cfg = global_cfg b.mount(url_str, web_obj) service_define = config_dict.get("service") if service_define != None: for (key, service_cls) in service_define.items(): print("loaded service: {0} ".format(str(service_cls))) service_obj = service_cls() service_obj.global_cfg = global_cfg service_list.append(service_obj) ''' 扫描模块存放目录, 加载扩展模块 ''' dir_list = os.listdir(MODULES_PATH) from dlutil import util_classutil as clsutil for directory in dir_list: path_tmp = "{0}/{1}".format(MODULES_PATH, directory) module_pack = MODULES_PATH.split('/')[-1] + "." + directory if os.path.isdir(path_tmp) and path_tmp[0:1] != ".": pack_name = clsutil.find_class_from_path(path_tmp, loadmodule, module_pack) wsgiAppAsResource = WSGIResource(reactor, wsgiThreadPool, b) site = server.Site(wsgiAppAsResource) reactor.listenTCP(port, site) reactor.run()
def twisted_adapter(host, port, application): from twisted.application import service, strports from twisted.web import server, http, wsgi from twisted.python.threadpool import ThreadPool from twisted.internet import reactor thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) resource = wsgi.WSGIResource(reactor, thread_pool, application) site = server.Site(resource) reactor.listenTCP(port, site) reactor.run()
class TestDonationAPI(TestCase): """ Tests for L{bdm.resource.DonationAPI}. """ def setUp(self): self.store = Store() self.threadPool = ThreadPool() self.threadPool.start() self.api = DonationAPI(self.store, 'nothing', self.threadPool) def tearDown(self): self.threadPool.stop() def test_serverStatsSuccess(self): """ L{serverStats} returns the expected dictionary results when passed a valid [IP, PORT]. """ def _cb(result): expected = [{ 'server_name': 'Test Server', 'map': 'testmap', 'player_count': 8, 'max_players': 16, 'online': True, 'location': 'ZA' }] self.assertEqual(expected, result) servers = [['1.1.1.1', 27015, "ZA"]] return self.api.serverStats(servers, querier=MockServerQuerier).addCallback(_cb) def test_serverStatsOffline(self): """ No exception is raised if the server is inaccesable, and the online status is set to C{False} """ def _cb(result): expected = [{ 'server_name': '1.1.1.2', 'online': False, 'location': 'ZA' }] self.assertEqual(expected, result) servers = [['1.1.1.2', 27015, "ZA"]] return self.api.serverStats(servers, querier=MockServerQuerier).addCallback(_cb)
class SameOrDifferentThread(TestCase): def setUp(self, *args, **kwargs): self.tp = ThreadPool(maxthreads=20) self.tp.start() self.addCleanup(self.tp.stop) super(SameOrDifferentThread, self).setUp(*args, **kwargs) def wsgi_thing(self, environ, start_response): start_response('200 OK', [('Content-type','text/plain')]) nameSpace.this_thread = threading.current_thread() @crosstown_traffic.follow_response(same_thread=self.use_same_thread) def long_thing_on_same_thread(): nameSpace.thread_that_is_supposed_to_be_the_same = threading.current_thread() logger.info("Finished async thing on same thread.") return "Nothing." def assert_that_threads_are_the_same(self): self.assertEqual(nameSpace.this_thread, nameSpace.thread_that_is_supposed_to_be_the_same ) def assert_that_threads_are_different(self): self.assertNotEqual(nameSpace.this_thread, nameSpace.thread_that_is_supposed_to_be_different) def request_same_or_different_thread_thread(self): hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing) request1 = DummyRequest('r1') request1.isSecure = lambda: False request1.content = "llamas" d = deferToThreadPool(reactor, self.tp, hr.render, request1) return d def test_that_threads_are_the_same(self): self.use_same_thread = True d = self.request_same_or_different_thread_thread() d.addCallback(lambda _: self.assert_that_threads_are_the_same) return d def test_that_threads_are_different(self): self.use_same_thread = False d = self.request_same_or_different_thread_thread() d.addCallback(lambda _: self.assert_that_threads_are_different) return d
def main(config_path): cfg = ConfigParser() cfg.read(config_path) # Start Twisted logging to console. log.startLogging(stderr) # Read database configuration options. db_url = cfg.get('database', 'url') # Read website configuration options. http_debug = cfg.getboolean('http', 'debug', fallback=False) http_host = cfg.get('http', 'host', fallback='localhost') http_port = cfg.getint('http', 'port', fallback=5000) http_pool = cfg.getint('http', 'pool_size', fallback=4) # Default to much saner database query defaults and always # commit and/or flush statements explicitly. # factory = sessionmaker(autocommit=False, autoflush=False) # Prepare database connection with table reflection. engine = create_engine(db_url) session = scoped_session(sessionmaker(autocommit=False, autoflush=False)) db = SQLSoup(engine, session=session) # Extract manager options, sans the pool_size we handle here. # pool_size = int(manager_opts.pop('pool_size', 2)) pool_size = 2 # Set the correct thread pool size for the manager. reactor.suggestThreadPoolSize(pool_size) # Prepare the website that will get exposed to the users. site = make_site(db, debug=http_debug) # Prepare WSGI site with a separate thread pool. pool = ThreadPool(http_pool, http_pool, 'http') site = Site(WSGIResource(reactor, pool, site)) pool.start() # Bind the website to it's address. reactor.listenTCP(http_port, site, interface=http_host) # Run the Twisted reactor until the user terminates us. reactor.run() # Kill the HTTP ThreadPool. pool.stop()
class FakeReactor: """ A fake reactor implementation which just supports enough reactor APIs for L{ThreadedResolver}. """ def __init__(self): self._clock = Clock() self.callLater = self._clock.callLater self._threadpool = ThreadPool() self._threadpool.start() self.getThreadPool = lambda: self._threadpool self._threadCalls = Queue() def callFromThread(self, callable: Callable[..., Any], *args, **kwargs): self._threadCalls.put((callable, args, kwargs)) def _runThreadCalls(self): f, args, kwargs = self._threadCalls.get() f(*args, **kwargs) def _stop(self): self._threadpool.stop() def getDelayedCalls(self): # IReactorTime.getDelayedCalls pass def seconds(self): # IReactorTime.seconds pass def callInThread(self, callable: Callable[..., Any], *args, **kwargs): # IReactorInThreads.callInThread pass def suggestThreadPoolSize(self, size): # IReactorThreads.suggestThreadPoolSize pass
def run_by_pool(): urls = [LINK_URL % n for n in range(1, PAGE_NUM + 1)] print (urls) # 5*20 最大100线程在运行 error_log("start:" + str(time.time())) pool = ThreadPool(minthreads=1, maxthreads=5) for url in urls: pool.callInThread(start, url, save_path=IMAGE_SAVE_BASEPATH) pool.start() while True: # 每20s判断一次线程池状态,没有线程正在运行则停止下载进程 time.sleep(20) if len(pool.working) == 0: pool.stop() error_log("end:" + str(time.time())) break
def add_site(config_func, ini_path="app.ini", relative_to="."): # Create and start a thread pool, pool = ThreadPool() pool.start() # ensuring that it will be stopped when the reactor shuts down reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) # init site app = loadapp("config:" + ini_path, relative_to=relative_to) resource = WSGIResource(reactor, pool, app) site = Site(resource) port = config_func("port", default=8080, _type=int) # just listen try: reactor.listenTCP(port, site) except Exception, e: reactor.stop()
class VMMasterServer(object): def __init__(self, reactor, port): self.reactor = reactor self.app = create_app() self.thread_pool = ThreadPool(maxthreads=config.THREAD_POOL_MAX) self.thread_pool.start() wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app) root_resource = RootResource(wsgi_resource) root_resource.putChild("proxy", ProxyResource(self.app)) site = Site(root_resource) site.protocol = HTTPChannelWithClient self.bind = self.reactor.listenTCP(port, site) log.info('Server is listening on %s ...' % port) def run(self): self.reactor.addSystemEventTrigger('before', 'shutdown', self.before_shutdown) self.reactor.run() del self def __del__(self): d = self.bind.stopListening() _block_on(d, 20) self.app.cleanup() self.thread_pool.stop() def wait_for_end_active_sessions(self): active_sessions = self.app.sessions.active() def wait_for(): while active_sessions: for session in active_sessions: if session.status in ('failed', 'succeed'): active_sessions.remove(session) time.sleep(1) log.info("Wait for end %s active session[s]:" " %s" % (len(active_sessions), active_sessions)) return deferToThread(wait_for, self).addBoth( lambda i: log.info("All active sessions has been completed")) @inlineCallbacks def before_shutdown(self): self.app.running = False yield self.wait_for_end_active_sessions()
def main(): # create thread pool used to serve WSGI requests thread_pool = ThreadPool() thread_pool.start() reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop) # create a Twisted Web WSGI resource for our Flask server wsgi_resource = WSGIResource(reactor, thread_pool, app) root_resource = WSGIRootResource(wsgi_resource, {}) factory = Site(root_resource) # mumudvb status thread statusThread = task.LoopingCall(mumudvbThread,"MuMuDVB thread") statusThread.start(10, False) reactor.listenTCP(port, factory) reactor.run()
def run_app_in_twisted(): globalLogBeginner.beginLoggingTo( [FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")]) threadpool = ThreadPool(maxthreads=30) wsgi_app = WSGIResource(reactor, threadpool, app) class ServerResource(Resource): isLeaf = True def __init__(self, wsgi): Resource.__init__(self) self._wsgi = wsgi def render(self, request): """ Adds headers to disable caching of api calls """ request.prepath = [] request.postpath = ['api'] + request.postpath[:] r = self._wsgi.render(request) request.responseHeaders.setRawHeaders( b'Cache-Control', [b'no-cache', b'no-store', b'must-revalidate']) request.responseHeaders.setRawHeaders(b'expires', [b'0']) return r # web-client files served from here base_resource = File('../client/dist') # api requests must go through /api base_resource.putChild('api', ServerResource(wsgi_app)) # downloadable files go here base_resource.putChild('file', File(config.SAVE_FOLDER)) site = Site(base_resource) # Start the threadpool now, shut it down when we're closing threadpool.start() reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop) endpoint = serverFromString(reactor, "tcp:port=" + str(config.PORT)) endpoint.listen(site) reactor.run()
class ThreadedRunner(SimpleRunner): """Run tests using a threadpool. Uses TwistedPython's thread pool""" def __init__(self, result_class): from twisted.python.threadpool import ThreadPool SimpleRunner.__init__(self, _threadclass(result_class)) self._pool = ThreadPool() self._pool.start() def run(self, fixture): assert not self._done self._pool.dispatch(None, fixture, self._result) def result(self): self._pool.stop() return SimpleRunner.result(self)
class TestDonationAPI(TestCase): """ Tests for L{bdm.resource.DonationAPI}. """ def setUp(self): self.store = Store() self.threadPool = ThreadPool() self.threadPool.start() self.api = DonationAPI(self.store, 'nothing', self.threadPool) def tearDown(self): self.threadPool.stop() def test_serverStatsSuccess(self): """ L{serverStats} returns the expected dictionary results when passed a valid [IP, PORT]. """ def _cb(result): expected = [{'server_name': 'Test Server', 'map': 'testmap', 'player_count': 8, 'max_players': 16, 'online': True, 'location': 'ZA'}] self.assertEqual(expected, result) servers = [['1.1.1.1', 27015, "ZA"]] return self.api.serverStats(servers, querier=MockServerQuerier).addCallback(_cb) def test_serverStatsOffline(self): """ No exception is raised if the server is inaccesable, and the online status is set to C{False} """ def _cb(result): expected = [{'server_name': '1.1.1.2', 'online': False, 'location':'ZA'}] self.assertEqual(expected, result) servers = [['1.1.1.2', 27015, "ZA"]] return self.api.serverStats(servers, querier=MockServerQuerier).addCallback(_cb)
def __init__(self, pool=None, minthreads=1, maxthreads=4, **kwargs): """Creates a twisted aware Session Notes ~~~~~ * If you provide both `pool` and `max_workers`, the latter is ignored and provided threadpool is used as is. """ requestsSession.__init__(self, **kwargs) self.ownPool = False if pool is None: self.ownPool = True pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads) # unclosed ThreadPool leads to reactor hangs at shutdown # this is a problem in many situation, so better enforce pool stop here reactor.addSystemEventTrigger("before", "shutdown", lambda:pool.stop()) self.pool = pool if self.ownPool: pool.start()
def test_integration(self): """ ``auto_threaded`` works with ``twisted.python.threads.ThreadPool``. """ from twisted.internet import reactor threadpool = ThreadPool(minthreads=1, name=self.id()) threadpool.start() self.addCleanup(threadpool.stop) spy = Spy() async_spy = AsyncSpy( reactor=reactor, threadpool=threadpool, provider=spy ) a = [object()] b = [object()] c = [object()] result = async_spy.method(a, b, c) result.addCallback(self.assertEqual, spy.method(a, b, c)) return result
def create(transport, path, config): personality = transport.worker.personality personality.WEB_SERVICE_CHECKERS['wsgi'](personality, config) reactor = transport.worker.components_shared['reactor'] if 'module' not in config: raise ApplicationError(u'crossbar.error.invalid_configuration', 'missing WSGI app module') if 'object' not in config: raise ApplicationError(u'crossbar.error.invalid_configuration', 'missing WSGI app object') # import WSGI app module and object mod_name = config['module'] try: mod = importlib.import_module(mod_name) except ImportError as e: raise ApplicationError(u'crossbar.error.invalid_configuration', 'WSGI app module "{}" import failed: {} - Python search path was {}'.format(mod_name, e, sys.path)) obj_name = config['object'] if obj_name not in mod.__dict__: raise ApplicationError(u'crossbar.error.invalid_configuration', 'WSGI app object "{}" not in module "{}"'.format(obj_name, mod_name)) else: app = getattr(mod, obj_name) # Create a thread-pool for running the WSGI requests in pool = ThreadPool(maxthreads=config.get('maxthreads', 20), minthreads=config.get('minthreads', 0), name='crossbar_wsgi_threadpool') reactor.addSystemEventTrigger('before', 'shutdown', pool.stop) pool.start() # Create a Twisted Web WSGI resource from the user's WSGI application object try: resource = WSGIResource(reactor, pool, app) if path == '/': resource = WSGIRootResource(resource, {}) except Exception as e: raise ApplicationError(u'crossbar.error.invalid_configuration', 'could not instantiate WSGI resource: {}'.format(e)) else: return RouterWebServiceWsgi(transport, path, config, resource)
def main(): """ Punto de entrada de la aplicacion. """ # Habilitamos la interfaz Web solo si hay conexion a la red local. if CONFIG["NC_ADDRESS"] is not None: # Creamos un thread pool para la aplicacion. threadPool = ThreadPool(maxthreads=CONFIG["SERVER_MAX_THREADS"]) threadPool.start() reactor.addSystemEventTrigger("before", "shutdown", threadPool.stop) app.mutex = threading.Lock() # Creamos los diferentes recursos que servira el servidor de Twisted. wsgiResource = WSGIResource(reactor, threadPool, app) staticResource = File(STATIC_FILES_PATH) tmpCSVResource = File("/tmp") webSocketFactory = WebSocketServerFactory( "ws://{0}:{1}".format(CONFIG["SERVER_INTERFACE"], CONFIG["SERVER_PORT"]) ) webSocketFactory.protocol = LibraServerProtocol webSocketResource = WebSocketResource(webSocketFactory) rootResource = WSGIRootResource( wsgiResource, {"static": staticResource, "ws": webSocketResource, "csv": tmpCSVResource} ) site = Site(rootResource) reactor.listenTCP(CONFIG["SERVER_PORT"], site) # Mostramos si hay conexion a la red local. if not CONFIG["DEVELOPMENT"]: if CONFIG["NC_ADDRESS"] is not None: lcd.showMessage("{0}".format(CONFIG["NC_ADDRESS"]), "{0}".format(CONFIG["SERVER_PORT"])) else: lcd.showMessage("Desconectado") reactor.run()
def _create_resource(self, path_config, nested=True): """ Creates child resource to be added to the parent. :param path_config: Configuration for the new child resource. :type path_config: dict :returns: Resource -- the new child resource """ # WAMP-WebSocket resource # if path_config['type'] == 'websocket': ws_factory = WampWebSocketServerFactory(self._router_session_factory, self.config.extra.cbdir, path_config, self._templates) # FIXME: Site.start/stopFactory should start/stop factories wrapped as Resources ws_factory.startFactory() return WebSocketResource(ws_factory) # Static file hierarchy resource # elif path_config['type'] == 'static': static_options = path_config.get('options', {}) if 'directory' in path_config: static_dir = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['directory'])) elif 'package' in path_config: if 'resource' not in path_config: raise ApplicationError(u"crossbar.error.invalid_configuration", "missing resource") try: mod = importlib.import_module(path_config['package']) except ImportError as e: emsg = "Could not import resource {} from package {}: {}".format(path_config['resource'], path_config['package'], e) self.log.error(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) else: try: static_dir = os.path.abspath(pkg_resources.resource_filename(path_config['package'], path_config['resource'])) except Exception as e: emsg = "Could not import resource {} from package {}: {}".format(path_config['resource'], path_config['package'], e) self.log.error(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) else: raise ApplicationError(u"crossbar.error.invalid_configuration", "missing web spec") static_dir = static_dir.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770 # create resource for file system hierarchy # if static_options.get('enable_directory_listing', False): static_resource_class = StaticResource else: static_resource_class = StaticResourceNoListing cache_timeout = static_options.get('cache_timeout', DEFAULT_CACHE_TIMEOUT) static_resource = static_resource_class(static_dir, cache_timeout=cache_timeout) # set extra MIME types # static_resource.contentTypes.update(EXTRA_MIME_TYPES) if 'mime_types' in static_options: static_resource.contentTypes.update(static_options['mime_types']) patchFileContentTypes(static_resource) # render 404 page on any concrete path not found # static_resource.childNotFound = Resource404(self._templates, static_dir) return static_resource # WSGI resource # elif path_config['type'] == 'wsgi': if not _HAS_WSGI: raise ApplicationError(u"crossbar.error.invalid_configuration", "WSGI unsupported") if 'module' not in path_config: raise ApplicationError(u"crossbar.error.invalid_configuration", "missing WSGI app module") if 'object' not in path_config: raise ApplicationError(u"crossbar.error.invalid_configuration", "missing WSGI app object") # import WSGI app module and object mod_name = path_config['module'] try: mod = importlib.import_module(mod_name) except ImportError as e: raise ApplicationError(u"crossbar.error.invalid_configuration", "WSGI app module '{}' import failed: {} - Python search path was {}".format(mod_name, e, sys.path)) else: obj_name = path_config['object'] if obj_name not in mod.__dict__: raise ApplicationError(u"crossbar.error.invalid_configuration", "WSGI app object '{}' not in module '{}'".format(obj_name, mod_name)) else: app = getattr(mod, obj_name) # Create a threadpool for running the WSGI requests in pool = ThreadPool(maxthreads=path_config.get("maxthreads", 20), minthreads=path_config.get("minthreads", 0), name="crossbar_wsgi_threadpool") self._reactor.addSystemEventTrigger('before', 'shutdown', pool.stop) pool.start() # Create a Twisted Web WSGI resource from the user's WSGI application object try: wsgi_resource = WSGIResource(self._reactor, pool, app) if not nested: wsgi_resource = WSGIRootResource(wsgi_resource, {}) except Exception as e: raise ApplicationError(u"crossbar.error.invalid_configuration", "could not instantiate WSGI resource: {}".format(e)) else: return wsgi_resource # Redirecting resource # elif path_config['type'] == 'redirect': redirect_url = path_config['url'].encode('ascii', 'ignore') return RedirectResource(redirect_url) # JSON value resource # elif path_config['type'] == 'json': value = path_config['value'] return JsonResource(value) # CGI script resource # elif path_config['type'] == 'cgi': cgi_processor = path_config['processor'] cgi_directory = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['directory'])) cgi_directory = cgi_directory.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770 return CgiDirectory(cgi_directory, cgi_processor, Resource404(self._templates, cgi_directory)) # WAMP-Longpoll transport resource # elif path_config['type'] == 'longpoll': path_options = path_config.get('options', {}) lp_resource = WampLongPollResource(self._router_session_factory, timeout=path_options.get('request_timeout', 10), killAfter=path_options.get('session_timeout', 30), queueLimitBytes=path_options.get('queue_limit_bytes', 128 * 1024), queueLimitMessages=path_options.get('queue_limit_messages', 100), debug=path_options.get('debug', False), debug_transport_id=path_options.get('debug_transport_id', None) ) lp_resource._templates = self._templates return lp_resource # Publisher resource (part of REST-bridge) # elif path_config['type'] == 'publisher': # create a vanilla session: the publisher will use this to inject events # publisher_session_config = ComponentConfig(realm=path_config['realm'], extra=None) publisher_session = ApplicationSession(publisher_session_config) # add the publisher session to the router # self._router_session_factory.add(publisher_session, authrole=path_config.get('role', 'anonymous')) # now create the publisher Twisted Web resource # return PublisherResource(path_config.get('options', {}), publisher_session) # Webhook resource (part of REST-bridge) # elif path_config['type'] == 'webhook': # create a vanilla session: the webhook will use this to inject events # webhook_session_config = ComponentConfig(realm=path_config['realm'], extra=None) webhook_session = ApplicationSession(webhook_session_config) # add the webhook session to the router # self._router_session_factory.add(webhook_session, authrole=path_config.get('role', 'anonymous')) # now create the webhook Twisted Web resource # return WebhookResource(path_config.get('options', {}), webhook_session) # Caller resource (part of REST-bridge) # elif path_config['type'] == 'caller': # create a vanilla session: the caller will use this to inject calls # caller_session_config = ComponentConfig(realm=path_config['realm'], extra=None) caller_session = ApplicationSession(caller_session_config) # add the calling session to the router # self._router_session_factory.add(caller_session, authrole=path_config.get('role', 'anonymous')) # now create the caller Twisted Web resource # return CallerResource(path_config.get('options', {}), caller_session) # File Upload resource # elif path_config['type'] == 'upload': upload_directory = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['directory'])) upload_directory = upload_directory.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770 if not os.path.isdir(upload_directory): emsg = "configured upload directory '{}' in file upload resource isn't a directory".format(upload_directory) self.log.error(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) if 'temp_directory' in path_config: temp_directory = os.path.abspath(os.path.join(self.config.extra.cbdir, path_config['temp_directory'])) temp_directory = temp_directory.encode('ascii', 'ignore') # http://stackoverflow.com/a/20433918/884770 else: temp_directory = os.path.abspath(tempfile.gettempdir()) temp_directory = os.path.join(temp_directory, 'crossbar-uploads') if not os.path.exists(temp_directory): os.makedirs(temp_directory) if not os.path.isdir(temp_directory): emsg = "configured temp directory '{}' in file upload resource isn't a directory".format(temp_directory) self.log.error(emsg) raise ApplicationError(u"crossbar.error.invalid_configuration", emsg) # file upload progress and finish events are published via this session # upload_session_config = ComponentConfig(realm=path_config['realm'], extra=None) upload_session = ApplicationSession(upload_session_config) self._router_session_factory.add(upload_session, authrole=path_config.get('role', 'anonymous')) self.log.info("File upload resource started. Uploads to {upl} using temp folder {tmp}.", upl=upload_directory, tmp=temp_directory) return FileUploadResource(upload_directory, temp_directory, path_config['form_fields'], upload_session, path_config.get('options', {})) # Generic Twisted Web resource # elif path_config['type'] == 'resource': try: klassname = path_config['classname'] self.log.debug("Starting class '{}'".format(klassname)) c = klassname.split('.') module_name, klass_name = '.'.join(c[:-1]), c[-1] module = importlib.import_module(module_name) make = getattr(module, klass_name) return make(path_config.get('extra', {})) except Exception as e: emsg = "Failed to import class '{}' - {}".format(klassname, e) self.log.error(emsg) self.log.error("PYTHONPATH: {pythonpath}", pythonpath=sys.path) raise ApplicationError(u"crossbar.error.class_import_failed", emsg) # Schema Docs resource # elif path_config['type'] == 'schemadoc': realm = path_config['realm'] if realm not in self.realm_to_id: raise ApplicationError(u"crossbar.error.no_such_object", "No realm with URI '{}' configured".format(realm)) realm_id = self.realm_to_id[realm] realm_schemas = self.realms[realm_id].session._schemas return SchemaDocResource(self._templates, realm, realm_schemas) # Nested subpath resource # elif path_config['type'] == 'path': nested_paths = path_config.get('paths', {}) if '/' in nested_paths: nested_resource = self._create_resource(nested_paths['/']) else: nested_resource = Resource404(self._templates, b'') # nest subpaths under the current entry # self._add_paths(nested_resource, nested_paths) return nested_resource else: raise ApplicationError(u"crossbar.error.invalid_configuration", "invalid Web path type '{}' in {} config".format(path_config['type'], 'nested' if nested else 'root'))
def start(self): reactor.addSystemEventTrigger('during', 'shutdown', self.stop) ThreadPool.start(self)