def __init__(self, app):
if (not os.environ.has_key("DJANGO_SETTINGS_MODULE")):
log.msg(u'DJANGO_SETTINGS_MODULE environment variable is not defined', logLevel=logging.WARN)
else:
log.msg(u'DJANGO_SETTINGS_MODULE = \'{}\''.format(os.environ.get("DJANGO_SETTINGS_MODULE")))
wsgi_path = app.split('.')
wsgi_module = import_module('.'.join(wsgi_path[:-1]))
wsgi_app = getattr(wsgi_module, wsgi_path[-1])
WSGIResource.__init__(self, reactor, reactor.getThreadPool(), wsgi_app)
class Root(Resource):
""" Twisted web root resource """
def __init__(self, plugin):
""" Constructor """
Resource.__init__(self)
self.wsgi = WSGIResource(plugin.service.reactor,
plugin.service.reactor.getThreadPool(),
plugin.app)
def getChild(self, child, request):
""" Get web child resource helper """
request.prepath.pop()
request.postpath.insert(0, child)
return self.wsgi
def render(self, request):
""" WSGI renderer helper """
return self.wsgi.render(request)
def __init__(self, plugin):
""" Constructor """
Resource.__init__(self)
self.wsgi = WSGIResource(plugin.service.reactor,
plugin.service.reactor.getThreadPool(),
plugin.app)
class Root( resource.Resource ):
"""Root resource that combines the two sites/entry points"""
def __init__(self, app):
resource.Resource.__init__(self)
self.wsgi = WSGIResource(reactor, reactor.getThreadPool(), app)
def getChild( self, child, request ):
# request.isLeaf = True
request.prepath.pop()
request.postpath.insert(0,child)
return self.wsgi
def render( self, request ):
"""Delegate to the WSGI resource"""
return self.wsgi.render( request )
def run_twisted():
factory = Site(WSGIResource(reactor, reactor.getThreadPool(), app))
reactor.listenTCP(8080, factory)
reactor.run()
def __init__(self, reactor, threadpool, htpasswd, path):
self.htpasswd = htpasswd
self.path = path
WSGIResource.__init__(
self, reactor, threadpool, self.middleware_auth(
self.tracApplication))
def make_server(port, app):
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
site = server.Site(resource)
reactor.listenTCP(port, site)
reactor.run()
def twisted(self, flask_app):
from twisted.internet import reactor
from twisted.internet.error import ReactorNotRunning
from twisted.web import server
from twisted.web.wsgi import WSGIResource
from twisted.application import service, strports
from twisted.scripts._twistd_unix import ServerOptions
from twisted.scripts._twistd_unix import UnixApplicationRunner
if self.conf['secure']:
self.conf['strport'] = 'ssl:{port}:'\
'interface={interface}:'\
'privateKey={priv_pem}:'\
'certKey={pub_pem}'.format(**self.conf)
else:
self.conf['strport'] = 'tcp:{port}:'\
'interface={interface}'.format(**self.conf)
# Options as in twistd command line utility
self.conf['twisted_opts'] = '--pidfile={pidfile} -no'.format(
**self.conf).split()
####################################################################
# See
# http://twistedmatrix.com/documents/current/web/howto/
# using-twistedweb.html
# (Serving WSGI Applications) for the basic ideas of the below code
####################################################################
def my_sigint(x, n):
try:
reactor.stop()
except ReactorNotRunning:
pass
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGINT, my_sigint)
resource = WSGIResource(reactor, reactor.getThreadPool(), flask_app)
class QuietSite(server.Site):
def log(*args, **kwargs):
'''Override the logging so that requests are not logged'''
pass
# Log only errors, not every page hit
site = QuietSite(resource)
# To log every single page hit, uncomment the following line
# site = server.Site(resource)
application = service.Application("Sage Notebook")
s = strports.service(native_str(self.conf['strport']), site)
self.open_page()
s.setServiceParent(application)
# This has to be done after sagenb.create_app is run
reactor.addSystemEventTrigger('before', 'shutdown', self.save_notebook)
# Run the application without .tac file
class AppRunner(UnixApplicationRunner):
'''
twisted application runner. The application is provided on init,
not read from file
'''
def __init__(self, app, conf):
super(self.__class__, self).__init__(conf)
self.app = app
def createOrGetApplication(self):
'''Overrides the reading of the application from file'''
return self.app
twisted_conf = ServerOptions()
twisted_conf.parseOptions(self.conf['twisted_opts'])
AppRunner(application, twisted_conf).run()
from twisted.internet import reactor
from twisted.web.resource import Resource
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from util.consumer import InferConsumer
from business_service import process_data
pivot = Pivot(int(config.get("max_task_num")))
infer_consumer = InferConsumer(pivot,
float(config.get("max_get_time")))
port = int(config.get("flask.server.port"))
work_thread_count = int(config.get('work.thread.count'))
site_root = str(config.get('server.root.site'))
reactor.suggestThreadPoolSize(int(work_thread_count))
flask_site = WSGIResource(reactor, reactor.getThreadPool(), app)
root = Resource()
root.putChild(
site_root if PYTHON_VERSION == 2 else bytes(site_root,
encoding='utf-8'),
flask_site)
logger.info(
"start app listen port:{} thread pool size:{} site root:{}".format(
port, work_thread_count, site_root))
reactor.listenTCP(port, Site(root))
# 把主线程设置为infer_consumer的守护线程，如果主线程结束了，infer_consumer线程就跟着结束
infer_consumer.setDaemon(True)
infer_consumer.start()
reactor.run()
def start_router_transport(self, id, config, details=None):
"""
Start a transport on this router.
:param id: The ID of the transport to start.
:type id: str
:param config: The transport configuration.
:type config: dict
"""
if self.debug:
log.msg(
"{}.start_router_transport".format(self.__class__.__name__),
id, config)
# prohibit starting a transport twice
#
if id in self.transports:
emsg = "ERROR: could not start transport - a transport with ID '{}'' is already running (or starting)".format(
id)
log.msg(emsg)
raise ApplicationError('crossbar.error.already_running', emsg)
# check configuration
#
try:
checkconfig.check_router_transport(config)
except Exception as e:
emsg = "ERROR: invalid router transport configuration ({})".format(
e)
log.msg(emsg)
raise ApplicationError("crossbar.error.invalid_configuration",
emsg)
else:
if self.debug:
log.msg("Starting {}-transport on router.".format(
config['type']))
# standalone WAMP-RawSocket transport
#
if config['type'] == 'rawsocket':
transport_factory = WampRawSocketServerFactory(
self._router_session_factory, config)
transport_factory.noisy = False
# standalone WAMP-WebSocket transport
#
elif config['type'] == 'websocket':
transport_factory = WampWebSocketServerFactory(
self._router_session_factory, self.config.extra.cbdir, config,
self._templates)
transport_factory.noisy = False
# Flash-policy file server pseudo transport
#
elif config['type'] == 'flashpolicy':
transport_factory = FlashPolicyFactory(
config.get('allowed_domain', None),
config.get('allowed_ports', None))
# WebSocket testee pseudo transport
#
elif config['type'] == 'websocket.testee':
transport_factory = WebSocketTesteeServerFactory(
config, self._templates)
# Stream testee pseudo transport
#
elif config['type'] == 'stream.testee':
transport_factory = StreamTesteeServerFactory()
# Twisted Web based transport
#
elif config['type'] == 'web':
options = config.get('options', {})
# create Twisted Web root resource
#
root_config = config['paths']['/']
root_type = root_config['type']
root_options = root_config.get('options', {})
# Static file hierarchy root resource
#
if root_type == 'static':
if 'directory' in root_config:
root_dir = os.path.abspath(
os.path.join(self.config.extra.cbdir,
root_config['directory']))
elif 'package' in root_config:
if 'resource' not in root_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing resource")
try:
mod = importlib.import_module(root_config['package'])
except ImportError as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
root_config['resource'], root_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration", emsg)
else:
try:
root_dir = os.path.abspath(
pkg_resources.resource_filename(
root_config['package'],
root_config['resource']))
except Exception as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
root_config['resource'],
root_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration", emsg)
else:
mod_version = getattr(mod, '__version__', '?.?.?')
log.msg(
"Loaded static Web resource '{}' from package '{} {}' (filesystem path {})"
.format(root_config['resource'],
root_config['package'], mod_version,
root_dir))
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing web spec")
root_dir = root_dir.encode(
'ascii',
'ignore') # http://stackoverflow.com/a/20433918/884770
if self.debug:
log.msg("Starting Web service at root directory {}".format(
root_dir))
# create resource for file system hierarchy
#
if root_options.get('enable_directory_listing', False):
static_resource_class = StaticResource
else:
static_resource_class = StaticResourceNoListing
cache_timeout = root_options.get('cache_timeout',
DEFAULT_CACHE_TIMEOUT)
root = static_resource_class(root_dir,
cache_timeout=cache_timeout)
# set extra MIME types
#
root.contentTypes.update(EXTRA_MIME_TYPES)
if 'mime_types' in root_options:
root.contentTypes.update(root_options['mime_types'])
patchFileContentTypes(root)
# render 404 page on any concrete path not found
#
root.childNotFound = Resource404(self._templates, root_dir)
# WSGI root resource
#
elif root_type == 'wsgi':
if not _HAS_WSGI:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI unsupported")
# wsgi_options = root_config.get('options', {})
if 'module' not in root_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing WSGI app module")
if 'object' not in root_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing WSGI app object")
# import WSGI app module and object
mod_name = root_config['module']
try:
mod = importlib.import_module(mod_name)
except ImportError as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI app module '{}' import failed: {} - Python search path was {}"
.format(mod_name, e, sys.path))
else:
obj_name = root_config['object']
if obj_name not in mod.__dict__:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI app object '{}' not in module '{}'".format(
obj_name, mod_name))
else:
app = getattr(mod, obj_name)
# create a Twisted Web WSGI resource from the user's WSGI application object
try:
wsgi_resource = WSGIResource(reactor,
reactor.getThreadPool(), app)
except Exception as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"could not instantiate WSGI resource: {}".format(e))
else:
# create a root resource serving everything via WSGI
root = WSGIRootResource(wsgi_resource, {})
# Redirecting root resource
#
elif root_type == 'redirect':
redirect_url = root_config['url'].encode('ascii', 'ignore')
root = RedirectResource(redirect_url)
# Publisher resource (part of REST-bridge)
#
elif root_type == 'publisher':
# create a vanilla session: the publisher will use this to inject events
#
publisher_session_config = ComponentConfig(
realm=root_config['realm'], extra=None)
publisher_session = ApplicationSession(
publisher_session_config)
# add the publishing session to the router
#
self._router_session_factory.add(publisher_session,
authrole=root_config.get(
'role', 'anonymous'))
# now create the publisher Twisted Web resource and add it to resource tree
#
root = PublisherResource(root_config.get('options', {}),
publisher_session)
# Caller resource (part of REST-bridge)
#
elif root_type == 'caller':
# create a vanilla session: the caller will use this to inject calls
#
caller_session_config = ComponentConfig(
realm=root_config['realm'], extra=None)
caller_session = ApplicationSession(caller_session_config)
# add the calling session to the router
#
self._router_session_factory.add(caller_session,
authrole=root_config.get(
'role', 'anonymous'))
# now create the caller Twisted Web resource and add it to resource tree
#
root = CallerResource(root_config.get('options', {}),
caller_session)
# Generic Twisted Web resource
#
elif root_type == 'resource':
try:
klassname = root_config['classname']
if self.debug:
log.msg("Starting class '{}'".format(klassname))
c = klassname.split('.')
module_name, klass_name = '.'.join(c[:-1]), c[-1]
module = importlib.import_module(module_name)
make = getattr(module, klass_name)
root = make(root_config.get('extra', {}))
except Exception as e:
emsg = "Failed to import class '{}' - {}".format(
klassname, e)
log.msg(emsg)
log.msg("PYTHONPATH: {}".format(sys.path))
raise ApplicationError(
"crossbar.error.class_import_failed", emsg)
# Invalid root resource
#
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"invalid Web root path type '{}'".format(root_type))
# create Twisted Web resources on all non-root paths configured
#
self.add_paths(root, config.get('paths', {}))
# create the actual transport factory
#
transport_factory = Site(root)
transport_factory.noisy = False
# Web access logging
#
if not options.get('access_log', False):
transport_factory.log = lambda _: None
# Traceback rendering
#
transport_factory.displayTracebacks = options.get(
'display_tracebacks', False)
# HSTS
#
if options.get('hsts', False):
if 'tls' in config['endpoint']:
hsts_max_age = int(options.get('hsts_max_age', 31536000))
transport_factory.requestFactory = createHSTSRequestFactory(
transport_factory.requestFactory, hsts_max_age)
else:
log.msg(
"Warning: HSTS requested, but running on non-TLS - skipping HSTS"
)
# enable Hixie-76 on Twisted Web
#
if options.get('hixie76_aware', False):
transport_factory.protocol = HTTPChannelHixie76Aware # needed if Hixie76 is to be supported
# Unknown transport type
#
else:
# should not arrive here, since we did check_transport() in the beginning
raise Exception("logic error")
# create transport endpoint / listening port from transport factory
#
d = create_listening_port_from_config(config['endpoint'],
transport_factory,
self.config.extra.cbdir, reactor)
def ok(port):
self.transports[id] = RouterTransport(id, config,
transport_factory, port)
if self.debug:
log.msg(
"Router transport '{}'' started and listening".format(id))
return
def fail(err):
emsg = "ERROR: cannot listen on transport endpoint ({})".format(
err.value)
log.msg(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
d.addCallbacks(ok, fail)
return d
$ time curl -s "http://localhost:9757/block?seconds=10" > /dev/null & \
time curl -s "http://localhost:9757/block?seconds=10" > /dev/null &
[1] 27537
[2] 27538
real 0m10.031s
user 0m0.008s
sys 0m0.007s
real 0m10.038s
user 0m0.006s
sys 0m0.006s
'''
host = '0.0.0.0'
port = 9757
if __name__ == '__main__':
application = initialize([SomeService])
wsgi_application = WsgiApplication(application)
resource = WSGIResource(reactor, reactor, wsgi_application)
site = Site(resource)
reactor.listenTCP(port, site, interface=host)
logging.info('listening on: %s:%d' % (host, port))
logging.info('wsdl is at: http://%s:%d/?wsdl' % (host, port))
sys.exit(reactor.run())
from twisted.internet import reactor
from twisted.web.wsgi import WSGIResource
from .. import exposition, REGISTRY
MetricsResource = lambda registry=REGISTRY: WSGIResource(
reactor, reactor.getThreadPool(), exposition.make_wsgi_app(registry))
# A standalone function that can be bound to a service.
def add(a, b):
return a + b
# Create a dictionary mapping the service namespaces to a function
# or class instance
services = {'example': example(), 'myadd': add}
# Create and start a thread pool,
wsgiThreadPool = ThreadPool()
wsgiThreadPool.start()
# ensuring that it will be stopped when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', wsgiThreadPool.stop)
# PyAMF gateway
gateway = WSGIGateway(services,
logger=logging,
expose_request=False,
debug=True)
# Create the WSGI resource
wsgiAppAsResource = WSGIResource(reactor, wsgiThreadPool, gateway)
site = server.Site(wsgiAppAsResource)
server = strports.service('tcp:8080', site)
# Hooks for twistd
application = service.Application('PyAMF Sample Remoting Server')
server.setServiceParent(application)
def setConfiguration(self, config):
for dashboard in config:
dashboard['app'].buildbot_api = self
resource = WSGIResource(reactor, reactor.getThreadPool(),
dashboard['app'])
self.resource.putChild(unicode2bytes(dashboard['name']), resource)
def get_resource(request):
return WSGIResource(reactor, reactor.getThreadPool(), httpbin.app)
# clear other users' stdin
def disableInput(self):
for sid in self.sessionids.copy():
SOCKETIO.emit('replbusy', namespace='/', room=sid)
def enableInput(self):
for sid in self.sessionids.copy():
SOCKETIO.emit('replready', namespace='/', room=sid)
#def emitStdout(self):
# TODO make this into a main interact() method?
###########
# twisted #
###########
# this is based on https://gist.github.com/ianschenck/977379a91154fe264897
# but I had trouble getting the reloader to work without zombie processes
RESOURCE = WSGIResource(REACTOR, REACTOR.getThreadPool(), FLASK)
SITE = Site(RESOURCE)
# TODO write a function to clean up all threads in SESSIONS on shutdown
# REACTOR.addSystemEventTrigger('before', 'shutdown', cleanup_sessions)
LOGGER.debug('creating main twisted reactor on port %s' % CONFIG['port'])
REACTOR.listenTCP(CONFIG['port'], SITE)
# TODO which signal handlers?
LOGGER.debug('running reactor')
REACTOR.run(installSignalHandlers=True)
ilsc.init_database()
appBackend.init_schedulers()
scheduler.add_job(appBackend.cleanup_everything, 'cron', id=f"leon_der_profi", hour=appConfig.app['cleancron'], minute=0)
scheduler.start()
if appConfig.app['cleanonstart']:
appBackend.checkout_all()
appBackend.cleanup_everything()
#appBackend.inject_random_userdata()#just for testing
try:
if appConfig.http['usessl']:
priv = appConfig.http['serverpriv']
cacert = appConfig.http['servercacert']
certdata = ssl.DefaultOpenSSLContextFactory(priv, cacert)
reactor.listenSSL(appConfig.http['port'], Site(WSGIResource(reactor, reactor.getThreadPool(), flaskApp)),certdata)
reactor.listenSSL(appConfig.websocket['port'], appBackend.get_websocket_site(),certdata)
else:
reactor.listenTCP(appConfig.http['port'], Site(WSGIResource(reactor, reactor.getThreadPool(), flaskApp)))
reactor.listenTCP(appConfig.websocket['port'], appBackend.get_websocket_site())
except Exception as e:
logging.warning(str(e))
print(e)
if __name__ == "__main__":
try:
reactor.run()
except Exception as e:
logging.warning(str(e))
print(e)
def start_router_transport(self, id, config, details=None):
"""
Start a transport on this router.
:param id: The ID of the transport to start.
:type id: str
:param config: The transport configuration.
:type config: dict
"""
if self.debug:
log.msg(
"{}.start_router_transport".format(self.__class__.__name__),
id, config)
## prohibit starting a transport twice
##
if id in self.transports:
emsg = "ERROR: could not start transport - a transport with ID '{}'' is already running (or starting)".format(
id)
log.msg(emsg)
raise ApplicationError('crossbar.error.already_running', emsg)
## check configuration
##
try:
checkconfig.check_router_transport(config)
except Exception as e:
emsg = "ERROR: invalid router transport configuration ({})".format(
e)
log.msg(emsg)
raise ApplicationError("crossbar.error.invalid_configuration",
emsg)
else:
if self.debug:
log.msg("Starting {}-transport on router.".format(
config['type']))
## standalone WAMP-RawSocket transport
##
if config['type'] == 'rawsocket':
transport_factory = CrossbarWampRawSocketServerFactory(
self.session_factory, config)
transport_factory.noisy = False
## standalone WAMP-WebSocket transport
##
elif config['type'] == 'websocket':
transport_factory = CrossbarWampWebSocketServerFactory(
self.session_factory, self.config.extra.cbdir, config,
self._templates)
transport_factory.noisy = False
## Twisted Web based transport
##
elif config['type'] == 'web':
options = config.get('options', {})
## create Twisted Web root resource
##
root_config = config['paths']['/']
root_type = root_config['type']
root_options = root_config.get('options', {})
## Static file hierarchy root resource
##
if root_type == 'static':
if 'directory' in root_config:
root_dir = os.path.abspath(
os.path.join(self.config.extra.cbdir,
root_config['directory']))
elif 'package' in root_config:
if not 'resource' in root_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing resource")
try:
mod = importlib.import_module(root_config['package'])
except ImportError:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
path_config['resource'], path_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration", emsg)
else:
try:
root_dir = os.path.abspath(
pkg_resources.resource_filename(
root_config['package'],
root_config['resource']))
except Exception as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
path_config['resource'],
path_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration", emsg)
else:
mod_version = getattr(mod, '__version__', '?.?.?')
log.msg(
"Loaded static Web resource '{}' from package '{} {}' (filesystem path {})"
.format(root_config['resource'],
root_config['package'], mod_version,
root_dir))
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing web spec")
root_dir = root_dir.encode(
'ascii',
'ignore') # http://stackoverflow.com/a/20433918/884770
if self.debug:
log.msg("Starting Web service at root directory {}".format(
root_dir))
## create resource for file system hierarchy
##
if root_options.get('enable_directory_listing', False):
root = File(root_dir)
else:
root = FileNoListing(root_dir)
## set extra MIME types
##
root.contentTypes.update(EXTRA_MIME_TYPES)
if 'mime_types' in root_options:
root.contentTypes.update(root_options['mime_types'])
patchFileContentTypes(root)
## render 404 page on any concrete path not found
##
root.childNotFound = Resource404(self._templates, root_dir)
## WSGI root resource
##
elif root_type == 'wsgi':
if not _HAS_WSGI:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI unsupported")
wsgi_options = root_config.get('options', {})
if not 'module' in root_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing module")
if not 'object' in root_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing object")
try:
mod = importlib.import_module(root_config['module'])
except ImportError:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"module import failed")
else:
if not root_config['object'] in mod.__dict__:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"object not in module")
else:
app = getattr(mod, root_config['object'])
## create a Twisted Web WSGI resource from the user's WSGI application object
try:
wsgi_resource = WSGIResource(reactor,
reactor.getThreadPool(), app)
except Exception as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"could not instantiate WSGI resource: {}".format(e))
else:
## create a root resource serving everything via WSGI
root = WSGIRootResource(wsgi_resource, {})
## Redirecting root resource
##
elif root_type == 'redirect':
redirect_url = root_config['url'].encode('ascii', 'ignore')
root = RedirectResource(redirect_url)
## Invalid root resource
##
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"invalid Web root path type '{}'".format(root_type))
## create Twisted Web resources on all non-root paths configured
##
for path in sorted(config.get('paths', [])):
if path != "/":
path_config = config['paths'][path]
## websocket_echo
## websocket_testee
## s3mirror
## websocket_stdio
##
## WAMP-WebSocket resource
##
if path_config['type'] == 'websocket':
ws_factory = CrossbarWampWebSocketServerFactory(
self.session_factory, self.config.extra.cbdir,
path_config, self._templates)
## FIXME: Site.start/stopFactory should start/stop factories wrapped as Resources
ws_factory.startFactory()
ws_resource = WebSocketResource(ws_factory)
root.putChild(path, ws_resource)
## Static file hierarchy resource
##
elif path_config['type'] == 'static':
static_options = path_config.get('options', {})
if 'directory' in path_config:
static_dir = os.path.abspath(
os.path.join(self.config.extra.cbdir,
path_config['directory']))
elif 'package' in path_config:
if not 'resource' in path_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing resource")
try:
mod = importlib.import_module(
path_config['package'])
except ImportError as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
path_config['resource'],
path_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration",
emsg)
else:
try:
static_dir = os.path.abspath(
pkg_resources.resource_filename(
path_config['package'],
path_config['resource']))
except Exception as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
path_config['resource'],
path_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration",
emsg)
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing web spec")
static_dir = static_dir.encode(
'ascii', 'ignore'
) # http://stackoverflow.com/a/20433918/884770
## create resource for file system hierarchy
##
if static_options.get('enable_directory_listing',
False):
static_resource = File(static_dir)
else:
static_resource = FileNoListing(static_dir)
## set extra MIME types
##
static_resource.contentTypes.update(EXTRA_MIME_TYPES)
if 'mime_types' in static_options:
static_resource.contentTypes.update(
static_options['mime_types'])
patchFileContentTypes(static_resource)
## render 404 page on any concrete path not found
##
static_resource.childNotFound = Resource404(
self._templates, static_dir)
root.putChild(path, static_resource)
## WSGI resource
##
elif path_config['type'] == 'wsgi':
if not _HAS_WSGI:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI unsupported")
wsgi_options = path_config.get('options', {})
if not 'module' in path_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing module")
if not 'object' in path_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing object")
try:
mod = importlib.import_module(
path_config['module'])
except ImportError as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"module import failed - {}".format(e))
else:
if not path_config['object'] in mod.__dict__:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"object not in module")
else:
app = getattr(mod, path_config['object'])
## create a Twisted Web WSGI resource from the user's WSGI application object
try:
wsgi_resource = WSGIResource(
reactor, reactor.getThreadPool(), app)
except Exception as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"could not instantiate WSGI resource: {}".
format(e))
else:
root.putChild(path, wsgi_resource)
## Redirecting resource
##
elif path_config['type'] == 'redirect':
redirect_url = path_config['url'].encode(
'ascii', 'ignore')
redirect_resource = RedirectResource(redirect_url)
root.putChild(path, redirect_resource)
## JSON value resource
##
elif path_config['type'] == 'json':
value = path_config['value']
json_resource = JsonResource(value)
root.putChild(path, json_resource)
## CGI script resource
##
elif path_config['type'] == 'cgi':
cgi_processor = path_config['processor']
cgi_directory = os.path.abspath(
os.path.join(self.config.extra.cbdir,
path_config['directory']))
cgi_directory = cgi_directory.encode(
'ascii', 'ignore'
) # http://stackoverflow.com/a/20433918/884770
cgi_resource = CgiDirectory(
cgi_directory, cgi_processor,
Resource404(self._templates, cgi_directory))
root.putChild(path, cgi_resource)
## WAMP-Longpoll transport resource
##
elif path_config['type'] == 'longpoll':
log.msg("Web path type 'longpoll' not implemented")
## Pusher resource
##
elif path_config['type'] == 'pusher':
## create a vanilla session: the pusher will use this to inject events
##
pusher_session_config = ComponentConfig(
realm=path_config['realm'], extra=None)
pusher_session = ApplicationSession(
pusher_session_config)
## add the pushing session to the router
##
self.session_factory.add(pusher_session)
## now create the pusher Twisted Web resource and add it to resource tree
##
pusher_resource = PusherResource(
path_config.get('options', {}), pusher_session)
root.putChild(path, pusher_resource)
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"invalid Web path type '{}'".format(
path_config['type']))
## create the actual transport factory
##
transport_factory = Site(root)
transport_factory.noisy = False
## Web access logging
##
if not options.get('access_log', False):
transport_factory.log = lambda _: None
## Traceback rendering
##
transport_factory.displayTracebacks = options.get(
'display_tracebacks', False)
## HSTS
##
if options.get('hsts', False):
if 'tls' in config['endpoint']:
hsts_max_age = int(options.get('hsts_max_age', 31536000))
transport_factory.requestFactory = createHSTSRequestFactory(
transport_factory.requestFactory, hsts_max_age)
else:
log.msg(
"Warning: HSTS requested, but running on non-TLS - skipping HSTS"
)
## enable Hixie-76 on Twisted Web
##
if options.get('hixie76_aware', False):
transport_factory.protocol = HTTPChannelHixie76Aware # needed if Hixie76 is to be supported
else:
## should not arrive here, since we did check_transport() in the beginning
raise Exception("logic error")
## create transport endpoint / listening port from transport factory
##
d = create_listening_port_from_config(config['endpoint'],
transport_factory,
self.config.extra.cbdir, reactor)
def ok(port):
self.transports[id] = RouterTransport(id, config,
transport_factory, port)
if self.debug:
log.msg(
"Router transport '{}'' started and listening".format(id))
return
def fail(err):
emsg = "ERROR: cannot listen on transport endpoint ({})".format(
err.value)
log.msg(emsg)
raise ApplicationError("crossbar.error.cannot_listen", emsg)
d.addCallbacks(ok, fail)
return d
def run(self):
"""Override Process run method to provide a custom wrapper for the API.
This provides a continuous loop for watching the service while keeping
an ear open to the main process from openContentPlatform, listening for
any interrupt requests.
"""
## Setup requested log handler
try:
## Twisted imports here to avoid issues with epoll on Linux
from twisted.internet import reactor, ssl
from twisted.python.filepath import FilePath
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from twisted.python.threadpool import ThreadPool
print('Starting {}'.format(self.serviceName))
self.getLocalLogger()
self.logger.info('Starting {}'.format(self.serviceName))
self.logger.info('Setting up service communication...')
## Setup shared resources for our WSGIResource instances to use
self.getSharedLogger()
self.getSharedDbPool()
## Create a PID file for system administration purposes
pidEntryService(self.serviceName, env, self.pid)
## Reference the magic WSGI throwable from the module using Hug
application = serviceCommunication.__hug_wsgi__
## Setup the WSGI to be hosted through Twisted's web server
wsgiThreadPool = ThreadPool()
wsgiThreadPool.start()
## For some reason the system event wasn't working all the time,
## so I'm adding an explicit wsgiThreadPool.stop() below as well,
## which was needed before reactor.stop() would properly cleanup.
reactor.addSystemEventTrigger('after', 'shutdown',
wsgiThreadPool.stop)
resource = WSGIResource(reactor, wsgiThreadPool, application)
self.logger.info('calling listener on {}:{}.'.format(
str(self.serviceEndpoint), self.listeningPort))
if self.useCertificates:
## Use TLS to encrypt the communication
certData = FilePath(
os.path.join(
env.privateInternalCertPath,
self.globalSettings.get(
'ocpCertificateCaFile'))).getContent()
certificate = ssl.PrivateCertificate.loadPEM(certData)
reactor.listenSSL(self.listeningPort, Site(resource),
certificate.options())
else:
## Plain text communication
reactor.listenTCP(self.listeningPort,
Site(resource),
interface=self.serviceEndpoint)
## Normally we'd just call reactor.run() here and let twisted handle
## the wait loop while watching for signals. The problem is that we
## need openContentPlatform (parent process) to manage this process.
## So this is a bit hacky in that I'm using the reactor code, but I
## am manually calling what would be called if I just called run():
reactor.startRunning()
## Start event wait loop
while reactor._started and not self.shutdownEvent.is_set():
try:
## Four lines from twisted.internet.main.mainloop:
reactor.runUntilCurrent()
t2 = reactor.timeout()
t = reactor.running and t2
reactor.doIteration(t)
except:
exception = traceback.format_exception(
sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
self.logger.error('Exception in {}: {}'.format(
self.serviceName, str(exception)))
break
if self.shutdownEvent.is_set():
self.logger.info('Process received shutdownEvent')
with suppress(Exception):
wsgiThreadPool.stop()
with suppress(Exception):
reactor.stop()
except:
exception = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
self.logger.error('Exception in {}: {}'.format(
self.serviceName, str(exception)))
## Cleanup
pidRemoveService(self.serviceName, env, self.pid)
self.logger.info('Stopped {}'.format(self.serviceName))
print('Stopped {}'.format(self.serviceName))
## end run
return
from prometheus_client import make_wsgi_app
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from twisted.web.resource import Resource
from twisted.internet import reactor
metrics_resource = WSGIResource(
reactor, reactor.getThreadPool(), make_wsgi_app())
class HelloWorld(Resource):
isLeaf = False
def render_GET(self, request):
return b"Hello World"
root = HelloWorld()
root.putChild(b'metrics', metrics_resource)
reactor.listenTCP(8000, Site(root))
reactor.run()
def installApplication(self, application):
"""Install the WSGI application into the Twisted site.
It's installed as a child with path "MAAS". This matches the default
front-end configuration (i.e. Apache) so that there's no need to force
script names.
"""
# Setup resources to process paths that twisted handles.
metadata = Resource()
metadata.putChild(b"status", StatusHandlerResource(self.status_worker))
maas = Resource()
maas.putChild(b"metadata", metadata)
maas.putChild(
b"ws", WebSocketsResource(lookupProtocolForFactory(self.websocket))
)
# /MAAS/r/{path} and /MAAS/l/{path} are all resolved by the new MAAS UI
# react app, and legacy angularjs app respectively.
# If any paths do not match then its routed to index.html in the new
# UI code as it uses HTML 5 routing.
maas.putChild(b"r", DefaultFallbackFile(settings.STATIC_ROOT))
maas.putChild(b"l", DefaultFallbackFile(settings.STATIC_ROOT))
# Redirect /MAAS to react app
maas.putChild(b"", Redirect(b"/MAAS/r/"))
# Setup static resources
maas.putChild(
b"assets",
NoListingFile(os.path.join(settings.STATIC_ROOT, "assets")),
)
maas.putChild(
b"machine-resources",
NoListingFile(
get_root_path() / "usr/share/maas/machine-resources"
),
)
# Setup static docs
maas.putChild(
b"docs",
DocsFallbackFile(os.path.join(settings.STATIC_ROOT, "docs")),
)
root = Resource()
root.putChild(b"", Redirect(b"MAAS/"))
root.putChild(b"MAAS", maas)
# Setup the resources to process paths that django handles.
underlay_maas = ResourceOverlay(
WSGIResource(reactor, self.threadpool, application)
)
underlay_root = Resource()
underlay_root.putChild(b"MAAS", underlay_maas)
underlay_site = Site(
underlay_root, logFormatter=reducedWebLogFormatter
)
underlay_site.requestFactory = CleanPathRequest
# Setup the main resource as the twisted handler and the underlay
# resource as the django handler.
self.site.resource = root
self.site.underlay = underlay_site
except:
traceback.print_exc()
err = HTTPError( 500, ''.join( traceback.format_exception_only( sys.exc_info()[0], sys.exc_info()[1] ) ),
{ '_error': { 'traceback': traceback.format_tb( sys.exc_info()[2] ) } } )
wsgi_cb( err.status, [ ('Content-Type','application/json') ] )
return err.result()
if __name__=="__main__":
params = namedParameters()
port = params.get('socket', 8080)
from twisted.web import server
from twisted.web.wsgi import WSGIResource
from twisted.python.threadpool import ThreadPool
from twisted.python import log
from twisted.internet import reactor
from twisted.application import service, strports
# Create and start a thread pool,
wsgiThreadPool = ThreadPool()
wsgiThreadPool.start()
# ensuring that it will be stopped when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', wsgiThreadPool.stop)
reactor.listenTCP( port, server.Site( WSGIResource(reactor, wsgiThreadPool, wsgi_application) ) )
log.startLogging( log.FileLogObserver( sys.stderr ) )
reactor.run()
def main(connection_string=DB_CONNECTION_STRING):
# configure logging
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
#logging.getLogger('sqlalchemy.engine.base.Engine').setLevel(logging.DEBUG)
from spynepi.const import FILES_PATH
RootService.FILES_ROOT = os.path.abspath(FILES_PATH)
index_app = MyApplication([RootService, IndexService],
"http://usefulinc.com/ns/doap#",
in_protocol=HttpRpc(),
out_protocol=HtmlTable())
rdf_app = MyApplication([RdfService],
"http://usefulinc.com/ns/doap#",
in_protocol=HttpRpc(),
out_protocol=XmlDocument())
html_app = MyApplication([HtmlService],
"http://usefulinc.com/ns/doap#",
in_protocol=HttpRpc(),
out_protocol=HttpRpc())
db_handle = init_database(connection_string)
class UserDefinedContext(object):
def __init__(self):
self.session = db_handle.Session()
def close(self):
self.session.close()
# this is called after validation
def _on_method_call(ctx):
ctx.udc = UserDefinedContext()
# this is called once all data is sent to the client.
def _on_method_return_object(ctx):
ctx.udc.session.commit()
def _on_wsgi_close(ctx):
if ctx.udc is not None:
ctx.udc.close()
for app in index_app, rdf_app, html_app:
app.event_manager.add_listener('method_call', _on_method_call)
app.event_manager.add_listener('method_return_object',
_on_method_return_object)
wsgi_index = WsgiApplication(index_app)
wsgi_rdf = WsgiApplication(rdf_app)
wsgi_html = WsgiApplication(html_app)
for a in wsgi_index, wsgi_rdf, wsgi_html:
a.event_manager.add_listener('wsgi_close', _on_wsgi_close)
url_map = Map([
Rule("/", endpoint=wsgi_index),
Rule("/<project_name>", endpoint=wsgi_html),
Rule("/<project_name>/", endpoint=wsgi_html),
Rule("/<project_name>/doap.rdf", endpoint=wsgi_rdf),
Rule("/<project_name>/<version>", endpoint=wsgi_html),
Rule("/<project_name>/<version>/", endpoint=wsgi_html),
Rule("/<project_name>/<version>/doap.rdf", endpoint=wsgi_rdf),
Rule("/files/<project_name>/<version>/<download>", endpoint=wsgi_html),
])
resource = WSGIResource(reactor, reactor, TWsgiApplication(url_map))
site = Site(resource)
reactor.listenTCP(PORT, site)
logging.info('listening on: %s:%d' % (HOST, PORT))
logging.info('wsdl is at: http://%s:%d/?wsdl' % (HOST, PORT))
sys.exit(reactor.run())
def page_alive():
return render
config.add_route('dir', '/dir')
#config.add_route('item', '/item')
config.add_route('get_config', '/get_config')
config.add_route('write_config', '/write_config')
if __name__ == "__main__":
log.startLogging(sys.stdout)
# create a Twisted Web resource for our WebSocket server
wsFactory = WebSocketServerFactory(u"ws://127.0.0.1:8080")
wsFactory.protocol = EchoServerProtocol
wsResource = WebSocketResource(wsFactory)
# create a Twisted Web WSGI resource for our Flask server
wsgiResource = WSGIResource(reactor, reactor.getThreadPool(), app)
# create a root resource serving everything via WSGI/Flask, but
# the path "/ws" served by our WebSocket stuff
rootResource = WSGIRootResource(wsgiResource, {b'ws': wsResource})
# create a Twisted Web Site and run everything
site = Site(rootResource)
reactor.listenTCP(8080, site)
reactor.run()
def start(self):
"""Start the Web Server """
self.site = Site(WSGIResource(reactor, reactor.getThreadPool(), self.app))
self.port = reactor.listenTCP(self.server.config.webport, self.site)
def create_resource(self, path_config):
"""
Creates child resource to be added to the parent.
:param path_config: Configuration for the new child resource.
:type path_config: dict
:returns: Resource -- the new child resource
"""
# WAMP-WebSocket resource
#
if path_config['type'] == 'websocket':
ws_factory = WampWebSocketServerFactory(
self._router_session_factory, self.config.extra.cbdir,
path_config, self._templates)
# FIXME: Site.start/stopFactory should start/stop factories wrapped as Resources
ws_factory.startFactory()
return WebSocketResource(ws_factory)
# Static file hierarchy resource
#
elif path_config['type'] == 'static':
static_options = path_config.get('options', {})
if 'directory' in path_config:
static_dir = os.path.abspath(
os.path.join(self.config.extra.cbdir,
path_config['directory']))
elif 'package' in path_config:
if 'resource' not in path_config:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"missing resource")
try:
mod = importlib.import_module(path_config['package'])
except ImportError as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
path_config['resource'], path_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration", emsg)
else:
try:
static_dir = os.path.abspath(
pkg_resources.resource_filename(
path_config['package'],
path_config['resource']))
except Exception as e:
emsg = "ERROR: could not import resource '{}' from package '{}' - {}".format(
path_config['resource'], path_config['package'], e)
log.msg(emsg)
raise ApplicationError(
"crossbar.error.invalid_configuration", emsg)
else:
raise ApplicationError("crossbar.error.invalid_configuration",
"missing web spec")
static_dir = static_dir.encode(
'ascii',
'ignore') # http://stackoverflow.com/a/20433918/884770
# create resource for file system hierarchy
#
if static_options.get('enable_directory_listing', False):
static_resource_class = StaticResource
else:
static_resource_class = StaticResourceNoListing
cache_timeout = static_options.get('cache_timeout',
DEFAULT_CACHE_TIMEOUT)
static_resource = static_resource_class(
static_dir, cache_timeout=cache_timeout)
# set extra MIME types
#
static_resource.contentTypes.update(EXTRA_MIME_TYPES)
if 'mime_types' in static_options:
static_resource.contentTypes.update(
static_options['mime_types'])
patchFileContentTypes(static_resource)
# render 404 page on any concrete path not found
#
static_resource.childNotFound = Resource404(
self._templates, static_dir)
return static_resource
# WSGI resource
#
elif path_config['type'] == 'wsgi':
if not _HAS_WSGI:
raise ApplicationError("crossbar.error.invalid_configuration",
"WSGI unsupported")
# wsgi_options = path_config.get('options', {})
if 'module' not in path_config:
raise ApplicationError("crossbar.error.invalid_configuration",
"missing WSGI app module")
if 'object' not in path_config:
raise ApplicationError("crossbar.error.invalid_configuration",
"missing WSGI app object")
# import WSGI app module and object
mod_name = path_config['module']
try:
mod = importlib.import_module(mod_name)
except ImportError as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI app module '{}' import failed: {} - Python search path was {}"
.format(mod_name, e, sys.path))
else:
obj_name = path_config['object']
if obj_name not in mod.__dict__:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"WSGI app object '{}' not in module '{}'".format(
obj_name, mod_name))
else:
app = getattr(mod, obj_name)
# create a Twisted Web WSGI resource from the user's WSGI application object
try:
wsgi_resource = WSGIResource(reactor, reactor.getThreadPool(),
app)
except Exception as e:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"could not instantiate WSGI resource: {}".format(e))
else:
return wsgi_resource
# Redirecting resource
#
elif path_config['type'] == 'redirect':
redirect_url = path_config['url'].encode('ascii', 'ignore')
return RedirectResource(redirect_url)
# JSON value resource
#
elif path_config['type'] == 'json':
value = path_config['value']
return JsonResource(value)
# CGI script resource
#
elif path_config['type'] == 'cgi':
cgi_processor = path_config['processor']
cgi_directory = os.path.abspath(
os.path.join(self.config.extra.cbdir,
path_config['directory']))
cgi_directory = cgi_directory.encode(
'ascii',
'ignore') # http://stackoverflow.com/a/20433918/884770
return CgiDirectory(cgi_directory, cgi_processor,
Resource404(self._templates, cgi_directory))
# WAMP-Longpoll transport resource
#
elif path_config['type'] == 'longpoll':
path_options = path_config.get('options', {})
lp_resource = WampLongPollResource(
self._router_session_factory,
timeout=path_options.get('request_timeout', 10),
killAfter=path_options.get('session_timeout', 30),
queueLimitBytes=path_options.get('queue_limit_bytes',
128 * 1024),
queueLimitMessages=path_options.get('queue_limit_messages',
100),
debug=path_options.get('debug', False),
debug_transport_id=path_options.get('debug_transport_id',
None))
lp_resource._templates = self._templates
return lp_resource
# Publisher resource (part of REST-bridge)
#
elif path_config['type'] == 'publisher':
# create a vanilla session: the publisher will use this to inject events
#
publisher_session_config = ComponentConfig(
realm=path_config['realm'], extra=None)
publisher_session = ApplicationSession(publisher_session_config)
# add the publisher session to the router
#
self._router_session_factory.add(publisher_session,
authrole=path_config.get(
'role', 'anonymous'))
# now create the publisher Twisted Web resource
#
return PublisherResource(path_config.get('options', {}),
publisher_session)
# Caller resource (part of REST-bridge)
#
elif path_config['type'] == 'caller':
# create a vanilla session: the caller will use this to inject calls
#
caller_session_config = ComponentConfig(realm=path_config['realm'],
extra=None)
caller_session = ApplicationSession(caller_session_config)
# add the calling session to the router
#
self._router_session_factory.add(caller_session,
authrole=path_config.get(
'role', 'anonymous'))
# now create the caller Twisted Web resource
#
return CallerResource(path_config.get('options', {}),
caller_session)
# File Upload resource
#
elif path_config['type'] == 'upload':
upload_directory = os.path.abspath(
os.path.join(self.config.extra.cbdir,
path_config['directory']))
upload_directory = upload_directory.encode(
'ascii',
'ignore') # http://stackoverflow.com/a/20433918/884770
if not os.path.isdir(upload_directory):
emsg = "configured upload directory '{}' in file upload resource isn't a directory".format(
upload_directory)
log.msg(emsg)
raise ApplicationError("crossbar.error.invalid_configuration",
emsg)
if 'temp_directory' in path_config:
temp_directory = os.path.abspath(
os.path.join(self.config.extra.cbdir,
path_config['temp_directory']))
temp_directory = temp_directory.encode(
'ascii',
'ignore') # http://stackoverflow.com/a/20433918/884770
else:
temp_directory = os.path.abspath(tempfile.gettempdir())
temp_directory = os.path.join(temp_directory,
'crossbar-uploads')
if not os.path.exists(temp_directory):
os.makedirs(temp_directory)
if not os.path.isdir(temp_directory):
emsg = "configured temp directory '{}' in file upload resource isn't a directory".format(
temp_directory)
log.msg(emsg)
raise ApplicationError("crossbar.error.invalid_configuration",
emsg)
# file upload progress and finish events are published via this session
#
upload_session_config = ComponentConfig(realm=path_config['realm'],
extra=None)
upload_session = ApplicationSession(upload_session_config)
self._router_session_factory.add(upload_session,
authrole=path_config.get(
'role', 'anonymous'))
return FileUploadResource(upload_directory, temp_directory,
path_config['form_fields'],
upload_session,
path_config.get('options', {}))
# Generic Twisted Web resource
#
elif path_config['type'] == 'resource':
try:
klassname = path_config['classname']
if self.debug:
log.msg("Starting class '{}'".format(klassname))
c = klassname.split('.')
module_name, klass_name = '.'.join(c[:-1]), c[-1]
module = importlib.import_module(module_name)
make = getattr(module, klass_name)
return make(path_config.get('extra', {}))
except Exception as e:
emsg = "Failed to import class '{}' - {}".format(klassname, e)
log.msg(emsg)
log.msg("PYTHONPATH: {}".format(sys.path))
raise ApplicationError("crossbar.error.class_import_failed",
emsg)
# Schema Docs resource
#
elif path_config['type'] == 'schemadoc':
realm = path_config['realm']
if realm not in self.realm_to_id:
raise ApplicationError(
"crossbar.error.no_such_object",
"No realm with URI '{}' configured".format(realm))
realm_id = self.realm_to_id[realm]
realm_schemas = self.realms[realm_id].session._schemas
return SchemaDocResource(self._templates, realm, realm_schemas)
# Nested subpath resource
#
elif path_config['type'] == 'path':
nested_paths = path_config.get('paths', {})
if '/' in nested_paths:
nested_resource = self.create_resource(nested_paths['/'])
else:
nested_resource = Resource()
# nest subpaths under the current entry
#
self.add_paths(nested_resource, nested_paths)
return nested_resource
else:
raise ApplicationError(
"crossbar.error.invalid_configuration",
"invalid Web path type '{}'".format(path_config['type']))
def __init__(self, app): resource.Resource.__init__(self) self.wsgi = WSGIResource(reactor, reactor.getThreadPool(), app)
def createService(sname, config):
global monitor_threads, monitors, servicename
try:
application = connexion.FlaskApp(__name__,
specification_dir='swagger/')
flask_app = application.app
flask_app.url_map.strict_slashes = False
anchore_engine.subsys.metrics.init_flask_metrics(
flask_app, servicename=servicename)
application.add_api('swagger.yaml')
except Exception as err:
traceback.print_exc()
raise err
try:
myconfig = config['services'][sname]
servicename = sname
except Exception as err:
raise err
try:
kick_timer = int(myconfig['cycle_timer_seconds'])
except:
kick_timer = 1
doapi = False
try:
if myconfig['listen'] and myconfig['port'] and myconfig[
'endpoint_hostname']:
doapi = True
except:
doapi = False
kwargs = {}
kwargs['kick_timer'] = kick_timer
kwargs['monitors'] = monitors
kwargs['monitor_threads'] = monitor_threads
kwargs['servicename'] = servicename
if doapi:
# start up flask service
flask_site = WSGIResource(reactor,
reactor.getThreadPool(),
application=flask_app)
realroot = Resource()
realroot.putChild(
b"v1",
anchore_engine.services.common.getAuthResource(
flask_site, sname, config))
realroot.putChild(b"health",
anchore_engine.services.common.HealthResource())
# this will rewrite any calls that do not have an explicit version to the base path before being processed by flask
root = rewrite.RewriterResource(realroot, default_version_rewrite)
ret_svc = anchore_engine.services.common.createServiceAPI(
root, sname, config)
# start up the monitor as a looping call
lc = LoopingCall(anchore_engine.services.common.monitor, **kwargs)
lc.start(1)
else:
# start up the monitor as a timer service
svc = internet.TimerService(1, anchore_engine.services.common.monitor,
**kwargs)
svc.setName(sname)
ret_svc = svc
return (ret_svc)
# Put all top level module instances here that need to get stuff done
def top_ticker():
services.radio.tick()
services.player.tick()
services.scheduler.tick()
root = static.Data("", "text/plain")
root.putChild("files", static.File("../files"))
root.putChild("static", static.File("../static"))
#simple = Simple()
#simple.putChild("hello", Hello())
#root.putChild("api", simple)
radio_controller = web.RadioController(services)
root.putChild("radio", radio_controller)
schedule_controller = web.ScheduleController(services)
root.putChild("schedule", schedule_controller)
root.putChild(
"dav", WSGIResource(reactor, reactor.getThreadPool(), wsgidav_application))
root.putChild("home", web.HomePage())
site = server.Site(root)
ticker = task.LoopingCall(top_ticker)
ticker.start(0.250) # call every 250 ms
reactor.listenTCP(port, site)
reactor.run()
import sys, threading
from twisted.python import log
from twisted.internet import reactor
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
log.startLogging(sys.stdout)
def foo(tid):
print("Main thread {} called from thread {}".format(
threading.current_thread().ident, tid))
def application(environ, start_response):
tid = threading.current_thread().ident
print("Request on thread: {}".format(tid))
reactor.callFromThread(foo, tid)
start_response('200 OK', [('Content-type', 'text/plain')])
return ['Hello, world!']
resource = WSGIResource(reactor, reactor.getThreadPool(), application)
site = Site(resource)
site.log = lambda _: None
reactor.listenTCP(8080, site)
print("Maint thread: {}".format(threading.current_thread().ident))
reactor.run()
def prepare_twisted_service():
LOG.info('Prepare twisted services')
# check all peers
all_peers = {}
# check running mode
if not CONF.standalone:
# rabbitmq factory
rabbit_mq_factory = PikaFactory(url=CONF.rabbit_mq.rabbit_url)
rabbit_mq_factory.peer_list = CONF.bgp.running_config.keys()
rabbit_mq_factory.connect()
# mongodb connection
if CONF.database.use_replica:
mongo_connection = MongoApi(
connection_url=CONF.database.connection,
db_name=CONF.database.dbname,
use_replica=CONF.database.use_replica,
replica_name=CONF.database.replica_name,
read_preference=CONF.database.read_preference,
write_concern=CONF.database.write_concern,
w_timeout=CONF.database.write_concern_timeout)
else:
mongo_connection = MongoApi(
connection_url=CONF.database.connection,
db_name=CONF.database.dbname)
# check api bind host
if CONF.rest.bind_host == '0.0.0.0':
LOG.error(
'please use the exactly rest host ip address when not running in standalone mode'
)
sys.exit()
# TODO load channel filter and peer policy
else:
rabbit_mq_factory = None
mongo_connection = None
for peer in CONF.bgp.running_config:
LOG.info('Get peer %s configuration', peer)
if not CONF.standalone:
if CONF.bgp.running_config[peer]['local_addr'] == '0.0.0.0':
LOG.error(
'please use the exactly local bgp ip address when not running in standalone mode'
)
sys.exit()
if CONF.message.write_disk:
msg_file_path_for_peer = os.path.join(CONF.message.write_dir,
peer.lower())
if not os.path.exists(msg_file_path_for_peer):
os.makedirs(msg_file_path_for_peer)
LOG.info('Create dir %s for peer %s', msg_file_path_for_peer,
peer)
LOG.info('BGP message file path is %s', msg_file_path_for_peer)
else:
msg_file_path_for_peer = None
LOG.info('Create BGPPeering instance')
afi_safi_list = [
bgp_cons.AFI_SAFI_STR_DICT[afi_safi]
for afi_safi in CONF.bgp.running_config[peer]['afi_safi']
]
CONF.bgp.running_config[peer]['afi_safi'] = afi_safi_list
CONF.bgp.running_config[peer]['capability']['local'][
'afi_safi'] = afi_safi_list
bgp_peering = BGPPeering(
myasn=CONF.bgp.running_config[peer]['local_as'],
myaddr=CONF.bgp.running_config[peer]['local_addr'],
peerasn=CONF.bgp.running_config[peer]['remote_as'],
peeraddr=CONF.bgp.running_config[peer]['remote_addr'],
tag=CONF.bgp.running_config[peer]['tag'],
afisafi=CONF.bgp.running_config[peer]['afi_safi'],
msgpath=msg_file_path_for_peer,
md5=CONF.bgp.running_config[peer]['md5'],
channel=rabbit_mq_factory,
mongo_conn=mongo_connection)
all_peers[peer] = bgp_peering
CONF.bgp.running_config[peer]['factory'] = bgp_peering
# register to database and check agent role
if not CONF.standalone:
register_to_db(peer_ip=peer, mongo_api=mongo_connection)
if not CONF.bgp.running_config[peer]['tag']:
LOG.error(
'Please point out the role tag(SRC,DST or BOTH)for not running in standalone mode'
)
sys.exit()
load_channel_filter_from_db(peer_ip=peer,
mongo_api=mongo_connection)
# Starting api server
if sys.version_info[0] == 2:
from twisted.web.wsgi import WSGIResource
LOG.info("Prepare RESTAPI service")
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
site = Site(resource)
try:
reactor.listenTCP(CONF.rest.bind_port,
site,
interface=CONF.rest.bind_host)
LOG.info("serving RESTAPI on http://%s:%s", CONF.rest.bind_host,
CONF.rest.bind_port)
except Exception as e:
LOG.error(e, exc_info=True)
sys.exit()
for peer in all_peers:
LOG.info('start peer, peer address=%s', peer)
all_peers[peer].automatic_start()
reactor.run()
usage(1)
use_encryption = CRYPTO_AVAILABLE
port = cereconf.SAPINTEGRATION_SERVICE_PORT
logfile = cereconf.SAPINTEGRATION_SERVICE_LOGFILE
for opt, val in opts:
if opt in ('-l', '--logfile'):
logfile = val
elif opt in ('-p', '--port'):
port = int(val)
elif opt in ('--unencrypted', ):
use_encryption = False
## TBD: Use Cerebrum logger instead?
# Init twisted logger
log_observer = startLogging(file(logfile, 'w'))
# Run service
service = Application([SAPIntegrationServer], 'tns')
resource = WSGIResource(reactor, reactor.getThreadPool(), service)
root = Resource()
root.putChild('SOAP', resource)
if use_encryption:
# TODO: we need to set up SSL properly
sslcontext = ssl.DefaultOpenSSLContextFactory(
cereconf.SSL_PRIVATE_KEY_FILE, cereconf.SSL_CERTIFICATE_FILE)
reactor.listenSSL(int(port), Site(root), contextFactory=sslcontext)
else:
reactor.listenTCP(int(port), Site(root))
reactor.run()
def main():
frame_thread = threading.Thread(target=frame_worker)
frame_thread.start()
server = 'CherryPy'
if server == 'Twisted':
input('This will cause a giant memory leak, continue?')
from twisted.internet import reactor
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
site = Site(resource)
reactor.listenTCP(5000, site)
reactor.run()
elif server == 'CherryPy':
import cherrypy
from paste.translogger import TransLogger
max_concurrent_streams = 10
# Enable WSGI access logging via Paste
app_logged = TransLogger(app)
# Mount the WSGI callable object (app) on the root directory
cherrypy.tree.graft(app_logged, '/')
# Set the configuration of the web server
cherrypy.config.update({
'engine.autoreload.on':
False,
'checker.on':
False,
'tools.log_headers.on':
False,
'request.show_tracebacks':
False,
'request.show_mismatched_params':
False,
'log.screen':
False,
'server.socket_port':
5000,
'server.socket_host':
'0.0.0.0',
'server.thread_pool':
max_concurrent_streams - 1,
'server.socket_queue_size':
max_concurrent_streams - 1,
'server.accepted_queue_size':
-1,
})
# Start the CherryPy WSGI web server
cherrypy.engine.start()
cherrypy.engine.block()
elif server == 'Flask':
app.run(host='0.0.0.0', port=5000)
wsgi_cb(
str(code) + ' ' + httpStatusCodes.get(code, 'ERROR'), headers.items())
return result
if __name__ == "__main__":
params = namedParameters()
port = params.get('socket', 8080)
from twisted.web import server
from twisted.web.wsgi import WSGIResource
from twisted.python.threadpool import ThreadPool
from twisted.python import log
from twisted.internet import reactor
from twisted.application import service, strports
# Create and start a thread pool,
wsgiThreadPool = ThreadPool(1, 1)
wsgiThreadPool.start()
# ensuring that it will be stopped when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', wsgiThreadPool.stop)
reactor.listenTCP(
port,
server.Site(WSGIResource(reactor, wsgiThreadPool, wsgi_application)))
log.startLogging(log.FileLogObserver(sys.stderr))
reactor.run()
def get_WHAT_resource(deployment_type, port=None):
'''
Pseudo factory that returns the proper Resource object for the WHAT.
Takes a deployment type and (for development) a port number.
Returns a tuple (Twisted Resource, Twisted Application, Twisted Server)
(This function can almost get away with returning just the resource, but we sometimes need to daemonize the server).
'''
print "Deployment type: %s" % deployment_type
if not port: #For development, port will have been specified.
if deployment_type == "production":
from settings.production import PORT as PRODUCTION_SERVER_PORT
port = PRODUCTION_SERVER_PORT
#TODO: Integrate SSL.
elif deployment_type == "staging":
from settings.staging import PORT as STAGING_SERVER_PORT
port = STAGING_SERVER_PORT
else: #If somehow the deployment type wasn't specified properly, let's crash now rather than wait for a problem in django.
wsgiThreadPool.stop()
exit('Deployment type must be "production," "staging," or "development." \nIf you need help, talk to the Dev / NetOps satchem.')
application = service.Application('SlashRoot WHAT in %s' % deployment_type)
# Create and start a thread pool,
wsgiThreadPool = ThreadPool()
#The pool will stop when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', wsgiThreadPool.stop)
what_server = service.MultiService()
tps = ThreadPoolService(wsgiThreadPool)
tps.setServiceParent(what_server)
#Use django's WSGIHandler to create the resource.
what_django_resource = WSGIResource(reactor, tps.pool, WSGIHandler())
root = Root(what_django_resource)
#Now we need to handle static media.
#Servce Django media files off of /media:
if deployment_type == "development":
#TODO: Grab static media locations for WHAT files and django admin files from local settings and use them here.
#raise RuntimeError('Dominick, fix this.')
#Maybe we want to hardcode production and staging paths. Maybe we don't.
admin_static = MediaService(os.path.join(os.path.abspath("."), resources.DEVELOPMENT_ADMIN_MEDIA))
staticrsrc = MediaService(os.path.join(os.path.abspath("."), "%s/static" % PROJECT_ROOT))
else:
#Maybe we want to hardcode production and staging paths. Maybe we don't.
admin_static = MediaService(os.path.join(os.path.abspath("."), resources.PRODUCTION_ADMIN_MEDIA))
staticrsrc = MediaService(os.path.join(os.path.abspath("."), resources.STATIC_PRODUCTION))
#Now that we have the static media locations, add them to the root.
root.putChild("media_admin", admin_static)
root.putChild("media", staticrsrc)
main_site = server.Site(root)
internet.TCPServer(port, main_site).setServiceParent(what_server)
#what_server = strports.service('tcp:%s' % port, server.Site(resource)) #TODO: Use port number from kwarg
return what_django_resource, application, what_server
def __init__(self, reactor, threadpool, htpasswd, path):
self.htpasswd = htpasswd
self.path = path
WSGIResource.__init__(
self, reactor, threadpool, self.middleware_auth(
self.tracApplication))
def __init__(self, reactor, threadpool, path):
self.path = path
WSGIResource.__init__(
self, reactor, threadpool, self.tracApplication)
