def setup(args=None):
"""
This is the main setup function to establish the TCP listening logic for
the API server. This code also takes into account development or unit test mode.
"""
# Setup API server
api = ParadropAPIServer(reactor)
api.putChild('internal', Base(apiinternal, allowNone=True))
site = Site(api, timeout=None)
# Development mode
if(args and args.development):
thePort = settings.PDFCD_PORT + 10000
out.info('Using DEVELOPMENT variables')
# Disable sending the error traceback to the client
site.displayTracebacks = True
elif(args and args.unittest):
thePort = settings.PDFCD_PORT + 20000
out.info('Running under unittest mode')
site.displayTracebacks = True
else:
thePort = settings.PDFCD_PORT
site.displayTracebacks = False
initializeSystem()
# Setup the port we listen on
reactor.listenTCP(thePort, site)
# Never return from here
reactor.run()
def setup(self, interface: str, port: int) -> None:
logger.info("Starting Internal API HTTP server on %s:%d", interface, port)
root = Resource()
matrix = Resource()
root.putChild(b"_matrix", matrix)
identity = Resource()
matrix.putChild(b"identity", identity)
internal = Resource()
identity.putChild(b"internal", internal)
authenticated_bind = AuthenticatedBindThreePidServlet(self.sydent)
internal.putChild(b"bind", authenticated_bind)
authenticated_unbind = AuthenticatedUnbindThreePidServlet(self.sydent)
internal.putChild(b"unbind", authenticated_unbind)
factory = Site(root)
factory.displayTracebacks = False
self.sydent.reactor.listenTCP(
port,
factory,
backlog=50, # taken from PosixReactorBase.listenTCP
interface=interface,
)
def _create_web_factory(self, config):
options = config.get('options', {})
# create Twisted Web root resource
if '/' in config['paths']:
root_config = config['paths']['/']
root = self._create_resource(root_config, nested=False)
else:
root = Resource404(self._templates, b'')
# create Twisted Web resources on all non-root paths configured
self._add_paths(root, config.get('paths', {}))
# create the actual transport factory
transport_factory = Site(root)
transport_factory.noisy = False
# Web access logging
if not options.get('access_log', False):
transport_factory.log = lambda _: None
# Traceback rendering
transport_factory.displayTracebacks = options.get('display_tracebacks', False)
# HSTS
if options.get('hsts', False):
if 'tls' in config['endpoint']:
hsts_max_age = int(options.get('hsts_max_age', 31536000))
transport_factory.requestFactory = createHSTSRequestFactory(transport_factory.requestFactory, hsts_max_age)
else:
self.log.warn("Warning: HSTS requested, but running on non-TLS - skipping HSTS")
return transport_factory
def run(port, sqlite_conn, index_html, static_root):
#--------------------------------------------------------------------------
# log to stdout
#--------------------------------------------------------------------------
from twisted.python import log
import sys
log.startLogging(sys.stdout)
#--------------------------------------------------------------------------
# database
#--------------------------------------------------------------------------
from twisted.enterprise import adbapi
pool = adbapi.ConnectionPool('sqlite3', sqlite_conn, check_same_thread=False,
cp_min=1, cp_max=1)
data_model = DataModel(pool)
#--------------------------------------------------------------------------
# url/resource mapping
#--------------------------------------------------------------------------
root = Resource()
root.putChild('', File(index_html))
root.putChild('static', File(static_root))
root.putChild('todos', Todos(data_model))
#--------------------------------------------------------------------------
# serve it
#--------------------------------------------------------------------------
from twisted.internet import reactor
from twisted.web.server import Site
site = Site(root)
site.displayTracebacks = False
reactor.listenTCP(port, site)
reactor.run()
def _create_web_factory(self, config):
options = config.get('options', {})
# create Twisted Web root resource
if '/' in config['paths']:
root_config = config['paths']['/']
root = self._create_resource(root_config, nested=False)
else:
root = Resource404(self._templates, b'')
# create Twisted Web resources on all non-root paths configured
self._add_paths(root, config.get('paths', {}))
# create the actual transport factory
transport_factory = Site(root)
transport_factory.noisy = False
# Web access logging
if not options.get('access_log', False):
transport_factory.log = lambda _: None
# Traceback rendering
transport_factory.displayTracebacks = options.get('display_tracebacks', False)
# HSTS
if options.get('hsts', False):
if 'tls' in config['endpoint']:
hsts_max_age = int(options.get('hsts_max_age', 31536000))
transport_factory.requestFactory = createHSTSRequestFactory(transport_factory.requestFactory, hsts_max_age)
else:
self.log.warn("Warning: HSTS requested, but running on non-TLS - skipping HSTS")
return transport_factory
def Service(options):
consumer = QueueConsumer(
consumer_queue=ns.MASTER_QUEUE,
pool_size=ns.MAX_MESSAGES,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT
)
publisher = QueuePublisher(
declare_strategy=1,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT
)
resource = MasterResource()
resource.init(consumer, publisher)
site = Site(resource)
site.displayTracebacks = ns.DEBUG
return internet.TCPServer(int(options['port']), site, \
interface=options['iface'])
def make_site(resource, site_logs_path):
site = Site(
resource,
logPath=site_logs_path.path,
logFormatter=_LogFormatter(datetime.utcnow).json_access_log,
)
site.displayTracebacks = False
return site
def make_site(resource, site_logs_path):
site = Site(
resource,
logPath=site_logs_path.path,
logFormatter=_LogFormatter(datetime.utcnow).json_access_log,
)
site.displayTracebacks = False
return site
def StartStorageService(config, block_store):
try:
http_port = config['StorageService']['HttpPort']
http_host = config['StorageService']['Host']
worker_threads = config['StorageService'].get('WorkerThreads', 8)
reactor_threads = config['StorageService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on port %s', http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
block = Resource()
block.putChild(
b'get',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlockApp(block_store))))
block.putChild(
b'gets',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlocksApp(block_store))))
block.putChild(
b'store',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(StoreBlocksApp(block_store))))
block.putChild(
b'list',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(ListBlocksApp(block_store))))
block.putChild(
b'check',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(CheckBlocksApp(block_store))))
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(block_store))))
root.putChild(b'block', block)
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def get_site(resource, logging=False):
"""
:param resource: A :class:`twisted.web.resource.Resource` object.
:return: a :class:`Site` that can be run
"""
site = Site(resource)
site.displayTracebacks = False
site.requestFactory = MimicLoggingRequest if logging else MimicRequest
return site
def get_site(resource, logging=False):
"""
:param resource: A :class:`twisted.web.resource.Resource` object.
:return: a :class:`Site` that can be run
"""
site = Site(resource)
site.displayTracebacks = False
site.requestFactory = MimicLoggingRequest if logging else MimicRequest
return site
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')
]
cassandra_cluster = LoggingCQLClient(
RoundRobinCassandraCluster(seed_endpoints,
config_value('cassandra.keyspace')),
log.bind(system='otter.silverberg'))
set_store(CassScalingGroupCollection(cassandra_cluster))
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler') and not config_value('mock'):
scheduler_service = SchedulerService(
int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')), cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def site(self, displayTracebacks=True, **kwargs):
"""
A :twisted:`web.server.Site` that will serve me.
"""
site = Site(self.app.resource(), **kwargs)
site.displayTracebacks = displayTracebacks
return site
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
# Try to configure graylog and airbrake.
if config_value('graylog'):
if GraylogUDPPublisher is not None:
log.addObserver(
make_observer_chain(
GraylogUDPPublisher(**config_value('graylog')), False))
else:
warnings.warn("There is a configuration option for Graylog, but "
"txgraylog is not installed.")
if config_value('airbrake'):
if AirbrakeLogObserver is not None:
airbrake = AirbrakeLogObserver(
config_value('airbrake.api_key'),
config_value('environment'),
use_ssl=True
)
airbrake.start()
else:
warnings.warn("There is a configuration option for Airbrake, but "
"txairbrake is not installed.")
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'))
set_store(CassScalingGroupCollection(cassandra_cluster))
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler'):
scheduler_service = SchedulerService(int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')),
cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace')), log.bind(system='otter.silverberg'))
set_store(CassScalingGroupCollection(cassandra_cluster))
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(
config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler') and not config_value('mock'):
scheduler_service = SchedulerService(int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')),
cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def secureSite(_environ=environ):
"""Builds the secure (HTTPS, port 443) site.
"""
root = File(_environ["STATIC_PATH"])
root.putChild("subscribe", SubscribeResource("*****@*****.**"))
site = Site(root)
site.displayTracebacks = False
site.requestFactory = _withHSTS(site.requestFactory)
return site
def setup_gitter_oauth(api, port, debug=False):
"""Register the OAuth website with Twisted.
"""
root = Resource()
root.putChild('', Index(api.bot_fullname))
root.putChild('auth_gitter', Redirect(api))
root.putChild('callback', Callback(api))
site = Site(root)
site.displayTracebacks = debug
site.logRequest = True
reactor.listenTCP(port, site)
def makeService(self, options):
store = Store(options['dbdir'])
siteService = IService(store)
site = Site(
RootResource(store=store,
trackingID=options['tracking-id']))
site.displayTracebacks = not options['notracebacks']
siteService.addService(
strports.service(options['port'], site, reactor=reactor))
return siteService
def makeService(config):
"""
Set up the otter-api service.
"""
s = MultiService()
core = MimicCore.fromPlugins(Clock())
root = MimicRoot(core)
site = Site(root.app.resource())
site.displayTracebacks = False
service(config['listen'], site).setServiceParent(s)
return s
def run(
self,
host=None,
port=None,
logFile=None,
endpoint_description=None,
displayTracebacks=True,
):
"""
Run a minimal twisted.web server on the specified C{port}, bound to the
interface specified by C{host} and logging to C{logFile}.
This function will run the default reactor for your platform and so
will block the main thread of your application. It should be the last
thing your klein application does.
@param host: The hostname or IP address to bind the listening socket
to. "0.0.0.0" will allow you to listen on all interfaces, and
"127.0.0.1" will allow you to listen on just the loopback
interface.
@type host: str
@param port: The TCP port to accept HTTP requests on.
@type port: int
@param logFile: The file object to log to, by default C{sys.stdout}
@type logFile: file object
@param endpoint_description: specification of endpoint. Must contain
protocol, port and interface. May contain other optional arguments,
e.g. to use SSL: "ssl:443:privateKey=key.pem:certKey=crt.pem"
@type endpoint_description: str
@param displayTracebacks: Weather a processing error will result in
a page displaying the traceback with debugging information or not.
@type displayTracebacks: bool
"""
if logFile is None:
logFile = sys.stdout
log.startLogging(logFile)
if not endpoint_description:
endpoint_description = "tcp:port={0}:interface={1}".format(
port, host)
endpoint = endpoints.serverFromString(reactor, endpoint_description)
site = Site(self.resource())
site.displayTracebacks = displayTracebacks
endpoint.listen(site)
reactor.run()
def StartService(config):
# load the eservice database
try:
dbfile_name = config['Service']['EnclaveServiceDatabaseFile']
except KeyError:
logger.error(
'missing required configuration for enclave service database')
sys.exit(-1)
try:
eservice_db.load_database(dbfile_name)
except Exception as e:
logger.error('error loading eservice database from file %s', str(e))
sys.exit(-1)
try:
http_port = config['Service']['HttpPort']
http_host = config['Service']['Host']
worker_threads = config['Service'].get('WorkerThreads', 8)
reactor_threads = config['Service'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on host %s, port %s', http_host, http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
flask_app = toxaway.views.register(config)
flask_site = WSGIResource(reactor, reactor.getThreadPool(), flask_app)
root = Resource()
root.putChild(b'shutdown', ShutdownResource())
root.putChild(b'flask', flask_site)
files = config.get('StaticContent', {})
logger.info('files=%s', files)
for key, val in files.items():
logger.info('map <%s> to <%s>', key, val)
root.putChild(key.encode(), File(val.encode()))
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def make_site(basefp, config):
resource = JinjaHandler('index.html')
resource.putChild('static', File('content/static'))
resource.putChild('blog', File('content/blog'))
resource.putChild('collect-email', CollectEmailHandler(basefp))
resource.putChild('signup', Redirect("/"))
resource.putChild('submit-subscription', SubmitSubscriptionHandler(basefp))
resource.putChild('support', Redirect("https://leastauthority.zendesk.com/home"))
site = Site(resource, logPath=basefp.child('sitelogs').path)
site.displayTracebacks = True
return site
def StartEnclaveService(config, enclave):
try:
http_port = config['EnclaveService']['HttpPort']
http_host = config['EnclaveService']['Host']
storage_url = config['StorageService']['URL']
worker_threads = config['EnclaveService'].get('WorkerThreads', 8)
reactor_threads = config['EnclaveService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('enclave service started on %s:%s', http_host, http_port)
logger.info('verifying_key: %s', enclave.verifying_key)
logger.info('encryption_key: %s', enclave.encryption_key)
logger.info('enclave_id: %s', enclave.enclave_id)
logger.info('storage service: %s', storage_url)
thread_pool = ThreadPool(minthreads=1, maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(enclave, storage_url))))
root.putChild(
b'initialize',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InitializeApp(enclave))))
root.putChild(
b'invoke',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InvokeApp(enclave))))
root.putChild(
b'verify',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(VerifyApp(enclave))))
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def make_site(basefp, config):
resource = JinjaHandler('index.html')
resource.putChild('static', File('content/static'))
resource.putChild('blog', File('content/blog'))
resource.putChild('collect-email', CollectEmailHandler(basefp, config.products))
resource.putChild('signup', SignupHandler(config.products))
resource.putChild('devpay-complete', DevPayPurchaseHandler(basefp, config.products))
resource.putChild('activation-request', ActivationRequestHandler(basefp, config.products))
resource.putChild('support', Redirect("https://leastauthority.zendesk.com/home"))
site = Site(resource, logPath=basefp.child('sitelogs').path)
site.displayTracebacks = False
return site
def startService(self):
if self.port_s is not None:
endpoint = serverFromString(reactor, self.port_s)
d = endpoint.listen(self.site)
d.addCallback(self.register_port, 'app')
if self.auth_info_endpoint_s is not None:
authInfoApp = AuthInfoApp()
self.authInfoApp = authInfoApp
authInfoSite = Site(authInfoApp.app.resource())
authInfoSite.displayTracebacks = self.site.displayTracebacks
endpoint = serverFromString(reactor, self.auth_info_endpoint_s)
d2 = endpoint.listen(authInfoSite)
d2.addCallback(self.register_port, 'authInfoSite')
def startService(self):
if self.port_s is not None:
endpoint = serverFromString(reactor, self.port_s)
d = endpoint.listen(self.site)
d.addCallback(self.register_port, 'app')
if self.auth_info_endpoint_s is not None:
authInfoApp = AuthInfoApp()
self.authInfoApp = authInfoApp
authInfoSite = Site(authInfoApp.app.resource())
authInfoSite.displayTracebacks = self.site.displayTracebacks
endpoint = serverFromString(reactor, self.auth_info_endpoint_s)
d2 = endpoint.listen(authInfoSite)
d2.addCallback(self.register_port, 'authInfoSite')
def startMimic():
"""
Setup the mimic application using steps similar to
:obj:`mimic.tap.makeService' and start listening for requests.
"""
clock = Clock()
core = MimicCore.fromPlugins(clock)
root = MimicRoot(core, clock)
site = Site(root.app.resource())
site.displayTracebacks = False
endpoint = serverFromString(reactor,
b"tcp:{0}:interface=127.0.0.1".format(_PORT))
endpoint.listen(site)
def makeService(config):
"""
Set up the otter-api service.
"""
s = MultiService()
port_offset = 8900
for klein_obj in (mimic_api.MimicPresetApi(), auth_api.AuthApi(),
nova_api.NovaApi(), loadbalancer_api.LoadBalancerApi()):
site = Site(klein_obj.app.resource())
api_service = service(str(port_offset), site)
api_service.setServiceParent(s)
site.displayTracebacks = False
port_offset += 1
return s
def setup(args=None):
"""
This is the main setup function to establish the TCP listening logic for
the API server. This code also takes into account development or unit test mode.
"""
# Setup API server
api = ParadropAPIServer(reactor)
api.putChild('internal', Base(apiinternal, allowNone=True))
site = Site(api, timeout=None)
# Setup local root of the website
root = static.File("/var/lib/apps/paradrop/www")
website = Site(root)
portalPort = 80
# Development mode
if(args and args.development):
thePort = settings.PDFCD_PORT + 10000
site.displayTracebacks = True
elif (args and args.local):
portalPort = 8080
thePort = settings.PDFCD_PORT + 10000
elif(args and args.unittest):
thePort = settings.PDFCD_PORT + 20000
site.displayTracebacks = True
else:
thePort = settings.PDFCD_PORT
site.displayTracebacks = False
initializeSystem()
# Setup the port we listen on
reactor.listenTCP(thePort, site)
reactor.listenTCP(portalPort, website)
# Never return from here
reactor.run(installSignalHandlers=0)
def makeService(config):
"""
Set up the otter-api service.
"""
s = MultiService()
if config['realtime']:
from twisted.internet import reactor as clock
else:
clock = Clock()
core = MimicCore.fromPlugins(clock)
root = MimicRoot(core, clock)
site = Site(root.app.resource())
site.displayTracebacks = False
service(config['listen'], site).setServiceParent(s)
return s
def makeService(config):
"""
Set up the service.
"""
from twisted.internet import reactor
s = MultiService()
bloc = Bloc(reactor, float(config["timeout"]), float(config["settle"]))
s.addService(bloc)
site = Site(bloc.app.resource())
site.displayTracebacks = False
# The Twisted code currently (v16.6.0, 17.1.0) compares the type of
# this argument to 'str' in order to determine how to handle it.
description = str(config['listen'])
s.addService(service(description, site))
return s
def startMimic():
"""
Setup the mimic application using steps similar to
:obj:`mimic.tap.makeService' and start listening for requests.
"""
clock = Clock()
core = MimicCore.fromPlugins(clock)
root = MimicRoot(core, clock)
site = Site(root.app.resource())
site.displayTracebacks = False
endpoint = serverFromString(
reactor,
b"tcp:{0}:interface=127.0.0.1".format(_PORT)
)
endpoint.listen(site)
def __init__(self,
bridge,
port,
homeserver_url,
homeserver_domain,
botname,
token_as,
token_hs,
debug=False):
self.bridge = bridge
self.homeserver_url = homeserver_url
self.homeserver_domain = homeserver_domain
self.token_as = token_as
self.token_hs = token_hs
if botname[0] == '@':
botname = botname[1:]
if ':' in botname:
botname, domain = botname.split(':', 1)
if domain != homeserver_domain:
raise ValueError("Bot domain doesn't match homeserver")
self.bot_username = botname
self.bot_fullname = '@%s:%s' % (botname, homeserver_domain)
# Create virtual user for bot
if not self.bridge.virtualuser_exists('gitter'):
log.info("Creating user gitter")
d = self.matrix_request('POST', '_matrix/client/r0/register', {
'type': 'm.login.application_service',
'username': '******'
})
self.bridge.add_virtualuser('gitter')
d.addErrback(
Errback(
log, "Error creating user 'gitter' for the "
"bridge; usage over federated rooms "
"might not work correctly"))
root = Resource()
root.putChild('transactions', Transaction(self))
root.putChild('users', Users(self))
site = Site(root)
site.displayTracebacks = debug
site.logRequest = True
reactor.listenTCP(port, site)
def _create_web_factory(self, config, is_secure):
options = config.get('options', {})
# create Twisted Web root resource
if '/' in config['paths']:
root_config = config['paths']['/']
root = self._create_resource(root_config, nested=False)
else:
root = Resource404(self._templates, b'')
# create Twisted Web resources on all non-root paths configured
self._add_paths(root, config.get('paths', {}))
# create the actual transport factory
transport_factory = Site(
root,
timeout=options.get('client_timeout', None),
)
transport_factory.noisy = False
# we override this factory so that we can inject
# _LessNoisyHTTPChannel to avoid info-level logging on timing
# out web clients (which happens all the time).
def channel_protocol_factory():
return _GenericHTTPChannelProtocol(_LessNoisyHTTPChannel())
transport_factory.protocol = channel_protocol_factory
# Web access logging
if not options.get('access_log', False):
transport_factory.log = lambda _: None
# Traceback rendering
transport_factory.displayTracebacks = options.get('display_tracebacks', False)
# HSTS
if options.get('hsts', False):
if is_secure:
hsts_max_age = int(options.get('hsts_max_age', 31536000))
transport_factory.requestFactory = createHSTSRequestFactory(transport_factory.requestFactory, hsts_max_age)
else:
self.log.warn("Warning: HSTS requested, but running on non-TLS - skipping HSTS")
return transport_factory
def setup(self, interface, port):
logger.info("Starting Internal API HTTP server on %s:%d", interface, port)
root = Resource()
matrix = Resource()
root.putChild('_matrix', matrix)
identity = Resource()
matrix.putChild('identity', identity)
internal = Resource()
identity.putChild('internal', internal)
authenticated_bind = AuthenticatedBindThreePidServlet(self.sydent)
internal.putChild('bind', authenticated_bind)
factory = Site(root)
factory.displayTracebacks = False
twisted.internet.reactor.listenTCP(port, factory, interface=interface)
def run(self):
'''
Starts the application.
'''
log_info('Starting application')
resource = HttIpResource(app=self)
site_factory = Site(resource)
site_factory.displayTracebacks = False
try:
sckt = self.get_socket()
except EnvironmentError as e:
sys.stderr.write(str(e) + '\n')
sys.exit(1)
self.listen(sckt, reactor, site_factory)
self.maybe_stop_later()
reactor.run()
def setup(self, interface, port):
logger.info("Starting Internal API HTTP server on %s:%d", interface, port)
root = Resource()
matrix = Resource()
root.putChild('_matrix', matrix)
identity = Resource()
matrix.putChild('identity', identity)
internal = Resource()
identity.putChild('internal', internal)
authenticated_bind = AuthenticatedBindThreePidServlet(self.sydent)
internal.putChild('bind', authenticated_bind)
factory = Site(root)
factory.displayTracebacks = False
self.sydent.reactor.listenTCP(port, factory, interface=interface)
def make_web_service(self):
"""
Create and return the web service site.
"""
web_realm = webauth.WebRealm.make_instance(self.reactor)
portal = Portal(web_realm)
checker = webauth.WerewolfWebCredChecker.make_instance()
portal.registerChecker(checker)
resources = WebResources.make_instance(
self.reactor,
portal)
root = resources.app.resource()
site = Site(root)
session_length = self.session_length
def sessionFactory(site, uid, reactor=None):
s = Session(site, uid, reactor=reactor)
s.sessionTimeout = session_length
return s
site.sessionFactory = sessionFactory
site.displayTracebacks = self.displayTracebacks
return site
def Service(options):
consumer = QueueConsumer(consumer_queue=ns.MASTER_QUEUE,
pool_size=ns.MAX_MESSAGES,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT)
publisher = QueuePublisher(declare_strategy=1,
spec_path=ns.SPEC_PATH,
vhost=ns.RABBITMQ_VHOST,
username=ns.RABBITMQ_USERNAME,
password=ns.RABBITMQ_PASSWORD,
host=ns.RABBITMQ_HOST,
port=ns.RABBITMQ_PORT)
resource = MasterResource()
resource.init(consumer, publisher)
site = Site(resource)
site.displayTracebacks = ns.DEBUG
return internet.TCPServer(int(options['port']), site, \
interface=options['iface'])
def StartStorageService(config, block_store, service_keys) :
try :
http_port = config['StorageService']['HttpPort']
http_host = config['StorageService']['Host']
worker_threads = config['StorageService'].get('WorkerThreads', 8)
reactor_threads = config['StorageService'].get('ReactorThreads', 8)
except KeyError as ke :
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on port %s', http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
block = Resource()
for (wsgi_verb, wsgi_app) in wsgi_block_operation_map.items() :
logger.info('add handler for %s', wsgi_verb)
verb = wsgi_verb.encode('utf8')
app = AppWrapperMiddleware(wsgi_app(config, block_store, service_keys))
block.putChild(verb, WSGIResource(reactor, thread_pool, app))
root = Resource()
root.putChild(b'info', WSGIResource(reactor, thread_pool, AppWrapperMiddleware(InfoApp(config, service_keys))))
root.putChild(b'block', block)
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor, http_port, backlog=32, interface=http_host)
endpoint.listen(site)
def __init__(self, xml):
#get protobufcfg from main config
protostreamcfg = manager.config.getConfig('iweb')
#ip
ip = manager.config.getValue(protostreamcfg, 'ip', default='0.0.0.0')
#port
port = int(manager.config.getValue(protostreamcfg, 'port', default=37016))
#sockjsport
sockjsport = int(manager.config.getValue(protostreamcfg, 'sockjsport', default=37017))
apicfg= protostreamcfg.find('apikeys').findall('key')
apiKeys = []
for apikeycfg in apicfg:
apiKeys.append(apikeycfg.get("value"))
proto = ProtoBufResource(manager.messages.getProto, manager.messages.proccessMessage, apiKeys)
site = Site(proto)
site.displayTracebacks = False
reactor.listenTCP(port, site, 50)
log.info("setting up protobufresource server on ", ip, ":", port)
reactor.listenTCP(sockjsport, SockJSFactory(WebSockFactory()), interface=ip)
log.info("setting up sockjs server on ", ip, ":", sockjsport)
def create_service():
class ServerContextFactory:
def getContext(self):
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_certificate_file(config.SERVER_CERT_PATH)
ctx.use_privatekey_file(config.SERVER_KEY_PATH)
return ctx
root = ServiceRoot()
siteFactory = Site(root, logPath='data/access_log')
if config.production:
siteFactory.displayTracebacks = False
if config.ENABLE_SSL:
server = internet.SSLServer(config.LISTEN_PORT,
siteFactory,
ServerContextFactory(),
interface=config.LISTEN_HOST)
else:
server = internet.TCPServer(config.LISTEN_PORT,
siteFactory,
interface=config.LISTEN_HOST)
return server
def addWebServer(config, distributor):
"""Set up a web server for HTTP(S)-based bridge distribution.
:type config: :class:`bridgedb.persistent.Conf`
:param config: A configuration object from
:mod:`bridgedb.Main`. Currently, we use these options::
HTTP_UNENCRYPTED_PORT
HTTP_UNENCRYPTED_BIND_IP
HTTP_USE_IP_FROM_FORWARDED_HEADER
HTTPS_N_BRIDGES_PER_ANSWER
HTTPS_INCLUDE_FINGERPRINTS
HTTPS_KEY_FILE
HTTPS_CERT_FILE
HTTPS_PORT
HTTPS_BIND_IP
HTTPS_USE_IP_FROM_FORWARDED_HEADER
HTTPS_ROTATION_PERIOD
RECAPTCHA_ENABLED
RECAPTCHA_PUB_KEY
RECAPTCHA_SEC_KEY
RECAPTCHA_REMOTEIP
GIMP_CAPTCHA_ENABLED
GIMP_CAPTCHA_DIR
GIMP_CAPTCHA_HMAC_KEYFILE
GIMP_CAPTCHA_RSA_KEYFILE
SERVER_PUBLIC_FQDN
CSP_ENABLED
CSP_REPORT_ONLY
CSP_INCLUDE_SELF
:type distributor: :class:`bridgedb.https.distributor.HTTPSDistributor`
:param distributor: A bridge distributor.
:raises SystemExit: if the servers cannot be started.
:rtype: :api:`twisted.web.server.Site`
:returns: A webserver.
"""
captcha = None
fwdHeaders = config.HTTP_USE_IP_FROM_FORWARDED_HEADER
numBridges = config.HTTPS_N_BRIDGES_PER_ANSWER
fprInclude = config.HTTPS_INCLUDE_FINGERPRINTS
logging.info("Starting web servers...")
setFQDN(config.SERVER_PUBLIC_FQDN)
index = IndexResource()
options = OptionsResource()
howto = HowtoResource()
robots = static.File(os.path.join(TEMPLATE_DIR, 'robots.txt'))
assets = static.File(os.path.join(TEMPLATE_DIR, 'assets/'))
keys = static.Data(bytes(strings.BRIDGEDB_OPENPGP_KEY), 'text/plain')
csp = CSPResource(enabled=config.CSP_ENABLED,
includeSelf=config.CSP_INCLUDE_SELF,
reportViolations=config.CSP_REPORT_ONLY,
useForwardedHeader=fwdHeaders)
root = CustomErrorHandlingResource()
root.putChild('', index)
root.putChild('robots.txt', robots)
root.putChild('keys', keys)
root.putChild('assets', assets)
root.putChild('options', options)
root.putChild('howto', howto)
root.putChild('maintenance', maintenance)
root.putChild('error', resource500)
root.putChild(CSPResource.reportURI, csp)
if config.RECAPTCHA_ENABLED:
publicKey = config.RECAPTCHA_PUB_KEY
secretKey = config.RECAPTCHA_SEC_KEY
captcha = partial(ReCaptchaProtectedResource,
remoteIP=config.RECAPTCHA_REMOTEIP)
elif config.GIMP_CAPTCHA_ENABLED:
# Get the master HMAC secret key for CAPTCHA challenges, and then
# create a new HMAC key from it for use on the server.
captchaKey = crypto.getKey(config.GIMP_CAPTCHA_HMAC_KEYFILE)
hmacKey = crypto.getHMAC(captchaKey, "Captcha-Key")
# Load or create our encryption keys:
secretKey, publicKey = crypto.getRSAKey(config.GIMP_CAPTCHA_RSA_KEYFILE)
captcha = partial(GimpCaptchaProtectedResource,
hmacKey=hmacKey,
captchaDir=config.GIMP_CAPTCHA_DIR)
if config.HTTPS_ROTATION_PERIOD:
count, period = config.HTTPS_ROTATION_PERIOD.split()
sched = ScheduledInterval(count, period)
else:
sched = Unscheduled()
bridges = BridgesResource(distributor, sched, numBridges, fwdHeaders,
includeFingerprints=fprInclude)
if captcha:
# Protect the 'bridges' page with a CAPTCHA, if configured to do so:
protected = captcha(publicKey=publicKey,
secretKey=secretKey,
useForwardedHeader=fwdHeaders,
protectedResource=bridges)
root.putChild('bridges', protected)
logging.info("Protecting resources with %s." % captcha.func.__name__)
else:
root.putChild('bridges', bridges)
site = Site(root)
site.displayTracebacks = False
if config.HTTP_UNENCRYPTED_PORT: # pragma: no cover
ip = config.HTTP_UNENCRYPTED_BIND_IP or ""
port = config.HTTP_UNENCRYPTED_PORT or 80
try:
reactor.listenTCP(port, site, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started HTTP server on %s:%d" % (str(ip), int(port)))
if config.HTTPS_PORT: # pragma: no cover
ip = config.HTTPS_BIND_IP or ""
port = config.HTTPS_PORT or 443
try:
from twisted.internet.ssl import DefaultOpenSSLContextFactory
factory = DefaultOpenSSLContextFactory(config.HTTPS_KEY_FILE,
config.HTTPS_CERT_FILE)
reactor.listenSSL(port, site, factory, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started HTTPS server on %s:%d" % (str(ip), int(port)))
return site
request.setHeader(b"content-type", b"text/html; charset=utf-8")
return self.page
def getChild(self, chnam, request):
return self
class NoResource(ErrorResource):
def __init__(self):
ErrorResource.__init__(self, NOT_FOUND)
application = Application("static-server")
plainSite = Site(RedirectResource())
plainSite.displayTracebacks = False
plainService6 = TCPServer(INSECURE_PORT, plainSite, interface='::')
plainService6.setServiceParent(application)
files = File(DOCUMENTS.path)
files.childNotFound = NoResource()
secureSite = Site(HSTSResource(files))
secureSite.displayTracebacks = False
secureService6 = SSLServer(SECURE_PORT,
secureSite,
ctxFactory,
interface='::')
secureService6.setServiceParent(application)
def addMoatServer(config, distributor):
"""Set up a web server for moat bridge distribution.
:type config: :class:`bridgedb.persistent.Conf`
:param config: A configuration object from
:mod:`bridgedb.main`. Currently, we use these options::
GIMP_CAPTCHA_DIR
SERVER_PUBLIC_FQDN
SUPPORTED_TRANSPORTS
MOAT_DIST
MOAT_DIST_VIA_MEEK_ONLY
MOAT_TLS_CERT_FILE
MOAT_TLS_KEY_FILE
MOAT_SERVER_PUBLIC_ROOT
MOAT_HTTPS_IP
MOAT_HTTPS_PORT
MOAT_HTTP_IP
MOAT_HTTP_PORT
MOAT_BRIDGES_PER_ANSWER
MOAT_TRANSPORT_PREFERENCE_LIST
MOAT_USE_IP_FROM_FORWARDED_HEADER
MOAT_SKIP_LOOPBACK_ADDRESSES
MOAT_ROTATION_PERIOD
MOAT_GIMP_CAPTCHA_HMAC_KEYFILE
MOAT_GIMP_CAPTCHA_RSA_KEYFILE
:type distributor: :class:`bridgedb.distributors.moat.distributor.MoatDistributor`
:param distributor: A bridge distributor.
:raises SystemExit: if the servers cannot be started.
:rtype: :api:`twisted.web.server.Site`
:returns: A webserver.
"""
captcha = None
fwdHeaders = config.MOAT_USE_IP_FROM_FORWARDED_HEADER
numBridges = config.MOAT_BRIDGES_PER_ANSWER
skipLoopback = config.MOAT_SKIP_LOOPBACK_ADDRESSES
logging.info("Starting moat servers...")
setFQDN(config.SERVER_PUBLIC_FQDN)
setRoot(config.MOAT_SERVER_PUBLIC_ROOT)
setSupportedTransports(config.SUPPORTED_TRANSPORTS)
setPreferredTransports(config.MOAT_TRANSPORT_PREFERENCE_LIST)
# Get the master HMAC secret key for CAPTCHA challenges, and then
# create a new HMAC key from it for use on the server.
captchaKey = crypto.getKey(config.MOAT_GIMP_CAPTCHA_HMAC_KEYFILE)
hmacKey = crypto.getHMAC(captchaKey, "Moat-Captcha-Key")
# Load or create our encryption keys:
secretKey, publicKey = crypto.getRSAKey(config.MOAT_GIMP_CAPTCHA_RSA_KEYFILE)
sched = Unscheduled()
if config.MOAT_ROTATION_PERIOD:
count, period = config.MOAT_ROTATION_PERIOD.split()
sched = ScheduledInterval(count, period)
sitePublicDir = getRoot()
meek = CustomErrorHandlingResource()
moat = CustomErrorHandlingResource()
fetch = CaptchaFetchResource(hmacKey, publicKey, secretKey,
config.GIMP_CAPTCHA_DIR,
fwdHeaders, skipLoopback)
check = CaptchaCheckResource(distributor, sched, numBridges,
hmacKey, publicKey, secretKey,
fwdHeaders, skipLoopback)
moat.putChild("fetch", fetch)
moat.putChild("check", check)
meek.putChild("moat", moat)
root = CustomErrorHandlingResource()
root.putChild("meek", meek)
root.putChild("moat", moat)
site = Site(root)
site.displayTracebacks = False
if config.MOAT_HTTP_PORT: # pragma: no cover
ip = config.MOAT_HTTP_IP or ""
port = config.MOAT_HTTP_PORT or 80
try:
reactor.listenTCP(port, site, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started Moat HTTP server on %s:%d" % (str(ip), int(port)))
if config.MOAT_HTTPS_PORT: # pragma: no cover
ip = config.MOAT_HTTPS_IP or ""
port = config.MOAT_HTTPS_PORT or 443
try:
from twisted.internet.ssl import DefaultOpenSSLContextFactory
factory = DefaultOpenSSLContextFactory(config.MOAT_TLS_KEY_FILE,
config.MOAT_TLS_CERT_FILE)
reactor.listenSSL(port, site, factory, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started Moat TLS server on %s:%d" % (str(ip), int(port)))
return site
loglevel=loglevel)
def main():
logger.info('路由模块已启动')
reactor.run() # @UndefinedVariable
if __name__ == '__main__':
loglevels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARN': logging.WARN,
'ERROR': logging.ERROR,
'FATAL': logging.FATAL
}
router_cfg = load_router_cfg('espush.ini')
initlog(level=loglevels[router_cfg.loglevel])
# router svc
gwsvc_factory = JSONServerFactory()
listenport = router_cfg.listenport
reactor.listenTCP(listenport, gwsvc_factory) # @UndefinedVariable
gl_routetable = RouteTable()
# web interface
web_factory = Site(web_init())
web_factory.displayTracebacks = False
apiport = router_cfg.apiport
reactor.listenTCP(apiport, web_factory) # @UndefinedVariable
write_pid_file()
main()
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
s = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace')),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
get_consistency = partial(
get_consistency_level,
default=config_value('cassandra.default_consistency'),
exceptions=config_value('cassandra.consistency_exceptions'))
store = CassScalingGroupCollection(cassandra_cluster, reactor,
get_consistency)
admin_store = CassAdmin(cassandra_cluster, get_consistency)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
RetryingAuthenticator(
reactor,
ImpersonatingAuthenticator(
config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
max_retries=config_value('identity.max_retries'),
retry_interval=config_value('identity.retry_interval')),
cache_ttl)
supervisor = SupervisorService(authenticator.authenticate_tenant,
region, coiterate)
supervisor.setServiceParent(s)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
s.addService(FunctionalService(stop=partial(call_after_supervisor,
cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check,
es_host=config_value('elasticsearch.host'))
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(s)
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
kz_client = TxKazooClient(hosts=config_value('zookeeper.hosts'),
threads=threads, txlog=log.bind(system='kazoo'))
d = kz_client.start()
def on_client_ready(_):
# Setup scheduler service after starting
scheduler = setup_scheduler(s, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Set the client after starting
# NOTE: There is small amount of time when the start is not finished
# and the kz_client is not set in which case policy execution and group
# delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
s.addService(FunctionalService(stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return s
def make_redirector_site(port):
# XXX It would be good to combine this with make_site so we didn't end up
# with http logs in multiple places. Not sure how to do that though.
site = Site(RedirectToHTTPS(port))
site.displayTracebacks = False
return site
url = request.args.get('imgur', None)
if not url:
return 'Please enter a valid imgur link.'
parsed_url = urlparse.urlparse(url)
if parsed_url.netloc != 'i.imgur.com':
return 'Please enter a valid imgur link.'
# Create a transaction that will be used until the user receives their image and presses OK to dismiss the transaction
# This way, if the user refreshes the page or
transaction_id = transaction_manager.addTransaction(Transaction(ImgurDownloadState(parsed_url.path[1:])))
session['imgur_id'] = transaction_id
return render_template('imgur/index.html', transaction_id=transaction_id)
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
static_resource = File(app.static_folder)
log.startLogging(sys.stdout)
ws = Resource()
ws.putChild('imgur', imgur_resource)
root_resource = WSGISiteResource(resource, {'static': static_resource, 'ws': ws})
site = Site(root_resource)
site.displayTracebacks = True
if __name__ == '__main__':
app.debug = True
log.startLogging(sys.stdout)
reactor.listenTCP(8000, site, interface='0.0.0.0')
reactor.run()
parsed_url = urlparse.urlparse(url)
if parsed_url.netloc != 'i.imgur.com':
return 'Please enter a valid imgur link.'
# Create a transaction that will be used until the user receives their image and presses OK to dismiss the transaction
# This way, if the user refreshes the page or
transaction_id = transaction_manager.addTransaction(
Transaction(ImgurDownloadState(parsed_url.path[1:])))
session['imgur_id'] = transaction_id
return render_template('imgur/index.html', transaction_id=transaction_id)
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
static_resource = File(app.static_folder)
log.startLogging(sys.stdout)
ws = Resource()
ws.putChild('imgur', imgur_resource)
root_resource = WSGISiteResource(resource, {
'static': static_resource,
'ws': ws
})
site = Site(root_resource)
site.displayTracebacks = True
if __name__ == '__main__':
app.debug = True
log.startLogging(sys.stdout)
reactor.listenTCP(8000, site, interface='0.0.0.0')
reactor.run()
def make_redirector_site(port):
site = Site(RedirectToHTTPS(port))
site.displayTracebacks = True
return site
def addWebServer(config, distributor):
"""Set up a web server for HTTP(S)-based bridge distribution.
:type config: :class:`bridgedb.persistent.Conf`
:param config: A configuration object from
:mod:`bridgedb.main`. Currently, we use these options::
HTTP_UNENCRYPTED_PORT
HTTP_UNENCRYPTED_BIND_IP
HTTP_USE_IP_FROM_FORWARDED_HEADER
HTTPS_N_BRIDGES_PER_ANSWER
HTTPS_INCLUDE_FINGERPRINTS
HTTPS_KEY_FILE
HTTPS_CERT_FILE
HTTPS_PORT
HTTPS_BIND_IP
HTTPS_USE_IP_FROM_FORWARDED_HEADER
HTTPS_ROTATION_PERIOD
RECAPTCHA_ENABLED
RECAPTCHA_PUB_KEY
RECAPTCHA_SEC_KEY
RECAPTCHA_REMOTEIP
GIMP_CAPTCHA_ENABLED
GIMP_CAPTCHA_DIR
GIMP_CAPTCHA_HMAC_KEYFILE
GIMP_CAPTCHA_RSA_KEYFILE
SERVER_PUBLIC_FQDN
CSP_ENABLED
CSP_REPORT_ONLY
CSP_INCLUDE_SELF
:type distributor: :class:`bridgedb.distributors.https.distributor.HTTPSDistributor`
:param distributor: A bridge distributor.
:raises SystemExit: if the servers cannot be started.
:rtype: :api:`twisted.web.server.Site`
:returns: A webserver.
"""
captcha = None
fwdHeaders = config.HTTP_USE_IP_FROM_FORWARDED_HEADER
numBridges = config.HTTPS_N_BRIDGES_PER_ANSWER
fprInclude = config.HTTPS_INCLUDE_FINGERPRINTS
logging.info("Starting web servers...")
setFQDN(config.SERVER_PUBLIC_FQDN)
index = IndexResource()
options = OptionsResource()
howto = HowtoResource()
robots = static.File(os.path.join(TEMPLATE_DIR, 'robots.txt'))
assets = static.File(os.path.join(TEMPLATE_DIR, 'assets/'))
keys = static.Data(bytes(strings.BRIDGEDB_OPENPGP_KEY), 'text/plain')
csp = CSPResource(enabled=config.CSP_ENABLED,
includeSelf=config.CSP_INCLUDE_SELF,
reportViolations=config.CSP_REPORT_ONLY,
useForwardedHeader=fwdHeaders)
root = CustomErrorHandlingResource()
root.putChild('', index)
root.putChild('robots.txt', robots)
root.putChild('keys', keys)
root.putChild('assets', assets)
root.putChild('options', options)
root.putChild('howto', howto)
root.putChild('maintenance', maintenance)
root.putChild('error', resource500)
root.putChild(CSPResource.reportURI, csp)
if config.RECAPTCHA_ENABLED:
publicKey = config.RECAPTCHA_PUB_KEY
secretKey = config.RECAPTCHA_SEC_KEY
captcha = partial(ReCaptchaProtectedResource,
remoteIP=config.RECAPTCHA_REMOTEIP)
elif config.GIMP_CAPTCHA_ENABLED:
# Get the master HMAC secret key for CAPTCHA challenges, and then
# create a new HMAC key from it for use on the server.
captchaKey = crypto.getKey(config.GIMP_CAPTCHA_HMAC_KEYFILE)
hmacKey = crypto.getHMAC(captchaKey, "Captcha-Key")
# Load or create our encryption keys:
secretKey, publicKey = crypto.getRSAKey(config.GIMP_CAPTCHA_RSA_KEYFILE)
captcha = partial(GimpCaptchaProtectedResource,
hmacKey=hmacKey,
captchaDir=config.GIMP_CAPTCHA_DIR)
if config.HTTPS_ROTATION_PERIOD:
count, period = config.HTTPS_ROTATION_PERIOD.split()
sched = ScheduledInterval(count, period)
else:
sched = Unscheduled()
bridges = BridgesResource(distributor, sched, numBridges, fwdHeaders,
includeFingerprints=fprInclude)
if captcha:
# Protect the 'bridges' page with a CAPTCHA, if configured to do so:
protected = captcha(publicKey=publicKey,
secretKey=secretKey,
useForwardedHeader=fwdHeaders,
protectedResource=bridges)
root.putChild('bridges', protected)
logging.info("Protecting resources with %s." % captcha.func.__name__)
else:
root.putChild('bridges', bridges)
site = Site(root)
site.displayTracebacks = False
if config.HTTP_UNENCRYPTED_PORT: # pragma: no cover
ip = config.HTTP_UNENCRYPTED_BIND_IP or ""
port = config.HTTP_UNENCRYPTED_PORT or 80
try:
reactor.listenTCP(port, site, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started HTTP server on %s:%d" % (str(ip), int(port)))
if config.HTTPS_PORT: # pragma: no cover
ip = config.HTTPS_BIND_IP or ""
port = config.HTTPS_PORT or 443
try:
from twisted.internet.ssl import DefaultOpenSSLContextFactory
factory = DefaultOpenSSLContextFactory(config.HTTPS_KEY_FILE,
config.HTTPS_CERT_FILE)
reactor.listenSSL(port, site, factory, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started HTTPS server on %s:%d" % (str(ip), int(port)))
return site
def render(self, request):
request.setResponseCode(self.status)
request.setHeader(b"content-type", b"text/html; charset=utf-8")
return self.page
def getChild(self, chnam, request):
return self
class NoResource(ErrorResource):
def __init__(self):
ErrorResource.__init__(self, NOT_FOUND)
application = Application("static-server")
plainSite = Site(RedirectResource())
plainSite.displayTracebacks = False
plainService6 = TCPServer(INSECURE_PORT, plainSite, interface='::')
plainService6.setServiceParent(application)
files = File(DOCUMENTS.path)
files.childNotFound = NoResource()
secureSite = Site(HSTSResource(files))
secureSite.displayTracebacks = False
secureService6 = SSLServer(SECURE_PORT, secureSite, ctxFactory, interface='::')
secureService6.setServiceParent(application)
def make_redirector_site(port):
site = Site(RedirectToHTTPS(port))
site.displayTracebacks = False
return site
def makeService(config):
"""
Set up the otter-api service.
"""
config = dict(config)
set_config_data(config)
parent = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'),
disconnect_on_cancel=True),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
store = CassScalingGroupCollection(
cassandra_cluster, reactor, config_value('limits.absolute.maxGroups'))
admin_store = CassAdmin(cassandra_cluster)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
service_configs = get_service_configs(config)
authenticator = generate_authenticator(reactor, config['identity'])
supervisor = SupervisorService(authenticator, region, coiterate,
service_configs)
supervisor.setServiceParent(parent)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
parent.addService(FunctionalService(stop=partial(
call_after_supervisor, cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check)
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(parent)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(parent)
# setup cloud feed
cf_conf = config.get('cloudfeeds', None)
if cf_conf is not None:
id_conf = deepcopy(config['identity'])
id_conf['strategy'] = 'single_tenant'
add_to_fanout(CloudFeedsObserver(
reactor=reactor,
authenticator=generate_authenticator(reactor, id_conf),
tenant_id=cf_conf['tenant_id'],
region=region,
service_configs=service_configs))
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
disable_logs = config_value('zookeeper.no_logs')
threadpool = ThreadPool(maxthreads=threads)
sync_kz_client = KazooClient(
hosts=config_value('zookeeper.hosts'),
# Keep trying to connect until the end of time with
# max interval of 10 minutes
connection_retry=dict(max_tries=-1, max_delay=600),
logger=None if disable_logs else TxLogger(log.bind(system='kazoo'))
)
kz_client = TxKazooClient(reactor, threadpool, sync_kz_client)
# Don't timeout. Keep trying to connect forever
d = kz_client.start(timeout=None)
def on_client_ready(_):
dispatcher = get_full_dispatcher(reactor, authenticator, log,
get_service_configs(config),
kz_client, store, supervisor,
cassandra_cluster)
# Setup scheduler service after starting
scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Give dispatcher to Otter REST object
otter.dispatcher = dispatcher
# Set the client after starting
# NOTE: There is small amount of time when the start is
# not finished and the kz_client is not set in which case
# policy execution and group delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
parent.addService(FunctionalService(
stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
setup_converger(
parent, kz_client, dispatcher,
config_value('converger.interval') or 10,
config_value('converger.build_timeout') or 3600,
config_value('converger.limited_retry_iterations') or 10,
config_value('converger.step_limits') or {})
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return parent
def addMoatServer(config, distributor):
"""Set up a web server for moat bridge distribution.
:type config: :class:`bridgedb.persistent.Conf`
:param config: A configuration object from
:mod:`bridgedb.main`. Currently, we use these options::
GIMP_CAPTCHA_DIR
SERVER_PUBLIC_FQDN
SUPPORTED_TRANSPORTS
MOAT_DIST
MOAT_DIST_VIA_MEEK_ONLY
MOAT_TLS_CERT_FILE
MOAT_TLS_KEY_FILE
MOAT_SERVER_PUBLIC_ROOT
MOAT_HTTPS_IP
MOAT_HTTPS_PORT
MOAT_HTTP_IP
MOAT_HTTP_PORT
MOAT_BRIDGES_PER_ANSWER
MOAT_TRANSPORT_PREFERENCE_LIST
MOAT_USE_IP_FROM_FORWARDED_HEADER
MOAT_SKIP_LOOPBACK_ADDRESSES
MOAT_ROTATION_PERIOD
MOAT_GIMP_CAPTCHA_HMAC_KEYFILE
MOAT_GIMP_CAPTCHA_RSA_KEYFILE
:type distributor: :class:`bridgedb.distributors.moat.distributor.MoatDistributor`
:param distributor: A bridge distributor.
:raises SystemExit: if the servers cannot be started.
:rtype: :api:`twisted.web.server.Site`
:returns: A webserver.
"""
captcha = None
fwdHeaders = config.MOAT_USE_IP_FROM_FORWARDED_HEADER
numBridges = config.MOAT_BRIDGES_PER_ANSWER
skipLoopback = config.MOAT_SKIP_LOOPBACK_ADDRESSES
logging.info("Starting moat servers...")
setFQDN(config.SERVER_PUBLIC_FQDN)
setRoot(config.MOAT_SERVER_PUBLIC_ROOT)
setSupportedTransports(config.SUPPORTED_TRANSPORTS)
setPreferredTransports(config.MOAT_TRANSPORT_PREFERENCE_LIST)
# Get the master HMAC secret key for CAPTCHA challenges, and then
# create a new HMAC key from it for use on the server.
captchaKey = crypto.getKey(config.MOAT_GIMP_CAPTCHA_HMAC_KEYFILE)
hmacKey = crypto.getHMAC(captchaKey, "Moat-Captcha-Key")
# Load or create our encryption keys:
secretKey, publicKey = crypto.getRSAKey(
config.MOAT_GIMP_CAPTCHA_RSA_KEYFILE)
sched = Unscheduled()
if config.MOAT_ROTATION_PERIOD:
count, period = config.MOAT_ROTATION_PERIOD.split()
sched = ScheduledInterval(count, period)
sitePublicDir = getRoot()
meek = CustomErrorHandlingResource()
moat = CustomErrorHandlingResource()
fetch = CaptchaFetchResource(hmacKey, publicKey, secretKey,
config.GIMP_CAPTCHA_DIR, fwdHeaders,
skipLoopback)
check = CaptchaCheckResource(distributor, sched, numBridges, hmacKey,
publicKey, secretKey, fwdHeaders,
skipLoopback)
moat.putChild(b"fetch", fetch)
moat.putChild(b"check", check)
meek.putChild(b"moat", moat)
root = CustomErrorHandlingResource()
root.putChild(b"meek", meek)
root.putChild(b"moat", moat)
site = Site(root)
site.displayTracebacks = False
if config.MOAT_HTTP_PORT: # pragma: no cover
ip = config.MOAT_HTTP_IP or ""
port = config.MOAT_HTTP_PORT or 80
try:
reactor.listenTCP(port, site, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started Moat HTTP server on %s:%d" %
(str(ip), int(port)))
if config.MOAT_HTTPS_PORT: # pragma: no cover
ip = config.MOAT_HTTPS_IP or ""
port = config.MOAT_HTTPS_PORT or 443
try:
from twisted.internet.ssl import DefaultOpenSSLContextFactory
factory = DefaultOpenSSLContextFactory(config.MOAT_TLS_KEY_FILE,
config.MOAT_TLS_CERT_FILE)
reactor.listenSSL(port, site, factory, interface=ip)
except CannotListenError as error:
raise SystemExit(error)
logging.info("Started Moat TLS server on %s:%d" % (str(ip), int(port)))
return site
