def main():
define('ip', default=None, help='IP to listen on')
define('port', default=None, help='port to listen on')
define('socket', default=None, help='unix socket path to bind (instead of ip:port)')
define('group', default='', help='comma separated group list for new users `students,other`')
define('skeldir', default='', help='skeleton directory that will be used for new homedirs')
define('shell', default='/bin/bash', help='default shell')
parse_command_line()
if not options.socket and not (options.port):
options.socket = '/var/run/restuser.sock'
app = web.Application(
[(r'/([^/]+)', UserHandler)],
group=options.group,
skeldir=options.skeldir,
shell=options.shell)
if options.socket:
socket = bind_unix_socket(options.socket, mode=0o600)
server = HTTPServer(app)
server.add_socket(socket)
else:
app.listen(options.port, options.ip)
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("\ninterrupted\n", file=sys.stderr)
return
def start(self):
self.tornadoApp=tornado.web.Application(self.handlers)
self.tornadoApp.server=self
if self.ssl_crt and self.ssl_key:
import ssl
ssl_options = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_options.load_cert_chain(self.ssl_crt,self.ssl_key)
else:
ssl_options = None
server = HTTPServer(self.tornadoApp, ssl_options=ssl_options)
if self.port:
server.listen(int(self.port))
#self.tornadoApp.listen(int(self.port))
else:
socket_path = os.path.join(gnrConfigPath(), 'sockets', '%s.tornado'%self.instance_name)
main_socket = bind_unix_socket(socket_path)
#server = HTTPServer(self.tornadoApp)
server.add_socket(main_socket)
debug_socket_path = os.path.join(gnrConfigPath(), 'sockets', '%s_debug.tornado'%self.instance_name)
debug_socket = bind_unix_socket(debug_socket_path)
debug_server = GnrDebugServer(self.io_loop)
#debug_server.listen(8888)
debug_server.add_socket(debug_socket)
debug_server.application = self.tornadoApp
signal.signal(signal.SIGTERM, self.onSignal)
signal.signal(signal.SIGINT, self.onSignal)
self.io_loop.start()
def main():
define('socket',
default='/var/run/ophubuser.sock',
help='unix socket path to bind')
define('usersdir',
default='/home/user-notebooks',
help='Shared user notebook directory')
define(
'notebookdir',
default='/home/USERNAME/notebooks',
help=
'path of a notebook directory for each user. "USERNAME" will be replaced by the name of user'
)
parse_command_line()
app = web.Application([(r'/mount/([^/]+)', MountHandler)],
notebookdir=options.notebookdir,
usersdir=options.usersdir)
socket = bind_unix_socket(options.socket, mode=0o600)
server = HTTPServer(app)
server.add_socket(socket)
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("\ninterrupted\n", file=sys.stderr)
return
def main():
define('ip', default=None, help='IP to listen on')
define('port', default=None, help='port to listen on')
define('socket',
default=None,
help='unix socket path to bind (instead of ip:port)')
define('group',
default='',
help='comma separated group list for new users `students,other`')
define('skeldir',
default='',
help='skeleton directory that will be used for new homedirs')
define('shell', default='/bin/bash', help='default shell')
parse_command_line()
if not options.socket and not (options.port):
options.socket = '/var/run/restuser.sock'
app = web.Application([(r'/([^/]+)', UserHandler)],
group=options.group,
skeldir=options.skeldir,
shell=options.shell)
if options.socket:
socket = bind_unix_socket(options.socket, mode=0o600)
server = HTTPServer(app)
server.add_socket(socket)
else:
app.listen(options.port, options.ip)
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("\ninterrupted\n", file=sys.stderr)
return
class TestWebSocketBase(AsyncTestCase):
def setUp(self):
super(TestWebSocketBase, self).setUp()
self.application = Application()
self.server = HTTPServer(self.application)
self.socket, self.port = testing.bind_unused_port()
self.server.add_socket(self.socket)
self.instance = WebSocketBase(
self.application,
HTTPServerRequest(
method="GET",
uri='/',
version="HTTP/1.0",
headers=HTTPHeaders(),
body=BytesIO(),
host=None,
files=None,
connection=HTTP1Connection(stream=IOStream(socket.socket()),
is_client=False),
start_line=RequestStartLine(method='GET',
path='/',
version='HTTP/1.1'),
))
self.instance.open()
def test_configure(self):
keepalive_timeout = randint(999, 9999)
client_timeout = randint(999, 9999)
WebSocketBase.configure(keepalive_timeout=keepalive_timeout,
client_timeout=client_timeout)
self.assertEqual(WebSocketBase._CLIENT_TIMEOUT, client_timeout)
self.assertEqual(WebSocketBase._KEEPALIVE_PING_TIMEOUT,
keepalive_timeout)
def test_authorize(self):
self.assertEqual(self.instance.authorize(), True)
def test_execute(self):
resp = self.instance._execute([])
self.assertTrue(isinstance(resp, Future))
resp = self.instance._execute(None)
self.assertTrue(isinstance(resp, Future))
def test_allowdraft76(self):
self.assertEqual(self.instance.allow_draft76(), True)
def send_message(self, msg):
return self.instance.on_message(json.dumps(msg))
@gen_test
def test_on_message(self):
try:
yield self.send_message({})
except Exception as e:
self.assertEqual(type(e), AssertionError)
result = yield self.send_message({'serial': 999, 'type': 'call'})
result = yield self.send_message({'serial': 999, 'type': 'callback'})
result = yield self.send_message({'serial': 999, 'type': 'error'})
class WebTest(AsyncTestCase):
def setUp(self):
super(WebTest, self).setUp()
self.application = Application()
self.server = HTTPServer(self.application)
self.socket, self.port = testing.bind_unused_port()
self.server.add_socket(self.socket)
self.static_path = os.path.join(os.path.dirname(wsrpc.__file__), 'static')
@coroutine
def fetch(self, filename):
response = yield AsyncHTTPClient().fetch("http://localhost:{0.port}/static/{1}".format(self, filename))
self.assertTrue(response.code, 200)
self.assertEqual(
response.body.decode('utf-8'),
codecs.open(os.path.join(self.static_path, filename), 'r', 'utf-8').read()
)
@gen_test
def test_wsrpc_js(self):
yield self.fetch('wsrpc.js')
@gen_test
def test_q_js(self):
yield self.fetch('q.js')
@gen_test
def test_q_min_js(self):
yield self.fetch('q.min.js')
@gen_test
def test_wsrpc_min_js(self):
yield self.fetch('wsrpc.min.js')
class SyncHTTPClientTest(unittest.TestCase):
def setUp(self):
if IOLoop.configured_class().__name__ == 'TwistedIOLoop':
# TwistedIOLoop only supports the global reactor, so we can't have
# separate IOLoops for client and server threads.
raise unittest.SkipTest(
'Sync HTTPClient not compatible with TwistedIOLoop')
self.server_ioloop = IOLoop()
@gen.coroutine
def init_server():
sock, self.port = bind_unused_port()
app = Application([('/', HelloWorldHandler)])
self.server = HTTPServer(app)
self.server.add_socket(sock)
self.server_ioloop.run_sync(init_server)
self.server_thread = threading.Thread(target=self.server_ioloop.start)
self.server_thread.start()
self.http_client = HTTPClient()
def tearDown(self):
def stop_server():
self.server.stop()
# Delay the shutdown of the IOLoop by several iterations because
# the server may still have some cleanup work left when
# the client finishes with the response (this is noticeable
# with http/2, which leaves a Future with an unexamined
# StreamClosedError on the loop).
@gen.coroutine
def slow_stop():
# The number of iterations is difficult to predict. Typically,
# one is sufficient, although sometimes it needs more.
for i in range(5):
yield
self.server_ioloop.stop()
self.server_ioloop.add_callback(slow_stop)
self.server_ioloop.add_callback(stop_server)
self.server_thread.join()
self.http_client.close()
self.server_ioloop.close(all_fds=True)
def get_url(self, path):
return 'http://127.0.0.1:%d%s' % (self.port, path)
def test_sync_client(self):
response = self.http_client.fetch(self.get_url('/'))
self.assertEqual(b'Hello world!', response.body)
def test_sync_client_error(self):
# Synchronous HTTPClient raises errors directly; no need for
# response.rethrow()
with self.assertRaises(HTTPError) as assertion:
self.http_client.fetch(self.get_url('/notfound'))
self.assertEqual(assertion.exception.code, 404)
def unix_socket_server(application, options):
""" unix socket server """
log.info("Binding application to unix socket {0}".format(options.unix_socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
class SyncHTTPClientTest(unittest.TestCase):
def setUp(self):
self.server_ioloop = IOLoop()
event = threading.Event()
@gen.coroutine
def init_server():
sock, self.port = bind_unused_port()
app = Application([("/", HelloWorldHandler)])
self.server = HTTPServer(app)
self.server.add_socket(sock)
event.set()
def start():
self.server_ioloop.run_sync(init_server)
self.server_ioloop.start()
self.server_thread = threading.Thread(target=start)
self.server_thread.start()
event.wait()
self.http_client = HTTPClient()
def tearDown(self):
def stop_server():
self.server.stop()
# Delay the shutdown of the IOLoop by several iterations because
# the server may still have some cleanup work left when
# the client finishes with the response (this is noticeable
# with http/2, which leaves a Future with an unexamined
# StreamClosedError on the loop).
@gen.coroutine
def slow_stop():
# The number of iterations is difficult to predict. Typically,
# one is sufficient, although sometimes it needs more.
for i in range(5):
yield
self.server_ioloop.stop()
self.server_ioloop.add_callback(slow_stop)
self.server_ioloop.add_callback(stop_server)
self.server_thread.join()
self.http_client.close()
self.server_ioloop.close(all_fds=True)
def get_url(self, path):
return "http://127.0.0.1:%d%s" % (self.port, path)
def test_sync_client(self):
response = self.http_client.fetch(self.get_url("/"))
self.assertEqual(b"Hello world!", response.body)
def test_sync_client_error(self):
# Synchronous HTTPClient raises errors directly; no need for
# response.rethrow()
with self.assertRaises(HTTPError) as assertion:
self.http_client.fetch(self.get_url("/notfound"))
self.assertEqual(assertion.exception.code, 404)
def setup_server():
# Config tornado.curl_httpclient to use NullHandler
tornado_logger = logging.getLogger('tornado.curl_httpclient')
tornado_logger.addHandler(logging.NullHandler())
tornado_logger.propagate = False
settings = dict(
autoreload=True,
secret="ElasticKube",
)
configure(settings)
handlers = [
(r"/api/v1/auth/providers", AuthProvidersHandler),
(r"/api/v1/auth/signup", SignupHandler),
(r"/api/v1/auth/login", PasswordHandler),
(r"/api/v1/auth/google", GoogleOAuth2LoginHandler),
(r"/api/v1/ws", MainWebSocketHandler),
(r"/icons/(?P<entity_id>[^\/]+)\/(?P<chart_id>[^\/]+)", IconGenerator)
]
application = Application(handlers, **settings)
server = HTTPServer(application)
socket = bind_unix_socket("/var/run/elastickube-api.sock", mode=0777)
server.add_socket(socket)
IOLoop.current().add_callback(initialize, settings)
def main(arguments=None):
"""Runs thumbor server with the specified arguments."""
server_parameters = get_server_parameters(arguments)
lookup_paths = [os.curdir, expanduser("~"), "/etc/", dirname(__file__)]
config = Config.load(
server_parameters.config_path,
conf_name="thumbor.conf",
lookup_paths=lookup_paths,
)
logging.basicConfig(
level=getattr(logging, server_parameters.log_level.upper()),
format=config.THUMBOR_LOG_FORMAT,
datefmt=config.THUMBOR_LOG_DATE_FORMAT,
)
importer = Importer(config)
importer.import_modules()
if importer.error_handler_class is not None:
importer.error_handler = importer.error_handler_class(config)
if server_parameters.security_key is None:
server_parameters.security_key = config.SECURITY_KEY
if not isinstance(server_parameters.security_key, basestring):
raise RuntimeError(
"No security key was found for this instance of thumbor. "
+ "Please provide one using the conf file or a security key file."
)
context = Context(server=server_parameters, config=config, importer=importer)
application = importer.import_class(server_parameters.app_class)(context)
server = HTTPServer(application)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is None:
with open(context.server.fd, "r") as sock:
fd_number = sock.fileno()
sock = socket.fromfd(fd_number, socket.AF_UNIX, socket.SOCK_STREAM)
server.add_socket(sock)
else:
server.bind(context.server.port, context.server.ip)
server.start(1)
try:
logging.debug(
"thumbor running at %s:%d" % (context.server.ip, context.server.port)
)
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
print("-- thumbor closed by user interruption --")
def main():
parse_command_line()
sock = socket.fromfd(options.fd, socket.AF_UNIX, socket.SOCK_STREAM)
server = HTTPServer(application)
server.add_socket(sock)
server.start(1)
tornado.ioloop.IOLoop.instance().start()
def main(arguments=None):
'''Runs thumbor server with the specified arguments.'''
server_parameters = get_server_parameters(arguments)
lookup_paths = [os.curdir, expanduser('~'), '/etc/', dirname(__file__)]
config = Config.load(server_parameters.config_path,
conf_name='thumbor.conf',
lookup_paths=lookup_paths)
logging.basicConfig(level=getattr(logging,
server_parameters.log_level.upper()),
format=config.THUMBOR_LOG_FORMAT,
datefmt=config.THUMBOR_LOG_DATE_FORMAT)
importer = Importer(config)
importer.import_modules()
if importer.error_handler_class is not None:
importer.error_handler = importer.error_handler_class(config)
if server_parameters.security_key is None:
server_parameters.security_key = config.SECURITY_KEY
if not isinstance(server_parameters.security_key, basestring):
raise RuntimeError(
'No security key was found for this instance of thumbor. ' +
'Please provide one using the conf file or a security key file.')
context = Context(server=server_parameters,
config=config,
importer=importer)
application = importer.import_class(server_parameters.app_class)(context)
server = HTTPServer(application)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is None:
with open(context.server.fd, 'r') as sock:
fd_number = sock.fileno()
sock = socket.fromfd(fd_number, socket.AF_INET | socket.AF_INET6,
socket.SOCK_STREAM)
server.add_socket(sock)
else:
server.bind(context.server.port, context.server.ip)
server.start(1)
try:
logging.debug('thumbor running at %s:%d' %
(context.server.ip, context.server.port))
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
print
print "-- thumbor closed by user interruption --"
def main():
application = Application([(r"/", SearchHandler)])
server = HTTPServer(application)
socket = bind_unix_socket(config.get("app:main",
"cherubplay.socket_search"),
mode=0777)
server.add_socket(socket)
IOLoop.instance().start()
class HttpEndpoint(object):
def __init__(self, uri='127.0.0.1:5000', backlog=128, ssl_options=None):
# uri should be a list
if isinstance(uri, six.string_types):
self.uri = uri.split(",")
else:
self.uri = uri
self.backlog = backlog
self.ssl_options = ssl_options
self.server = None
self.loop = None
self.io_loop = None
def __str__(self):
return ",".join(self.uri)
def start(self, loop, app):
for uri in self.uri:
logger.info('Start new endpoint at http://%s', uri)
self.loop = loop
self.app = app
self.io_loop = IOLoop(_loop=loop)
self._start_server()
def _start_server(self):
self.server = HTTPServer(self.app,
io_loop=self.io_loop,
ssl_options=self.ssl_options)
# bind the handler to needed interface
for uri in self.uri:
addr = parse_address(uri)
if isinstance(addr, six.string_types):
sock = netutil.bind_unix_socket(addr)
elif is_ipv6(addr[0]):
sock = netutil.bind_sockets(addr[1],
address=addr[0],
family=socket.AF_INET6,
backlog=self.backlog)
else:
sock = netutil.bind_sockets(addr[1], backlog=self.backlog)
if isinstance(sock, list):
for s in sock:
self.server.add_socket(s)
else:
self.server.add_socket(sock)
# start the server
self.server.start()
def stop(self):
self.server.stop()
self.io_loop.close()
def restart(self):
self.server.stop()
self._start_server()
def setUp(self):
super(AsyncHTTPTestCaseTest, self).setUp()
# Bind a second port.
sock, port = bind_unused_port()
app = Application()
server = HTTPServer(app, **self.get_httpserver_options())
server.add_socket(sock)
self.second_port = port
self.second_server = server
class HttpEndpoint(object):
def __init__(self, uri='127.0.0.1:5000', backlog=128,
ssl_options=None):
# uri should be a list
if isinstance(uri, six.string_types):
self.uri = uri.split(",")
else:
self.uri = uri
self.backlog = backlog
self.ssl_options = ssl_options
self.server = None
self.loop = None
self.io_loop = None
def __str__(self):
return ",".join(self.uri)
def start(self, loop, app):
for uri in self.uri:
logger.info('Start new endpoint at http://%s', uri)
self.loop = loop
self.app = app
self.io_loop = IOLoop(_loop=loop)
self._start_server()
def _start_server(self):
self.server = HTTPServer(self.app, io_loop=self.io_loop,
ssl_options=self.ssl_options)
# bind the handler to needed interface
for uri in self.uri:
addr = parse_address(uri)
if isinstance(addr, six.string_types):
sock = netutil.bind_unix_socket(addr)
elif is_ipv6(addr[0]):
sock = netutil.bind_sockets(addr[1], address=addr[0],
family=socket.AF_INET6, backlog=self.backlog)
else:
sock = netutil.bind_sockets(addr[1], backlog=self.backlog)
if isinstance(sock, list):
for s in sock:
self.server.add_socket(s)
else:
self.server.add_socket(sock)
# start the server
self.server.start()
def stop(self):
self.server.stop()
self.io_loop.close()
def restart(self):
self.server.stop()
self._start_server()
def setUp(self):
super(AsyncHTTPTestCaseTest, self).setUp()
# Bind a second port.
sock, port = bind_unused_port()
app = Application()
server = HTTPServer(app, **self.get_httpserver_options())
server.add_socket(sock)
self.second_port = port
self.second_server = server
def run_server(application, context):
server = HTTPServer(application)
define('unix_socket',
group='webserver',
default=thumbor_socket,
help='Path to unix socket to bind')
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
server.start(1)
def listen(self, port, address="", **kwargs):
if options.fd:
from tornado.httpserver import HTTPServer
sock = socket.fromfd(options.fd, socket.AF_INET, socket.SOCK_STREAM)
server = HTTPServer(self, **kwargs)
server.add_socket(sock)
else:
super(Application, self).listen(port, address, **kwargs)
def _init(index, crc, settings_path, server_id):
global log, game_server_process, _loop
game_server_process = GameProcess(index, crc, settings_path, server_id, loop=_loop)
settings = game_server_process.settings
setup_logger(settings)
log = game_server_process.logger = getLogger('process')
tornado_port = game_server_process.ports['tornado']
unix_socket = game_server_process.sockets['tornado']
environment_variables = setup_game_server(settings, tornado_port)
environment_variables['data_format'] = settings['data_format']
environment_variables['game_settings'] = settings
environment_variables['logger'] = log
environment_variables['server_process'] = game_server_process
handlers = [
(r'/crossdomain.xml', CrossDomainHandler),
(r'/data/(.*)', StaticDataHandler, environment_variables),
(r'/', GameServerHandler, environment_variables),
]
app_settings = {}
if settings['development_mode']:
log.info('Development mode on')
development_handlers = [
(r'/static/(.+)', DevelopmentStaticHandler, {
'path': [
norm_path('engine', 'social', 'static'),
norm_path('..', 'static'),
norm_path('..', 'data')
]
}),
# THIS IS TEMPORARY
(r'/(.+)', DevelopmentStaticHandler, {
'path': [
norm_path('engine', 'social', 'static'),
norm_path('..', 'static')
]
})
]
handlers += development_handlers
app_settings['debug'] = True
game_server_application = Application(handlers, app_settings)
if unix_socket:
server = HTTPServer(game_server_application)
log.info('Tornado running through socket {0}'.format(unix_socket))
socket = tornado.netutil.bind_unix_socket(unix_socket, mode=0o757)
server.add_socket(socket)
else:
log.info('Tornado listening to {0}'.format(tornado_port))
game_server_application.listen(tornado_port, address=settings['machine_address'])
log.info('game.{} : Game server started.'.format(server_id))
game_server_process.start()
class TestWebSocketBase(AsyncTestCase):
def setUp(self):
super(TestWebSocketBase, self).setUp()
self.application = Application()
self.server = HTTPServer(self.application)
self.socket, self.port = testing.bind_unused_port()
self.server.add_socket(self.socket)
self.instance = WebSocketBase(self.application, HTTPServerRequest(
method="GET",
uri='/',
version="HTTP/1.0",
headers=HTTPHeaders(),
body=BytesIO(),
host=None,
files=None,
connection=HTTP1Connection(
stream=IOStream(socket.socket()),
is_client=False
),
start_line=RequestStartLine(method='GET', path='/', version='HTTP/1.1'),
))
self.instance.open()
def test_configure(self):
keepalive_timeout = randint(999, 9999)
client_timeout = randint(999, 9999)
WebSocketBase.configure(keepalive_timeout=keepalive_timeout, client_timeout=client_timeout)
self.assertEqual(WebSocketBase._CLIENT_TIMEOUT, client_timeout)
self.assertEqual(WebSocketBase._KEEPALIVE_PING_TIMEOUT, keepalive_timeout)
def test_authorize(self):
self.assertEqual(self.instance.authorize(), True)
def test_execute(self):
resp = self.instance._execute([])
self.assertTrue(isinstance(resp, Future))
resp = self.instance._execute(None)
self.assertTrue(isinstance(resp, Future))
def test_allowdraft76(self):
self.assertEqual(self.instance.allow_draft76(), True)
def send_message(self, msg):
return self.instance.on_message(json.dumps(msg))
@gen_test
def test_on_message(self):
try:
yield self.send_message({})
except Exception as e:
self.assertEqual(type(e), AssertionError)
result = yield self.send_message({'serial': 999, 'type': 'call'})
result = yield self.send_message({'serial': 999, 'type': 'callback'})
result = yield self.send_message({'serial': 999, 'type': 'error'})
def run_server(application, context):
server = HTTPServer(application)
define(
'unix_socket',
group='webserver',
default=thumbor_socket,
help='Path to unix socket to bind')
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
server.start(1)
def listen(self, uds_path = None, port = None, *args, **kwargs):
if None == uds_path:
return super().listen(port, *args, **kwargs)
if (0 != len(args)) or ('address' in kwargs) or ('port' in kwargs):
# These are mutually exclusive arguments.
raise ValueError("uds_path and network address or port specified")
s = tornado.netutil.bind_unix_socket(uds_path)
from tornado.httpserver import HTTPServer
server = HTTPServer(self)
server.add_socket(s)
def run(): # pragma: no cover
if not CHORD_URLS_SET:
print(
f"[{SERVICE_NAME} {datetime.utcnow()}] No CHORD URLs given, terminating..."
)
exit(1)
server = HTTPServer(application)
server.add_socket(bind_unix_socket(SERVICE_SOCKET))
tornado.ioloop.IOLoop.current().start()
def main():
app = Application([('/', HelloHandler)])
server = HTTPServer(app)
server.add_socket(bind_unix_socket(SOCKPATH))
resolver = UnixResolver(resolver=Resolver())
AsyncHTTPClient.configure(None, resolver=resolver)
response = yield AsyncHTTPClient().fetch('http://unixsocket/')
print response.body
def listen(self, port, address="", **kwargs):
if options.fd:
from tornado.httpserver import HTTPServer
sock = socket.fromfd(options.fd, socket.AF_INET,
socket.SOCK_STREAM)
server = HTTPServer(self, **kwargs)
server.add_socket(sock)
else:
super(Application, self).listen(port, address, **kwargs)
def start(self):
self.pool = self.pool_executor_cls(max_workers=self.max_workers)
if not self.options.unix_socket:
self.listen(self.options.RPORT, address=self.options.address)
else:
from tornado.netutil import bind_unix_socket
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket)
server.add_socket(socket)
self.started = True
self.io_loop.start()
def main(arguments=None):
'''Runs thumbor server with the specified arguments.'''
server_parameters = get_server_parameters(arguments)
lookup_paths = [os.curdir,
expanduser('~'),
'/etc/',
dirname(__file__)]
config = Config.load(server_parameters.config_path, conf_name='thumbor.conf', lookup_paths=lookup_paths)
logging.basicConfig(
level=getattr(logging, server_parameters.log_level.upper()),
format=config.THUMBOR_LOG_FORMAT,
datefmt=config.THUMBOR_LOG_DATE_FORMAT
)
importer = Importer(config)
importer.import_modules()
if importer.error_handler_class is not None:
importer.error_handler = importer.error_handler_class(config)
if server_parameters.security_key is None:
server_parameters.security_key = config.SECURITY_KEY
if not isinstance(server_parameters.security_key, basestring):
raise RuntimeError(
'No security key was found for this instance of thumbor. ' +
'Please provide one using the conf file or a security key file.')
context = Context(server=server_parameters, config=config, importer=importer)
application = importer.import_class(server_parameters.app_class)(context)
server = HTTPServer(application)
if context.server.fd is not None:
sock = socket.fromfd(context.server.fd,
socket.AF_INET | socket.AF_INET6,
socket.SOCK_STREAM)
server.add_socket(sock)
else:
server.bind(context.server.port, context.server.ip)
server.start(1)
try:
logging.debug('thumbor running at %s:%d' % (context.server.ip, context.server.port))
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
print
print "-- thumbor closed by user interruption --"
def get_server(loop, received):
io_loop = IOLoop(_loop=loop)
test_handlers = [(r'/([^/]+)', TestHandler)]
app = Application(test_handlers, received=received)
server = HTTPServer(app, io_loop=io_loop)
[sock] = netutil.bind_sockets(TEST_PORT, address=TEST_HOST)
server.add_socket(sock)
return server
def start_server(app_io_loop, app_start_lock):
# initialize db by hand because we need it in the worker thread
config.db_conf = {'type': 'sqlite', 'url': ':memory:'}
db_init()
load_dev_fixtures(config.db)
app = mkapp()
server = HTTPServer(app, io_loop=app_io_loop)
server.add_socket(unused_port[0])
app_io_loop.add_callback(callback=app_start_lock.release)
app_io_loop.start()
def run():
parse_command_line()
if options.socket:
print('Listening on {}'.format(options.socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.socket, mode=0777)
server.add_socket(socket)
else:
print('Listening on {}:{}'.format(options.addr, options.port))
application.listen(options.port, address=options.addr)
application.debug = not production
tornado.ioloop.IOLoop.instance().start()
def main():
application = Application([(r"/(.*)/", ChatHandler)])
server = HTTPServer(application)
socket = bind_unix_socket(config.get("app:main", "cherubplay.socket_chat"), mode=0777)
server.add_socket(socket)
signal.signal(signal.SIGTERM, sig_handler)
signal.signal(signal.SIGINT, sig_handler)
ioloop.start()
def run():
parse_command_line()
if options.socket:
print('Listening on {}'.format(options.socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.socket, mode=0o777)
server.add_socket(socket)
else:
print('Listening on {}:{}'.format(options.addr, options.port))
application.listen(options.port, address=options.addr)
application.debug = not production
tornado.ioloop.IOLoop.instance().start()
def main():
application = Application([(r"/(.*)/", ChatHandler)])
server = HTTPServer(application)
socket = bind_unix_socket(config.get("app:main", "cherubplay.socket_chat"), mode=0o777)
server.add_socket(socket)
signal.signal(signal.SIGTERM, sig_handler)
signal.signal(signal.SIGINT, sig_handler)
ioloop.start()
class SyncHTTPClientTest(unittest.TestCase):
def setUp(self):
if IOLoop.configured_class().__name__ in ('TwistedIOLoop',
'AsyncIOMainLoop'):
# TwistedIOLoop only supports the global reactor, so we can't have
# separate IOLoops for client and server threads.
# AsyncIOMainLoop doesn't work with the default policy
# (although it could with some tweaks to this test and a
# policy that created loops for non-main threads).
raise unittest.SkipTest(
'Sync HTTPClient not compatible with TwistedIOLoop or '
'AsyncIOMainLoop')
self.server_ioloop = IOLoop()
sock, self.port = bind_unused_port()
app = Application([('/', HelloWorldHandler)])
self.server = HTTPServer(app, io_loop=self.server_ioloop)
self.server.add_socket(sock)
self.server_thread = threading.Thread(target=self.server_ioloop.start)
self.server_thread.start()
self.http_client = HTTPClient()
def tearDown(self):
def stop_server():
self.server.stop()
# Delay the shutdown of the IOLoop by one iteration because
# the server may still have some cleanup work left when
# the client finishes with the response (this is noticable
# with http/2, which leaves a Future with an unexamined
# StreamClosedError on the loop).
self.server_ioloop.add_callback(self.server_ioloop.stop)
self.server_ioloop.add_callback(stop_server)
self.server_thread.join()
self.http_client.close()
self.server_ioloop.close(all_fds=True)
def get_url(self, path):
return 'http://127.0.0.1:%d%s' % (self.port, path)
def test_sync_client(self):
response = self.http_client.fetch(self.get_url('/'))
self.assertEqual(b'Hello world!', response.body)
def test_sync_client_error(self):
# Synchronous HTTPClient raises errors directly; no need for
# response.rethrow()
with self.assertRaises(HTTPError) as assertion:
self.http_client.fetch(self.get_url('/notfound'))
self.assertEqual(assertion.exception.code, 404)
def main():
app = Application(debug=options.debug)
server = HTTPServer(app, xheaders=True)
if options.fd:
import socket
sock = socket.fromfd(options.fd, socket.AF_INET, socket.SOCK_STREAM)
server.add_socket(sock)
server.listen(options.port)
IOLoop.instance().start()
def get_server(loop, received):
io_loop = IOLoop(_loop=loop)
test_handlers = [
(r'/([^/]+)', TestHandler)
]
app = Application(test_handlers, received=received)
server = HTTPServer(app, io_loop=io_loop)
[sock] = netutil.bind_sockets(TEST_PORT, address=TEST_HOST)
server.add_socket(sock)
return server
def start(self):
self.events.start()
if self.options.inspect:
self.state.start()
if self.options.unix_socket:
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket)
server.add_socket(socket)
else:
self.listen(self.options.port, address=self.options.address,
ssl_options=self.ssl, xheaders=self.options.xheaders)
self.io_loop.start()
class SyncHTTPClientTest(unittest.TestCase):
def setUp(self):
if IOLoop.configured_class().__name__ in ('TwistedIOLoop',
'AsyncIOMainLoop'):
# TwistedIOLoop only supports the global reactor, so we can't have
# separate IOLoops for client and server threads.
# AsyncIOMainLoop doesn't work with the default policy
# (although it could with some tweaks to this test and a
# policy that created loops for non-main threads).
raise unittest.SkipTest(
'Sync HTTPClient not compatible with TwistedIOLoop or '
'AsyncIOMainLoop')
self.server_ioloop = IOLoop()
sock, self.port = bind_unused_port()
app = Application([('/', HelloWorldHandler)])
self.server = HTTPServer(app, io_loop=self.server_ioloop)
self.server.add_socket(sock)
self.server_thread = threading.Thread(target=self.server_ioloop.start)
self.server_thread.start()
self.http_client = HTTPClient()
def tearDown(self):
def stop_server():
self.server.stop()
# Delay the shutdown of the IOLoop by one iteration because
# the server may still have some cleanup work left when
# the client finishes with the response (this is noticable
# with http/2, which leaves a Future with an unexamined
# StreamClosedError on the loop).
self.server_ioloop.add_callback(self.server_ioloop.stop)
self.server_ioloop.add_callback(stop_server)
self.server_thread.join()
self.http_client.close()
self.server_ioloop.close(all_fds=True)
def get_url(self, path):
return 'http://127.0.0.1:%d%s' % (self.port, path)
def test_sync_client(self):
response = self.http_client.fetch(self.get_url('/'))
self.assertEqual(b'Hello world!', response.body)
def test_sync_client_error(self):
# Synchronous HTTPClient raises errors directly; no need for
# response.rethrow()
with self.assertRaises(HTTPError) as assertion:
self.http_client.fetch(self.get_url('/notfound'))
self.assertEqual(assertion.exception.code, 404)
def run(self, args):
# locale must be set before importing UDM!
log_init('/dev/stdout', args.debug)
language = str(Locale(args.language))
locale.setlocale(locale.LC_MESSAGES, language)
os.umask(
0o077) # FIXME: should probably be changed, this is what UMC sets
# The UMC-Server and module processes are clearing environment variables
os.environ.clear()
os.environ['PATH'] = '/bin:/sbin:/usr/bin:/usr/sbin'
os.environ['LANG'] = language
import univention.admin.modules as udm_modules
udm_modules.update()
from univention.admin.rest.module import Application
application = Application(serve_traceback=ucr.is_true(
'directory/manager/rest/show-tracebacks', True))
server = HTTPServer(application)
if args.port:
server.bind(args.port)
server.start(args.cpus)
if args.unix_socket:
socket = bind_unix_socket(args.unix_socket)
server.add_socket(socket)
signal.signal(signal.SIGTERM, partial(self.signal_handler_stop,
server))
signal.signal(signal.SIGINT, partial(self.signal_handler_stop, server))
signal.signal(signal.SIGHUP, self.signal_handler_reload)
channel = logging.StreamHandler()
channel.setFormatter(
tornado.log.LogFormatter(
fmt=
'%(color)s%(asctime)s %(levelname)10s (%(process)9d) :%(end_color)s %(message)s',
datefmt='%d.%m.%y %H:%M:%S'))
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(channel)
try:
tornado.ioloop.IOLoop.current().start()
except (SystemExit, KeyboardInterrupt):
raise
except:
CORE.error(traceback.format_exc())
raise
class APICollectorBot(CollectorBot):
"""Collect data by exposing a HTTP API interface"""
name: str = "API"
port: int = 5000
_collector_empty_process: bool = True
provider: str = "APICollector"
_is_multithreadable: bool = False
use_socket = False
socket_path = '/tmp/imq_api_default_socket'
_server: Optional['HTTPServer'] = None
_unix_socket: Optional[socket.socket] = None
_eventLoopThread: Optional[Thread] = None
def init(self):
if IOLoop is None:
raise MissingDependencyError("tornado")
app = Application(self.request_handler, [
("/intelmq/push", MainHandler),
])
if self.use_socket:
self.server = HTTPServer(app)
self._unix_socket = bind_unix_socket(self.socket_path)
self.server.add_socket(self._unix_socket)
else:
self.server = app.listen(self.port)
self._eventLoopThread = Thread(target=IOLoop.current().start)
self._eventLoopThread.daemon = True
self._eventLoopThread.start()
def request_handler(self, data):
report = self.new_report()
report.add("raw", data)
self.send_message(report)
def process(self):
pass
def shutdown(self):
if self.server:
# Closes the server and the socket, prevents address already in use
self.server.stop()
loop = IOLoop.current()
if loop:
loop.add_callback(loop.stop)
if self._eventLoopThread:
self._eventLoopThread.join()
def main():
""" entry point for the application """
root = os.path.dirname(__file__)
# get the command line options
options = command_line_options()
handler = setup_logging()
# import the handler file, this will fill out the route.get_routes() call.
import pyjojo.handlers
# setup the application
log.info("Setting up the application")
application = create_application(options.debug)
# unix domain socket
if options.unix_socket:
log.info("Binding application to unix socket {0}".format(
options.unix_socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
# https server
elif options.certfile and options.keyfile:
log.info("Binding application to unix socket {0}".format(
options.unix_socket))
server = HTTPServer(application,
ssl_options={
"certfile": options.certfile,
"keyfile": options.keyfile,
"ciphers": "HIGH,MEDIUM"
})
server.bind(options.port, options.address)
server.start()
# http server
else:
log.warn(
"Application is running in HTTP mode, this is insecure. Pass in the --certfile and --keyfile to use SSL."
)
server = HTTPServer(application)
server.bind(options.port, options.address)
server.start()
# start the ioloop
log.info("Starting the IOLoop")
IOLoop.instance().start()
class UnixSocketTest(AsyncTestCase):
"""HTTPServers can listen on Unix sockets too.
Why would you want to do this? Nginx can proxy to backends listening
on unix sockets, for one thing (and managing a namespace for unix
sockets can be easier than managing a bunch of TCP port numbers).
Unfortunately, there's no way to specify a unix socket in a url for
an HTTP client, so we have to test this by hand.
"""
def setUp(self):
super(UnixSocketTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(self.sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
self.server = HTTPServer(app)
self.server.add_socket(sock)
self.stream = IOStream(socket.socket(socket.AF_UNIX))
self.stream.connect(self.sockfile, self.stop)
self.wait()
def tearDown(self):
self.stream.close()
self.io_loop.run_sync(self.server.close_all_connections)
self.server.stop()
shutil.rmtree(self.tmpdir)
super(UnixSocketTest, self).tearDown()
def test_unix_socket(self):
self.stream.write(b"GET /hello HTTP/1.0\r\n\r\n")
self.stream.read_until(b"\r\n", self.stop)
response = self.wait()
self.assertEqual(response, b"HTTP/1.1 200 OK\r\n")
self.stream.read_until(b"\r\n\r\n", self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
self.stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b"Hello world")
def test_unix_socket_bad_request(self):
# Unix sockets don't have remote addresses so they just return an
# empty string.
with ExpectLog(gen_log, "Malformed HTTP message from"):
self.stream.write(b"garbage\r\n\r\n")
self.stream.read_until_close(self.stop)
response = self.wait()
self.assertEqual(response, b"HTTP/1.1 400 Bad Request\r\n\r\n")
class UnixSocketTest(AsyncTestCase):
"""HTTPServers can listen on Unix sockets too.
Why would you want to do this? Nginx can proxy to backends listening
on unix sockets, for one thing (and managing a namespace for unix
sockets can be easier than managing a bunch of TCP port numbers).
Unfortunately, there's no way to specify a unix socket in a url for
an HTTP client, so we have to test this by hand.
"""
def setUp(self):
super(UnixSocketTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(self.sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
self.server = HTTPServer(app, io_loop=self.io_loop)
self.server.add_socket(sock)
self.stream = IOStream(socket.socket(socket.AF_UNIX),
io_loop=self.io_loop)
self.stream.connect(self.sockfile, self.stop)
self.wait()
def tearDown(self):
self.stream.close()
self.server.stop()
shutil.rmtree(self.tmpdir)
super(UnixSocketTest, self).tearDown()
def test_unix_socket(self):
self.stream.write(b"GET /hello HTTP/1.0\r\n\r\n")
self.stream.read_until(b"\r\n", self.stop)
response = self.wait()
self.assertEqual(response, b"HTTP/1.0 200 OK\r\n")
self.stream.read_until(b"\r\n\r\n", self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
self.stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b"Hello world")
def test_unix_socket_bad_request(self):
# Unix sockets don't have remote addresses so they just return an
# empty string.
with ExpectLog(gen_log, "Malformed HTTP message from"):
self.stream.write(b"garbage\r\n\r\n")
self.stream.read_until_close(self.stop)
response = self.wait()
self.assertEqual(response, b"")
def start_server(future):
settings = future.result()
handlers = [(r"/api/v1/auth/providers", AuthProvidersHandler),
(r"/api/v1/auth/signup", SignupHandler),
(r"/api/v1/auth/login", PasswordHandler),
(r"/api/v1/auth/google", GoogleOAuth2LoginHandler),
(r"/api/v1/ws", MainWebSocketHandler),
(r"/icons/(?P<entity_id>[^\/]+)\/(?P<chart_id>[^\/]+)",
IconGenerator)]
application = Application(handlers, **settings)
server = HTTPServer(application)
socket = bind_unix_socket("/var/run/elastickube-api.sock", mode=0777)
server.add_socket(socket)
class ServerProcess(Process):
def __init__(self, test_app, test_sock):
self.test_app = test_app
self.sock = test_sock
super().__init__(daemon=True)
def run(self):
app = self.test_app()
self.server = HTTPServer(app)
self.server.add_socket(self.sock)
IOLoop.current().start()
def terminate(self):
self.server.stop()
self.sock.close()
super().terminate()
def start(self):
self.events.start()
if not self.options.unix_socket:
self.listen(self.options.port, address=self.options.address,
ssl_options=self.ssl_options,
xheaders=self.options.xheaders)
else:
from tornado.netutil import bind_unix_socket
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket, mode=0o777)
server.add_socket(socket)
self.started = True
self.update_workers()
self.io_loop.start()
def run_server(application, context):
server = HTTPServer(application)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is None:
with open(context.server.fd, 'r') as sock:
fd_number = sock.fileno()
sock = socket.fromfd(fd_number, socket.AF_INET | socket.AF_INET6,
socket.SOCK_STREAM)
server.add_socket(sock)
else:
server.bind(context.server.port, context.server.ip)
server.start(1)
def cmd_web(unix_socket, debug):
app = make_app(debug=debug)
server = HTTPServer(app)
if os.path.exists(unix_socket):
try:
r = requests.get('http+unix://{0}'.format(unix_socket.replace('/', '%2F')))
if r.text.strip() == 'atx.taskqueue':
print 'Already listening'
return
except:
print 'Unlink unix socket'
os.unlink(unix_socket)
socket = bind_unix_socket(unix_socket)
server.add_socket(socket)
IOLoop.current().start()
def run_server(application, context):
server = HTTPServer(application)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is None:
with open(context.server.fd, 'r') as sock:
fd_number = sock.fileno()
sock = socket.fromfd(fd_number,
socket.AF_INET | socket.AF_INET6,
socket.SOCK_STREAM)
server.add_socket(sock)
else:
server.bind(context.server.port, context.server.ip)
server.start(1)
def start(self):
self.pool = self.pool_executor_cls(max_workers=self.max_workers)
self.events.start()
if not self.options.unix_socket:
self.listen(self.options.port, address=self.options.address,
ssl_options=self.ssl_options, xheaders=self.options.xheaders)
else:
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket)
server.add_socket(socket)
self.io_loop.add_future(
control.ControlHandler.update_workers(app=self),
callback=lambda x: logger.debug('Successfully updated worker cache'))
self.started = True
self.io_loop.start()
def setUp(self):
if IOLoop.configured_class().__name__ == 'TwistedIOLoop':
# TwistedIOLoop only supports the global reactor, so we can't have
# separate IOLoops for client and server threads.
raise unittest.SkipTest(
'Sync HTTPClient not compatible with TwistedIOLoop')
self.server_ioloop = IOLoop()
sock, self.port = bind_unused_port()
app = Application([('/', HelloWorldHandler)])
server = HTTPServer(app, io_loop=self.server_ioloop)
server.add_socket(sock)
self.server_thread = threading.Thread(target=self.server_ioloop.start)
self.server_thread.start()
self.http_client = HTTPClient()
def start_server(future):
settings = future.result()
handlers = [
(r"/api/v1/auth/providers", AuthProvidersHandler),
(r"/api/v1/auth/signup", SignupHandler),
(r"/api/v1/auth/login", PasswordHandler),
(r"/api/v1/auth/google", GoogleOAuth2LoginHandler),
(r"/api/v1/ws", MainWebSocketHandler),
(r"/icons/(?P<entity_id>[^\/]+)\/(?P<chart_id>[^\/]+)", IconGenerator)
]
application = Application(handlers, **settings)
server = HTTPServer(application)
socket = bind_unix_socket("/var/run/elastickube-api.sock", mode=0777)
server.add_socket(socket)
def test_unix_socket(self):
sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
server = HTTPServer(app, io_loop=self.io_loop)
server.add_socket(sock)
stream = IOStream(socket.socket(socket.AF_UNIX), io_loop=self.io_loop)
stream.connect(sockfile, self.stop)
self.wait()
stream.write(b("GET /hello HTTP/1.0\r\n\r\n"))
stream.read_until(b("\r\n"), self.stop)
response = self.wait()
self.assertEqual(response, b("HTTP/1.0 200 OK\r\n"))
stream.read_until(b("\r\n\r\n"), self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b("Hello world"))
class ProxyTestServer(object):
def __init__(self,):
self.server_ioloop = IOLoop()
self.access_count = 0
@tornado.gen.coroutine
def init_server():
sock, self.port = bind_unused_port()
app = Application([('/(.*)', ProxyTestHandler, dict(server=self))])
self.server = HTTPServer(app)
self.server.add_socket(sock)
self.server_ioloop.run_sync(init_server)
self.server_thread = threading.Thread(target=self.server_ioloop.start)
def start(self):
self.server_thread.start()
def stop(self):
def stop_server():
self.server.stop()
@tornado.gen.coroutine
def slow_stop():
for i in range(5):
yield
self.server_ioloop.stop()
self.server_ioloop.add_callback(slow_stop)
self.server_ioloop.add_callback(stop_server)
self.server_thread.join()
self.server_ioloop.close(all_fds=True)
def get_access_count(self):
return self.access_count
def clear_access_count(self):
self.access_count = 0
