def start(self):
self.tornadoApp=tornado.web.Application(self.handlers)
self.tornadoApp.server=self
if self.ssl_crt and self.ssl_key:
import ssl
ssl_options = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_options.load_cert_chain(self.ssl_crt,self.ssl_key)
else:
ssl_options = None
server = HTTPServer(self.tornadoApp, ssl_options=ssl_options)
if self.port:
server.listen(int(self.port))
#self.tornadoApp.listen(int(self.port))
else:
socket_path = os.path.join(gnrConfigPath(), 'sockets', '%s.tornado'%self.instance_name)
main_socket = bind_unix_socket(socket_path)
#server = HTTPServer(self.tornadoApp)
server.add_socket(main_socket)
debug_socket_path = os.path.join(gnrConfigPath(), 'sockets', '%s_debug.tornado'%self.instance_name)
debug_socket = bind_unix_socket(debug_socket_path)
debug_server = GnrDebugServer(self.io_loop)
#debug_server.listen(8888)
debug_server.add_socket(debug_socket)
debug_server.application = self.tornadoApp
signal.signal(signal.SIGTERM, self.onSignal)
signal.signal(signal.SIGINT, self.onSignal)
self.io_loop.start()
def setUp(self):
super(HTTPUnixClientTestCase, self).setUp()
self.socket_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
"test_socket")
self.http_server = self.get_http_server()
sock = netutil.bind_unix_socket(self.socket_path)
self.http_server.add_sockets([sock])
def __init__(self, unixsocket, io_loop=None):
super(RuntimeMock, self).__init__(io_loop=io_loop or ioloop.IOLoop.current())
self.io_loop = io_loop or ioloop.IOLoop.current()
self.actions = list()
self.counter = 1
self.endpoint = unixsocket
self.add_socket(netutil.bind_unix_socket(unixsocket))
def unix_socket_server(application, options):
""" unix socket server """
log.info("Binding application to unix socket {0}".format(options.unix_socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
def launch(self):
import signal
import tornado.httpserver
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
if os.name == "posix":
signal.signal(signal.SIGTSTP, self.sig_handler)
self.http_server = tornado.httpserver.HTTPServer(self.application)
if firenado.conf.app['socket'] or self.socket:
from tornado.netutil import bind_unix_socket
socket_path = firenado.conf.app['socket']
if self.socket:
socket_path = self.socket
socket = bind_unix_socket(socket_path)
self.http_server.add_socket(socket)
logger.info("Firenado listening at socket ""%s" %
socket.getsockname())
else:
addresses = self.addresses
if addresses is None:
addresses = firenado.conf.app['addresses']
port = self.port
if port is None:
port = firenado.conf.app['port']
for address in addresses:
self.http_server.listen(port, address)
logger.info("Firenado listening at ""http://%s:%s" % (address,
port))
logger.info("Firenado server started successfully.")
tornado.ioloop.IOLoop.instance().start()
def setup_server():
# Config tornado.curl_httpclient to use NullHandler
tornado_logger = logging.getLogger('tornado.curl_httpclient')
tornado_logger.addHandler(logging.NullHandler())
tornado_logger.propagate = False
settings = dict(
autoreload=True,
secret="ElasticKube",
)
configure(settings)
handlers = [
(r"/api/v1/auth/providers", AuthProvidersHandler),
(r"/api/v1/auth/signup", SignupHandler),
(r"/api/v1/auth/login", PasswordHandler),
(r"/api/v1/auth/google", GoogleOAuth2LoginHandler),
(r"/api/v1/ws", MainWebSocketHandler),
(r"/icons/(?P<entity_id>[^\/]+)\/(?P<chart_id>[^\/]+)", IconGenerator)
]
application = Application(handlers, **settings)
server = HTTPServer(application)
socket = bind_unix_socket("/var/run/elastickube-api.sock", mode=0777)
server.add_socket(socket)
IOLoop.current().add_callback(initialize, settings)
def run_server(
django_host='http://127.0.0.1:8000',
listen_address='0.0.0.0',
listen_port=22000,
unix_socket=None,
socket_mode=0o600,
socket_group=None):
subscribed_channels_url = django_host + reverse('luna_websockets_channels')
BrokerConnection.subscribed_channels_url = subscribed_channels_url
BrokerRouter = sockjs.tornado.SockJSRouter(BrokerConnection, '/ws')
app = tornado.web.Application(BrokerRouter.urls)
if unix_socket:
server = tornado.httpserver.HTTPServer(app)
socket = bind_unix_socket(unix_socket, mode=socket_mode)
if socket_group is not None:
gid = grp.getgrnam(socket_group).gr_gid
os.chown(unix_socket, -1, gid)
server.add_socket(socket)
logging.info('Listening on %s' % unix_socket)
else:
app.listen(listen_port, address=listen_address)
logging.info('Listening on %s:%d' % (listen_address, listen_port))
tornado.ioloop.IOLoop.instance().start()
def launch(self):
import signal
# TODO: Resolve module if doesn't exists
if firenado.conf.app['pythonpath']:
sys.path.append(firenado.conf.app['pythonpath'])
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
if os.name == "posix":
signal.signal(signal.SIGTSTP, self.sig_handler)
self.application = TornadoApplication(debug=firenado.conf.app['debug'])
self.http_server = tornado.httpserver.HTTPServer(
self.application)
if firenado.conf.app['socket']:
from tornado.netutil import bind_unix_socket
socket = bind_unix_socket(firenado.conf.app['socket'])
self.http_server.add_socket(socket)
else:
addresses = firenado.conf.app['addresses']
port = firenado.conf.app['port']
for address in addresses:
self.http_server.listen(port, address)
logger.info("Firenado listening at ""http://%s:%s" % (address,
port))
tornado.ioloop.IOLoop.instance().start()
def bind(server, port, usocket, address=None):
"""Make server listen on port (inet socket).
If given, prefer `usocket`, path to a unix socket.
The latter is useful for reverse proxying.
If listening on a inet socket, `address` might be
given. `address` may be either an IP address or hostname.
If it's a hostname, the server will listen on all IP addresses
associated with the name. If not given (and not listening on a unix
socket) will listen on all available interfaces."""
# If we have a unix socket path
if usocket:
LOG.info('Starting on unix socket %r', usocket)
try:
socket = bind_unix_socket(usocket, mode=options.usocket_mode)
os.chown(usocket, options.usocket_uid, options.usocket_gid)
except OSError as error:
LOG.error('Cannot create unix socket: %r', error)
else:
server.add_socket(socket)
LOG.info('Started')
else:
LOG.info('Starting on port %d', port)
sockets = bind_sockets(port, address=address)
server.add_sockets(sockets)
LOG.info('Started')
def main():
define('ip', default=None, help='IP to listen on')
define('port', default=None, help='port to listen on')
define('socket', default=None, help='unix socket path to bind (instead of ip:port)')
define('group', default='', help='comma separated group list for new users `students,other`')
define('skeldir', default='', help='skeleton directory that will be used for new homedirs')
define('shell', default='/bin/bash', help='default shell')
parse_command_line()
if not options.socket and not (options.port):
options.socket = '/var/run/restuser.sock'
app = web.Application(
[(r'/([^/]+)', UserHandler)],
group=options.group,
skeldir=options.skeldir,
shell=options.shell)
if options.socket:
socket = bind_unix_socket(options.socket, mode=0o600)
server = HTTPServer(app)
server.add_socket(socket)
else:
app.listen(options.port, options.ip)
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("\ninterrupted\n", file=sys.stderr)
return
def unix_socket(path, u, g):
uid = pwd.getpwnam(u).pw_uid
gid = grp.getgrnam(g).gr_gid
with netutil.bind_unix_socket(path) as socket:
os.chown(path, uid, gid)
try:
yield socket
finally:
os.remove(path)
def listen(self, port, address=""):
"""Bind to a unix domain socket and add to self.
Note that port in our case actually contains the uds file name"""
# Create a Unix domain socket and bind
socket = bind_unix_socket(port)
# Add to self
self.add_socket(socket)
def standalone():
"""
Initializes Tornado and our application. Forks worker processes to handle
requests. Does not return until all child processes exit normally.
"""
# Parse arguments
parser = argparse.ArgumentParser(description="Ellis web server")
parser.add_argument("--background", action="store_true", help="Detach and run server in background")
args = parser.parse_args()
# We don't initialize logging until we fork because we want each child to
# have its own logging and it's awkward to reconfigure logging that is
# defined by the parent.
application = create_application()
if args.background:
# Get a new logfile, rotating the old one if present.
err_log_name = os.path.join(settings.LOGS_DIR, settings.LOG_FILE_PREFIX + "-err.log")
try:
os.rename(err_log_name, err_log_name + ".old")
except OSError:
pass
# Fork into background.
utils.daemonize(err_log_name)
utils.install_sigusr1_handler(settings.LOG_FILE_PREFIX)
# Drop a pidfile.
pid = os.getpid()
with open(settings.PID_FILE, "w") as pidfile:
pidfile.write(str(pid) + "\n")
# Fork off a child process per core. In the parent process, the
# fork_processes call blocks until the children exit.
num_processes = settings.TORNADO_PROCESSES_PER_CORE * tornado.process.cpu_count()
task_id = tornado.process.fork_processes(num_processes)
if task_id is not None:
logging_config.configure_logging(settings.LOG_LEVEL, settings.LOGS_DIR, settings.LOG_FILE_PREFIX, task_id)
# We're a child process, start up.
_log.info("Process %s starting up", task_id)
connection.init_connection()
http_server = httpserver.HTTPServer(application)
unix_socket = bind_unix_socket(settings.HTTP_UNIX + "-" + str(task_id),
0666);
http_server.add_socket(unix_socket)
homestead.ping()
background.start_background_worker_io_loop()
io_loop = tornado.ioloop.IOLoop.instance()
io_loop.start()
else:
# This shouldn't happen since the children should run their IOLoops
# forever.
_log.critical("Children all exited")
def setUp(self):
super(UnixSocketTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(self.sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
self.server = HTTPServer(app)
self.server.add_socket(sock)
self.stream = IOStream(socket.socket(socket.AF_UNIX))
self.io_loop.run_sync(lambda: self.stream.connect(self.sockfile))
def run_server(application, context):
server = HTTPServer(application)
define(
'unix_socket',
group='webserver',
default=thumbor_socket,
help='Path to unix socket to bind')
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
server.start(1)
def run():
parse_command_line()
if options.socket:
print('Listening on {}'.format(options.socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.socket, mode=0777)
server.add_socket(socket)
else:
print('Listening on {}:{}'.format(options.addr, options.port))
application.listen(options.port, address=options.addr)
application.debug = not production
tornado.ioloop.IOLoop.instance().start()
def main():
application = Application([(r"/(.*)/", ChatHandler)])
server = HTTPServer(application)
socket = bind_unix_socket(config.get("app:main", "cherubplay.socket_chat"), mode=0o777)
server.add_socket(socket)
signal.signal(signal.SIGTERM, sig_handler)
signal.signal(signal.SIGINT, sig_handler)
ioloop.start()
def start(self):
self.events.start()
if self.options.inspect:
self.state.start()
if self.options.unix_socket:
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket)
server.add_socket(socket)
else:
self.listen(self.options.port, address=self.options.address,
ssl_options=self.ssl, xheaders=self.options.xheaders)
self.io_loop.start()
def build_listener(address, backlog=0):
"""address: tuple ('0.0.0.0', 1234) or unix domain socket path such as '/tmp/tmp.sock'"""
address_type = get_address_type(address)
if address_type == 'tcp':
host, port = address
socks = netutil.bind_sockets(port, address=host, backlog=backlog)
return socks
if address_type == 'uds':
sock = netutil.bind_unix_socket(address, mode=0o600, backlog=backlog)
return [sock]
def __init__(self, usocket, starting_point, allow_design):
"""Used to run the plugins that are installed based on the starting point
@param socket Unix socket you are watching
@param starting_point The starting point of the plugins
"""
self.queue = sundaytasks.utils.get_plugins()
self.extensions = sundaytasks.utils.get_extensions()
self.starting_point = starting_point
self.instance = IOLoop.instance()
self._allow_design = allow_design
unix_socket = netutil.bind_unix_socket(usocket)
netutil.add_accept_handler(unix_socket, self.accept)
def bind_sockets(addr, backlog=128, allows_unix_socket=False):
# initialize the socket
addr = parse_address(addr)
if isinstance(addr, six.string_types):
if not allows_unix_socket:
raise RuntimeError("unix addresses aren't supported")
sock = netutil.bind_unix_socket(addr)
elif is_ipv6(addr[0]):
sock = netutil.bind_sockets(addr[1], address=addr[0],
family=socket.AF_INET6, backlog=backlog)
else:
sock = netutil.bind_sockets(addr[1], backlog=backlog)
return sock
def start(self):
try:
self.unix_socket = netutil.bind_unix_socket(self.getConfigurationValue('path_to_socket'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Will not start module %s. Could not create unix socket %s. Exception: %s, Error: %s." % (self.__class__.__name__, self.getConfigurationValue('path_to_socket'), etype, evalue))
return
try:
self.server = SocketServer(gp_module=self)
self.server.add_socket(self.unix_socket)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not access socket %s. Exception: %s, Error: %s" % (self.getConfigurationValue("path_to_socket"), etype, evalue))
return
self.running = True
def main():
server_port = int(options.base_port)
print("Starting server 0.0.0.0:%d" % server_port)
wsgi_app = tornado.wsgi.WSGIContainer(web.wsgi.application)
app = tornado.web.Application(
[('.*', tornado.web.FallbackHandler, dict(fallback=wsgi_app)),],
debug=options.debug
)
if options.unix_socket:
server = tornado.httpserver.HTTPServer(app)
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
else:
app.listen(server_port)
tornado.ioloop.IOLoop.instance().start()
def cmd_web(unix_socket, debug):
app = make_app(debug=debug)
server = HTTPServer(app)
if os.path.exists(unix_socket):
try:
r = requests.get('http+unix://{0}'.format(unix_socket.replace('/', '%2F')))
if r.text.strip() == 'atx.taskqueue':
print 'Already listening'
return
except:
print 'Unlink unix socket'
os.unlink(unix_socket)
socket = bind_unix_socket(unix_socket)
server.add_socket(socket)
IOLoop.current().start()
def run():
app = Application([
(r"/", RedirectHandler, {"url": "/s/index.html"}),
(r"/s/(.*)", StaticFileHandler, {"path": staticfilepath}),
(r"/word", WordsHandler),
(r'/word/(.*)', WordHandler)
], debug=False)
server = HTTPServer(app)
if "socket" == siteconfig["bindtype"]:
# mode 438 means 0o666
socket = netutil.bind_unix_socket(siteconfig["socketpath"], mode=438)
tornado.process.fork_processes(0)
server.add_sockets([socket])
else:
server.listen(siteconfig["port"])
IOLoop.current().start()
def start_server(future):
settings = future.result()
handlers = [
(r"/api/v1/auth/providers", AuthProvidersHandler),
(r"/api/v1/auth/signup", SignupHandler),
(r"/api/v1/auth/login", PasswordHandler),
(r"/api/v1/auth/google", GoogleOAuth2LoginHandler),
(r"/api/v1/ws", MainWebSocketHandler),
(r"/icons/(?P<entity_id>[^\/]+)\/(?P<chart_id>[^\/]+)", IconGenerator)
]
application = Application(handlers, **settings)
server = HTTPServer(application)
socket = bind_unix_socket("/var/run/elastickube-api.sock", mode=0777)
server.add_socket(socket)
def start(self):
self.pool = self.pool_executor_cls(max_workers=self.max_workers)
self.events.start()
if not self.options.unix_socket:
self.listen(self.options.port, address=self.options.address,
ssl_options=self.ssl_options, xheaders=self.options.xheaders)
else:
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket)
server.add_socket(socket)
self.io_loop.add_future(
control.ControlHandler.update_workers(app=self),
callback=lambda x: logger.debug('Successfully updated worker cache'))
self.started = True
self.io_loop.start()
def test_unix_socket(self):
sockfile = os.path.join(self.tmpdir, "test.sock")
sock = netutil.bind_unix_socket(sockfile)
app = Application([("/hello", HelloWorldRequestHandler)])
server = HTTPServer(app, io_loop=self.io_loop)
server.add_socket(sock)
stream = IOStream(socket.socket(socket.AF_UNIX), io_loop=self.io_loop)
stream.connect(sockfile, self.stop)
self.wait()
stream.write(b("GET /hello HTTP/1.0\r\n\r\n"))
stream.read_until(b("\r\n"), self.stop)
response = self.wait()
self.assertEqual(response, b("HTTP/1.0 200 OK\r\n"))
stream.read_until(b("\r\n\r\n"), self.stop)
headers = HTTPHeaders.parse(self.wait().decode('latin1'))
stream.read_bytes(int(headers["Content-Length"]), self.stop)
body = self.wait()
self.assertEqual(body, b("Hello world"))
def main():
""" entry point for the application """
root = os.path.dirname(__file__)
# get the command line options
options = command_line_options()
handler = setup_logging()
# import the handler file, this will fill out the route.get_routes() call.
import pyjojo.handlers
# setup the application
log.info("Setting up the application")
application = create_application(options.debug)
# unix domain socket
if options.unix_socket:
log.info("Binding application to unix socket {0}".format(options.unix_socket))
server = HTTPServer(application)
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
# https server
elif options.certfile and options.keyfile:
log.info("Binding application to unix socket {0}".format(options.unix_socket))
server = HTTPServer(application, ssl_options={
"certfile": options.certfile,
"keyfile": options.keyfile,
"ciphers": "HIGH,MEDIUM"
})
server.bind(options.port, options.address)
server.start()
# http server
else:
log.warn("Application is running in HTTP mode, this is insecure. Pass in the --certfile and --keyfile to use SSL.")
server = HTTPServer(application)
server.bind(options.port, options.address)
server.start()
# start the ioloop
log.info("Starting the IOLoop")
IOLoop.instance().start()
def launch(self):
import signal
# TODO: Resolve module if doesn't exists
if firenado.conf.app['pythonpath']:
sys.path.append(firenado.conf.app['pythonpath'])
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
signal.signal(signal.SIGTSTP, self.sig_handler)
self.application = TornadoApplication(debug=firenado.conf.app['debug'])
self.http_server = tornado.httpserver.HTTPServer(
self.application)
if firenado.conf.app['socket']:
from tornado.netutil import bind_unix_socket
socket = bind_unix_socket(firenado.conf.app['socket'])
self.http_server.add_socket(socket)
else:
self.http_server.listen(firenado.conf.app['port'])
tornado.ioloop.IOLoop.instance().start()
def launch_server():
state = State(get_client(options.hdfs_host, options.use_kerberos))
application = tornado.web.Application([
(r"/v1/completetions", GetCompletetionsHandler, dict(state=state)),
(r"/v1/list", ListHandler, dict(state=state)),
])
if options.unix_socket:
sockets = [netutil.bind_unix_socket(options.unix_socket)]
elif options.port_file:
sockets = netutil.bind_sockets(0, address=options.local_host)
port = sockets[0].getsockname()[1]
with open(options.port_file, "w+") as port_file:
port_file.write(str(port))
else:
sys.stderr.write("Use either --unix_socket or --port_file\n")
return
server = httpserver.HTTPServer(application)
server.add_sockets(sockets)
tornado.ioloop.IOLoop.current().start()
def start(self):
self.pool = self.pool_executor_cls(max_workers=self.max_workers)
self.events.start()
if not self.options.unix_socket:
self.listen(self.options.port,
address=self.options.address,
ssl_options=self.ssl_options,
xheaders=self.options.xheaders)
else:
from tornado.netutil import bind_unix_socket
server = HTTPServer(self)
socket = bind_unix_socket(self.options.unix_socket, mode=0o777)
server.add_socket(socket)
self.io_loop.add_future(
control.ControlHandler.update_workers(app=self),
callback=lambda x: logger.debug('Successfully updated worker cache'
))
self.started = True
self.io_loop.start()
def main():
define('ip', default=None, help='IP to listen on')
define('port', default=None, help='port to listen on')
define('socket', default=None, help='unix socket path to bind (instead of ip:port)')
parse_command_line()
if not options.socket and not (options.port):
options.socket = '/var/run/restuser.sock'
app = web.Application([(r'/([^/]+)', UserHandler)])
if options.socket:
socket = bind_unix_socket(options.socket, mode=0o600)
server = HTTPServer(app)
server.add_socket(socket)
else:
app.listen(options.port, options.ip)
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("\ninterrupted\n", file=sys.stderr)
return
def run_server(application, context):
server = HTTPServer(application, xheaders=True)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is not None:
# TODO: replace with socket.socket(fileno=fd_number) when we require Python>=3.7
sock = socket_from_fd(fd_number, True)
else:
sock = bind_unix_socket(context.server.fd)
server.add_socket(sock)
logging.debug("thumbor starting at fd %s", context.server.fd)
else:
server.bind(context.server.port, context.server.ip)
logging.debug("thumbor starting at %s:%d", context.server.ip,
context.server.port)
server.start(context.server.processes)
return server
def run_server(application, context):
server = HTTPServer(application, xheaders=True)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is not None:
sock = socket.socket(fileno=fd_number) # pylint: disable=too-many-function-args
else:
sock = bind_unix_socket(context.server.fd)
server.add_socket(sock)
logging.debug("thumbor starting at fd %s", context.server.fd)
else:
server.bind(context.server.port, context.server.ip)
logging.debug(
"thumbor starting at %s:%d", context.server.ip, context.server.port
)
server.start(context.server.processes)
return server
def run_server(application, context):
server = HTTPServer(application, xheaders=True)
if context.server.fd is not None:
fd_number = get_as_integer(context.server.fd)
if fd_number is not None:
sock = socket.fromfd(fd_number, socket.AF_INET | socket.AF_INET6,
socket.SOCK_STREAM)
else:
sock = bind_unix_socket(context.server.fd)
server.add_socket(sock)
logging.debug("thumbor starting at fd %s", context.server.fd)
else:
server.bind(context.server.port, context.server.ip)
logging.debug("thumbor starting at %s:%d", context.server.ip,
context.server.port)
server.start(context.server.processes)
return server
def start_server(future):
settings = future.result()
handlers = [
(r"/api/v1/auth/providers", AuthProvidersHandler),
(r"/api/v1/auth/signup", SignupHandler),
(r"/api/v1/auth/request-invite", RequestInviteHandler),
(r"/api/v1/auth/reset-password", ResetPasswordHandler),
(r"/api/v1/auth/change-password", ChangePasswordHandler),
(r"/api/v1/auth/login", PasswordHandler),
(r"/api/v1/auth/google", GoogleOAuth2LoginHandler),
(r"/api/v1/auth/saml", Saml2LoginHandler),
(r"/api/v1/auth/saml/metadata", Saml2MetadataHandler),
(r"/api/v1/ws", MainWebSocketHandler),
(r"/icons/(?P<entity_id>[^\/]+)\/(?P<chart_id>[^\/]+)", IconGenerator)
]
application = Application(handlers, **settings)
server = HTTPServer(application)
socket = bind_unix_socket("/var/run/elastickube-api.sock", mode=0777)
server.add_socket(socket)
def run(self, args):
# locale must be set before importing UDM!
log_init('/dev/stdout', args.debug)
language = str(Locale(args.language))
locale.setlocale(locale.LC_MESSAGES, language)
os.umask(
0o077) # FIXME: should probably be changed, this is what UMC sets
# The UMC-Server and module processes are clearing environment variables
os.environ.clear()
os.environ['PATH'] = '/bin:/sbin:/usr/bin:/usr/sbin'
os.environ['LANG'] = language
import univention.admin.modules as udm_modules
udm_modules.update()
from univention.management.modules.udm.module import Application
application = Application(serve_traceback=True, autoreload=True)
server = HTTPServer(application)
if args.port:
server.listen(args.port)
if args.unix_socket:
socket = bind_unix_socket(args.unix_socket)
server.add_socket(socket)
signal.signal(signal.SIGTERM, partial(self.signal_handler_stop,
server))
signal.signal(signal.SIGINT, partial(self.signal_handler_stop, server))
signal.signal(signal.SIGHUP, self.signal_handler_reload)
tornado.log.enable_pretty_logging()
try:
tornado.ioloop.IOLoop.current().start()
except (SystemExit, KeyboardInterrupt):
raise
except:
CORE.error(traceback.format_exc())
raise
def _start_server(self):
self.server = HTTPServer(self.app, io_loop=self.io_loop,
ssl_options=self.ssl_options)
# bind the handler to needed interface
for uri in self.uri:
addr = parse_address(uri)
if isinstance(addr, six.string_types):
sock = netutil.bind_unix_socket(addr)
elif is_ipv6(addr[0]):
sock = netutil.bind_sockets(addr[1], address=addr[0],
family=socket.AF_INET6, backlog=self.backlog)
else:
sock = netutil.bind_sockets(addr[1], backlog=self.backlog)
if isinstance(sock, list):
for s in sock:
self.server.add_socket(s)
else:
self.server.add_socket(sock)
# start the server
self.server.start()
def run_server(application, context):
server = HTTPServer(application, xheaders=True)
if hasattr(socket, "AF_UNIX") and context.server.fd is not None:
# for unix only, avoid error on windows.
fd_number = get_as_integer(context.server.fd)
if fd_number is not None:
# TODO: replace with socket.socket(fileno=fd_number) when we require Python>=3.7
sock = socket_from_fd(fd_number, True) # pylint: disable=too-many-function-args
else:
sock = bind_unix_socket(context.server.fd)
server.add_socket(sock)
logging.debug("thumbor starting at fd %s", context.server.fd)
else:
server.bind(context.server.port, context.server.ip)
logging.debug("thumbor starting at %s:%d", context.server.ip,
context.server.port)
server.start(context.server.processes)
return server
def main():
logging.basicConfig()
#logging.disable("INFO")
tornado.options.parse_command_line()
ioloop = tornado.ioloop.IOLoop.instance()
# XXX must create new ports
# tornado.process.fork_processes(5, max_restarts=100)
dnspython_version = dns.version.version
logging.warn("dnspython version %s" % dnspython_version)
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(options.port)
unix_socket = bind_unix_socket("/tmp/fdserver.sock",
mode=0777,
backlog=10000)
http_server.add_socket(unix_socket)
# Trigger timeout - will cause the blocking process to die.
# ioloop.set_blocking_signal_threshold(2, _log_blocking)
# ioloop.set_blocking_log_threshold(2)
ioloop.start()
def main():
from tornado import options
opts = options.OptionParser()
opts.define("version",
type=bool,
help="show version and exit",
callback=show_version)
opts.define("locators",
default=["localhost:10053"],
type=str,
multiple=True,
help="comma-separated endpoints of locators")
opts.define("cache",
default=DEFAULT_SERVICE_CACHE_COUNT,
type=int,
help="count of instances per service")
opts.define(
"config",
help="path to configuration file",
type=str,
callback=lambda path: opts.parse_config_file(path, final=False))
opts.define("count",
default=1,
type=int,
help="count of tornado processes")
opts.define("port", default=8080, type=int, help="listening port number")
opts.define(
"endpoints",
default=["tcp://localhost:8080"],
type=str,
multiple=True,
help=
"Specify endpoints to bind on: prefix unix:// or tcp:// should be used"
)
opts.define("request_header",
default="X-Request-Id",
type=str,
help="header used as a trace id")
opts.define("forcegen_request_header",
default=False,
type=bool,
help="enable force generation of the request header")
opts.define("sticky_header",
default="X-Cocaine-Sticky",
type=str,
help="sticky header name")
opts.define("gcstats",
default=False,
type=bool,
help="print garbage collector stats to stderr")
# tracing options
opts.define("tracing_chance",
default=DEFAULT_TRACING_CHANCE,
type=float,
help="default chance for an app to be traced")
opts.define("configuration_service",
default="unicorn",
type=str,
help="name of configuration service")
opts.define(
"tracing_conf_path",
default="/zipkin_sampling",
type=str,
help="path to the configuration nodes in the configuration service")
# various logging options
opts.define(
"logging",
default="info",
help=("Set the Python log level. If 'none', tornado won't touch the "
"logging configuration."),
metavar="debug|info|warning|error|none")
opts.define(
"log_to_stderr",
type=bool,
default=None,
help=("Send log output to stderr. "
"By default use stderr if --log_file_prefix is not set and "
"no other logging is configured."))
opts.define("log_file_prefix",
type=str,
default=None,
metavar="PATH",
help=("Path prefix for log file"))
opts.define("datefmt",
type=str,
default="%z %d/%b/%Y:%H:%M:%S",
help="datefmt")
opts.define("generallogfmt",
type=str,
help="log format of general logging system",
default=DEFAULT_GENERAL_LOGFORMAT)
opts.define("accesslogfmt",
type=str,
help="log format of access logging system",
default=DEFAULT_ACCESS_LOGFORMAT)
opts.define("logframework",
type=bool,
default=False,
help="enable logging various framework messages")
opts.define("fingerscrossed",
type=bool,
default=True,
help="enable lazy logging")
# util server
opts.define("utilport",
default=8081,
type=int,
help="listening port number for an util server")
opts.define("utiladdress",
default="127.0.0.1",
type=str,
help="address for an util server")
opts.define("enableutil",
default=False,
type=bool,
help="enable util server")
opts.define("so_reuseport",
default=True,
type=bool,
help="use SO_REUSEPORT option")
opts.parse_command_line()
enable_logging(opts)
logger = logging.getLogger("cocaine.proxy.general")
use_reuseport = False
endpoints = Endpoints(opts.endpoints)
sockets = []
if endpoints.has_unix:
logger.info("Start binding on unix sockets")
for path in endpoints.unix:
logger.info("Binding on %s", path)
sockets.append(bind_unix_socket(path, mode=0o666))
if opts.so_reuseport:
if not support_reuseport():
logger.warning("Your system doesn't support SO_REUSEPORT."
" Bind and fork mechanism will be used")
else:
logger.info("SO_REUSEPORT will be used")
use_reuseport = True
if not use_reuseport and endpoints.has_tcp:
logger.info("Start binding on tcp sockets")
for endpoint in endpoints.tcp:
logger.info("Binding on %s:%d", endpoint.host, endpoint.port)
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port, address=endpoint.host)
logger.info(
"Listening %s",
' '.join(str("%s:%s" % s.getsockname()[:2]) for s in socks))
sockets.extend(socks)
if opts.enableutil:
utilsockets = bind_sockets(opts.utilport, address=opts.utiladdress)
logger.info(
"Util server is listening on %s",
' '.join(str("%s:%s" % s.getsockname()[:2]) for s in utilsockets))
try:
if opts.count != 1:
process.fork_processes(opts.count)
if opts.gcstats:
enable_gc_stats()
if use_reuseport and endpoints.has_tcp:
logger.info("Start binding on tcp sockets")
for endpoint in endpoints.tcp:
logger.info("Binding on %s:%d", endpoint.host, endpoint.port)
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port,
address=endpoint.host,
reuse_port=True)
logger.info(
"Listening %s", ' '.join(
str("%s:%s" % s.getsockname()[:2]) for s in socks))
sockets.extend(socks)
proxy = CocaineProxy(
locators=opts.locators,
cache=opts.cache,
request_id_header=opts.request_header,
sticky_header=opts.sticky_header,
forcegen_request_header=opts.forcegen_request_header,
default_tracing_chance=opts.tracing_chance)
server = HTTPServer(proxy)
server.add_sockets(sockets)
if opts.enableutil:
utilsrv = HTTPServer(UtilServer(proxy=proxy))
utilsrv.add_sockets(utilsockets)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
pass
print('closing connection')
return
print('received_message')
chunk = sock.recv(1024)
print(chunk)
def handle_connection(sock, fd, events):
print('handling new connection')
try:
(connection, address) = sock.accept()
except socket.error, e:
if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
return
raise
message_callback = functools.partial(handle_message, connection)
IOLoop.instance().add_handler(connection.fileno(), message_callback,
IOLoop.ERROR | IOLoop.READ)
unix_socket = bind_unix_socket('pacemaker-cloud-cped')
accept_callback = functools.partial(handle_connection, unix_socket)
IOLoop.instance().add_handler(unix_socket.fileno(), accept_callback,
IOLoop.READ)
IOLoop.instance().start()
if __name__ == '__main__':
main()
def main():
from tornado import options
default_general_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(message)s"
default_access_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(trace_id)s\t%(message)s"
opts = options.OptionParser()
opts.define("version",
type=bool,
help="show version and exit",
callback=show_version)
opts.define("locators",
default=["localhost:10053"],
type=str,
multiple=True,
help="comma-separated endpoints of locators")
opts.define("cache",
default=DEFAULT_SERVICE_CACHE_COUNT,
type=int,
help="count of instances per service")
opts.define(
"config",
help="path to configuration file",
type=str,
callback=lambda path: opts.parse_config_file(path, final=False))
opts.define("count",
default=1,
type=int,
help="count of tornado processes")
opts.define(
"endpoints",
default=["tcp://localhost:8080"],
type=str,
multiple=True,
help=
"Specify endpoints to bind on: prefix unix:// or tcp:// should be used"
)
opts.define("request_header",
default="X-Request-Id",
type=str,
help="header used as a trace id")
opts.define("forcegen_request_header",
default=False,
type=bool,
help="enable force generation of the request header")
opts.define("sticky_header",
default="X-Cocaine-Sticky",
type=str,
help="sticky header name")
opts.define("gcstats",
default=False,
type=bool,
help="print garbage collector stats to stderr")
opts.define("srwconfig", default="", type=str, help="path to srwconfig")
opts.define("allow_json_rpc",
default=True,
type=bool,
help="allow JSON RPC module")
# tracing options
opts.define("tracing_chance",
default=DEFAULT_TRACING_CHANCE,
type=float,
help="default chance for an app to be traced")
opts.define("configuration_service",
default="unicorn",
type=str,
help="name of configuration service")
opts.define(
"tracing_conf_path",
default="/zipkin_sampling",
type=str,
help="path to the configuration nodes in the configuration service")
# various logging options
opts.define(
"logging",
default="info",
help=("Set the Python log level. If 'none', tornado won't touch the "
"logging configuration."),
metavar="debug|info|warning|error|none")
opts.define("log_to_cocaine",
default=False,
type=bool,
help="log to cocaine")
opts.define(
"log_to_stderr",
type=bool,
default=None,
help=("Send log output to stderr. "
"By default use stderr if --log_file_prefix is not set and "
"no other logging is configured."))
opts.define("log_file_prefix",
type=str,
default=None,
metavar="PATH",
help=("Path prefix for log file"))
opts.define("datefmt",
type=str,
default="%z %d/%b/%Y:%H:%M:%S",
help="datefmt")
opts.define("generallogfmt",
type=str,
help="log format of general logging system",
default=default_general_logformat)
opts.define("accesslogfmt",
type=str,
help="log format of access logging system",
default=default_access_logformat)
opts.define("logframework",
type=bool,
default=False,
help="enable logging various framework messages")
# util server
opts.define("utilport",
default=8081,
type=int,
help="listening port number for an util server")
opts.define("utiladdress",
default="127.0.0.1",
type=str,
help="address for an util server")
opts.define("enableutil",
default=False,
type=bool,
help="enable util server")
opts.parse_command_line()
srw_config = None
if opts.srwconfig:
try:
srw_config = load_srw_config(opts.srwconfig)
except Exception as err:
print("unable to load SRW config: %s" % err)
exit(1)
use_reuseport = hasattr(socket, "SO_REUSEPORT")
endpoints = Endpoints(opts.endpoints)
sockets = []
for path in endpoints.unix:
sockets.append(bind_unix_socket(path, mode=0o666))
if not use_reuseport:
for endpoint in endpoints.tcp:
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port, address=endpoint.host)
sockets.extend(socks)
if opts.enableutil:
utilsockets = bind_sockets(opts.utilport, address=opts.utiladdress)
try:
if opts.count != 1:
process.fork_processes(opts.count)
enable_logging(opts)
if opts.gcstats:
enable_gc_stats()
if use_reuseport:
for endpoint in endpoints.tcp:
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port,
address=endpoint.host,
reuse_port=True)
sockets.extend(socks)
proxy = CocaineProxy(
locators=opts.locators,
cache=opts.cache,
request_id_header=opts.request_header,
sticky_header=opts.sticky_header,
forcegen_request_header=opts.forcegen_request_header,
default_tracing_chance=opts.tracing_chance,
srw_config=srw_config,
allow_json_rpc=opts.allow_json_rpc)
server = HTTPServer(proxy)
server.add_sockets(sockets)
if opts.enableutil:
utilsrv = HTTPServer(UtilServer(proxy=proxy))
utilsrv.add_sockets(utilsockets)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
pass
def runCouchPotato(options,
base_path,
args,
data_dir=None,
log_dir=None,
Env=None,
desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
old_db_path = os.path.join(data_dir, 'couchpotato.db')
# Remove database folder if both exists
if os.path.isdir(db_path) and os.path.isfile(old_db_path):
db = SuperThreadSafeDatabase(db_path)
db.open()
db.destroy()
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(
os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename),
arcname='database/%s' %
os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
session = cfscrape.create_scraper()
session.max_redirects = 5
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('http_opener', session)
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default=False, type='bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default=False, type='bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in [
'enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado',
'requests'
]:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get(
'desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or
options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'),
'a',
500000,
10,
encoding=Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
# Check soft-chroot dir exists:
try:
# Load Soft-Chroot
soft_chroot = Env.get('softchroot')
soft_chroot_dir = Env.setting('soft_chroot',
section='core',
default=None,
type='unicode')
soft_chroot.initialize(soft_chroot_dir)
except SoftChrootInitError as exc:
log.error(exc)
return
except:
log.error('Unable to check whether SOFT-CHROOT is defined')
return
# Check available space
try:
total_space, available_space = getFreeSpace(data_dir)
if available_space < 100:
log.error(
'Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
return
except:
log.error('Failed getting diskspace: %s', traceback.format_exc())
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') +
'/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value=api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default='0.0.0.0')
host6 = Env.setting('host6', default='::')
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default=5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'host6': host6 if host6 and len(host6) > 0 else '::',
'ssl_cert': Env.setting('ssl_cert', default=None),
'ssl_key': Env.setting('ssl_key', default=None),
}
# Load the app
application = Application(
[],
log_function=lambda x: None,
debug=config['use_reloader'],
gzip=True,
cookie_secret=api_key,
login_url='%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(
".*$",
[
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {
"url": web_base + 'docs/'
}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(
".*$",
[('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {
'path':
sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))
})])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root=sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order=True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def reload_hook():
fireEvent('app.shutdown')
add_reload_hook(reload_hook)
# Some logging and fire load event
try:
log.info('Starting server on port %(port)s', config)
except:
pass
fireEventAsync('app.load')
ssl_options = None
if config['ssl_cert'] and config['ssl_key']:
ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
}
server = HTTPServer(application,
no_keep_alive=True,
ssl_options=ssl_options)
try_restart = True
restart_tries = 5
while try_restart:
try:
if config['host'].startswith('unix:'):
server.add_socket(bind_unix_socket(config['host'][5:]))
else:
server.listen(config['port'], config['host'])
if Env.setting('ipv6', default=False):
try:
server.listen(config['port'], config['host6'])
except:
log.info2('Tried to bind to IPV6 but failed')
loop.start()
server.close_all_connections()
server.stop()
loop.close(all_fds=True)
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info(
'Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds',
(config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except ValueError:
return
except:
pass
raise
try_restart = False
def get(self):
self.write("Hello from Tornado")
application = tornado.web.Application([
(r"/hello", HelloHandler),
],
debug=False)
define('listen_address', default='127.0.0.1', help='Listen address')
define('listen_port', default=8080, help='Listen port')
define('unix_socket', default=None, help='Path to unix socket to bind')
define('backlog_size', default=8191, help='Size of the listen backlog')
if __name__ == "__main__":
tornado.options.parse_command_line()
if options.unix_socket:
from tornado.netutil import bind_unix_socket
from tornado.httpserver import HTTPServer
server = HTTPServer(application)
socket = bind_unix_socket(options.unix_socket,
backlog=options.backlog_size)
server.add_socket(socket)
print "Listening on %s" % options.unix_socket
else:
application.listen(options.listen_port, address=options.listen_address)
print "Listening on http://%s:%s" % (options.listen_address,
options.listen_port)
tornado.ioloop.IOLoop.instance().start()
def make_app():
return Application(handlers=route.get_routes(),
template_path=os.path.join(os.path.dirname(__file__),
'template'),
static_path=os.path.join(os.path.dirname(__file__),
'../public'),
debug=DEBUG_MODE,
autoreload=False)
if __name__ == '__main__':
application = make_app()
if UNIX_SOCKET:
# Windows does not support bind_unix_socket
from tornado.httpserver import HTTPServer
from tornado.netutil import bind_unix_socket
server = HTTPServer(application)
socket = bind_unix_socket(UNIX_SOCKET, mode=0o666)
server.add_socket(socket)
server_info = 'Server(%s)' % UNIX_SOCKET
else:
application.listen(LISTEN_PORT, xheaders=True)
server_info = 'Server(Port: %d)' % LISTEN_PORT
logging.info('%s start at %s' % (server_info, str(datetime.utcnow())))
IOLoop.instance().start()
logging.info('%s stop at %s' % (server_info, str(datetime.utcnow())))
async def start(self):
"""Start the whole thing.
Overridden to make service.start() asynchronous."""
self.io_loop = loop = IOLoop.current()
if self.subapp:
self.subapp.start()
loop.stop()
return
if self.generate_config:
self.write_config_file()
loop.stop()
return
if self.generate_certs:
self.load_config_file(self.config_file)
if not self.internal_ssl:
self.log.warning(
"You'll need to enable `internal_ssl` "
"in the `jupyterhub_config` file to use "
"these certs."
)
self.internal_ssl = True
self.init_internal_ssl()
self.log.info(
"Certificates written to directory `{}`".format(
self.internal_certs_location
)
)
loop.stop()
return
# start the proxy
if self.proxy.should_start:
try:
await self.proxy.start()
except Exception:
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
else:
self.log.info("Not starting proxy")
# verify that we can talk to the proxy before listening.
# avoids delayed failure if we can't talk to the proxy
await self.proxy.get_all_routes()
ssl_context = make_ssl_context(
self.internal_ssl_key,
self.internal_ssl_cert,
cafile=self.internal_ssl_ca,
check_hostname=False,
)
# start the webserver
self.http_server = tornado.httpserver.HTTPServer(
self.tornado_application,
ssl_options=ssl_context,
xheaders=True,
trusted_downstream=self.trusted_downstream_ips,
)
bind_url = urlparse(self.hub.bind_url)
try:
if bind_url.scheme.startswith('unix+'):
from tornado.netutil import bind_unix_socket
socket = bind_unix_socket(unquote(bind_url.netloc))
self.http_server.add_socket(socket)
else:
ip = bind_url.hostname
port = bind_url.port
if not port:
if bind_url.scheme == 'https':
port = 443
else:
port = 80
self.http_server.listen(port, address=ip)
self.log.info("Hub API listening on %s", self.hub.bind_url)
if self.hub.url != self.hub.bind_url:
self.log.info("Private Hub API connect url %s", self.hub.url)
except Exception:
self.log.error("Failed to bind hub to %s", self.hub.bind_url)
raise
# start the service(s)
for service_name, service in self._service_map.items():
msg = (
'%s at %s' % (service_name, service.url)
if service.url
else service_name
)
if service.managed:
self.log.info("Starting managed service %s", msg)
try:
await service.start()
except Exception:
self.log.critical(
"Failed to start service %s", service_name, exc_info=True
)
self.exit(1)
else:
self.log.info("Adding external service %s", msg)
if service.url:
tries = 10 if service.managed else 1
for i in range(tries):
try:
ssl_context = make_ssl_context(
self.internal_ssl_key,
self.internal_ssl_cert,
cafile=self.internal_ssl_ca,
)
await Server.from_orm(service.orm.server).wait_up(
http=True, timeout=1, ssl_context=ssl_context
)
except TimeoutError:
if service.managed:
status = await service.spawner.poll()
if status is not None:
self.log.error(
"Service %s exited with status %s",
service_name,
status,
)
break
else:
break
else:
self.log.error(
"Cannot connect to %s service %s at %s. Is it running?",
service.kind,
service_name,
service.url,
)
await self.proxy.check_routes(self.users, self._service_map)
if self.service_check_interval and any(
s.url for s in self._service_map.values()
):
pc = PeriodicCallback(
self.check_services_health, 1e3 * self.service_check_interval
)
pc.start()
if self.last_activity_interval:
pc = PeriodicCallback(
self.update_last_activity, 1e3 * self.last_activity_interval
)
self.last_activity_callback = pc
pc.start()
self.log.info("JupyterHub is now running at %s", self.proxy.public_url)
# Use atexit for Windows, it doesn't have signal handling support
if os.name == "nt":
atexit.register(self.atexit)
# register cleanup on both TERM and INT
self.init_signal()
self._start_future.set_result(None)
def start_server(args):
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.netutil import bind_unix_socket
logging.info('Run on host %s:%i', args.host, args.port)
http_server = HTTPServer(WSGIContainer(app))
http_server.listen(args.port, args.host)
if args.control_unix_socket:
control_server = HTTPServer(WSGIContainer(control_app))
socket = bind_unix_socket(args.control_unix_socket, mode=0o666)
control_server.add_socket(socket)
logging.info('Run control server on unix socket %s',
args.control_unix_socket)
global NGINX_TEMPLATE_GEN, PORT_MANAGER, LOCK
NGINX_TEMPLATE_GEN = NginxTemplateGenerator(
nginx_port=args.nginx_port,
nginx_conf_file=args.nginx_conf_file,
nginx_pid_file=args.nginx_pid_file,
controller_port=args.port)
def nginx_regenerate_conf():
NGINX_TEMPLATE_GEN.generate(PORT_MANAGER)
NGINX_TEMPLATE_GEN.reload_nginx_config()
PORT_MANAGER = PortManager(
port_registered_cb=lambda rport: nginx_regenerate_conf(),
port_unregistered_cb=lambda rport: nginx_regenerate_conf())
nginx_regenerate_conf()
def on_ports_changed(new_ports):
with LOCK:
PORT_MANAGER.update_accessible_ports(new_ports)
PortWatcher.start_instance(ports_changed_cb=on_ports_changed,
wait_delay=args.wait_delay)
loop = IOLoop.current()
def stop_ioloop():
logging.info('Stopping IOLoop')
loop.stop()
def signal_term_handler(signal, frame):
print('Got signal {}, exiting'.format(signal), file=sys.stderr)
stop_ioloop()
sys.exit(0)
def on_exit():
if args.control_unix_socket:
os.unlink(args.control_unix_socket)
signal.signal(signal.SIGTERM, signal_term_handler)
signal.signal(signal.SIGINT, signal_term_handler)
atexit.register(on_exit)
loop.start()
def main():
""" Starts the AdminServer. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
parser = argparse.ArgumentParser(
prog='appscale-admin', description='Manages AppScale-related processes')
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
serve_parser = subparsers.add_parser(
'serve', description='Starts the server that manages AppScale processes')
serve_parser.add_argument(
'-p', '--port', type=int, default=constants.DEFAULT_PORT,
help='The port to listen on')
serve_parser.add_argument(
'-v', '--verbose', action='store_true', help='Output debug-level logging')
subparsers.add_parser(
'summary', description='Lists AppScale processes running on this machine')
restart_parser = subparsers.add_parser(
'restart',
description='Restart AppScale processes running on this machine')
restart_parser.add_argument('service', nargs='+',
help='The process or service ID to restart')
args = parser.parse_args()
if args.command == 'summary':
table = sorted(list(get_combined_services().items()))
print(tabulate(table, headers=['Service', 'State']))
sys.exit(0)
if args.command == 'restart':
socket_path = urlquote(ServiceManagerHandler.SOCKET_PATH, safe='')
session = requests_unixsocket.Session()
response = session.post(
'http+unix://{}/'.format(socket_path),
data={'command': 'restart', 'arg': [args.service]})
response.raise_for_status()
return
if args.verbose:
logger.setLevel(logging.DEBUG)
options.define('secret', appscale_info.get_secret())
options.define('login_ip', appscale_info.get_login_ip())
options.define('private_ip', appscale_info.get_private_ip())
options.define('load_balancers', appscale_info.get_load_balancer_ips())
acc = appscale_info.get_appcontroller_client()
ua_client = UAClient(appscale_info.get_db_master_ip(), options.secret)
zk_client = KazooClient(
hosts=','.join(appscale_info.get_zk_node_ips()),
connection_retry=ZK_PERSISTENT_RECONNECTS)
zk_client.start()
version_update_lock = zk_client.Lock(constants.VERSION_UPDATE_LOCK_NODE)
thread_pool = ThreadPoolExecutor(4)
monit_operator = MonitOperator()
all_resources = {
'acc': acc,
'ua_client': ua_client,
'zk_client': zk_client,
'version_update_lock': version_update_lock,
'thread_pool': thread_pool
}
if options.private_ip in appscale_info.get_taskqueue_nodes():
logger.info('Starting push worker manager')
GlobalPushWorkerManager(zk_client, monit_operator)
service_manager = ServiceManager(zk_client)
service_manager.start()
app = web.Application([
('/oauth/token', OAuthHandler, {'ua_client': ua_client}),
('/v1/apps/([a-z0-9-]+)/services/([a-z0-9-]+)/versions', VersionsHandler,
{'ua_client': ua_client, 'zk_client': zk_client,
'version_update_lock': version_update_lock, 'thread_pool': thread_pool}),
('/v1/projects', ProjectsHandler, all_resources),
('/v1/projects/([a-z0-9-]+)', ProjectHandler, all_resources),
('/v1/apps/([a-z0-9-]+)/services/([a-z0-9-]+)', ServiceHandler,
all_resources),
('/v1/apps/([a-z0-9-]+)/services/([a-z0-9-]+)/versions/([a-z0-9-]+)',
VersionHandler, all_resources),
('/v1/apps/([a-z0-9-]+)/operations/([a-z0-9-]+)', OperationsHandler,
{'ua_client': ua_client}),
('/api/cron/update', UpdateCronHandler,
{'acc': acc, 'zk_client': zk_client, 'ua_client': ua_client}),
('/api/datastore/index/add', UpdateIndexesHandler,
{'zk_client': zk_client, 'ua_client': ua_client}),
('/api/queue/update', UpdateQueuesHandler,
{'zk_client': zk_client, 'ua_client': ua_client})
])
logger.info('Starting AdminServer')
app.listen(args.port)
management_app = web.Application([
('/', ServiceManagerHandler, {'service_manager': service_manager})])
management_server = HTTPServer(management_app)
management_socket = bind_unix_socket(ServiceManagerHandler.SOCKET_PATH)
management_server.add_socket(management_socket)
io_loop = IOLoop.current()
io_loop.start()
def main(argv):
global opt
signal.signal(signal.SIGTERM, handle_sigterm)
UnixAddress = namedtuple('UnixAddress', ['path'])
InetAddress = namedtuple('InetAddress', ['host', 'port'])
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) > 1:
o.fatal("at most one argument expected")
if len(extra) == 0:
address = InetAddress(host='127.0.0.1', port=8080)
else:
bind_url = extra[0]
if bind_url.startswith('unix://'):
address = UnixAddress(path=bind_url[len('unix://'):])
else:
addr_parts = extra[0].split(':', 1)
if len(addr_parts) == 1:
host = '127.0.0.1'
port = addr_parts[0]
else:
host, port = addr_parts
try:
port = int(port)
except (TypeError, ValueError) as ex:
o.fatal('port must be an integer, not %r' % port)
address = InetAddress(host=host, port=port)
git.check_repo_or_die()
settings = dict(
debug=1,
template_path=resource_path(b'web').decode('utf-8'),
static_path=resource_path(b'web/static').decode('utf-8'),
)
# Disable buffering on stdout, for debug messages
try:
sys.stdout._line_buffering = True
except AttributeError:
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
application = tornado.web.Application([
(r"(?P<path>/.*)", BupRequestHandler, dict(repo=LocalRepo())),
], **settings)
http_server = HTTPServer(application)
io_loop_pending = IOLoop.instance()
if isinstance(address, InetAddress):
sockets = tornado.netutil.bind_sockets(address.port, address.host)
http_server.add_sockets(sockets)
print('Serving HTTP on %s:%d...' % sockets[0].getsockname()[0:2])
if opt.browser:
browser_addr = 'http://' + address[0] + ':' + str(address[1])
io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr))
elif isinstance(address, UnixAddress):
unix_socket = bind_unix_socket(address.path)
http_server.add_socket(unix_socket)
print('Serving HTTP on filesystem socket %r' % address.path)
else:
log('error: unexpected address %r', address)
sys.exit(1)
io_loop = io_loop_pending
io_loop.start()
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
repo = from_opts(opt, reverse=False)
application = tornado.web.Application([
(r"(?P<path>/.*)", BupRequestHandler, dict(repo=repo)),
], **settings)
http_server = HTTPServer(application)
io_loop_pending = IOLoop.instance()
if isinstance(address, InetAddress):
sockets = tornado.netutil.bind_sockets(address.port, address.host)
http_server.add_sockets(sockets)
print('Serving HTTP on %s:%d...' % sockets[0].getsockname())
if opt.browser:
browser_addr = 'http://' + address[0] + ':' + str(address[1])
io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr))
elif isinstance(address, UnixAddress):
unix_socket = bind_unix_socket(address.path)
http_server.add_socket(unix_socket)
print('Serving HTTP on filesystem socket %r' % address.path)
else:
log('error: unexpected address %r', address)
sys.exit(1)
io_loop = io_loop_pending
io_loop.start()
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
sys.exit(1)
from tornado.options import options, define
from tornado.netutil import bind_unix_socket
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write("Hello, world")
application = tornado.web.Application([
(r"/", MainHandler),
])
define('listen_address',
group='webserver',
default='127.0.0.1',
help='Listen address')
define('listen_port', group='webserver', default=8888, help='Listen port')
define('unix_socket',
group='webserver',
default=None,
help='Path to unix socket to bind')
if __name__ == "__main__":
if options.unix_socket:
server = HTTPServer(application)
socket = bind_unix_socket(options.unix_socket)
server.add_socket(socket)
else:
application.listen(options.listen_port, address=options.listen_address)
tornado.ioloop.IOLoop.instance().start()
import tornado.ioloop
import tornado.web
from tornado import httpserver
from tornado.netutil import bind_unix_socket
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write('hello world')
def make_app():
return tornado.web.Application([
(r'/', MainHandler),
])
if __name__ == '__main__':
app = make_app()
server = httpserver.HTTPServer(app, xheaders=True)
socket = bind_unix_socket('/tmp/main_tornado.sock', mode=0o666)
server.add_socket(socket)
tornado.ioloop.IOLoop.current().start()
[root@localhost unix_socket]# python main.py
# # 开启另外一个窗口来访问.
# [root@localhost simple_socket]# curl --unix-socket /tmp/main_tornado.sock http://localhost/
#
# # 显示信息
# Hello, world
List all the volumes registered with the plugin. Since Ceph is distributed and CephFS instances
are typically created in a distributed manner, this endpoint calls the CephFS API to get all the
existing CephFS instances in the system.
"""
def initialize(self):
self.__ceph = Ceph()
async def post(self):
_, vols, _ = await self.__ceph.list_volumes()
self.write(json_encode(dict(Volumes=list(vols), Err='')))
if '__main__' == __name__:
app = tornado.web.Application([
(r'/Plugin.Activate', ActivateHandler),
(r'/VolumeDriver.Capabilities', VolumeCapabilitiesHandler),
(r'/VolumeDriver.Create', VolumeCreateHandler),
(r'/VolumeDriver.Remove', VolumeRemoveHandler),
(r'/VolumeDriver.Mount', VolumeMountHandler),
(r'/VolumeDriver.Unmount', VolumeUnmountHandler),
(r'/VolumeDriver.Path', VolumePathHandler),
(r'/VolumeDriver.Get', VolumeGetHandler),
(r'/VolumeDriver.List', VolumeListHandler),
])
server = HTTPServer(app)
socket = bind_unix_socket('/run/docker/plugins/cephfs.sock')
server.listen(8888)
server.add_socket(socket)
tornado.ioloop.IOLoop.instance().start()
def __init__(self,
applications: Mapping[str, Application | ModifyDoc]
| Application | ModifyDoc,
io_loop: IOLoop | None = None,
http_server_kwargs: Dict[str, Any] | None = None,
**kwargs: Any) -> None:
''' Create a ``Server`` instance.
Args:
applications (dict[str, Application] or Application or callable) :
A mapping from URL paths to Application instances, or a single
Application to put at the root URL.
The Application is a factory for Documents, with a new Document
initialized for each Session. Each application is identified
by a path that corresponds to a URL, like "/" or "/myapp"
If a single Application is provided, it is mapped to the URL
path "/" automatically.
As a convenience, a callable may also be provided, in which
an Application will be created for it using
``FunctionHandler``.
io_loop (IOLoop, optional) :
An explicit Tornado ``IOLoop`` to run Bokeh Server code on. If
None, ``IOLoop.current()`` will be used (default: None)
http_server_kwargs (dict, optional) :
Extra arguments passed to ``tornado.httpserver.HTTPServer``.
E.g. ``max_buffer_size`` to specify the maximum upload size.
More details can be found at:
http://www.tornadoweb.org/en/stable/httpserver.html#http-server
If None, no extra arguments are passed (default: None)
Additionally, the following options may be passed to configure the
operation of ``Server``:
.. bokeh-options:: _ServerOpts
:module: bokeh.server.server
Any remaining keyword arguments will be passed as-is to
``BokehTornado``.
'''
log.info("Starting Bokeh server version %s (running on Tornado %s)" %
(__version__, tornado_version))
opts = _ServerOpts(kwargs)
if opts.num_procs > 1 and io_loop is not None:
raise RuntimeError(
"Setting both num_procs and io_loop in Server is incompatible. Use BaseServer to coordinate an explicit IOLoop and multi-process HTTPServer"
)
if opts.num_procs > 1 and sys.platform == "win32":
raise RuntimeError("num_procs > 1 not supported on Windows")
if opts.unix_socket and sys.platform == "win32":
raise RuntimeError("Unix sockets are not supported on windows.")
if http_server_kwargs is None:
http_server_kwargs = {}
http_server_kwargs.setdefault('xheaders', opts.use_xheaders)
if opts.ssl_certfile:
log.info("Configuring for SSL termination")
import ssl
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain(certfile=opts.ssl_certfile,
keyfile=opts.ssl_keyfile,
password=opts.ssl_password)
http_server_kwargs['ssl_options'] = context
if opts.unix_socket:
sockets = [netutil.bind_unix_socket(opts.unix_socket)]
self._unix_socket = opts.unix_socket
self._address, self._port = None, None
extra_websocket_origins = []
else:
sockets, self._port = bind_sockets(opts.address, opts.port)
self._address = opts.address
extra_websocket_origins = create_hosts_allowlist(
opts.allow_websocket_origin, self.port)
try:
tornado_app = BokehTornado(
applications,
extra_websocket_origins=extra_websocket_origins,
prefix=opts.prefix,
index=opts.index,
websocket_max_message_size_bytes=opts.
websocket_max_message_size,
**kwargs)
if opts.num_procs != 1:
assert all(
app_context.application.safe_to_fork
for app_context in tornado_app.applications.values()
), ('User application code has run before attempting to start '
'multiple processes. This is considered an unsafe operation.'
)
http_server = HTTPServer(tornado_app, **http_server_kwargs)
http_server.start(opts.num_procs)
http_server.add_sockets(sockets)
except Exception:
for s in sockets:
s.close()
raise
# Can only refer to IOLoop after HTTPServer.start() is called, see #5524
if io_loop is None:
io_loop = IOLoop.current()
super().__init__(io_loop, tornado_app, http_server)
class StaticPageHandler(RequestHandler):
def initialize(self, page):
self.template = os.path.join(settings.template_path, page + '.html')
def get(self):
self.render(self.template)
application = Application([
url(r'/static/(.*)', StaticFileHandler, {'path': settings.static_path}),
url(r"/?", HomePageHandler, name='home'),
url(r"/m/(.{10})/?", MoodPageHandler, name='mood'),
url(r'/privacy/?', StaticPageHandler, {'page': 'privacy'}),
url(r'/tos/?', StaticPageHandler, {'page': 'tos'}),
],
debug=settings.debug)
if '__main__' == __name__:
if settings.debug:
print('Starting debug server on {}'.format(settings.debug_port))
application.listen(settings.debug_port)
IOLoop.instance().start()
else:
print('Starting production server.')
socket = bind_unix_socket(settings.socket, mode=0o777)
fork_processes(None)
server = HTTPServer(application)
server.add_socket(socket)
IOLoop.instance().start()
if not access_token:
raise tornado.web.HTTPError(403)
rsp_2 = ss.get("https://api.github.com/user", headers={
"Authorization": f"token {access_token}",
})
token = jwt.encode({
"role": "q",
"name": rsp_2.json()["login"],
}, 'x' * 32, algorithm='HS256').decode()
self.write(tpl_script.format(authorization=rsp_2.text, jwt=token))
app = tornado.web.Application([
(r"/gh", Api),
])
if __name__ == '__main__':
from tornado.options import define, parse_command_line, options
from tornado.httpserver import HTTPServer
from tornado.netutil import bind_unix_socket
from tornado.ioloop import IOLoop
define("sock", default=".sock")
parse_command_line()
server = HTTPServer(app, xheaders=True)
server.add_socket(bind_unix_socket(options.sock, 0o666))
IOLoop.current().start()
else:
from tornado.wsgi import WSGIAdapter
application = WSGIAdapter(app)
# Start up tornado app.
application = make_app()
if args.unix_socket == '':
# Original branch
application.listen(args.port)
print(
'Tornado Starting: \nPoint your web browser to http://localhost:%d/ \nCtrl-C to stop'
% (args.port))
if not args.noBrowser:
webbrowser.open_new_tab('http://localhost:%d' % (args.port))
else:
unix_socket_path = os.path.abspath(args.unix_socket)
server = HTTPServer(application)
server_socket = bind_unix_socket(unix_socket_path)
server.add_socket(server_socket)
if not args.noBrowser:
print(
"Not opening web browser; use ssh port forwarding, and connect to the destination of the forward"
)
print(f"Running web server on UNIX domain port {unix_socket_path}")
print(
"To see the page, use ssh to forward this port to an IP port, for example:"
)
print(
f"$ ssh -L 127.0.0.1:8888:{unix_socket_path} {socket.gethostname()}"
)
print(
"Then connect to http://127.0.0.1:8888 from the host running the ssh client"
)
