def main():
# options
parse_command_line()
# Locale
load_gettext_translations(settings.I18N_PATH, "luoyun")
set_default_locale('zh_CN')
logging.info("starting torando web server")
if settings.IPV4_ONLY:
import socket
sockets = bind_sockets(options.port, family=socket.AF_INET)
else:
sockets = bind_sockets(options.port)
if not settings.DEBUG:
import tornado.process
tornado.process.fork_processes(0)
application = Application()
server = HTTPServer(application, xheaders=True)
server.add_sockets(sockets)
IOLoop.instance().start()
def server_start(self):
logging.info('server starting...')
logger_module.logger.load_config()
#tornado把默认的根logger加了handler
#把根logger的handler去除，然后重新绑定在tornado的logger下
logging.getLogger().handlers = []
enable_pretty_logging(None, logging.getLogger('tornado'))
#服务启动
from tornado.netutil import bind_sockets
if self.settings.IPV4_ONLY:
import socket
sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
else:
sockets = bind_sockets(options.port, options.address)
self.print_settings_info()
http_server = tornado.httpserver.HTTPServer(self.application)
http_server.add_sockets(sockets)
logging.info('tornado server started. listen port: %s ,host address: %s' % (options.port, options.address))
tornado.ioloop.IOLoop.instance().start()
def build_listener(address, backlog=0):
address_type = get_address_type(address)
if address_type == 'tcp':
host, port = address
socks = netutil.bind_sockets(port, address=host, backlog=backlog)
return socks
if address_type == 'uds':
socks = netutil.bind_sockets(address, mode=0o600, backlog=backlog)
return [socks]
def main():
taskID = process.fork_processes(4)
if taskID == 0:
app = httpserver.HTTPServer(makeFrontend())
app.add_sockets(netutil.bind_sockets(BASE_PORT))
logging.info("Front end is listening on " + str(BASE_PORT))
else:
port = BASE_PORT + taskID
app = httpserver.HTTPServer(makeBackend(port))
app.add_sockets(netutil.bind_sockets(port))
logging.info("Back end %d listening on %d" % (taskID, port))
IOLoop.current().start()
def bind_sockets(addr, backlog=128, allows_unix_socket=False):
# initialize the socket
addr = parse_address(addr)
if isinstance(addr, six.string_types):
if not allows_unix_socket:
raise RuntimeError("unix addresses aren't supported")
sock = netutil.bind_unix_socket(addr)
elif is_ipv6(addr[0]):
sock = netutil.bind_sockets(addr[1], address=addr[0],
family=socket.AF_INET6, backlog=backlog)
else:
sock = netutil.bind_sockets(addr[1], backlog=backlog)
return sock
def server_start(self):
#服务启动
if self.settings.IPV4_ONLY:
import socket
from tornado.netutil import bind_sockets
sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
else:
sockets = bind_sockets(options.port, options.address)
self.print_settings_info()
http_server = tornado.httpserver.HTTPServer(self.application)
http_server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
print 'tornado server started. listen port: %s ,host address: %s' % (options.port, options.address)
def start_game():
''' Main entry point for the application '''
cache_actions()
sockets = netutil.bind_sockets(8888)
#if process.task_id() == None:
# tornado.process.fork_processes(-1, max_restarts = 10)
server = HTTPServer(application)
server.add_sockets(sockets)
io_loop = IOLoop.instance()
session_manager = SessionManager.Instance()
if process.task_id() == None:
scoring = PeriodicCallback(scoring_round, application.settings['ticks'], io_loop = io_loop)
session_clean_up = PeriodicCallback(session_manager.clean_up, application.settings['clean_up_timeout'], io_loop = io_loop)
scoring.start()
session_clean_up.start()
try:
for count in range(3, 0, -1):
logging.info("The game will begin in ... %d" % (count,))
sleep(1)
logging.info("Good hunting!")
io_loop.start()
except KeyboardInterrupt:
if process.task_id() == 0:
print '\r[!] Shutdown Everything!'
session_clean_up.stop()
io_loop.stop()
def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128):
"""Binds this server to the given port on the given address.
To start the server, call start(). If you want to run this server
in a single process, you can call listen() as a shortcut to the
sequence of bind() and start() calls.
Address may be either an IP address or hostname. If it's a hostname,
the server will listen on all IP addresses associated with the
name. Address may be an empty string or None to listen on all
available interfaces. Family may be set to either socket.AF_INET
or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise
both will be used if available.
The ``backlog`` argument has the same meaning as for
``socket.listen()``.
This method may be called multiple times prior to start() to listen
on multiple ports or interfaces.
"""
sockets = netutil.bind_sockets(port, address=address,
family=family, backlog=backlog)
if self._started:
self.add_sockets(sockets)
else:
self._pending_sockets.extend(sockets)
def start_server():
''' Main entry point for the application '''
if options.debug:
logging.warn("%sDebug mode is enabled; DO NOT USE THIS IN PRODUCTION%s" % (
bold + R, W
))
# Setup server object
if options.ssl:
server = HTTPServer(app,
ssl_options={
"certfile": options.certfile,
"keyfile": options.keyfile,
},
xheaders=options.x_headers
)
else:
server = HTTPServer(app, xheaders=options.x_headers)
sockets = netutil.bind_sockets(options.listen_port)
server.add_sockets(sockets)
try:
io_loop.start()
except KeyboardInterrupt:
sys.stdout.write('\r' + WARN + 'Shutdown Everything!\n')
except:
logging.exception("Main i/o loop threw exception")
finally:
io_loop.stop()
_exit(0)
def test_100_continue(self):
# testing if httpclient is able to skip 100 continue responses.
# to test without httpserver implementation, using
# raw response as same as httpclient_test.test_chunked_close.
port = get_unused_port()
(sock,) = netutil.bind_sockets(port, address="127.0.0.1")
with closing(sock):
def write_response(stream, request_data):
stream.write(b("""\
HTTP/1.1 100 Continue
HTTP/1.1 200 OK
Content-Length: 6
hjkl
""").replace(b("\n"), b("\r\n")), callback=stream.close)
def accept_callback(conn, address):
# fake an HTTP server using chunked encoding where the final chunks
# and connection close all happen at once
stream = IOStream(conn, io_loop=self.io_loop)
stream.read_until(b("\r\n\r\n"),
functools.partial(write_response, stream))
netutil.add_accept_handler(sock, accept_callback, self.io_loop)
self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop,
headers={"Expect": "100-continue"})
resp = self.wait()
resp.rethrow()
self.assertEqual(resp.body, b("hjkl\r\n"))
def main():
numProcs = inventory.NUM_INDEX_SHARDS + inventory.NUM_DOC_SHARDS + 1
taskID = process.fork_processes(numProcs, max_restarts=0)
port = inventory.BASE_PORT + taskID
if taskID == 0:
app = httpserver.HTTPServer(tornado.web.Application([
(r"/search", Web),
(r"/upload", UploadHandler),
(r"/(.*)", IndexDotHTMLAwareStaticFileHandler, dict(path=SETTINGS['static_path']))
], **SETTINGS))
logging.info("Front end is listening on " + str(port))
else:
if taskID <= inventory.NUM_INDEX_SHARDS:
shardIx = taskID - 1
#data = pickle.load(open("data/index%d.pkl" % (shardIx), "r"))
inverted_path = os.path.join(os.getcwd(),"../assignment5/df_jobs/%d.out" % (shardIx))
logging.info("Inverted file path: %s" % inverted_path)
data = pickle.load(open(inverted_path ,'r'))
idf_path = os.path.join(os.getcwd(), "../assignment5/idf_jobs/0.out")
logIDF = pickle.load(open(idf_path,'r'))
app = httpserver.HTTPServer(web.Application([(r"/index", index.Index, dict(data=data, logIDF=logIDF))]))
logging.info("Index shard %d listening on %d" % (shardIx, port))
else:
shardIx = taskID - inventory.NUM_INDEX_SHARDS - 1
#data = pickle.load(open("data/doc%d.pkl" % (shardIx), "r"))
doc_path = os.path.join(os.getcwd(),"../assignment5/i_df_jobs/%d.out" % (shardIx))
logging.info("Doc Server path %s" % doc_path)
data = pickle.load(open(doc_path, "r"))
app = httpserver.HTTPServer(web.Application([(r"/doc", doc.Doc, dict(data=data))]))
logging.info("Doc shard %d listening on %d" % (shardIx, port))
app.add_sockets(netutil.bind_sockets(port))
IOLoop.current().start()
def __init__(self, family):
super(TestTCPServer, self).__init__()
self.streams = []
self.queue = Queue()
sockets = bind_sockets(None, 'localhost', family)
self.add_sockets(sockets)
self.port = sockets[0].getsockname()[1]
def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128,
reuse_port=False):
"""Binds this server to the given port on the given address.
To start the server, call `start`. If you want to run this server
in a single process, you can call `listen` as a shortcut to the
sequence of `bind` and `start` calls.
Address may be either an IP address or hostname. If it's a hostname,
the server will listen on all IP addresses associated with the
name. Address may be an empty string or None to listen on all
available interfaces. Family may be set to either `socket.AF_INET`
or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise
both will be used if available.
The ``backlog`` argument has the same meaning as for
`socket.listen <socket.socket.listen>`. The ``reuse_port`` argument
has the same meaning as for `.bind_sockets`.
This method may be called multiple times prior to `start` to listen
on multiple ports or interfaces.
.. versionchanged:: 4.4
Added the ``reuse_port`` argument.
"""
sockets = bind_sockets(port, address=address, family=family,
backlog=backlog, reuse_port=reuse_port)
if self._started:
self.add_sockets(sockets)
else:
self._pending_sockets.extend(sockets)
def load_httpserver(self, sockets=None, **kwargs):
if not sockets:
from tornado.netutil import bind_sockets
if settings.IPV4_ONLY:
import socket
sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
else:
sockets = bind_sockets(options.port, options.address)
http_server = tornado.httpserver.HTTPServer(self.application, **kwargs)
http_server.add_sockets(sockets)
self.httpserver = http_server
return self.httpserver
def main():
tornado.options.parse_command_line()
http_server = tornado.httpserver.HTTPServer(Application(),xheaders=True)
sockets = bind_sockets(options.port, family=2)
#http_server.listen(options.port)
http_server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
def main():
script_path = os.path.dirname(os.path.realpath(__file__))
config_path = os.path.join(script_path, 'etc/harvest.cfg')
config = ConfigParser()
config.read(config_path)
sockets = bind_sockets(config.get('server', 'port'),
config.get('server', 'address'))
fork_processes(config.getint('server', 'instances'))
datastore = DataStore(config.get('datastore', 'host'),
config.getint('datastore', 'port'),
config.get('datastore', 'username'),
config.get('datastore', 'password'),
config.get('datastore', 'database'))
app = Application([(r"/rpc/store", Handler,
{'datastore': datastore,
'api_key': config.get('server', 'api_key')})])
server = HTTPServer(app,
no_keep_alive=config.get('server', 'no_keep_alive'),
ssl_options={
'certfile': config.get('server', 'certfile'),
'keyfile': config.get('server', 'keyfile')})
server.add_sockets(sockets)
IOLoop.instance().start()
def initialize(self, cfg):
self.cfg = cfg
# Check we have a certificate and key defines
if not cfg.ssl_cert_file or not cfg.ssl_key_file:
log.error('a valid ssl_cert_file and ssl_key_file are required, '
'aborting')
sys.exit(1)
# Set up the SSL context
try:
ssl_context = ssl.create_default_context(
purpose=ssl.Purpose.CLIENT_AUTH,
cafile=cfg.ssl_ca_file)
except IOError as e:
log.error('cannot read ssl_ca_file: {}'.format(e.args[1]))
sys.exit(1)
self.ssl_context = ssl_context
# Enable client certificate verification if wanted
if cfg.ssl_verify_client:
ssl_context.verify_mode = ssl.CERT_REQUIRED
# Load our own certificate into the server
try:
ssl_context.load_cert_chain(
certfile=cfg.ssl_cert_file,
keyfile=cfg.ssl_key_file)
except IOError as e:
log.error('cannot read ssl_cert_file or ssl_key_file: {}'
.format(e.args[1]))
sys.exit(1)
self.app = NrpeApplication(cfg)
# Set up the HTTPServer instance
super(NrpeHTTPServer, self).initialize(
self.app, no_keep_alive=True, ssl_options=ssl_context,
idle_connection_timeout=cfg.connection_timeout,
body_timeout=cfg.connection_timeout)
# Because Tornado unconditionally sets IPV6_V6ONLY on IPv6 sockets, we
# need a means to preserve old behaviour. This should only be an issue
# when listening to '::' for IPv6 any-address. Tornado will listen both
# the IPv4 and IPv6 any-address when the bind address is blank, so just
# set the address to be empty if we encounter '::'.
server_address = cfg.server_address
if server_address == '::':
server_address = ''
try:
self.sockets = bind_sockets(port=cfg.server_port,
address=server_address)
except socket.error as e:
log.error('failed to bind socket: {}'.format(e.args[1]))
sys.exit(1)
# Prevent tornado from logging HTTP requests
if not self.cfg.debug:
logging.getLogger('tornado.access').disabled = True
def start_server():
''' Main entry point for the application '''
sockets = netutil.bind_sockets(config.listen_port)
server = HTTPServer(app)
server.add_sockets(sockets)
io_loop = IOLoop.instance()
scoring = PeriodicCallback(
scoring_round, int(5 * 60 * 1000), io_loop=io_loop
)
scoring.start()
try:
sys.stdout.write("\r" + INFO + "The game has begun, good hunting!\n")
if config.debug:
sys.stdout.write(WARN + "WARNING: Debug mode is enabled.\n")
sys.stdout.flush()
game_history = GameHistory.Instance()
history_callback = PeriodicCallback(
game_history.take_snapshot, int(60 * 1000), io_loop=io_loop
)
history_callback.start()
io_loop.start()
except KeyboardInterrupt:
print('\r' + WARN + 'Shutdown Everything!')
except:
logging.exception("Main i/o loop threw exception")
finally:
io_loop.stop()
if config.debug and \
raw_input(PROMPT + "Flush Memcache? [Y/n]: ").lower() == 'y':
print(INFO + 'Flushing cache ...'),
FileCache.flush()
print('OK')
_exit(0)
def __init__(self, family):
super(TestTCPServer, self).__init__()
self.streams = [] # type: List[IOStream]
self.queue = Queue() # type: Queue[IOStream]
sockets = bind_sockets(0, "localhost", family)
self.add_sockets(sockets)
self.port = sockets[0].getsockname()[1]
def start(self):
self.__ssl.guarantee_valid_certs()
log.pcsd.info("Starting server...")
self.__server = HTTPServer(
self.__make_app(self),
ssl_options=self.__ssl.create_context()
)
# It is necessary to bind sockets for every new HTTPServer since
# HTTPServer.stop calls sock.close() inside.
sockets = []
for address in self.__bind_addresses:
log.pcsd.info(
"Binding socket for address '%s' and port '%s'",
address if address is not None else "*",
self.__port
)
sockets.extend(bind_sockets(self.__port, address))
self.__server.add_sockets(sockets)
log.pcsd.info("Server is listening")
self.__server_is_running = True
return self
def test_chunked_close(self):
# test case in which chunks spread read-callback processing
# over several ioloop iterations, but the connection is already closed.
port = get_unused_port()
(sock,) = netutil.bind_sockets(port, address="127.0.0.1")
with closing(sock):
def write_response(stream, request_data):
stream.write(b("""\
HTTP/1.1 200 OK
Transfer-Encoding: chunked
1
1
1
2
0
""").replace(b("\n"), b("\r\n")), callback=stream.close)
def accept_callback(conn, address):
# fake an HTTP server using chunked encoding where the final chunks
# and connection close all happen at once
stream = IOStream(conn, io_loop=self.io_loop)
stream.read_until(b("\r\n\r\n"),
functools.partial(write_response, stream))
netutil.add_accept_handler(sock, accept_callback, self.io_loop)
self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
resp = self.wait()
resp.rethrow()
self.assertEqual(resp.body, b("12"))
def start_server():
''' Main entry point for the application '''
if config.debug:
logging.warn("Debug mode is enabled; some security measures will be ignored")
# Setup server object
if config.use_ssl:
server = HTTPServer(app,
ssl_options={
"certfile": config.certfile,
"keyfile": config.keyfile,
},
xheaders=config.x_headers
)
else:
server = HTTPServer(app, xheaders=config.x_headers)
sockets = netutil.bind_sockets(config.listen_port)
server.add_sockets(sockets)
try:
io_loop.start()
except KeyboardInterrupt:
sys.stdout.write('\r' + WARN + 'Shutdown Everything!\n')
except:
logging.exception("Main i/o loop threw exception")
finally:
io_loop.stop()
_exit(0)
def bind_unused_port():
"""Binds a server socket to an available port on localhost.
Returns a tuple (socket, port).
"""
[sock] = netutil.bind_sockets(None, 'localhost', family=socket.AF_INET)
port = sock.getsockname()[1]
return sock, port
def start_server():
''' Main entry point for the application '''
sockets = netutil.bind_sockets(config.listen_port)
if config.use_ssl:
server = HTTPServer(
app,
ssl_options={
"certfile": config.certfile,
"keyfile": config.keyfile,
},
xheaders=config.x_headers
)
else:
server = HTTPServer(
app,
xheaders=config.x_headers
)
server.add_sockets(sockets)
io_loop = IOLoop.instance()
try:
io_loop.start()
except KeyboardInterrupt:
logging.warn("Keyboard interrupt, shutdown everything!")
except:
logging.exception("Main I/O Loop threw an excetion!")
finally:
io_loop.stop()
def make_iostream_pair(self, **kwargs):
port = get_unused_port()
[listener] = netutil.bind_sockets(port, '127.0.0.1',
family=socket.AF_INET)
streams = [None, None]
def accept_callback(connection, address):
streams[0] = self._make_server_iostream(connection, **kwargs)
if isinstance(streams[0], SSLIOStream):
# HACK: The SSL handshake won't complete (and
# therefore the client connect callback won't be
# run)until the server side has tried to do something
# with the connection. For these tests we want both
# sides to connect before we do anything else with the
# connection, so we must cause some dummy activity on the
# server. If this turns out to be useful for real apps
# it should have a cleaner interface.
streams[0]._add_io_state(IOLoop.READ)
self.stop()
def connect_callback():
streams[1] = client_stream
self.stop()
netutil.add_accept_handler(listener, accept_callback,
io_loop=self.io_loop)
client_stream = self._make_client_iostream(socket.socket(), **kwargs)
client_stream.connect(('127.0.0.1', port),
callback=connect_callback)
self.wait(condition=lambda: all(streams))
self.io_loop.remove_handler(listener.fileno())
listener.close()
return streams
def start_server():
''' Main entry point for the application '''
if options.debug:
logging.warn("%sDebug mode is enabled; DO NOT USE THIS IN PRODUCTION%s" % (
bold + R, W
))
if options.autostart_game:
logging.info("The game is about to begin, good hunting!")
app.settings['history_callback'].start()
if options.use_bots:
app.settings['score_bots_callback'].start()
# Setup server object
if options.ssl:
server = HTTPServer(app,
ssl_options={
"certfile": options.certfile,
"keyfile": options.keyfile,
},
xheaders=options.x_headers
)
else:
server = HTTPServer(app, xheaders=options.x_headers)
sockets = netutil.bind_sockets(options.listen_port, options.listen_interface)
server.add_sockets(sockets)
Scoreboard.now(app)
try:
io_loop.start()
except KeyboardInterrupt:
sys.stdout.write('\r' + WARN + 'Shutdown Everything!\n')
except:
logging.exception("Main i/o loop threw exception")
finally:
io_loop.stop()
_exit(0)
def main():
for back in BACKENDS:
print back
app = httpserver.HTTPServer(makeFrontend())
app.add_sockets(netutil.bind_sockets(FRONTEND_PORT))
logging.info("Front end is listening on " + str(FRONTEND_PORT))
IOLoop.current().start()
def bind(server, port, usocket, address=None):
"""Make server listen on port (inet socket).
If given, prefer `usocket`, path to a unix socket.
The latter is useful for reverse proxying.
If listening on a inet socket, `address` might be
given. `address` may be either an IP address or hostname.
If it's a hostname, the server will listen on all IP addresses
associated with the name. If not given (and not listening on a unix
socket) will listen on all available interfaces."""
# If we have a unix socket path
if usocket:
LOG.info('Starting on unix socket %r', usocket)
try:
socket = bind_unix_socket(usocket, mode=options.usocket_mode)
os.chown(usocket, options.usocket_uid, options.usocket_gid)
except OSError as error:
LOG.error('Cannot create unix socket: %r', error)
else:
server.add_socket(socket)
LOG.info('Started')
else:
LOG.info('Starting on port %d', port)
sockets = bind_sockets(port, address=address)
server.add_sockets(sockets)
LOG.info('Started')
def make_http_server(loop, request_handler):
server = HTTPServer(request_handler, io_loop=loop)
sockets = bind_sockets(0, HOST, family=FAMILY)
assert len(sockets) == 1
server.add_sockets(sockets)
sock = sockets[0]
return sock
def build_socket(self):
"""Builds the listener socket. If `self.port` is not set, uses an
OS-assigned port. Note that the `host` and `port` values are updated
by the `start` method.
"""
[sock] = bind_sockets(self.port, self.host, family=socket.AF_INET)
host, port = sock.getsockname()
return sock, port, host
def start_server():
''' Main entry point for the application '''
task_event_consumer = TaskEventConsumer()
# Application setup
app = Application(
[
# Static Handlers - Serves static CSS, JS and images
(r'/static/(.*\.(css|js|png|jpg|jpeg|svg|ttf|html|json))',
StaticFileHandler, {
'path': 'static/'
}),
# Home page serving SPA app
(r'/', HomePageHandler),
# Monitor Socket
(r'/connect/monitor', MonitorSocketHandler),
# Error Handlers -
(r'/403', ForbiddenHandler),
# Catch all 404 page
(r'(.*)', NotFoundHandler),
],
# Randomly generated secret key
cookie_secret=urandom(32).encode('hex'),
# Request that does not pass @authorized will be
# redirected here
forbidden_url='/403',
# Requests that does not pass @authenticated will be
# redirected here
login_url='/login',
# Template directory
template_path='templates/',
# Debug mode
debug=config.debug,
task_event_consumer=task_event_consumer)
sockets = netutil.bind_sockets(config.listen_port)
if config.use_ssl:
server = HTTPServer(app,
ssl_options={
"certfile": config.certfile,
"keyfile": config.keyfile,
},
xheaders=config.x_headers)
else:
server = HTTPServer(app, xheaders=config.x_headers)
server.add_sockets(sockets)
io_loop = IOLoop.instance()
try:
io_loop.add_callback(task_event_consumer.connect)
io_loop.start()
except KeyboardInterrupt:
logging.warn("Keyboard interrupt, shutdown everything!")
except:
logging.exception("Main I/O Loop threw an excetion!")
finally:
io_loop.stop()
def run_server( port: int, address: str="", jobs: int=1, user: str=None, workers: int=0) -> None:
""" Run the server
:param port: port number
:param address: interface address to bind to (default to any)
:param jobs: Number of jobs to fork (default to 1: i.e no fork)
:user: User to setuid after opening ports (default no setuid)
"""
import traceback
from tornado.netutil import bind_sockets
from tornado.httpserver import HTTPServer
kwargs = {}
ipcaddr = configure_ipc_addresses(workers)
application = None
server = None
if user:
setuid(user)
def close_sockets(sockets):
for sock in sockets:
sock.close()
worker_pool = None
broker_pr = None
broker_client = None
# Setup ssl config
if confservice.getboolean('server','ssl'):
LOGGER.info("SSL enabled")
kwargs['ssl_options'] = create_ssl_options()
# Run
try:
# Fork processes
if jobs > 1:
sockets = bind_sockets(port, address=address)
if process.fork_processes(jobs) is None: # We are in the main process
close_sockets(sockets)
broker_pr = create_broker_process(ipcaddr)
worker_pool = create_worker_pool(workers) if workers>0 else None
set_signal_handlers()
# Note that manage_processes(...) never return in main process
# and call exit(0) which will be caught by SystemExit exception
task_id = process.manage_processes(max_restarts=5, logger=LOGGER)
assert False, "Not Reached"
else:
broker_pr = create_broker_process(ipcaddr)
worker_pool = create_worker_pool(workers) if workers>0 else None
sockets = bind_sockets(port, address=address)
# Install asyncio event loop after forking
# This is why we do not use server.bind/server.start
tornado.platform.asyncio.AsyncIOMainLoop().install()
LOGGER.info("Running server on port %s:%s", address, port)
application = Application(ipcaddr)
# Init HTTP server
server = HTTPServer(application, **kwargs)
server.add_sockets(sockets)
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM, lambda: loop.stop())
loop.run_forever()
except Exception:
traceback.print_exc()
if process.task_id() is not None:
# Let a chance to the child process to
# restart
raise
else:
# Make sure that child processes are terminated
print("Terminating child processes", file=sys.stderr)
process.terminate_childs()
except (KeyboardInterrupt, SystemExit) as e:
pass
# Teardown
if server is not None:
server.stop()
if application is not None:
application.terminate()
application = None
loop = asyncio.get_event_loop()
if not loop.is_closed():
loop.close()
print("{}: Server instance stopped".format(os.getpid()), file=sys.stderr)
if process.task_id() is None:
if worker_pool:
print("Stopping workers")
worker_pool.terminate()
worker_pool.join()
if broker_pr:
print("Stopping broker")
broker_pr.terminate()
broker_pr.join()
print("Server shutdown", file=sys.stderr)
#The documents are then scored. Each document's score is the inner product (a.k.a. dot product,
#effectively correlation here) of its vector and the query vector. In addition, scores should be
#biased so that documents with the query terms in their title receive especially high scores.
ScoredListing = []
for docID in MyMap:
score = dotProd(MyMap[docID], QueryVector)
ScoredListing.append([score, docID]) ## Bias comes from tfs itself
#Finally, the K highest-scoring documents are written out as a JSON-encoded list of (docID, score) pairs.
ScoredListing.sort(reverse=True) ## check so sorted by score
topPostings = ScoredListing[:10]
TP = []
for posting in topPostings:
TP.append([posting[1], posting[0]])
#print 'Length from ' + str(self._port)
#print len(ScoredListing)
self.write(json.JSONEncoder().encode({"postings": TP}))
if __name__ == "__main__":
task_id = process.fork_processes(inventory.num_ind)
I_BASE_PORT = inventory.port1
#application.listen(I_BASE_PORT)
port = I_BASE_PORT + task_id
app = httpserver.HTTPServer(
web.Application([web.url(r"/index", IndexHandler, dict(port=port))]))
app.add_sockets(netutil.bind_sockets(port))
print('indexServer' + str(task_id) + ' at port: ' + str(port))
tornado.ioloop.IOLoop.current().start()
处理文本数据
:param stream:
:param send_to:
:param msg_len:
:return:
"""
data = yield stream.read_bytes(msg_len, partial=True)
data = struct.unpack("!" + str(msg_len) + "s", data)
logger.debug("data:%s", data)
try:
# 打包数据,数据格式:数据发送者+数据类型+数据长度+数据体
binary_msg = struct.pack("!IBI" + str(msg_len) + "s", sender, 1,
msg_len, data[0])
# 发送数据
yield stream.write(binary_msg)
logger.debug("=" * 25)
except KeyError:
# 将离线消息保存到数据库
pass
@gen.coroutine
def handle_pic_stream(self, stream, sender, msg_len):
pass
if __name__ == '__main__':
sockets = bind_sockets(ChatServer.PORT)
server = ChatServer()
server.add_sockets(sockets)
IOLoop.current().start()
def start_pyrox(config):
if config is None:
raise ConfigurationError('No configuration object passed in')
# Log some important things
if config.routing.upstream_hosts is not None:
_LOG.info('Upstream targets are: {}'.format(
[dst for dst in config.routing.upstream_hosts]))
# Set bind host
bind_host = config.core.bind_host.split(':')
if len(bind_host) != 2:
raise ConfigurationError('bind_host must have a port specified')
# Bind the sockets in the main process
sockets = None
try:
sockets = bind_sockets(port=bind_host[1], address=bind_host[0])
except Exception as ex:
_LOG.exception(ex)
return
# Bind the server port(s)
_LOG.info('Pyrox listening on: http://{0}:{1}'.format(
bind_host[0], bind_host[1]))
# Are we trying to profile Pyrox?
if config.core.enable_profiling:
_LOG.warning("""
**************************************************************************
Notice: Pyrox Profiling Mode Enabled
You have enabled Pyrox with profiling enabled in the Pyrox config. This
will restrict Pyrox to one resident process. It is not recommended that
you run Pyrox in production with this feature enabled.
**************************************************************************
""")
start_proxy(sockets, config)
return
# Number of processess to spin
num_processes = config.core.processes
if num_processes <= 0:
num_processes = cpu_count()
global _active_children_pids
for i in range(num_processes):
pid = os.fork()
if pid == 0:
_LOG.info('Starting process {}'.format(i))
start_proxy(sockets, config)
sys.exit(0)
else:
_active_children_pids.append(pid)
# Take over SIGTERM and SIGINT
signal.signal(signal.SIGTERM, stop_parent)
signal.signal(signal.SIGINT, signal.SIG_IGN)
while len(_active_children_pids):
try:
pid, status = os.wait()
except OSError as oserr:
if oserr.errno != errno.EINTR:
_LOG.exception(oserr)
continue
except Exception as ex:
_LOG.exception(ex)
continue
_LOG.info('Child process {} exited with status {}'.format(pid, status))
_active_children_pids.remove(pid)
try:
data_dict = json.loads(data.decode())
# get result
ht = Hunter()
rh = RedisHandler()
# If can't find data in redis, fetch it from mysql
result_dict = {}
result_dict = rh.outhash(data_dict)
if not result_dict:
result_dict = ht.get_data(data_dict)
# if need to change voltage data, use the statement below
# result_dict = data_magic(result_dict)
data_send = json.dumps(result_dict)
yield stream.write(struct.pack('i',len(data_send))+data_send.encode())
except KeyError:
logging.info('** Data Error **')
except StreamClosedError:
logging.info("Server finished")
logging.info("*"*25)
del DataServer.clients[address]
break
if __name__ == '__main__':
server = DataServer()
sockets = bind_sockets(30105)
server.add_sockets(sockets)
IOLoop.current().start()
start = time.time()
print(urls)
client = httpclient.AsyncHTTPClient()
for url in urls:
request = httpclient.HTTPRequest(url=url)
result = client.fetch(request, callback=self.finish_response)
print('result', result)
results.append(result)
self.finish('hello world')
print('response results:', results)
print('post spend time:', time.time() - start)
def finish_response(self, response):
pass
@concurrent.run_on_executor()
def handle(self, request):
response = requests.get(request.url)
return response.content
if __name__ == '__main__':
app = web.Application([
(r'/handle_request', NewHandleRequest)
])
sockets = netutil.bind_sockets(6666, reuse_port=True)
process.fork_processes(os.cpu_count() * 2)
server = NewHttpServer(app)
server.add_sockets(sockets)
IOLoop.current().start()
def _open(self, host, port, **kwargs):
# Note: does not get called if host is False. That way we can
# run Flexx in e.g. JLab's application.
# Hook Tornado up with asyncio. Flexx' BaseServer makes sure
# that the correct asyncio event loop is current (for this thread).
# http://www.tornadoweb.org/en/stable/asyncio.html
# todo: Since Tornado v5.0 asyncio is autom used, deprecating AsyncIOMainLoop
self._io_loop = AsyncIOMainLoop()
# I am sorry for this hack, but Tornado wont work otherwise :(
# I wonder how long it will take before this will bite me back. I guess
# we will be alright as long as there is no other Tornado stuff going on.
IOLoop._current.instance = None
self._io_loop.make_current()
# handle ssl, wether from configuration or given args
if config.ssl_certfile:
if 'ssl_options' not in kwargs:
kwargs['ssl_options'] = {}
if 'certfile' not in kwargs['ssl_options']:
kwargs['ssl_options']['certfile'] = config.ssl_certfile
if config.ssl_keyfile:
if 'ssl_options' not in kwargs:
kwargs['ssl_options'] = {}
if 'keyfile' not in kwargs['ssl_options']:
kwargs['ssl_options']['keyfile'] = config.ssl_keyfile
if config.tornado_debug:
app_kwargs = dict(debug=True)
else:
app_kwargs = dict()
# Create tornado application
self._app = Application([
(r"/flexx/ws/(.*)", WSHandler),
(r"/flexx/(.*)", MainHandler),
(r"/(.*)", AppHandler),
], **app_kwargs)
self._app._io_loop = self._io_loop
# Create tornado server, bound to our own ioloop
if tornado.version_info < (5, ):
kwargs['io_loop'] = self._io_loop
self._server = HTTPServer(self._app, **kwargs)
# Start server (find free port number if port not given)
if port:
# Turn port into int, use hashed port number if a string was given
try:
port = int(port)
except ValueError:
port = port_hash(port)
self._server.listen(port, host)
else:
# Try N ports in a repeatable range (easier, browser history, etc.)
prefered_port = port_hash('Flexx')
for i in range(8):
port = prefered_port + i
try:
self._server.listen(port, host)
break
except (OSError, IOError):
pass # address already in use
else:
# Ok, let Tornado figure out a port
[sock] = netutil.bind_sockets(None,
host,
family=socket.AF_INET)
self._server.add_sockets([sock])
port = sock.getsockname()[1]
# Notify address, so its easy to e.g. copy and paste in the browser
self._serving = self._app._flexx_serving = host, port
proto = 'http'
if 'ssl_options' in kwargs:
proto = 'https'
# This string 'Serving apps at' is our 'ready' signal and is tested for.
logger.info('Serving apps at %s://%s:%i/' % (proto, host, port))
def start_server():
""" Main entry point for the application """
if options.debug:
logging.warn(
"%sDebug mode is enabled; DO NOT USE THIS IN PRODUCTION%s" %
(bold + R, W))
locale.set_default_locale("en_US")
locale.load_translations("locale")
if options.autostart_game:
logging.info("The game is about to begin, good hunting!")
app.settings["game_started"] = True
app.settings["history_callback"].start()
if options.use_bots:
app.settings["score_bots_callback"].start()
# Setup server object
if options.ssl:
server = HTTPServer(
app,
ssl_options={
"certfile": options.certfile,
"keyfile": options.keyfile
},
xheaders=options.x_headers,
)
else:
server = HTTPServer(app, xheaders=options.x_headers)
try:
sockets = netutil.bind_sockets(options.listen_port,
options.listen_interface)
except (OSError, IOError) as err:
logging.error("Problem binding socket to port %s",
str(options.listen_port))
if err.errno == 13:
pypath = sys.executable
if os_path.islink(pypath):
pypath = os_path.realpath(pypath)
logging.error(
"Possible Fix: sudo setcap CAP_NET_BIND_SERVICE=+eip %s",
pypath)
elif err.errno == 98:
logging.error(
"The port may be in use by an existing service. RTB already running?"
)
else:
logging.error(err)
sys.exit()
server.add_sockets(sockets)
try:
Scoreboard.update_gamestate(app)
except OperationalError as err:
if "Table definition has changed" in str(err):
logging.info(
"Table definitions have changed -restarting RootTheBox.")
return "restart"
else:
logging.error("There was a problem starting RootTheBox. Error: " +
str(err))
try:
io_loop.start()
except KeyboardInterrupt:
sys.stdout.write("\r" + WARN + "Shutdown Everything!\n")
except:
logging.exception("Main i/o loop threw exception")
finally:
io_loop.stop()
_exit(0)
def main():
from tornado import options
default_general_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(message)s"
default_access_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(trace_id)s\t%(message)s"
opts = options.OptionParser()
opts.define("version",
type=bool,
help="show version and exit",
callback=show_version)
opts.define("locators",
default=["localhost:10053"],
type=str,
multiple=True,
help="comma-separated endpoints of locators")
opts.define("cache",
default=DEFAULT_SERVICE_CACHE_COUNT,
type=int,
help="count of instances per service")
opts.define(
"config",
help="path to configuration file",
type=str,
callback=lambda path: opts.parse_config_file(path, final=False))
opts.define("count",
default=1,
type=int,
help="count of tornado processes")
opts.define(
"endpoints",
default=["tcp://localhost:8080"],
type=str,
multiple=True,
help=
"Specify endpoints to bind on: prefix unix:// or tcp:// should be used"
)
opts.define("request_header",
default="X-Request-Id",
type=str,
help="header used as a trace id")
opts.define("forcegen_request_header",
default=False,
type=bool,
help="enable force generation of the request header")
opts.define("sticky_header",
default="X-Cocaine-Sticky",
type=str,
help="sticky header name")
opts.define("gcstats",
default=False,
type=bool,
help="print garbage collector stats to stderr")
opts.define("srwconfig", default="", type=str, help="path to srwconfig")
opts.define("allow_json_rpc",
default=True,
type=bool,
help="allow JSON RPC module")
# tracing options
opts.define("tracing_chance",
default=DEFAULT_TRACING_CHANCE,
type=float,
help="default chance for an app to be traced")
opts.define("configuration_service",
default="unicorn",
type=str,
help="name of configuration service")
opts.define(
"tracing_conf_path",
default="/zipkin_sampling",
type=str,
help="path to the configuration nodes in the configuration service")
# various logging options
opts.define(
"logging",
default="info",
help=("Set the Python log level. If 'none', tornado won't touch the "
"logging configuration."),
metavar="debug|info|warning|error|none")
opts.define("log_to_cocaine",
default=False,
type=bool,
help="log to cocaine")
opts.define(
"log_to_stderr",
type=bool,
default=None,
help=("Send log output to stderr. "
"By default use stderr if --log_file_prefix is not set and "
"no other logging is configured."))
opts.define("log_file_prefix",
type=str,
default=None,
metavar="PATH",
help=("Path prefix for log file"))
opts.define("datefmt",
type=str,
default="%z %d/%b/%Y:%H:%M:%S",
help="datefmt")
opts.define("generallogfmt",
type=str,
help="log format of general logging system",
default=default_general_logformat)
opts.define("accesslogfmt",
type=str,
help="log format of access logging system",
default=default_access_logformat)
opts.define("logframework",
type=bool,
default=False,
help="enable logging various framework messages")
# util server
opts.define("utilport",
default=8081,
type=int,
help="listening port number for an util server")
opts.define("utiladdress",
default="127.0.0.1",
type=str,
help="address for an util server")
opts.define("enableutil",
default=False,
type=bool,
help="enable util server")
opts.define("client_id",
default=0,
type=int,
help="client id used for authentication")
opts.define("client_secret",
default='',
type=str,
help="client secret used for authentication")
opts.parse_command_line()
srw_config = None
if opts.srwconfig:
try:
srw_config = load_srw_config(opts.srwconfig)
except Exception as err:
print("unable to load SRW config: %s" % err)
exit(1)
use_reuseport = hasattr(socket, "SO_REUSEPORT")
endpoints = Endpoints(opts.endpoints)
sockets = []
for path in endpoints.unix:
sockets.append(bind_unix_socket(path, mode=0o666))
if not use_reuseport:
for endpoint in endpoints.tcp:
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port, address=endpoint.host)
sockets.extend(socks)
if opts.enableutil:
utilsockets = bind_sockets(opts.utilport, address=opts.utiladdress)
try:
if opts.count != 1:
process.fork_processes(opts.count)
enable_logging(opts)
if opts.gcstats:
enable_gc_stats()
if use_reuseport:
for endpoint in endpoints.tcp:
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port,
address=endpoint.host,
reuse_port=True)
sockets.extend(socks)
proxy = CocaineProxy(
locators=opts.locators,
cache=opts.cache,
request_id_header=opts.request_header,
sticky_header=opts.sticky_header,
forcegen_request_header=opts.forcegen_request_header,
default_tracing_chance=opts.tracing_chance,
srw_config=srw_config,
allow_json_rpc=opts.allow_json_rpc,
client_id=opts.client_id,
client_secret=opts.client_secret)
server = HTTPServer(proxy)
server.add_sockets(sockets)
if opts.enableutil:
utilsrv = HTTPServer(UtilServer(proxy=proxy))
utilsrv.add_sockets(utilsockets)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
pass
def _open(self, host, port, **kwargs):
# Note: does not get called if host is False. That way we can
# run Flexx in e.g. JLab's application.
# handle ssl, wether from configuration or given args
if config.ssl_certfile:
if 'ssl_options' not in kwargs:
kwargs['ssl_options'] = {}
if 'certfile' not in kwargs['ssl_options']:
kwargs['ssl_options']['certfile'] = config.ssl_certfile
if config.ssl_keyfile:
if 'ssl_options' not in kwargs:
kwargs['ssl_options'] = {}
if 'keyfile' not in kwargs['ssl_options']:
kwargs['ssl_options']['keyfile'] = config.ssl_keyfile
if config.tornado_debug:
app_kwargs = dict(debug=True)
else:
app_kwargs = dict()
# Create tornado application
self._app = Application([
(r"/flexx/ws/(.*)", WSHandler),
(r"/flexx/(.*)", MainHandler),
(r"/(.*)", AppHandler),
], **app_kwargs)
# Create tornado server, bound to our own ioloop
self._server = HTTPServer(self._app, io_loop=self._loop, **kwargs)
# Start server (find free port number if port not given)
if port:
# Turn port into int, use hashed port number if a string was given
try:
port = int(port)
except ValueError:
port = port_hash(port)
self._server.listen(port, host)
else:
# Try N ports in a repeatable range (easier, browser history, etc.)
prefered_port = port_hash('Flexx')
for i in range(8):
port = prefered_port + i
try:
self._server.listen(port, host)
break
except (OSError, IOError):
pass # address already in use
else:
# Ok, let Tornado figure out a port
[sock] = netutil.bind_sockets(None,
host,
family=socket.AF_INET)
self._server.add_sockets([sock])
port = sock.getsockname()[1]
# Notify address, so its easy to e.g. copy and paste in the browser
self._serving = self._app._flexx_serving = host, port
proto = 'http'
if 'ssl_options' in kwargs:
proto = 'https'
logger.info('Serving apps at %s://%s:%i/' % (proto, host, port))
if options.more:
log.setLevel(DEBUG)
else:
log.setLevel(WARNING)
ioloop = IOLoop.instance()
if os.getenv('LISTEN_PID'):
log.info('Getting socket from systemd')
sck = socket.fromfd(
SYSTEMD_SOCKET_FD + 1, # Second socket in .socket file
socket.AF_INET6 if socket.has_ipv6 else socket.AF_INET,
socket.SOCK_STREAM)
sck.setblocking(0)
sck.listen(128)
sockets = [sck]
else:
log.info('Binding sockets')
sockets = bind_sockets(options.socket_port)
log.info('Accepting')
for sck in sockets:
add_accept_handler(sck, handle_connection, ioloop)
log.info('Listening')
http_server = SystemdHTTPServer(server)
http_server.listen(options.server_port)
log.info('Starting loop')
ioloop.start()
def test_multi_process(self):
self.assertFalse(IOLoop.initialized())
port = get_unused_port()
def get_url(path):
return "http://127.0.0.1:%d%s" % (port, path)
sockets = bind_sockets(port, "127.0.0.1")
# ensure that none of these processes live too long
signal.alarm(5) # master process
try:
id = fork_processes(3, max_restarts=3)
self.assertTrue(id is not None)
signal.alarm(5) # child processes
except SystemExit as e:
# if we exit cleanly from fork_processes, all the child processes
# finished with status 0
self.assertEqual(e.code, 0)
self.assertTrue(task_id() is None)
for sock in sockets:
sock.close()
return
try:
if id in (0, 1):
self.assertEqual(id, task_id())
server = HTTPServer(self.get_app())
server.add_sockets(sockets)
IOLoop.instance().start()
elif id == 2:
self.assertEqual(id, task_id())
for sock in sockets:
sock.close()
# Always use SimpleAsyncHTTPClient here; the curl
# version appears to get confused sometimes if the
# connection gets closed before it's had a chance to
# switch from writing mode to reading mode.
client = HTTPClient(SimpleAsyncHTTPClient)
def fetch(url, fail_ok=False):
try:
return client.fetch(get_url(url))
except HTTPError as e:
if not (fail_ok and e.code == 599):
raise
# Make two processes exit abnormally
fetch("/?exit=2", fail_ok=True)
fetch("/?exit=3", fail_ok=True)
# They've been restarted, so a new fetch will work
int(fetch("/").body)
# Now the same with signals
# Disabled because on the mac a process dying with a signal
# can trigger an "Application exited abnormally; send error
# report to Apple?" prompt.
#fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True)
#fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True)
#int(fetch("/").body)
# Now kill them normally so they won't be restarted
fetch("/?exit=0", fail_ok=True)
# One process left; watch it's pid change
pid = int(fetch("/").body)
fetch("/?exit=4", fail_ok=True)
pid2 = int(fetch("/").body)
self.assertNotEqual(pid, pid2)
# Kill the last one so we shut down cleanly
fetch("/?exit=0", fail_ok=True)
os._exit(0)
except Exception:
logging.error("exception in child process %d", id, exc_info=True)
raise
cookie_secret=COOKIE_SECRET,
template_path=TEMPLATE_PATH,
static_path=STATIC_PATH,
xsrf_cookies=XSRF_COOKIES,
autoreload=AUTO_RELOAD,
debug=DEBUG,
login_url=LOGIN_URL,
)
return app
if __name__ == "__main__":
if IPV4_ONLY:
import socket
sockets = bind_sockets(options.port, family=socket.AF_INET)
else:
sockets = bind_sockets(options.port)
if not DEBUG:
import tornado.process
tornado.process.fork_processes(0) # 0 表示按CPU数目创建相应数据的子进程
SSL = True
import ssl
ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
if OHHOPlatform.is_windows():
cert_file = os.path.join(os.path.abspath("."), "ca/server-cert.pem")
# # start indexer and save it properly
# from indexer import indexer
# print bcolors.HEADER + "====== PREPARATION ======" + bcolors.ENDC
# myindexer=indexer.Indexer(Idxservers, Docservers)
# myindexer.genNum()
# myindexer.getCategoryCorrelation('./constants/reducer_tmp_1000lines.txt', Idxservers, Docservers)
# start working
print "\n" + bcolors.HEADER + '====== START FORKING ======' + bcolors.ENDC
from backend import back
from frontend import front
uid = fork_processes(numIdx + numDoc + 1)
if uid == 0:
sockets = bind_sockets(Baseport)
myfront = front.FrontEndApp(Idxservers, Docservers)
server = myfront.app
elif uid < numIdx + 1:
sockets = bind_sockets(Idxservers[uid - 1].split(':')[-1])
myback_idx = back.BackEndApp('indexServer', uid - 1,
Idxservers[uid - 1].split(':')[-1])
server = myback_idx.app
elif uid < numIdx + numDoc + 1:
sockets = bind_sockets(Docservers[uid - numIdx - 1].split(':')[-1])
myback_doc = back.BackEndApp('docServer', uid - numIdx - 1,
Docservers[uid - numIdx - 1].split(':')[-1])
server = myback_doc.app
server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
url(r"/api/cost-estimation/(.*)",
CostEstimationAPIHandler,
name="api-cost-detail"),
url(r"/api/configurations",
ConfigurationListAPIHandler,
name="api-configuration-list"),
url(
r"/api/configuration/(.*)",
ConfigurationItemAPIHandler,
name="api-configuration-detail",
),
],
debug=DEBUG,
static_path=os.path.join(RESOURCE_FOLDER_PATH, "static"),
template_path=os.path.join(RESOURCE_FOLDER_PATH, "templates"),
)
sockets = bind_sockets(0)
server = HTTPServer(app)
server.add_sockets(sockets)
_, port = sockets[0].getsockname()
local_url = f"http://localhost:{port}"
log.info(f"Installer page ready on {local_url}")
if not DEBUG:
log.info("Should open automatically in browser...")
webbrowser.open_new(local_url)
tornado.ioloop.IOLoop.current().start()
API用于接收实时数据，同步到cmdb
resource_id: 单条资源唯一ID
resource_type: 资源类型
:return:
"""
data = json.loads(self.request.body)
resource_id = str(self.path_kwargs.get('rid', "")).strip()
resource_type = data.get("resource_type")
print "resource_type====", resource_type
print "resource_id====", resource_id
if resource_type == "recharge":
DBSync.send_recharge_log(id=resource_id)
if resource_type == "consume":
DBSync.send_consume_log(id=resource_id)
if resource_type == "new_realauth":
NewRealAuth.sync_new_realauth(id=resource_id)
# resource_type = self.get_argument("resource_type", "")
application = tornado.web.Application([
(r"/sync/(?P<rid>.+)$", SyncData),
])
if __name__ == "__main__":
sockets = bind_sockets(8888, "0.0.0.0")
tornado.process.fork_processes(0)
http_server = tornado.httpserver.HTTPServer(application, xheaders=True)
http_server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
def make_http_server(loop, request_handler):
server = HTTPServer(request_handler, io_loop=loop)
sockets = bind_sockets(0, HOST, family=FAMILY)
assert len(sockets) == 1
server.add_sockets(sockets)
return sockets[0].getsockname()
def patched_bind_unused_port(reuse_port=False):
sock = netutil.bind_sockets(None, '127.0.0.1', family=socket.AF_INET,
reuse_port=reuse_port)[0]
port = sock.getsockname()[1]
return sock, port
:param msg_len:
:return:
"""
data = yield stream.read_bytes(msg_len, partial=True)
data = struct.unpack("!"+str(msg_len)+"s", data)
logger.debug("data:%s", data)
try:
# 打包数据,数据格式:数据发送者+数据类型+数据长度+数据体
binary_msg = struct.pack("!IBI" + str(msg_len) + "s", sender, 1, msg_len, data[0])
# 发送数据
yield stream.write(binary_msg)
logger.debug("="*25)
except KeyError:
# 将离线消息保存到数据库
pass
@gen.coroutine
def handle_pic_stream(self, stream, sender, msg_len):
pass
if __name__ == '__main__':
sockets = bind_sockets(IoTTcpServer.PORT)
server = IoTTcpServer()
server.add_sockets(sockets)
try:
IOLoop.current().start()
except KeyboardInterrupt:
pass
url(r"/api/configurations",
ConfigurationListAPIHandler,
name="api-configuration-list"),
url(
r"/api/configuration/(.*)",
ConfigurationItemAPIHandler,
name="api-configuration-detail",
),
url(r"/gas_price/(.*)", GasPriceHandler, name="gas_price"),
],
debug=DEBUG,
static_path=os.path.join(RESOURCE_FOLDER_PATH, "static"),
template_path=os.path.join(RESOURCE_FOLDER_PATH, "templates"),
)
# port = (sum(ord(c) for c in "RAIDEN_WIZARD") + 1000) % 2 ** 16 - 1 = 1994
sockets = bind_sockets(1994, "localhost")
server = HTTPServer(app)
server.add_sockets(sockets)
_, port = sockets[0].getsockname()
local_url = f"http://localhost:{port}"
log.info(f"Installer page ready on {local_url}")
if not DEBUG:
log.info("Should open automatically in browser...")
recover_ld_library_env_path()
webbrowser.open_new(local_url)
tornado.ioloop.IOLoop.current().start()
# Load Config
csl = Consul(token="", host=APP_CONSUL_ADDR)
_, cfgs = csl.catalog.service('config')
try:
with urlopen("http://%s:%s/resources/support/config.yml" %
(cfgs[0]['Address'], cfgs[0]['ServicePort'])) as f:
conf = yaml.load(f)
app.config.update(conf)
except HTTPError as e:
logging.getLogger(__name__).error("Unable to find config file")
sys.exit(1)
finally:
csl = None
# Bind to Random socket
sockets = bind_sockets(0, '')
server = HTTPServer(tr)
server.add_sockets(sockets)
# Save port
for s in sockets:
app.app_port = s.getsockname()[:2][1]
# Register to consul & run !
try:
__register_to_consul()
signal.signal(signal.SIGTERM, __signaled_unregister_from_consul)
signal.signal(signal.SIGINT, __signaled_unregister_from_consul)
IOLoop.instance().start()
except KeyboardInterrupt:
__unregister_from_consul(None, None)
from utils.helper import prop
define('num_processes', default=4, help='sub-processes count', type=int)
if __name__ == '__main__':
os.chdir(os.path.dirname(os.path.realpath(__file__)))
opt.parse_command_line()
routes = c.handlers + c.views
app = Application(routes,
default_handler_class=c.InvalidPageHandler,
ui_modules=c.modules,
xsrf_cookies=True)
try:
ssl_options = not opt.debug and app.site.get('https') or None
server = HTTPServer(app, xheaders=True, ssl_options=ssl_options)
sockets = netutil.bind_sockets(opt.port, family=socket.AF_INET)
fork_id = 0 if opt.debug or os.name == 'nt' else process.fork_processes(
opt.num_processes)
server.add_sockets(sockets)
protocol = 'https' if ssl_options else 'http'
logging.info('Start the service #%d v%s on %s://localhost:%d' %
(fork_id, app.version, protocol, opt.port))
if fork_id == 0 and app.db and app.db.page.count_documents(
{}) == 0 and os.path.exists(app.config.get('data_path', '')):
script = 'python3 utils/add_pages.py --uri={0} --db_name={1} --json_path={2}/json'.format(
app.db_uri, app.config['database']['name'],
app.config['data_path'])
os.system(script)
os.system('ln -s %s static/ocr_img' % app.config['data_path'])
ioloop.IOLoop.current().start()
def listen(self,
port,
addr=None,
family=None,
debug=False,
reuse_port=True,
**kwargs):
from tornado import httpserver, log, ioloop
import struct
flags = 0
if hasattr(socket, 'TCP_DEFER_ACCEPT'):
flags |= socket.TCP_DEFER_ACCEPT
if hasattr(socket, 'TCP_QUICKACK'):
flags |= socket.TCP_QUICKACK
sockets = netutil.bind_sockets(port,
address=addr,
backlog=1048576,
reuse_port=reuse_port,
flags=None if flags == 0 else flags)
for sock in sockets:
if not sock.family == socket.AF_INET:
continue
#if hasattr(socket, 'SO_LINGER'):
# sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 0,0))
#sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, struct.pack('i', 0))
#sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, struct.pack('i', 0))
for i in range(16, 0, -1):
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF,
struct.pack('i', i * 1024 * 1024))
print('server sock.SO_RCVBUF: %dM' % (i))
break
except:
continue
for i in range(16, 0, -1):
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF,
struct.pack('i', i * 1024 * 1024))
print('server sock.SO_SNDBUF: %dM' % (i))
break
except:
continue
self.server = httpserver.HTTPServer(self,
no_keep_alive=False,
**kwargs)
self.server.add_sockets(sockets)
self.server.start(1)
try:
ioloop.IOLoop.current().start()
except (KeyboardInterrupt, SystemExit):
log.app_log.info("\nSystem Exit.\n")
sys.exit(0)
except:
log.app_log.error("System Exception:\n", exc_info=True)
sys.exit(-1)
finally:
self.on_close()
self.render("error.html")
def initialize(self, status_code):
self.set_status(status_code)
if len(sys.argv) > 1:
MAIN_SITE_PORT = int(sys.argv[1])
else:
MAIN_SITE_PORT = settings.SITE_PORT
tornado.web.ErrorHandler = PageNotFoundHandler
if __name__ == "__main__":
tornado.locale.load_translations(settings.settings['translations'])
application = tornado.web.Application(urls.url_mapping,
**settings.settings)
# bind signals
sockets = bind_sockets(MAIN_SITE_PORT)
if not settings.DEBUG and settings.OS == 'linux':
import tornado.process
tornado.process.fork_processes(0) # 0 表示按 CPU 数目创建相应数目的子进程
server = HTTPServer(application, xheaders=True)
server.add_sockets(sockets)
log.debug(settings.SITE_URL)
tornado.ioloop.IOLoop.instance().start()
# coding=utf-8
__author__ = 'Yuheng Chen'
from Logger.Logger import baseLogger
from tornado.ioloop import IOLoop
from Server.Server import BaseServer
from Settings import Settings
from tornado.netutil import bind_sockets
from tornado import process
import tornado
if __name__ == '__main__':
baseLogger.info('Server is running...')
server = BaseServer()
if Settings.MULTI_PROCESS:
server.bind(Settings.SERVER_PORT)
server.start(0)
for advanced_port in Settings.ADVANCED_SERVER_PORT:
# bind more port if set
tornado.process.fork_processes(0)
sockets = bind_sockets(advanced_port)
server.add_sockets(sockets)
else:
server.listen(Settings.SERVER_PORT)
IOLoop.instance().start()
def __init__(self, family):
super(TestTCPServer, self).__init__()
self.streams = []
sockets = bind_sockets(None, 'localhost', family)
self.add_sockets(sockets)
self.port = sockets[0].getsockname()[1]
def run_server(port: int,
address: str = "",
jobs: int = 1,
user: str = None,
workers: int = 0) -> None:
""" Run the server
:param port: port number
:param address: interface address to bind to (default to any)
:param jobs: Number of jobs to fork (default to 1: i.e no fork)
:user: User to setuid after opening ports (default no setuid)
"""
import traceback
from tornado.netutil import bind_sockets
from tornado.httpserver import HTTPServer
kwargs = {}
ipcaddr = configure_ipc_addresses(workers)
application = None
server = None
if user:
setuid(user)
worker_pool = None
broker_pr = None
# Setup ssl config
if confservice.getboolean('server', 'ssl'):
LOGGER.info("SSL enabled")
kwargs['ssl_options'] = create_ssl_options()
if confservice.getboolean('server', 'http_proxy'):
LOGGER.info("Proxy configuration enabled")
kwargs['xheaders'] = True
# Check for declared cache observers
declare_cache_observers()
try:
# Fork processes
# This is a *DEPRECATED* feature
if jobs > 1:
def close_sockets(sockets):
for sock in sockets:
sock.close()
print(("DEPRECATION WARNING: "
"the 'jobs' option is deprecated "
"and will be removed in near future"),
file=sys.stderr,
flush=True)
sockets = bind_sockets(port, address=address)
if process.fork_processes(
jobs) is None: # We are in the main process
close_sockets(sockets)
broker_pr = create_broker_process(ipcaddr)
worker_pool = create_poolserver(
workers) if workers > 0 else None
set_signal_handlers()
# Note that manage_processes(...) never return in main process
# and call exit(0) which will be caught by SystemExit exception
process.manage_processes(max_restarts=5, logger=LOGGER)
assert False, "Not Reached"
else:
broker_pr = create_broker_process(ipcaddr)
worker_pool = create_poolserver(workers) if workers > 0 else None
sockets = bind_sockets(port, address=address)
LOGGER.info("Running server on port %s:%s", address, port)
application = Application(ipcaddr)
# Init HTTP server
server = HTTPServer(application, **kwargs)
server.add_sockets(sockets)
management = None
# Activate management
if confservice['management'].getboolean('enabled'):
from .management.server import start_management_server
management = start_management_server(worker_pool, ipcaddr)
management.stats = application.stats
# Initialize pool supervisor
worker_pool.start_supervisor()
# Start cache observer
cache_observer = start_cache_observer()
# XXX This trigger a deprecation warning in python 3.10
# but there is no clear alternative with tornado atm
# See https://github.com/tornadoweb/tornado/issues/3033
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM, lambda: loop.stop())
LOGGER.info("Starting processing requests")
loop.run_forever()
except Exception:
traceback.print_exc()
if process.task_id() is not None:
# Let a chance to the child process to
# restart
raise
else:
# Make sure that child processes are terminated
print("Terminating child processes", file=sys.stderr, flush=True)
process.terminate_childs()
exit_code = 1
except KeyboardInterrupt:
print("Keyboard Interrupt", flush=True)
exit_code = 15
except SystemExit as exc:
print("Exiting with code:", exc.code, flush=True)
exit_code = exc.code
else:
exit_code = 0
# Teardown
if server is not None:
server.stop()
if management is not None:
management.terminate()
management = None
if application is not None:
application.terminate()
application = None
print("PID {}: Server instance stopped".format(os.getpid()),
flush=True)
if process.task_id() is None:
if cache_observer:
cache_observer.stop()
if worker_pool:
print("Stopping workers", flush=True)
worker_pool.terminate()
if broker_pr:
print("Stopping broker", flush=True)
broker_pr.terminate()
broker_pr.join()
print("Server shutdown", flush=True)
sys.exit(exit_code)
def listen(self, port, address=""):
sockets = bind_sockets(port, address=address)
self.add_sockets(sockets)
from starlette.templating import Jinja2Templates
from starlette.staticfiles import StaticFiles
from starlette.responses import JSONResponse
# Tornado imports
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.netutil import bind_sockets
from tornado.web import StaticFileHandler
# Jinja imports
from jinja2 import Environment, FileSystemLoader
# Startup options for Bokeh Server:
socket = bind_sockets(
int(os.environ["BOKEH_SERVER_PORT"]),
address="localhost",
)[0]
port = socket.getsockname()[1]
# Startup options for Starlette:
templates = Jinja2Templates(directory='templates')
# Bokeh Applications:
def bkapp(curdoc):
#import time
#time.sleep(5)
"""
df = sea_surface_temperature.copy()
source = ColumnDataSource(data=df)