Example #1
0
def main():

    # options
    parse_command_line()

    # Locale
    load_gettext_translations(settings.I18N_PATH, "luoyun")
    set_default_locale('zh_CN')

    logging.info("starting torando web server")

    if settings.IPV4_ONLY:
        import socket
        sockets = bind_sockets(options.port, family=socket.AF_INET)
    else:
        sockets = bind_sockets(options.port)

    if not settings.DEBUG:
        import tornado.process
        tornado.process.fork_processes(0)

    application = Application()
    server = HTTPServer(application, xheaders=True)
    server.add_sockets(sockets)
    IOLoop.instance().start()
Example #2
0
    def server_start(self):

        logging.info('server starting...')
        logger_module.logger.load_config()
        #tornado把默认的根logger加了handler
        #把根logger的handler去除,然后重新绑定在tornado的logger下
        logging.getLogger().handlers = []
        enable_pretty_logging(None, logging.getLogger('tornado'))

        #服务启动

        from tornado.netutil import bind_sockets

        if self.settings.IPV4_ONLY:
            import socket

            sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
        else:
            sockets = bind_sockets(options.port, options.address)

        self.print_settings_info()
        http_server = tornado.httpserver.HTTPServer(self.application)
        http_server.add_sockets(sockets)
        logging.info('tornado server started. listen port: %s ,host address: %s' % (options.port, options.address))
        tornado.ioloop.IOLoop.instance().start()
Example #3
0
def build_listener(address, backlog=0):
    address_type = get_address_type(address)
    if address_type == 'tcp':
        host, port = address
        socks = netutil.bind_sockets(port, address=host, backlog=backlog)
        return socks
    if address_type == 'uds':
        socks = netutil.bind_sockets(address, mode=0o600, backlog=backlog)
        return [socks]
Example #4
0
def main(): 
    taskID = process.fork_processes(4)
    if taskID == 0:
        app = httpserver.HTTPServer(makeFrontend())
        app.add_sockets(netutil.bind_sockets(BASE_PORT))
        logging.info("Front end is listening on " + str(BASE_PORT))
    else:
        port = BASE_PORT + taskID
        app = httpserver.HTTPServer(makeBackend(port))
        app.add_sockets(netutil.bind_sockets(port))
        logging.info("Back end %d listening on %d" % (taskID, port))
    IOLoop.current().start()
Example #5
0
def bind_sockets(addr, backlog=128, allows_unix_socket=False):
    # initialize the socket
    addr = parse_address(addr)
    if isinstance(addr, six.string_types):
        if not allows_unix_socket:
            raise RuntimeError("unix addresses aren't supported")

        sock = netutil.bind_unix_socket(addr)
    elif is_ipv6(addr[0]):
        sock = netutil.bind_sockets(addr[1], address=addr[0],
                family=socket.AF_INET6, backlog=backlog)
    else:
        sock = netutil.bind_sockets(addr[1], backlog=backlog)
    return sock
Example #6
0
    def server_start(self):
        #服务启动
        if self.settings.IPV4_ONLY:
            import socket
            from tornado.netutil import bind_sockets

            sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
        else:
            sockets = bind_sockets(options.port, options.address)

        self.print_settings_info()
        http_server = tornado.httpserver.HTTPServer(self.application)
        http_server.add_sockets(sockets)
        tornado.ioloop.IOLoop.instance().start()
        print 'tornado server started. listen port: %s ,host address: %s' % (options.port, options.address)
Example #7
0
def start_game():
    ''' Main entry point for the application '''
    cache_actions()
    sockets = netutil.bind_sockets(8888)
    #if process.task_id() == None:
    #    tornado.process.fork_processes(-1, max_restarts = 10)
    server = HTTPServer(application)
    server.add_sockets(sockets)
    io_loop = IOLoop.instance()
    session_manager = SessionManager.Instance()
    if process.task_id() == None:
        scoring = PeriodicCallback(scoring_round, application.settings['ticks'], io_loop = io_loop)
        session_clean_up = PeriodicCallback(session_manager.clean_up, application.settings['clean_up_timeout'], io_loop = io_loop)
        scoring.start()
        session_clean_up.start()
    try:
        for count in range(3, 0, -1):
            logging.info("The game will begin in ... %d" % (count,))
            sleep(1)
        logging.info("Good hunting!")
        io_loop.start()
    except KeyboardInterrupt:
        if process.task_id() == 0:
            print '\r[!] Shutdown Everything!'
            session_clean_up.stop()
            io_loop.stop()
Example #8
0
    def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128):
        """Binds this server to the given port on the given address.

        To start the server, call start(). If you want to run this server
        in a single process, you can call listen() as a shortcut to the
        sequence of bind() and start() calls.

        Address may be either an IP address or hostname.  If it's a hostname,
        the server will listen on all IP addresses associated with the
        name.  Address may be an empty string or None to listen on all
        available interfaces.  Family may be set to either socket.AF_INET
        or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise
        both will be used if available.

        The ``backlog`` argument has the same meaning as for 
        ``socket.listen()``.

        This method may be called multiple times prior to start() to listen
        on multiple ports or interfaces.
        """
        sockets = netutil.bind_sockets(port, address=address,
                                       family=family, backlog=backlog)
        if self._started:
            self.add_sockets(sockets)
        else:
            self._pending_sockets.extend(sockets)
Example #9
0
def start_server():
    ''' Main entry point for the application '''
    if options.debug:
        logging.warn("%sDebug mode is enabled; DO NOT USE THIS IN PRODUCTION%s" % (
            bold + R, W
        ))
    # Setup server object
    if options.ssl:
        server = HTTPServer(app,
                            ssl_options={
                                "certfile": options.certfile,
                                "keyfile": options.keyfile,
                            },
                            xheaders=options.x_headers
                            )
    else:
        server = HTTPServer(app, xheaders=options.x_headers)
    sockets = netutil.bind_sockets(options.listen_port)
    server.add_sockets(sockets)
    try:
        io_loop.start()
    except KeyboardInterrupt:
        sys.stdout.write('\r' + WARN + 'Shutdown Everything!\n')
    except:
        logging.exception("Main i/o loop threw exception")
    finally:
        io_loop.stop()
        _exit(0)
Example #10
0
    def test_100_continue(self):
        # testing if httpclient is able to skip 100 continue responses.
        # to test without httpserver implementation, using
        # raw response as same as httpclient_test.test_chunked_close.
        port = get_unused_port()
        (sock,) = netutil.bind_sockets(port, address="127.0.0.1")
        with closing(sock):
            def write_response(stream, request_data):
                stream.write(b("""\
HTTP/1.1 100 Continue

HTTP/1.1 200 OK
Content-Length: 6

hjkl
""").replace(b("\n"), b("\r\n")), callback=stream.close)
            def accept_callback(conn, address):
                # fake an HTTP server using chunked encoding where the final chunks
                # and connection close all happen at once
                stream = IOStream(conn, io_loop=self.io_loop)
                stream.read_until(b("\r\n\r\n"),
                                  functools.partial(write_response, stream))
            netutil.add_accept_handler(sock, accept_callback, self.io_loop)
            self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop,
                                   headers={"Expect": "100-continue"})
            resp = self.wait()
            resp.rethrow()
            self.assertEqual(resp.body, b("hjkl\r\n"))
Example #11
0
def main():
    numProcs = inventory.NUM_INDEX_SHARDS + inventory.NUM_DOC_SHARDS + 1
    taskID = process.fork_processes(numProcs, max_restarts=0)
    port = inventory.BASE_PORT + taskID
    if taskID == 0:
        app = httpserver.HTTPServer(tornado.web.Application([
                (r"/search", Web),
                (r"/upload", UploadHandler),
                (r"/(.*)", IndexDotHTMLAwareStaticFileHandler, dict(path=SETTINGS['static_path']))
            ], **SETTINGS))
        logging.info("Front end is listening on " + str(port))
    else:       
        if taskID <= inventory.NUM_INDEX_SHARDS:
            shardIx = taskID - 1
            #data = pickle.load(open("data/index%d.pkl" % (shardIx), "r"))
            inverted_path = os.path.join(os.getcwd(),"../assignment5/df_jobs/%d.out"  % (shardIx))
            logging.info("Inverted file path: %s" % inverted_path)
            data = pickle.load(open(inverted_path ,'r'))
            idf_path = os.path.join(os.getcwd(), "../assignment5/idf_jobs/0.out")
            logIDF = pickle.load(open(idf_path,'r'))
            app = httpserver.HTTPServer(web.Application([(r"/index", index.Index, dict(data=data, logIDF=logIDF))]))
            logging.info("Index shard %d listening on %d" % (shardIx, port))
        else:
            shardIx = taskID - inventory.NUM_INDEX_SHARDS - 1
            #data = pickle.load(open("data/doc%d.pkl" % (shardIx), "r"))
            doc_path = os.path.join(os.getcwd(),"../assignment5/i_df_jobs/%d.out" % (shardIx))
            logging.info("Doc Server path %s" % doc_path)
            data = pickle.load(open(doc_path, "r"))
            app = httpserver.HTTPServer(web.Application([(r"/doc", doc.Doc, dict(data=data))]))
            logging.info("Doc shard %d listening on %d" % (shardIx, port))
    app.add_sockets(netutil.bind_sockets(port))
    IOLoop.current().start()
Example #12
0
 def __init__(self, family):
     super(TestTCPServer, self).__init__()
     self.streams = []
     self.queue = Queue()
     sockets = bind_sockets(None, 'localhost', family)
     self.add_sockets(sockets)
     self.port = sockets[0].getsockname()[1]
Example #13
0
    def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128,
             reuse_port=False):
        """Binds this server to the given port on the given address.

        To start the server, call `start`. If you want to run this server
        in a single process, you can call `listen` as a shortcut to the
        sequence of `bind` and `start` calls.

        Address may be either an IP address or hostname.  If it's a hostname,
        the server will listen on all IP addresses associated with the
        name.  Address may be an empty string or None to listen on all
        available interfaces.  Family may be set to either `socket.AF_INET`
        or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise
        both will be used if available.

        The ``backlog`` argument has the same meaning as for
        `socket.listen <socket.socket.listen>`. The ``reuse_port`` argument
        has the same meaning as for `.bind_sockets`.

        This method may be called multiple times prior to `start` to listen
        on multiple ports or interfaces.

        .. versionchanged:: 4.4
           Added the ``reuse_port`` argument.
        """
        sockets = bind_sockets(port, address=address, family=family,
                               backlog=backlog, reuse_port=reuse_port)
        if self._started:
            self.add_sockets(sockets)
        else:
            self._pending_sockets.extend(sockets)
Example #14
0
    def load_httpserver(self, sockets=None, **kwargs):
        if not sockets:
            from tornado.netutil import bind_sockets

            if settings.IPV4_ONLY:
                import socket

                sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
            else:
                sockets = bind_sockets(options.port, options.address)

        http_server = tornado.httpserver.HTTPServer(self.application, **kwargs)

        http_server.add_sockets(sockets)
        self.httpserver = http_server
        return self.httpserver
Example #15
0
def main():
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(Application(),xheaders=True)
    sockets = bind_sockets(options.port, family=2)
    #http_server.listen(options.port)
    http_server.add_sockets(sockets)
    tornado.ioloop.IOLoop.instance().start()
Example #16
0
def main():
    script_path = os.path.dirname(os.path.realpath(__file__))
    config_path = os.path.join(script_path, 'etc/harvest.cfg')

    config = ConfigParser()
    config.read(config_path)

    sockets = bind_sockets(config.get('server', 'port'),
                           config.get('server', 'address'))
    fork_processes(config.getint('server', 'instances'))

    datastore = DataStore(config.get('datastore', 'host'),
                          config.getint('datastore', 'port'),
                          config.get('datastore', 'username'),
                          config.get('datastore', 'password'),
                          config.get('datastore', 'database'))

    app = Application([(r"/rpc/store", Handler,
                       {'datastore': datastore,
                        'api_key': config.get('server', 'api_key')})])

    server = HTTPServer(app,
                        no_keep_alive=config.get('server', 'no_keep_alive'),
                        ssl_options={
                            'certfile': config.get('server', 'certfile'),
                            'keyfile': config.get('server', 'keyfile')})

    server.add_sockets(sockets)
    IOLoop.instance().start()
Example #17
0
    def initialize(self, cfg):
        self.cfg = cfg

        # Check we have a certificate and key defines
        if not cfg.ssl_cert_file or not cfg.ssl_key_file:
            log.error('a valid ssl_cert_file and ssl_key_file are required, '
                      'aborting')
            sys.exit(1)

        # Set up the SSL context
        try:
            ssl_context = ssl.create_default_context(
                purpose=ssl.Purpose.CLIENT_AUTH,
                cafile=cfg.ssl_ca_file)
        except IOError as e:
            log.error('cannot read ssl_ca_file: {}'.format(e.args[1]))
            sys.exit(1)
        self.ssl_context = ssl_context

        # Enable client certificate verification if wanted
        if cfg.ssl_verify_client:
            ssl_context.verify_mode = ssl.CERT_REQUIRED

        # Load our own certificate into the server
        try:
            ssl_context.load_cert_chain(
                certfile=cfg.ssl_cert_file,
                keyfile=cfg.ssl_key_file)
        except IOError as e:
            log.error('cannot read ssl_cert_file or ssl_key_file: {}'
                      .format(e.args[1]))
            sys.exit(1)

        self.app = NrpeApplication(cfg)

        # Set up the HTTPServer instance
        super(NrpeHTTPServer, self).initialize(
            self.app, no_keep_alive=True, ssl_options=ssl_context,
            idle_connection_timeout=cfg.connection_timeout,
            body_timeout=cfg.connection_timeout)

        # Because Tornado unconditionally sets IPV6_V6ONLY on IPv6 sockets, we
        # need a means to preserve old behaviour. This should only be an issue
        # when listening to '::' for IPv6 any-address. Tornado will listen both
        # the IPv4 and IPv6 any-address when the bind address is blank, so just
        # set the address to be empty if we encounter '::'.
        server_address = cfg.server_address
        if server_address == '::':
            server_address = ''

        try:
            self.sockets = bind_sockets(port=cfg.server_port,
                                        address=server_address)
        except socket.error as e:
            log.error('failed to bind socket: {}'.format(e.args[1]))
            sys.exit(1)

        # Prevent tornado from logging HTTP requests
        if not self.cfg.debug:
            logging.getLogger('tornado.access').disabled = True
Example #18
0
def start_server():
    ''' Main entry point for the application '''
    sockets = netutil.bind_sockets(config.listen_port)
    server = HTTPServer(app)
    server.add_sockets(sockets)
    io_loop = IOLoop.instance()
    scoring = PeriodicCallback(
        scoring_round, int(5 * 60 * 1000), io_loop=io_loop
    )
    scoring.start()
    try:
        sys.stdout.write("\r" + INFO + "The game has begun, good hunting!\n")
        if config.debug:
            sys.stdout.write(WARN + "WARNING: Debug mode is enabled.\n")
        sys.stdout.flush()
        game_history = GameHistory.Instance()
        history_callback = PeriodicCallback(
            game_history.take_snapshot, int(60 * 1000), io_loop=io_loop
        )
        history_callback.start()
        io_loop.start()
    except KeyboardInterrupt:
        print('\r' + WARN + 'Shutdown Everything!')
    except:
      logging.exception("Main i/o loop threw exception")
    finally:
        io_loop.stop()
        if config.debug and \
                raw_input(PROMPT + "Flush Memcache? [Y/n]: ").lower() == 'y':
            print(INFO + 'Flushing cache ...'),
            FileCache.flush()
            print('OK')
        _exit(0)
Example #19
0
 def __init__(self, family):
     super(TestTCPServer, self).__init__()
     self.streams = []  # type: List[IOStream]
     self.queue = Queue()  # type: Queue[IOStream]
     sockets = bind_sockets(0, "localhost", family)
     self.add_sockets(sockets)
     self.port = sockets[0].getsockname()[1]
Example #20
0
    def start(self):
        self.__ssl.guarantee_valid_certs()

        log.pcsd.info("Starting server...")

        self.__server = HTTPServer(
            self.__make_app(self),
            ssl_options=self.__ssl.create_context()
        )

        # It is necessary to bind sockets for every new HTTPServer since
        # HTTPServer.stop calls sock.close() inside.
        sockets = []
        for address in self.__bind_addresses:
            log.pcsd.info(
                "Binding socket for address '%s' and port '%s'",
                address if address is not None else "*",
                self.__port
            )
            sockets.extend(bind_sockets(self.__port, address))

        self.__server.add_sockets(sockets)

        log.pcsd.info("Server is listening")
        self.__server_is_running = True
        return self
Example #21
0
    def test_chunked_close(self):
        # test case in which chunks spread read-callback processing
        # over several ioloop iterations, but the connection is already closed.
        port = get_unused_port()
        (sock,) = netutil.bind_sockets(port, address="127.0.0.1")
        with closing(sock):
            def write_response(stream, request_data):
                stream.write(b("""\
HTTP/1.1 200 OK
Transfer-Encoding: chunked

1
1
1
2
0

""").replace(b("\n"), b("\r\n")), callback=stream.close)
            def accept_callback(conn, address):
                # fake an HTTP server using chunked encoding where the final chunks
                # and connection close all happen at once
                stream = IOStream(conn, io_loop=self.io_loop)
                stream.read_until(b("\r\n\r\n"),
                                  functools.partial(write_response, stream))
            netutil.add_accept_handler(sock, accept_callback, self.io_loop)
            self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
            resp = self.wait()
            resp.rethrow()
            self.assertEqual(resp.body, b("12"))
Example #22
0
def start_server():
    ''' Main entry point for the application '''
    if config.debug:
        logging.warn("Debug mode is enabled; some security measures will be ignored")
    # Setup server object
    if config.use_ssl:
        server = HTTPServer(app,
            ssl_options={
                "certfile": config.certfile,
                "keyfile": config.keyfile,
            },
            xheaders=config.x_headers
        )
    else:
        server = HTTPServer(app, xheaders=config.x_headers)
    sockets = netutil.bind_sockets(config.listen_port)
    server.add_sockets(sockets)
    try:
        io_loop.start()
    except KeyboardInterrupt:
        sys.stdout.write('\r' + WARN + 'Shutdown Everything!\n')
    except:
        logging.exception("Main i/o loop threw exception")
    finally:
        io_loop.stop()
        _exit(0)
Example #23
0
def bind_unused_port():
    """Binds a server socket to an available port on localhost.
    Returns a tuple (socket, port).
    """
    [sock] = netutil.bind_sockets(None, 'localhost', family=socket.AF_INET)
    port = sock.getsockname()[1]
    return sock, port
Example #24
0
def start_server():
    ''' Main entry point for the application '''
    sockets = netutil.bind_sockets(config.listen_port)
    if config.use_ssl:
        server = HTTPServer(
            app, 
            ssl_options={
                "certfile": config.certfile,
                "keyfile": config.keyfile,
            }, 
            xheaders=config.x_headers
        )
    else:
        server = HTTPServer(
            app, 
            xheaders=config.x_headers
        )
    server.add_sockets(sockets)
    io_loop = IOLoop.instance()
    try:
        io_loop.start()
    except KeyboardInterrupt:
        logging.warn("Keyboard interrupt, shutdown everything!")
    except:
        logging.exception("Main I/O Loop threw an excetion!")
    finally:
        io_loop.stop()
    def make_iostream_pair(self, **kwargs):
        port = get_unused_port()
        [listener] = netutil.bind_sockets(port, '127.0.0.1',
                                          family=socket.AF_INET)
        streams = [None, None]

        def accept_callback(connection, address):
            streams[0] = self._make_server_iostream(connection, **kwargs)
            if isinstance(streams[0], SSLIOStream):
                # HACK: The SSL handshake won't complete (and
                # therefore the client connect callback won't be
                # run)until the server side has tried to do something
                # with the connection.  For these tests we want both
                # sides to connect before we do anything else with the
                # connection, so we must cause some dummy activity on the
                # server.  If this turns out to be useful for real apps
                # it should have a cleaner interface.
                streams[0]._add_io_state(IOLoop.READ)
            self.stop()

        def connect_callback():
            streams[1] = client_stream
            self.stop()
        netutil.add_accept_handler(listener, accept_callback,
                                   io_loop=self.io_loop)
        client_stream = self._make_client_iostream(socket.socket(), **kwargs)
        client_stream.connect(('127.0.0.1', port),
                              callback=connect_callback)
        self.wait(condition=lambda: all(streams))
        self.io_loop.remove_handler(listener.fileno())
        listener.close()
        return streams
Example #26
0
def start_server():
    ''' Main entry point for the application '''
    if options.debug:
        logging.warn("%sDebug mode is enabled; DO NOT USE THIS IN PRODUCTION%s" % (
            bold + R, W
        ))
    if options.autostart_game:
        logging.info("The game is about to begin, good hunting!")
        app.settings['history_callback'].start()
        if options.use_bots:
            app.settings['score_bots_callback'].start()
    # Setup server object
    if options.ssl:
        server = HTTPServer(app,
                            ssl_options={
                                "certfile": options.certfile,
                                "keyfile": options.keyfile,
                            },
                            xheaders=options.x_headers
                            )
    else:
        server = HTTPServer(app, xheaders=options.x_headers)
    sockets = netutil.bind_sockets(options.listen_port, options.listen_interface)
    server.add_sockets(sockets)
    Scoreboard.now(app)
    try:
        io_loop.start()
    except KeyboardInterrupt:
        sys.stdout.write('\r' + WARN + 'Shutdown Everything!\n')
    except:
        logging.exception("Main i/o loop threw exception")
    finally:
        io_loop.stop()
        _exit(0)
Example #27
0
def main():
    for back in BACKENDS:
        print back
    app = httpserver.HTTPServer(makeFrontend())
    app.add_sockets(netutil.bind_sockets(FRONTEND_PORT))
    logging.info("Front end is listening on " + str(FRONTEND_PORT))
    IOLoop.current().start()
Example #28
0
def bind(server, port, usocket, address=None):
    """Make server listen on port (inet socket).
    If given, prefer `usocket`, path to a unix socket.
    The latter is useful for reverse proxying.

    If listening on a inet socket, `address` might be
    given. `address` may be either an IP address or hostname.
    If it's a hostname, the server will listen on all IP addresses
    associated with the name. If not given (and not listening on a unix
    socket) will listen on all available interfaces."""

    # If we have a unix socket path
    if usocket:
        LOG.info('Starting on unix socket %r', usocket)
        try:
            socket = bind_unix_socket(usocket, mode=options.usocket_mode)
            os.chown(usocket, options.usocket_uid, options.usocket_gid)
        except OSError as error:
            LOG.error('Cannot create unix socket: %r', error)
        else:
            server.add_socket(socket)
            LOG.info('Started')
    else:
        LOG.info('Starting on port %d', port)
        sockets = bind_sockets(port, address=address)
        server.add_sockets(sockets)
        LOG.info('Started')
Example #29
0
def make_http_server(loop, request_handler):
    server = HTTPServer(request_handler, io_loop=loop)
    sockets = bind_sockets(0, HOST, family=FAMILY)
    assert len(sockets) == 1
    server.add_sockets(sockets)
    sock = sockets[0]
    return sock
Example #30
0
File: net.py Project: jsober/denim
 def build_socket(self):
     """Builds the listener socket. If `self.port` is not set, uses an
     OS-assigned port. Note that the `host` and `port` values are updated
     by the `start` method.
     """
     [sock] = bind_sockets(self.port, self.host, family=socket.AF_INET)
     host, port = sock.getsockname()
     return sock, port, host
Example #31
0
def start_server():
    ''' Main entry point for the application '''

    task_event_consumer = TaskEventConsumer()

    # Application setup
    app = Application(
        [

            # Static Handlers - Serves static CSS, JS and images
            (r'/static/(.*\.(css|js|png|jpg|jpeg|svg|ttf|html|json))',
             StaticFileHandler, {
                 'path': 'static/'
             }),

            # Home page serving SPA app
            (r'/', HomePageHandler),

            # Monitor Socket
            (r'/connect/monitor', MonitorSocketHandler),

            # Error Handlers -
            (r'/403', ForbiddenHandler),

            # Catch all 404 page
            (r'(.*)', NotFoundHandler),
        ],

        # Randomly generated secret key
        cookie_secret=urandom(32).encode('hex'),

        # Request that does not pass @authorized will be
        # redirected here
        forbidden_url='/403',

        # Requests that does not pass @authenticated  will be
        # redirected here
        login_url='/login',

        # Template directory
        template_path='templates/',

        # Debug mode
        debug=config.debug,
        task_event_consumer=task_event_consumer)

    sockets = netutil.bind_sockets(config.listen_port)
    if config.use_ssl:
        server = HTTPServer(app,
                            ssl_options={
                                "certfile": config.certfile,
                                "keyfile": config.keyfile,
                            },
                            xheaders=config.x_headers)
    else:
        server = HTTPServer(app, xheaders=config.x_headers)
    server.add_sockets(sockets)
    io_loop = IOLoop.instance()
    try:
        io_loop.add_callback(task_event_consumer.connect)
        io_loop.start()
    except KeyboardInterrupt:
        logging.warn("Keyboard interrupt, shutdown everything!")
    except:
        logging.exception("Main I/O Loop threw an excetion!")
    finally:
        io_loop.stop()
Example #32
0
def run_server( port: int, address: str="", jobs: int=1,  user: str=None, workers: int=0) -> None:
    """ Run the server

        :param port: port number
        :param address: interface address to bind to (default to any)
        :param jobs: Number of jobs to fork (default to 1: i.e no fork)
        :user: User to setuid after opening ports (default no setuid)
    """
    import traceback
    from tornado.netutil import bind_sockets
    from tornado.httpserver import HTTPServer

    kwargs = {}

    ipcaddr = configure_ipc_addresses(workers)

    application = None
    server = None

    if user:
       setuid(user)

    def close_sockets(sockets):
        for sock in sockets:
            sock.close()

    worker_pool   = None
    broker_pr     = None
    broker_client = None

    # Setup ssl config
    if confservice.getboolean('server','ssl'):
        LOGGER.info("SSL enabled")
        kwargs['ssl_options'] = create_ssl_options()

    # Run
    try:
        # Fork processes
        if jobs > 1:
            sockets = bind_sockets(port, address=address)
            if  process.fork_processes(jobs) is None: # We are in the main process
                close_sockets(sockets)
                broker_pr   = create_broker_process(ipcaddr)
                worker_pool = create_worker_pool(workers) if workers>0 else None
                set_signal_handlers()

                # Note that manage_processes(...) never return in main process 
                # and call exit(0) which will be caught by SystemExit exception
                task_id = process.manage_processes(max_restarts=5, logger=LOGGER)

                assert False, "Not Reached"
        else:
            broker_pr   = create_broker_process(ipcaddr)
            worker_pool = create_worker_pool(workers) if workers>0 else None
            sockets = bind_sockets(port, address=address)

        # Install asyncio event loop after forking
        # This is why we do not use server.bind/server.start
        tornado.platform.asyncio.AsyncIOMainLoop().install()

        LOGGER.info("Running server on port %s:%s", address, port)
    
        application = Application(ipcaddr)

        # Init HTTP server
        server = HTTPServer(application, **kwargs)
        server.add_sockets(sockets)
 
        loop = asyncio.get_event_loop()
        loop.add_signal_handler(signal.SIGTERM, lambda: loop.stop())
        loop.run_forever()
    except Exception:
        traceback.print_exc()
        if process.task_id() is not None:
            # Let a chance to the child process to 
            # restart
            raise
        else: 
            # Make sure that child processes are terminated
            print("Terminating child processes", file=sys.stderr)
            process.terminate_childs()
    except (KeyboardInterrupt, SystemExit) as e:
        pass

    # Teardown
    if server is not None:
        server.stop()
    if application is not None:
        application.terminate()
        application = None
        loop = asyncio.get_event_loop()
        if not loop.is_closed():
            loop.close()
        print("{}: Server instance stopped".format(os.getpid()), file=sys.stderr)

    if process.task_id() is None:
        if worker_pool:
            print("Stopping workers")
            worker_pool.terminate()
            worker_pool.join()
        if broker_pr:
            print("Stopping broker")
            broker_pr.terminate()
            broker_pr.join()
        print("Server shutdown", file=sys.stderr)
Example #33
0
        #The documents are then scored. Each document's score is the inner product (a.k.a. dot product,
        #effectively correlation here) of its vector and the query vector. In addition, scores should be
        #biased so that documents with the query terms in their title receive especially high scores.
        ScoredListing = []
        for docID in MyMap:
            score = dotProd(MyMap[docID], QueryVector)
            ScoredListing.append([score, docID])  ## Bias comes from tfs itself

        #Finally, the K highest-scoring documents are written out as a JSON-encoded list of (docID, score) pairs.
        ScoredListing.sort(reverse=True)  ## check so sorted by score
        topPostings = ScoredListing[:10]
        TP = []
        for posting in topPostings:
            TP.append([posting[1], posting[0]])
        #print 'Length from ' + str(self._port)
        #print len(ScoredListing)
        self.write(json.JSONEncoder().encode({"postings": TP}))


if __name__ == "__main__":

    task_id = process.fork_processes(inventory.num_ind)
    I_BASE_PORT = inventory.port1
    #application.listen(I_BASE_PORT)
    port = I_BASE_PORT + task_id
    app = httpserver.HTTPServer(
        web.Application([web.url(r"/index", IndexHandler, dict(port=port))]))
    app.add_sockets(netutil.bind_sockets(port))
    print('indexServer' + str(task_id) + ' at port: ' + str(port))
    tornado.ioloop.IOLoop.current().start()
Example #34
0
        处理文本数据
        :param stream:
        :param send_to:
        :param msg_len:
        :return:
        """
        data = yield stream.read_bytes(msg_len, partial=True)
        data = struct.unpack("!" + str(msg_len) + "s", data)
        logger.debug("data:%s", data)
        try:
            # 打包数据,数据格式:数据发送者+数据类型+数据长度+数据体
            binary_msg = struct.pack("!IBI" + str(msg_len) + "s", sender, 1,
                                     msg_len, data[0])
            # 发送数据
            yield stream.write(binary_msg)
            logger.debug("=" * 25)
        except KeyError:
            # 将离线消息保存到数据库
            pass

    @gen.coroutine
    def handle_pic_stream(self, stream, sender, msg_len):
        pass


if __name__ == '__main__':
    sockets = bind_sockets(ChatServer.PORT)
    server = ChatServer()
    server.add_sockets(sockets)
    IOLoop.current().start()
Example #35
0
def start_pyrox(config):
    if config is None:
        raise ConfigurationError('No configuration object passed in')

    # Log some important things
    if config.routing.upstream_hosts is not None:
        _LOG.info('Upstream targets are: {}'.format(
            [dst for dst in config.routing.upstream_hosts]))

    # Set bind host
    bind_host = config.core.bind_host.split(':')
    if len(bind_host) != 2:
        raise ConfigurationError('bind_host must have a port specified')

    # Bind the sockets in the main process
    sockets = None

    try:
        sockets = bind_sockets(port=bind_host[1], address=bind_host[0])
    except Exception as ex:
        _LOG.exception(ex)
        return

    # Bind the server port(s)
    _LOG.info('Pyrox listening on: http://{0}:{1}'.format(
        bind_host[0], bind_host[1]))

    # Are we trying to profile Pyrox?
    if config.core.enable_profiling:
        _LOG.warning("""
**************************************************************************
Notice: Pyrox Profiling Mode Enabled

You have enabled Pyrox with profiling enabled in the Pyrox config. This
will restrict Pyrox to one resident process. It is not recommended that
you run Pyrox in production with this feature enabled.
**************************************************************************
""")
        start_proxy(sockets, config)
        return

    # Number of processess to spin
    num_processes = config.core.processes

    if num_processes <= 0:
        num_processes = cpu_count()

    global _active_children_pids

    for i in range(num_processes):
        pid = os.fork()
        if pid == 0:
            _LOG.info('Starting process {}'.format(i))
            start_proxy(sockets, config)
            sys.exit(0)
        else:
            _active_children_pids.append(pid)

    # Take over SIGTERM and SIGINT
    signal.signal(signal.SIGTERM, stop_parent)
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    while len(_active_children_pids):
        try:
            pid, status = os.wait()
        except OSError as oserr:
            if oserr.errno != errno.EINTR:
                _LOG.exception(oserr)
            continue
        except Exception as ex:
            _LOG.exception(ex)
            continue

        _LOG.info('Child process {} exited with status {}'.format(pid, status))
        _active_children_pids.remove(pid)
Example #36
0
				try:
					data_dict = json.loads(data.decode())
					# get result
					ht = Hunter()
					rh = RedisHandler()
					# If can't find data in redis, fetch it from mysql
					result_dict = {}
					result_dict = rh.outhash(data_dict)
					if not result_dict:
						result_dict = ht.get_data(data_dict) 
					# if need to change voltage data, use the statement below
					# result_dict = data_magic(result_dict)
					data_send = json.dumps(result_dict)
					
					yield stream.write(struct.pack('i',len(data_send))+data_send.encode())
				except KeyError:
					logging.info('** Data Error **')

			except StreamClosedError:
				logging.info("Server finished")
				logging.info("*"*25)
				del DataServer.clients[address]
				break


if __name__ == '__main__':
	server = DataServer()
	sockets = bind_sockets(30105)
	server.add_sockets(sockets)
	IOLoop.current().start()
Example #37
0
        start = time.time()
        print(urls)
        client = httpclient.AsyncHTTPClient()

        for url in urls:
            request = httpclient.HTTPRequest(url=url)
            result = client.fetch(request, callback=self.finish_response)
            print('result', result)
            results.append(result)
        self.finish('hello world')
        print('response results:', results)
        print('post spend time:', time.time() - start)

    def finish_response(self, response):
        pass

    @concurrent.run_on_executor()
    def handle(self, request):
        response = requests.get(request.url)
        return response.content

if __name__ == '__main__':
    app = web.Application([
        (r'/handle_request', NewHandleRequest)
    ])
    sockets = netutil.bind_sockets(6666, reuse_port=True)
    process.fork_processes(os.cpu_count() * 2)
    server = NewHttpServer(app)
    server.add_sockets(sockets)
    IOLoop.current().start()
Example #38
0
    def _open(self, host, port, **kwargs):
        # Note: does not get called if host is False. That way we can
        # run Flexx in e.g. JLab's application.

        # Hook Tornado up with asyncio. Flexx' BaseServer makes sure
        # that the correct asyncio event loop is current (for this thread).
        # http://www.tornadoweb.org/en/stable/asyncio.html
        # todo: Since Tornado v5.0 asyncio is autom used, deprecating AsyncIOMainLoop
        self._io_loop = AsyncIOMainLoop()
        # I am sorry for this hack, but Tornado wont work otherwise :(
        # I wonder how long it will take before this will bite me back. I guess
        # we will be alright as long as there is no other Tornado stuff going on.
        IOLoop._current.instance = None
        self._io_loop.make_current()

        # handle ssl, wether from configuration or given args
        if config.ssl_certfile:
            if 'ssl_options' not in kwargs:
                kwargs['ssl_options'] = {}
            if 'certfile' not in kwargs['ssl_options']:
                kwargs['ssl_options']['certfile'] = config.ssl_certfile

        if config.ssl_keyfile:
            if 'ssl_options' not in kwargs:
                kwargs['ssl_options'] = {}
            if 'keyfile' not in kwargs['ssl_options']:
                kwargs['ssl_options']['keyfile'] = config.ssl_keyfile

        if config.tornado_debug:
            app_kwargs = dict(debug=True)
        else:
            app_kwargs = dict()
        # Create tornado application
        self._app = Application([
            (r"/flexx/ws/(.*)", WSHandler),
            (r"/flexx/(.*)", MainHandler),
            (r"/(.*)", AppHandler),
        ], **app_kwargs)
        self._app._io_loop = self._io_loop
        # Create tornado server, bound to our own ioloop
        if tornado.version_info < (5, ):
            kwargs['io_loop'] = self._io_loop
        self._server = HTTPServer(self._app, **kwargs)

        # Start server (find free port number if port not given)
        if port:
            # Turn port into int, use hashed port number if a string was given
            try:
                port = int(port)
            except ValueError:
                port = port_hash(port)
            self._server.listen(port, host)
        else:
            # Try N ports in a repeatable range (easier, browser history, etc.)
            prefered_port = port_hash('Flexx')
            for i in range(8):
                port = prefered_port + i
                try:
                    self._server.listen(port, host)
                    break
                except (OSError, IOError):
                    pass  # address already in use
            else:
                # Ok, let Tornado figure out a port
                [sock] = netutil.bind_sockets(None,
                                              host,
                                              family=socket.AF_INET)
                self._server.add_sockets([sock])
                port = sock.getsockname()[1]

        # Notify address, so its easy to e.g. copy and paste in the browser
        self._serving = self._app._flexx_serving = host, port
        proto = 'http'
        if 'ssl_options' in kwargs:
            proto = 'https'
        # This string 'Serving apps at' is our 'ready' signal and is tested for.
        logger.info('Serving apps at %s://%s:%i/' % (proto, host, port))
Example #39
0
def start_server():
    """ Main entry point for the application """
    if options.debug:
        logging.warn(
            "%sDebug mode is enabled; DO NOT USE THIS IN PRODUCTION%s" %
            (bold + R, W))
    locale.set_default_locale("en_US")
    locale.load_translations("locale")
    if options.autostart_game:
        logging.info("The game is about to begin, good hunting!")
        app.settings["game_started"] = True
        app.settings["history_callback"].start()
        if options.use_bots:
            app.settings["score_bots_callback"].start()
    # Setup server object
    if options.ssl:
        server = HTTPServer(
            app,
            ssl_options={
                "certfile": options.certfile,
                "keyfile": options.keyfile
            },
            xheaders=options.x_headers,
        )
    else:
        server = HTTPServer(app, xheaders=options.x_headers)
    try:
        sockets = netutil.bind_sockets(options.listen_port,
                                       options.listen_interface)
    except (OSError, IOError) as err:
        logging.error("Problem binding socket to port %s",
                      str(options.listen_port))
        if err.errno == 13:
            pypath = sys.executable
            if os_path.islink(pypath):
                pypath = os_path.realpath(pypath)
            logging.error(
                "Possible Fix: sudo setcap CAP_NET_BIND_SERVICE=+eip %s",
                pypath)
        elif err.errno == 98:
            logging.error(
                "The port may be in use by an existing service.  RTB already running?"
            )
        else:
            logging.error(err)
        sys.exit()
    server.add_sockets(sockets)
    try:
        Scoreboard.update_gamestate(app)
    except OperationalError as err:
        if "Table definition has changed" in str(err):
            logging.info(
                "Table definitions have changed -restarting RootTheBox.")
            return "restart"
        else:
            logging.error("There was a problem starting RootTheBox. Error: " +
                          str(err))
    try:
        io_loop.start()
    except KeyboardInterrupt:
        sys.stdout.write("\r" + WARN + "Shutdown Everything!\n")
    except:
        logging.exception("Main i/o loop threw exception")
    finally:
        io_loop.stop()
        _exit(0)
Example #40
0
def main():
    from tornado import options

    default_general_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(message)s"
    default_access_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(trace_id)s\t%(message)s"

    opts = options.OptionParser()
    opts.define("version",
                type=bool,
                help="show version and exit",
                callback=show_version)
    opts.define("locators",
                default=["localhost:10053"],
                type=str,
                multiple=True,
                help="comma-separated endpoints of locators")
    opts.define("cache",
                default=DEFAULT_SERVICE_CACHE_COUNT,
                type=int,
                help="count of instances per service")
    opts.define(
        "config",
        help="path to configuration file",
        type=str,
        callback=lambda path: opts.parse_config_file(path, final=False))
    opts.define("count",
                default=1,
                type=int,
                help="count of tornado processes")
    opts.define(
        "endpoints",
        default=["tcp://localhost:8080"],
        type=str,
        multiple=True,
        help=
        "Specify endpoints to bind on: prefix unix:// or tcp:// should be used"
    )
    opts.define("request_header",
                default="X-Request-Id",
                type=str,
                help="header used as a trace id")
    opts.define("forcegen_request_header",
                default=False,
                type=bool,
                help="enable force generation of the request header")
    opts.define("sticky_header",
                default="X-Cocaine-Sticky",
                type=str,
                help="sticky header name")
    opts.define("gcstats",
                default=False,
                type=bool,
                help="print garbage collector stats to stderr")
    opts.define("srwconfig", default="", type=str, help="path to srwconfig")
    opts.define("allow_json_rpc",
                default=True,
                type=bool,
                help="allow JSON RPC module")

    # tracing options
    opts.define("tracing_chance",
                default=DEFAULT_TRACING_CHANCE,
                type=float,
                help="default chance for an app to be traced")
    opts.define("configuration_service",
                default="unicorn",
                type=str,
                help="name of configuration service")
    opts.define(
        "tracing_conf_path",
        default="/zipkin_sampling",
        type=str,
        help="path to the configuration nodes in the configuration service")

    # various logging options
    opts.define(
        "logging",
        default="info",
        help=("Set the Python log level. If 'none', tornado won't touch the "
              "logging configuration."),
        metavar="debug|info|warning|error|none")
    opts.define("log_to_cocaine",
                default=False,
                type=bool,
                help="log to cocaine")
    opts.define(
        "log_to_stderr",
        type=bool,
        default=None,
        help=("Send log output to stderr. "
              "By default use stderr if --log_file_prefix is not set and "
              "no other logging is configured."))
    opts.define("log_file_prefix",
                type=str,
                default=None,
                metavar="PATH",
                help=("Path prefix for log file"))
    opts.define("datefmt",
                type=str,
                default="%z %d/%b/%Y:%H:%M:%S",
                help="datefmt")
    opts.define("generallogfmt",
                type=str,
                help="log format of general logging system",
                default=default_general_logformat)
    opts.define("accesslogfmt",
                type=str,
                help="log format of access logging system",
                default=default_access_logformat)
    opts.define("logframework",
                type=bool,
                default=False,
                help="enable logging various framework messages")

    # util server
    opts.define("utilport",
                default=8081,
                type=int,
                help="listening port number for an util server")
    opts.define("utiladdress",
                default="127.0.0.1",
                type=str,
                help="address for an util server")
    opts.define("enableutil",
                default=False,
                type=bool,
                help="enable util server")
    opts.define("client_id",
                default=0,
                type=int,
                help="client id used for authentication")
    opts.define("client_secret",
                default='',
                type=str,
                help="client secret used for authentication")
    opts.parse_command_line()

    srw_config = None
    if opts.srwconfig:
        try:
            srw_config = load_srw_config(opts.srwconfig)
        except Exception as err:
            print("unable to load SRW config: %s" % err)
            exit(1)

    use_reuseport = hasattr(socket, "SO_REUSEPORT")
    endpoints = Endpoints(opts.endpoints)
    sockets = []

    for path in endpoints.unix:
        sockets.append(bind_unix_socket(path, mode=0o666))

    if not use_reuseport:
        for endpoint in endpoints.tcp:
            # We have to bind before fork to distribute sockets to our forks
            socks = bind_sockets(endpoint.port, address=endpoint.host)
            sockets.extend(socks)

    if opts.enableutil:
        utilsockets = bind_sockets(opts.utilport, address=opts.utiladdress)

    try:
        if opts.count != 1:
            process.fork_processes(opts.count)

        enable_logging(opts)

        if opts.gcstats:
            enable_gc_stats()

        if use_reuseport:
            for endpoint in endpoints.tcp:
                # We have to bind before fork to distribute sockets to our forks
                socks = bind_sockets(endpoint.port,
                                     address=endpoint.host,
                                     reuse_port=True)
                sockets.extend(socks)

        proxy = CocaineProxy(
            locators=opts.locators,
            cache=opts.cache,
            request_id_header=opts.request_header,
            sticky_header=opts.sticky_header,
            forcegen_request_header=opts.forcegen_request_header,
            default_tracing_chance=opts.tracing_chance,
            srw_config=srw_config,
            allow_json_rpc=opts.allow_json_rpc,
            client_id=opts.client_id,
            client_secret=opts.client_secret)
        server = HTTPServer(proxy)
        server.add_sockets(sockets)

        if opts.enableutil:
            utilsrv = HTTPServer(UtilServer(proxy=proxy))
            utilsrv.add_sockets(utilsockets)

        tornado.ioloop.IOLoop.current().start()
    except KeyboardInterrupt:
        pass
Example #41
0
    def _open(self, host, port, **kwargs):
        # Note: does not get called if host is False. That way we can
        # run Flexx in e.g. JLab's application.

        # handle ssl, wether from configuration or given args
        if config.ssl_certfile:
            if 'ssl_options' not in kwargs:
                kwargs['ssl_options'] = {}
            if 'certfile' not in kwargs['ssl_options']:
                kwargs['ssl_options']['certfile'] = config.ssl_certfile

        if config.ssl_keyfile:
            if 'ssl_options' not in kwargs:
                kwargs['ssl_options'] = {}
            if 'keyfile' not in kwargs['ssl_options']:
                kwargs['ssl_options']['keyfile'] = config.ssl_keyfile

        if config.tornado_debug:
            app_kwargs = dict(debug=True)
        else:
            app_kwargs = dict()
        # Create tornado application
        self._app = Application([
            (r"/flexx/ws/(.*)", WSHandler),
            (r"/flexx/(.*)", MainHandler),
            (r"/(.*)", AppHandler),
        ], **app_kwargs)
        # Create tornado server, bound to our own ioloop
        self._server = HTTPServer(self._app, io_loop=self._loop, **kwargs)

        # Start server (find free port number if port not given)
        if port:
            # Turn port into int, use hashed port number if a string was given
            try:
                port = int(port)
            except ValueError:
                port = port_hash(port)
            self._server.listen(port, host)
        else:
            # Try N ports in a repeatable range (easier, browser history, etc.)
            prefered_port = port_hash('Flexx')
            for i in range(8):
                port = prefered_port + i
                try:
                    self._server.listen(port, host)
                    break
                except (OSError, IOError):
                    pass  # address already in use
            else:
                # Ok, let Tornado figure out a port
                [sock] = netutil.bind_sockets(None,
                                              host,
                                              family=socket.AF_INET)
                self._server.add_sockets([sock])
                port = sock.getsockname()[1]

        # Notify address, so its easy to e.g. copy and paste in the browser
        self._serving = self._app._flexx_serving = host, port
        proto = 'http'
        if 'ssl_options' in kwargs:
            proto = 'https'
        logger.info('Serving apps at %s://%s:%i/' % (proto, host, port))
Example #42
0
    if options.more:
        log.setLevel(DEBUG)
else:
    log.setLevel(WARNING)

ioloop = IOLoop.instance()

if os.getenv('LISTEN_PID'):
    log.info('Getting socket from systemd')
    sck = socket.fromfd(
        SYSTEMD_SOCKET_FD + 1,  # Second socket in .socket file
        socket.AF_INET6 if socket.has_ipv6 else socket.AF_INET,
        socket.SOCK_STREAM)
    sck.setblocking(0)
    sck.listen(128)
    sockets = [sck]
else:
    log.info('Binding sockets')
    sockets = bind_sockets(options.socket_port)

log.info('Accepting')
for sck in sockets:
    add_accept_handler(sck, handle_connection, ioloop)

log.info('Listening')
http_server = SystemdHTTPServer(server)
http_server.listen(options.server_port)

log.info('Starting loop')
ioloop.start()
Example #43
0
    def test_multi_process(self):
        self.assertFalse(IOLoop.initialized())
        port = get_unused_port()

        def get_url(path):
            return "http://127.0.0.1:%d%s" % (port, path)

        sockets = bind_sockets(port, "127.0.0.1")
        # ensure that none of these processes live too long
        signal.alarm(5)  # master process
        try:
            id = fork_processes(3, max_restarts=3)
            self.assertTrue(id is not None)
            signal.alarm(5)  # child processes
        except SystemExit as e:
            # if we exit cleanly from fork_processes, all the child processes
            # finished with status 0
            self.assertEqual(e.code, 0)
            self.assertTrue(task_id() is None)
            for sock in sockets:
                sock.close()
            return
        try:
            if id in (0, 1):
                self.assertEqual(id, task_id())
                server = HTTPServer(self.get_app())
                server.add_sockets(sockets)
                IOLoop.instance().start()
            elif id == 2:
                self.assertEqual(id, task_id())
                for sock in sockets:
                    sock.close()
                # Always use SimpleAsyncHTTPClient here; the curl
                # version appears to get confused sometimes if the
                # connection gets closed before it's had a chance to
                # switch from writing mode to reading mode.
                client = HTTPClient(SimpleAsyncHTTPClient)

                def fetch(url, fail_ok=False):
                    try:
                        return client.fetch(get_url(url))
                    except HTTPError as e:
                        if not (fail_ok and e.code == 599):
                            raise

                # Make two processes exit abnormally
                fetch("/?exit=2", fail_ok=True)
                fetch("/?exit=3", fail_ok=True)

                # They've been restarted, so a new fetch will work
                int(fetch("/").body)

                # Now the same with signals
                # Disabled because on the mac a process dying with a signal
                # can trigger an "Application exited abnormally; send error
                # report to Apple?" prompt.
                #fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True)
                #fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True)
                #int(fetch("/").body)

                # Now kill them normally so they won't be restarted
                fetch("/?exit=0", fail_ok=True)
                # One process left; watch it's pid change
                pid = int(fetch("/").body)
                fetch("/?exit=4", fail_ok=True)
                pid2 = int(fetch("/").body)
                self.assertNotEqual(pid, pid2)

                # Kill the last one so we shut down cleanly
                fetch("/?exit=0", fail_ok=True)

                os._exit(0)
        except Exception:
            logging.error("exception in child process %d", id, exc_info=True)
            raise
Example #44
0
        cookie_secret=COOKIE_SECRET,
        template_path=TEMPLATE_PATH,
        static_path=STATIC_PATH,
        xsrf_cookies=XSRF_COOKIES,
        autoreload=AUTO_RELOAD,
        debug=DEBUG,
        login_url=LOGIN_URL,
    )
    return app


if __name__ == "__main__":
    if IPV4_ONLY:
        import socket

        sockets = bind_sockets(options.port, family=socket.AF_INET)
    else:
        sockets = bind_sockets(options.port)
    if not DEBUG:
        import tornado.process

        tornado.process.fork_processes(0)  # 0 表示按CPU数目创建相应数据的子进程

    SSL = True

    import ssl

    ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)

    if OHHOPlatform.is_windows():
        cert_file = os.path.join(os.path.abspath("."), "ca/server-cert.pem")
# # start indexer and save it properly
# from indexer import indexer
# print bcolors.HEADER + "====== PREPARATION ======" + bcolors.ENDC
# myindexer=indexer.Indexer(Idxservers, Docservers)
# myindexer.genNum()
# myindexer.getCategoryCorrelation('./constants/reducer_tmp_1000lines.txt', Idxservers, Docservers)

# start working
print "\n" + bcolors.HEADER + '====== START FORKING ======' + bcolors.ENDC
from backend import back
from frontend import front

uid = fork_processes(numIdx + numDoc + 1)
if uid == 0:
    sockets = bind_sockets(Baseport)
    myfront = front.FrontEndApp(Idxservers, Docservers)
    server = myfront.app
elif uid < numIdx + 1:
    sockets = bind_sockets(Idxservers[uid - 1].split(':')[-1])
    myback_idx = back.BackEndApp('indexServer', uid - 1,
                                 Idxservers[uid - 1].split(':')[-1])
    server = myback_idx.app
elif uid < numIdx + numDoc + 1:
    sockets = bind_sockets(Docservers[uid - numIdx - 1].split(':')[-1])
    myback_doc = back.BackEndApp('docServer', uid - numIdx - 1,
                                 Docservers[uid - numIdx - 1].split(':')[-1])
    server = myback_doc.app

server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
Example #46
0
            url(r"/api/cost-estimation/(.*)",
                CostEstimationAPIHandler,
                name="api-cost-detail"),
            url(r"/api/configurations",
                ConfigurationListAPIHandler,
                name="api-configuration-list"),
            url(
                r"/api/configuration/(.*)",
                ConfigurationItemAPIHandler,
                name="api-configuration-detail",
            ),
        ],
        debug=DEBUG,
        static_path=os.path.join(RESOURCE_FOLDER_PATH, "static"),
        template_path=os.path.join(RESOURCE_FOLDER_PATH, "templates"),
    )

    sockets = bind_sockets(0)
    server = HTTPServer(app)
    server.add_sockets(sockets)

    _, port = sockets[0].getsockname()
    local_url = f"http://localhost:{port}"
    log.info(f"Installer page ready on {local_url}")

    if not DEBUG:
        log.info("Should open automatically in browser...")
        webbrowser.open_new(local_url)

    tornado.ioloop.IOLoop.current().start()
        API用于接收实时数据,同步到cmdb
        resource_id: 单条资源唯一ID
        resource_type: 资源类型
        :return:
        """
        data = json.loads(self.request.body)
        resource_id = str(self.path_kwargs.get('rid', "")).strip()
        resource_type = data.get("resource_type")
        print "resource_type====", resource_type
        print "resource_id====", resource_id
        if resource_type == "recharge":
            DBSync.send_recharge_log(id=resource_id)
        if resource_type == "consume":
            DBSync.send_consume_log(id=resource_id)
        if resource_type == "new_realauth":
            NewRealAuth.sync_new_realauth(id=resource_id)

        # resource_type = self.get_argument("resource_type", "")


application = tornado.web.Application([
    (r"/sync/(?P<rid>.+)$", SyncData),
])

if __name__ == "__main__":
    sockets = bind_sockets(8888, "0.0.0.0")
    tornado.process.fork_processes(0)
    http_server = tornado.httpserver.HTTPServer(application, xheaders=True)
    http_server.add_sockets(sockets)
    tornado.ioloop.IOLoop.instance().start()
Example #48
0
def make_http_server(loop, request_handler):
    server = HTTPServer(request_handler, io_loop=loop)
    sockets = bind_sockets(0, HOST, family=FAMILY)
    assert len(sockets) == 1
    server.add_sockets(sockets)
    return sockets[0].getsockname()
Example #49
0
def patched_bind_unused_port(reuse_port=False):
    sock = netutil.bind_sockets(None, '127.0.0.1', family=socket.AF_INET,
                                reuse_port=reuse_port)[0]
    port = sock.getsockname()[1]
    return sock, port
Example #50
0
        :param msg_len:
        :return:
        """
        data = yield stream.read_bytes(msg_len, partial=True)
        data = struct.unpack("!"+str(msg_len)+"s", data)
        logger.debug("data:%s", data)
        try:
            # 打包数据,数据格式:数据发送者+数据类型+数据长度+数据体
            binary_msg = struct.pack("!IBI" + str(msg_len) + "s", sender, 1, msg_len, data[0])
            # 发送数据
            yield stream.write(binary_msg)
            logger.debug("="*25)
        except KeyError:
            # 将离线消息保存到数据库
            pass

    @gen.coroutine
    def handle_pic_stream(self, stream, sender, msg_len):
        pass


if __name__ == '__main__':
    sockets = bind_sockets(IoTTcpServer.PORT)
    server = IoTTcpServer()
    server.add_sockets(sockets)
    try:
        IOLoop.current().start()
    except KeyboardInterrupt:
        pass

Example #51
0
            url(r"/api/configurations",
                ConfigurationListAPIHandler,
                name="api-configuration-list"),
            url(
                r"/api/configuration/(.*)",
                ConfigurationItemAPIHandler,
                name="api-configuration-detail",
            ),
            url(r"/gas_price/(.*)", GasPriceHandler, name="gas_price"),
        ],
        debug=DEBUG,
        static_path=os.path.join(RESOURCE_FOLDER_PATH, "static"),
        template_path=os.path.join(RESOURCE_FOLDER_PATH, "templates"),
    )

    # port = (sum(ord(c) for c in "RAIDEN_WIZARD") + 1000) % 2 ** 16 - 1 = 1994
    sockets = bind_sockets(1994, "localhost")
    server = HTTPServer(app)
    server.add_sockets(sockets)

    _, port = sockets[0].getsockname()
    local_url = f"http://localhost:{port}"
    log.info(f"Installer page ready on {local_url}")

    if not DEBUG:
        log.info("Should open automatically in browser...")
        recover_ld_library_env_path()
        webbrowser.open_new(local_url)

    tornado.ioloop.IOLoop.current().start()
Example #52
0
    # Load Config
    csl = Consul(token="", host=APP_CONSUL_ADDR)
    _, cfgs = csl.catalog.service('config')
    try:
        with urlopen("http://%s:%s/resources/support/config.yml" %
                     (cfgs[0]['Address'], cfgs[0]['ServicePort'])) as f:
            conf = yaml.load(f)
            app.config.update(conf)
    except HTTPError as e:
        logging.getLogger(__name__).error("Unable to find config file")
        sys.exit(1)
    finally:
        csl = None

    # Bind to Random socket
    sockets = bind_sockets(0, '')
    server = HTTPServer(tr)
    server.add_sockets(sockets)

    # Save port
    for s in sockets:
        app.app_port = s.getsockname()[:2][1]

    # Register to consul & run !
    try:
        __register_to_consul()
        signal.signal(signal.SIGTERM, __signaled_unregister_from_consul)
        signal.signal(signal.SIGINT, __signaled_unregister_from_consul)
        IOLoop.instance().start()
    except KeyboardInterrupt:
        __unregister_from_consul(None, None)
Example #53
0
from utils.helper import prop

define('num_processes', default=4, help='sub-processes count', type=int)

if __name__ == '__main__':
    os.chdir(os.path.dirname(os.path.realpath(__file__)))
    opt.parse_command_line()
    routes = c.handlers + c.views
    app = Application(routes,
                      default_handler_class=c.InvalidPageHandler,
                      ui_modules=c.modules,
                      xsrf_cookies=True)
    try:
        ssl_options = not opt.debug and app.site.get('https') or None
        server = HTTPServer(app, xheaders=True, ssl_options=ssl_options)
        sockets = netutil.bind_sockets(opt.port, family=socket.AF_INET)
        fork_id = 0 if opt.debug or os.name == 'nt' else process.fork_processes(
            opt.num_processes)
        server.add_sockets(sockets)
        protocol = 'https' if ssl_options else 'http'
        logging.info('Start the service #%d v%s on %s://localhost:%d' %
                     (fork_id, app.version, protocol, opt.port))
        if fork_id == 0 and app.db and app.db.page.count_documents(
            {}) == 0 and os.path.exists(app.config.get('data_path', '')):
            script = 'python3 utils/add_pages.py --uri={0} --db_name={1} --json_path={2}/json'.format(
                app.db_uri, app.config['database']['name'],
                app.config['data_path'])
            os.system(script)
            os.system('ln -s %s static/ocr_img' % app.config['data_path'])
        ioloop.IOLoop.current().start()
    def listen(self,
               port,
               addr=None,
               family=None,
               debug=False,
               reuse_port=True,
               **kwargs):
        from tornado import httpserver, log, ioloop
        import struct

        flags = 0
        if hasattr(socket, 'TCP_DEFER_ACCEPT'):
            flags |= socket.TCP_DEFER_ACCEPT
        if hasattr(socket, 'TCP_QUICKACK'):
            flags |= socket.TCP_QUICKACK

        sockets = netutil.bind_sockets(port,
                                       address=addr,
                                       backlog=1048576,
                                       reuse_port=reuse_port,
                                       flags=None if flags == 0 else flags)
        for sock in sockets:
            if not sock.family == socket.AF_INET:
                continue
            #if hasattr(socket, 'SO_LINGER'):
            #    sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 0,0))

            #sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, struct.pack('i', 0))
            #sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDTIMEO, struct.pack('i', 0))
            for i in range(16, 0, -1):
                try:
                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF,
                                    struct.pack('i', i * 1024 * 1024))
                    print('server sock.SO_RCVBUF: %dM' % (i))
                    break
                except:
                    continue
            for i in range(16, 0, -1):
                try:
                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF,
                                    struct.pack('i', i * 1024 * 1024))
                    print('server sock.SO_SNDBUF: %dM' % (i))
                    break
                except:
                    continue

        self.server = httpserver.HTTPServer(self,
                                            no_keep_alive=False,
                                            **kwargs)
        self.server.add_sockets(sockets)
        self.server.start(1)

        try:
            ioloop.IOLoop.current().start()
        except (KeyboardInterrupt, SystemExit):
            log.app_log.info("\nSystem Exit.\n")
            sys.exit(0)
        except:
            log.app_log.error("System Exception:\n", exc_info=True)
            sys.exit(-1)
        finally:
            self.on_close()
Example #55
0
        self.render("error.html")

    def initialize(self, status_code):
        self.set_status(status_code)


if len(sys.argv) > 1:
    MAIN_SITE_PORT = int(sys.argv[1])
else:
    MAIN_SITE_PORT = settings.SITE_PORT

tornado.web.ErrorHandler = PageNotFoundHandler

if __name__ == "__main__":

    tornado.locale.load_translations(settings.settings['translations'])
    application = tornado.web.Application(urls.url_mapping,
                                          **settings.settings)

    # bind signals
    sockets = bind_sockets(MAIN_SITE_PORT)

    if not settings.DEBUG and settings.OS == 'linux':
        import tornado.process
        tornado.process.fork_processes(0)  # 0 表示按 CPU 数目创建相应数目的子进程

    server = HTTPServer(application, xheaders=True)
    server.add_sockets(sockets)
    log.debug(settings.SITE_URL)
    tornado.ioloop.IOLoop.instance().start()
Example #56
0
# coding=utf-8
__author__ = 'Yuheng Chen'

from Logger.Logger import baseLogger
from tornado.ioloop import IOLoop
from Server.Server import BaseServer
from Settings import Settings
from tornado.netutil import bind_sockets
from tornado import process
import tornado

if __name__ == '__main__':
    baseLogger.info('Server is running...')
    server = BaseServer()

    if Settings.MULTI_PROCESS:
        server.bind(Settings.SERVER_PORT)
        server.start(0)
        for advanced_port in Settings.ADVANCED_SERVER_PORT:
            # bind more port if set
            tornado.process.fork_processes(0)
            sockets = bind_sockets(advanced_port)
            server.add_sockets(sockets)
    else:
        server.listen(Settings.SERVER_PORT)

    IOLoop.instance().start()
Example #57
0
 def __init__(self, family):
     super(TestTCPServer, self).__init__()
     self.streams = []
     sockets = bind_sockets(None, 'localhost', family)
     self.add_sockets(sockets)
     self.port = sockets[0].getsockname()[1]
Example #58
0
def run_server(port: int,
               address: str = "",
               jobs: int = 1,
               user: str = None,
               workers: int = 0) -> None:
    """ Run the server

        :param port: port number
        :param address: interface address to bind to (default to any)
        :param jobs: Number of jobs to fork (default to 1: i.e no fork)
        :user: User to setuid after opening ports (default no setuid)
    """
    import traceback
    from tornado.netutil import bind_sockets
    from tornado.httpserver import HTTPServer

    kwargs = {}

    ipcaddr = configure_ipc_addresses(workers)

    application = None
    server = None

    if user:
        setuid(user)

    worker_pool = None
    broker_pr = None

    # Setup ssl config
    if confservice.getboolean('server', 'ssl'):
        LOGGER.info("SSL enabled")
        kwargs['ssl_options'] = create_ssl_options()

    if confservice.getboolean('server', 'http_proxy'):
        LOGGER.info("Proxy configuration enabled")
        kwargs['xheaders'] = True

    # Check for declared cache observers
    declare_cache_observers()

    try:
        # Fork processes
        # This is a *DEPRECATED* feature
        if jobs > 1:

            def close_sockets(sockets):
                for sock in sockets:
                    sock.close()

            print(("DEPRECATION WARNING: "
                   "the 'jobs' option is deprecated "
                   "and will be removed in near future"),
                  file=sys.stderr,
                  flush=True)

            sockets = bind_sockets(port, address=address)
            if process.fork_processes(
                    jobs) is None:  # We are in the main process
                close_sockets(sockets)
                broker_pr = create_broker_process(ipcaddr)
                worker_pool = create_poolserver(
                    workers) if workers > 0 else None
                set_signal_handlers()

                # Note that manage_processes(...) never return in main process
                # and call exit(0) which will be caught by SystemExit exception
                process.manage_processes(max_restarts=5, logger=LOGGER)

                assert False, "Not Reached"
        else:
            broker_pr = create_broker_process(ipcaddr)
            worker_pool = create_poolserver(workers) if workers > 0 else None
            sockets = bind_sockets(port, address=address)

        LOGGER.info("Running server on port %s:%s", address, port)

        application = Application(ipcaddr)

        # Init HTTP server
        server = HTTPServer(application, **kwargs)
        server.add_sockets(sockets)

        management = None
        # Activate management
        if confservice['management'].getboolean('enabled'):
            from .management.server import start_management_server
            management = start_management_server(worker_pool, ipcaddr)
            management.stats = application.stats

        # Initialize pool supervisor
        worker_pool.start_supervisor()

        # Start cache observer
        cache_observer = start_cache_observer()

        # XXX This trigger a deprecation warning in python 3.10
        # but there is no clear alternative with tornado atm
        # See https://github.com/tornadoweb/tornado/issues/3033
        loop = asyncio.get_event_loop()
        loop.add_signal_handler(signal.SIGTERM, lambda: loop.stop())
        LOGGER.info("Starting processing requests")
        loop.run_forever()
    except Exception:
        traceback.print_exc()
        if process.task_id() is not None:
            # Let a chance to the child process to
            # restart
            raise
        else:
            # Make sure that child processes are terminated
            print("Terminating child processes", file=sys.stderr, flush=True)
            process.terminate_childs()
            exit_code = 1
    except KeyboardInterrupt:
        print("Keyboard Interrupt", flush=True)
        exit_code = 15
    except SystemExit as exc:
        print("Exiting with code:", exc.code, flush=True)
        exit_code = exc.code
    else:
        exit_code = 0

    # Teardown
    if server is not None:
        server.stop()
    if management is not None:
        management.terminate()
        management = None
    if application is not None:
        application.terminate()
        application = None
        print("PID {}: Server instance stopped".format(os.getpid()),
              flush=True)
    if process.task_id() is None:
        if cache_observer:
            cache_observer.stop()
        if worker_pool:
            print("Stopping workers", flush=True)
            worker_pool.terminate()
        if broker_pr:
            print("Stopping broker", flush=True)
            broker_pr.terminate()
            broker_pr.join()

    print("Server shutdown", flush=True)
    sys.exit(exit_code)
Example #59
0
 def listen(self, port, address=""):
     sockets = bind_sockets(port, address=address)
     self.add_sockets(sockets)
Example #60
0
from starlette.templating import Jinja2Templates
from starlette.staticfiles import StaticFiles
from starlette.responses import JSONResponse

# Tornado imports
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.netutil import bind_sockets
from tornado.web import StaticFileHandler

# Jinja imports
from jinja2 import Environment, FileSystemLoader

# Startup options for Bokeh Server:
socket = bind_sockets(
    int(os.environ["BOKEH_SERVER_PORT"]),
    address="localhost",
)[0]
port = socket.getsockname()[1]

# Startup options for Starlette:
templates = Jinja2Templates(directory='templates')


# Bokeh Applications:
def bkapp(curdoc):
    #import time
    #time.sleep(5)
    """
    df = sea_surface_temperature.copy()
    source = ColumnDataSource(data=df)