def _run_all_tests(self): log = [] def handle(socket, address): while True: data = socket.recv(1024) print('got %r' % data) if not data: break log.append(data) server = StreamServer(self.args[1], handle) server.start() try: conn = socket.create_connection(('127.0.0.1', 10011)) conn.sendall(b'msg1') sleep(0.1) # On Windows, SIGTERM actually abruptly terminates the process; # it can't be caught. However, CTRL_C_EVENT results in a KeyboardInterrupt # being raised, so we can shut down properly. self.popen.send_signal(getattr(signal, 'CTRL_C_EVENT') if hasattr(signal, 'CTRL_C_EVENT') else signal.SIGTERM) sleep(0.1) conn.sendall(b'msg2') conn.close() with gevent.Timeout(2.1): self.popen.wait() finally: server.close() self.assertEqual([b'msg1', b'msg2'], log)
def close(self): if self.closed: sys.exit('Multiple exit signals received - aborting.') else: log('Closing listener socket') self.filter_db.close() StreamServer.close(self)
def run(self): servers = [] ssl_args = {} if self.cfg.is_ssl: ssl_args = dict(server_side=True, **self.cfg.ssl_options) for s in self.sockets: s.setblocking(1) pool = Pool(self.worker_connections) if self.server_class is not None: environ = base_environ(self.cfg) environ.update({ "wsgi.multithread": True, "SERVER_SOFTWARE": VERSION, }) server = self.server_class( s, application=self.wsgi, spawn=pool, log=self.log, handler_class=self.wsgi_handler, environ=environ, **ssl_args) else: hfun = partial(self.handle, s) server = StreamServer(s, handle=hfun, spawn=pool, **ssl_args) server.start() servers.append(server) while self.alive: self.notify() gevent.sleep(1.0) try: # Stop accepting requests for server in servers: if hasattr(server, 'close'): # gevent 1.0 server.close() if hasattr(server, 'kill'): # gevent < 1.0 server.kill() # Handle current requests until graceful_timeout ts = time.time() while time.time() - ts <= self.cfg.graceful_timeout: accepting = 0 for server in servers: if server.pool.free_count() != server.pool.size: accepting += 1 # if no server is accepting a connection, we can exit if not accepting: return self.notify() gevent.sleep(1.0) # Force kill all active the handlers self.log.warning("Worker graceful timeout (pid:%s)" % self.pid) for server in servers: server.stop(timeout=1) except: pass
class GeventStreamServer(): def __init__(self, ip=ConfigHolder.getConfig().getProperty(BASE, SERVER_IP), port=int(ConfigHolder.getConfig().getProperty( BASE, SERVER_PORT)), maxCon=1000): self.pool = None # do not accept more than 10000 connections self.ip = ip self.port = port self.maxCon = maxCon self.inited = False def start(self): if not self.inited: log.info("启动GeventStreamServer服务,端口:%s............." % self.port) self.pool = Pool(self.maxCon) self.inited = True self.server = StreamServer((self.ip, self.port), handleRequest, spawn=self.pool) self.server.serve_forever() def connected(self): return not self.server.closed() def stop(self): self.server.close()
def test(self): log = [] def handle(socket, address): while True: data = socket.recv(1024) if not data: break log.append(data) server = StreamServer(self.args[1], handle) server.start() try: conn = socket.create_connection(self.get_address()) # make sure the connection is accepted at app level rather than at OS level # before sending a signal conn.sendall('msg1') gevent.sleep(0.1) self.process.send_signal(15) # now let's make sure the signal was received gevent.sleep(0.1) conn.sendall('msg2') conn.close() finally: server.close() with gevent.Timeout(0.1): self.process.wait() self.assertEqual(['msg1', 'msg2'], log)
def stop(self, *args, **kwargs): self.logger.info("Agent server {} stopping".format(self.listener)) StreamServer.close(self) for serv in self.server.agent_clients.values(): serv.stop() Component.stop(self) self.logger.info("Exit")
def _run_all_tests(self): log = [] def handle(socket, address): while True: data = socket.recv(1024) print 'got %r' % data if not data: break log.append(data) server = StreamServer(self.args[1], handle) server.start() try: conn = socket.create_connection(('127.0.0.1', 10011)) conn.sendall('msg1') sleep(0.1) self.popen.send_signal(15) sleep(0.1) conn.sendall('msg2') conn.close() with gevent.Timeout(0.1): self.popen.wait() finally: server.close() self.assertEqual(['msg1', 'msg2'], log)
def _run_all_tests(self): log = [] def handle(sock, _address): while True: data = sock.recv(1024) print('got %r' % data) if not data: break log.append(data) server = StreamServer(self.example_args[1], handle) server.start() try: conn = socket.create_connection(('127.0.0.1', 10011)) conn.sendall(b'msg1') sleep(0.1) # On Windows, SIGTERM actually abruptly terminates the process; # it can't be caught. However, CTRL_C_EVENT results in a KeyboardInterrupt # being raised, so we can shut down properly. self.popen.send_signal( getattr(signal, 'CTRL_C_EVENT', signal.SIGTERM)) sleep(0.1) conn.sendall(b'msg2') conn.close() with gevent.Timeout(2.1): self.popen.wait() finally: server.close() self.assertEqual([b'msg1', b'msg2'], log)
def close(self): if self.closed: sys.exit('Multiple exit signals received - aborting.') else: note('Closing socket') send_bye(self.socket) StreamServer.close()
def stop(self): """Shutting down the server. """ if self.closed: sys.exit("Multiple exit signals received - aborting.") else: j.logger.debug("Closing listener socket") StreamServer.close(self)
def run(self): servers = [] # init thrift transport&protocol objects self.tfactory = TTransport.TTransportFactoryBase() self.pfactory = TBinaryProtocolFactoryExt() for s in self.sockets: s.setblocking(1) pool = Pool(self.worker_connections) hfun = partial(self.handle, s) server = StreamServer(s, handle=hfun, spawn=pool) server.start() servers.append(server) try: while self.alive: self.notify() gevent.sleep(0.1) except KeyboardInterrupt: pass except: for server in servers: try: server.stop() except: pass raise try: # Stop accepting requests for server in servers: if hasattr(server, 'close'): # gevent 1.0 server.close() if hasattr(server, 'kill'): # gevent < 1.0 server.kill() # Handle current requests until graceful_timeout ts = time.time() while time.time() - ts <= self.cfg.graceful_timeout: accepting = 0 for server in servers: if server.pool.free_count() != server.pool.size: accepting += 1 # if no server is accepting a connection, we can exit if not accepting: return self.notify() gevent.sleep(1.0) # Force kill all active the handlers self.log.warning("Worker graceful timeout (pid:%s)" % self.pid) [server.stop(timeout=1) for server in servers] except: pass
def stop(self, *args, **kwargs): self.logger.info("Stratum server {address}:{port} stopping" .format(**self.config)) StreamServer.close(self) for client in self.clients.values(): client.stop() for serv in self.agent_servers: serv.stop() Component.stop(self)
def stop(self, *args, **kwargs): self.logger.info("Stratum server {} stopping".format(self.listener)) StreamServer.close(self) for serv in self.agent_servers: serv.stop() for client in self.clients.values(): client.stop() StreamServer.stop(self) Component.stop(self) self.logger.info("Exit")
def start_server(port=5000): # pragma: no cover """Start stream server.""" patch_all() sserver = StreamServer(('127.0.0.1', port), connection) print('Starting server on port {}'.format(port)) try: sserver.serve_forever() except KeyboardInterrupt: sserver.close() print('Server closed')
def server(): """server to continue serving.""" try: patch_all() server = StreamServer(('127.0.0.1', 8000), send_response) print('Serving on 8000') server.serve_forever() except KeyboardInterrupt: server.close() print('closed server') sys.exit()
def server(): # pragma: no cover """Instantiate a new server to serve forever.""" try: patch_all() s = StreamServer(('127.0.0.1', 3000), send_http_response) print('Starting server on port 3000') s.serve_forever() except KeyboardInterrupt: s.close() print('Server closed') sys.exit()
def bind_server(ip, port, spawn_limit): """ 创建服务 :return: """ try: server = StreamServer((ip, port), read_from_client, spawn=spawn_limit) # 创建新的服务器 # server.start() # 开始接受新的连接 server.serve_forever() #启动服务器,一直等待,直到终端或服务器停止 except Exception as e: print(str(e)) server.close() if server is not None and server.started else None
class MessageHandler(gevent.greenlet.Greenlet): def __init__(self): super(MessageHandler, self).__init__() self.server = StreamServer( self.bind_unix_listener(), self.read_socket ) def read_socket(self, sock, address): try: self.data = sock.recv(RECEIVE_DATA_SIZE) message = Message.from_json(self.data) Logger(message) except (socket_error, socket_timeout): exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception( exc_type, exc_value, exc_tb, file=sys.stderr) except (TypeError, ValueError, KeyError, AttributeError): sys.stderr.write( "Unable to log the message.%s\n" % self.data) exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception( exc_type, exc_value, exc_tb, file=sys.stderr) def _run(self): try: self.server.serve_forever() except (TypeError, BlockingIOError, socket_error, ValueError): exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception( exc_type, exc_value, exc_tb, file=sys.stderr) def stop(self): socket_path = tendrl_ns.config.data['logging_socket_path'] self.sock.close() if os.path.exists(socket_path): os.remove(socket_path) self.server.close() def bind_unix_listener(self): socket_path = tendrl_ns.config.data['logging_socket_path'] self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: if os.path.exists(socket_path): os.remove(socket_path) self.sock.setblocking(0) self.sock.bind(socket_path) self.sock.listen(50) except (TypeError, BlockingIOError, socket_error, ValueError): exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception( exc_type, exc_value, exc_tb, file=sys.stderr) return self.sock
class BroadcastServer: def __init__(self, message=None): self.message = message self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.clients = dict() def handler(self, client_socket, client_address): """Handles the incoming client connection""" self.clients[client_address] = client_socket logger.info('Incoming Connection from client {}:{}'.format( client_address[0], client_address[1])) client_socket.sendall( b'Welcome to the broadcast server! Type quit to exit.\r\n') # using a makefile because we want to use readline() rfileobj = client_socket.makefile(mode='rb') while True: line = rfileobj.readline() for client in self.clients.items(): if client[0] == client_address: text = b'You have entered: ' + line client_socket.sendall(text) else: text = 'client {} has entered data: {}'.format( client_address, line.decode()) client[1].sendall(text.encode()) if not line: logger.info("Client {}:{} disconnected".format( client_address[0], client_address[1])) break if line.strip().lower() == b'quit': logger.info("Client {}:{} quit".format(client_address[0], client_address[1])) break # client_socket.sendall(line) logger.info("Echoed %r from client %s:%s", line, client_address[0], client_address[1]) rfileobj.close() def start(self, host, port): """Starts the Broadcast Web server""" conn = (host, port) self.listener = StreamServer(conn, self.handler) try: logger.info('Starting BroadcastServer on {}'.format(conn)) self.listener.serve_forever() except: self.stop() def stop(self): """Stops the Broadcast Server""" logger.info('Stopping Broadcast Server') self.listener.close()
def bind_server(ip, port, spawn_limit): """ when called, this will bind a tcp server server using gevent for concurrency :return: """ try: server = StreamServer((ip, port), read_from_client, spawn=spawn_limit) # creates a new server # server.start() server.serve_forever() except Exception as e: print(str(e)) server.close() if server is not None and server.started else None
def bind_server(ip, port, spawn_limit): '''创建一个服务器 Arguments: ip {[type]} -- [description] port {[type]} -- [description] spawn_limit {[type]} -- [description] ''' try: server = StreamServer((ip, port), read_from_client, spawn=spawn_limit) # 创建一个服务器 server.serve_forever() # 启动服务器一直等待,知道终端或服务端终止 except Exception as e: print(str(e)) server.close() if server is not None and server.started else None
def main(): args = sys.argv[1:] config.parse_config(args) check_root_user() init_logger() setup_pid_file() from server import handle_sftp_session server = StreamServer( ('0.0.0.0', int(config.options.get('sftp_port', 2200))), handle_sftp_session) _logger.info('Solt SFTP server is running and waiting for connections...') try: server.serve_forever() except (SystemExit, KeyboardInterrupt): server.close()
def start_server(port): assert type(port) == int try: pool = Pool(256) server = StreamServer(('0.0.0.0', port), CacheProxy(Cache(cache_dir, **args)), spawn=pool) server.serve_forever() except Exception as e: print(e) server.close() del pool del server finally: server.close() del pool del server print("Closed Server")
def run(self): servers = [] ssl_args = {} for s in self.sockets: s.setblocking(1) pool = Pool(self.worker_connections) # TODO(benjamin): process handler hfun = partial(self.handler, s) server = StreamServer(s, handle=hfun, spawn=pool, **ssl_args) server.start() servers.append(server) while self.alive: self.notify() gevent.sleep(1.0) try: # Stop accepting requests for server in servers: server.close() # Handle current requests until graceful_timeout ts = time.time() while time.time() - ts <= self.config.GRACEFUL_TIMEOUT: accepting = 0 for server in servers: if server.pool.free_count() != server.pool.size: accepting += 1 # if no server is accepting a connection, we can exit if not accepting: return self.notify() gevent.sleep(1.0) # Force kill all active the handlers self.logger.warning("Worker graceful timeout (pid:%s)" % self.pid) [server.stop(timeout=1) for server in servers] except: pass
class ReportServer(Server): def __init__(self, port, task_queue): self._port = port self._serving = False self._serving_greenlet = None self._report_server = None self._logger = logging.getLogger(__name__) self._task_queue = task_queue def __report_receiver(self, sock, address): report = "" fp = sock.makefile("rb") while True: line = fp.readline() if not line: break report += line fp.flush() sock.shutdown(socket.SHUT_WR) sock.close() self._task_queue.put((address[0], report)) def __serve(self): self._logger.info("[ReportServer] initialized on port " + str(self._port) + " ...") self._report_server = StreamServer(('', self._port), self.__report_receiver) self._report_server.serve_forever() gevent.sleep(0) def start_server(self): if not self._serving: self._serving_greenlet = gevent.spawn(self.__serve) self._serving = True gevent.sleep(0) def stop_server(self): if self._serving: gevent.kill(self._serving_greenlet) self._serving = False self._report_server.close() self._logger.info("[ReportServer] shut down")
class Listener: def __init__(self, port, task_queue): self._port = port self._serving = False self._serving_greenlet = None self._node_server = None self._logger = logging.getLogger(__name__) self._task_queue = task_queue def __listener_receiver(self, sock, address): job = "" fp = sock.makefile() while True: line = fp.readline() if line: job += line fp.flush() else: break sock.shutdown(socket.SHUT_WR) sock.close() self._logger.debug("Received a job from " + address[0]) self._task_queue.put([address[0], job]) def __serve(self): self._logger.info("[Listener] initialized on port " + str(self._port) + " ...") self._node_server = StreamServer(('', self._port), self.__listener_receiver) self._node_server.serve_forever() def serve(self): if not self._serving: self._serving_greenlet = gevent.spawn(self.__serve) self._serving = True gevent.sleep(0) else: pass def stop(self): if self._serving: self._node_server.close()
class LibeventServer: """ Simple Libevent Based Server capable of handling multiple clients at a time. In python, we are using gevent which is python's port to Libevent library """ def __init__(self): self.server = None # Initialize later def handler(self, client_socket, client_address): """Handles the incoming client connection""" logger.info('Incoming Connection from client:', client_address) client_socket.sendall( b'Welcome to the echo server! Type quit to exit.\r\n') # using a makefile because we want to use readline() rfileobj = client_socket.makefile(mode='rb') while True: line = rfileobj.readline() if not line: logger.info("Client disconnected") break if line.strip().lower() == b'quit': logger.info("Client quit") break client_socket.sendall(line) logger.info("Echoed %r", line) rfileobj.close() def start(self, host, port): """Starts the Libevent Web server""" conn = (host, port) self.server = StreamServer(conn, self.handler) try: logger.info('Starting Libevent Server on port 16000') self.server.serve_forever() except: self.stop() def stop(self): """Stops the Libevent Server""" logger.info('Stopping Libevent Server') self.server.close()
def test(self): log = [] def handle(socket, address): while True: data = socket.recv(1024) log.append(data) if not data: break server = StreamServer('127.0.0.6:9999', handle) server.start() try: conn = socket.create_connection(('127.0.0.5', 9999)) conn.sendall('msg1') gevent.sleep(0.1) self.assertEqual(log, ['msg1']) conn.sendall('msg2') conn.close() finally: server.close()
class Listener: def __init__(self, port, task_queue): self._port = port self._serving = False self._serving_greenlet = None self._node_server = None self._logger = logging.getLogger(__name__) self._task_queue = task_queue def __listener_receiver(self, sock, address): job = "" fp = sock.makefile() while True: line = fp.readline() if line: job += line fp.flush() else: break sock.shutdown(socket.SHUT_WR) sock.close() self._logger.debug("Received a job from " + address[0]) self._task_queue.put([address[0], job]) def __serve(self): self._logger.info("[Listener] initialized on port " + str(self._port) + " ...") self._node_server = StreamServer(("", self._port), self.__listener_receiver) self._node_server.serve_forever() def serve(self): if not self._serving: self._serving_greenlet = gevent.spawn(self.__serve) self._serving = True gevent.sleep(0) else: pass def stop(self): if self._serving: self._node_server.close()
def _run_all_tests(self): log = [] def handle(socket, address): while True: data = socket.recv(1024) print('got %r' % data) if not data: break log.append(data) server = StreamServer(self.args[1], handle) server.start() try: conn = socket.create_connection(('127.0.0.1', 10011)) conn.sendall('msg1') sleep(0.1) self.popen.send_signal(15) sleep(0.1) try: conn.sendall('msg2') conn.close() except socket.error: if sys.platform != 'win32': raise # On Windows, signal/15 kills the process rather than actually sends a signal # so, sendall('msg2') fails with # error: [Errno 10054] An existing connection was forcibly closed by the remote host # XXX maybe it could be made working with CTRL_C_EVENT somehow? with gevent.Timeout(0.1): self.popen.wait() finally: server.close() if sys.platform == 'win32': self.assertEqual(['msg1'], log) else: self.assertEqual(['msg1', 'msg2'], log)
def _run_all_tests(self): log = [] def handle(socket, address): while True: data = socket.recv(1024) print('got %r' % data) if not data: break log.append(data) server = StreamServer(self.args[1], handle) server.start() try: conn = socket.create_connection(('127.0.0.1', 10011)) conn.sendall(b'msg1') sleep(0.1) self.popen.send_signal(15) sleep(0.1) try: conn.sendall(b'msg2') conn.close() except socket.error: if sys.platform != 'win32': raise # On Windows, signal/15 kills the process rather than actually sends a signal # so, sendall('msg2') fails with # error: [Errno 10054] An existing connection was forcibly closed by the remote host # XXX maybe it could be made working with CTRL_C_EVENT somehow? with gevent.Timeout(0.1): self.popen.wait() finally: server.close() if sys.platform == 'win32': self.assertEqual([b'msg1'], log) else: self.assertEqual([b'msg1', b'msg2'], log)
class Firehose(object): '''Listener object to receive data (Events, Detects or Audit) from a limacharlie.io Organization in push mode.''' def __init__(self, manager, listen_on, data_type, public_dest=None, name=None, ssl_cert=None, ssl_key=None, is_parse=True, max_buffer=1024, inv_id=None, tag=None, cat=None, sid=None): '''Create a listener and optionally register it with limacharlie.io automatically. If name is None, the Firehose will assume the Output is already created and will skip it's initialization and teardown. If public_dest is None and name is not None, initialization of the Output will use the dynamically detected public IP address of this host and port specified in listen_on. Args: manager (limacharlie.Manager obj): a Manager to use for interaction with limacharlie.io. listen_on (str): the interface and port to listen on for data from the cloud, ex: "1.2.3.4:443", "0.0.0.0:443", ":443". data_typer (str): the type of data received from the cloud as specified in Outputs (event, detect, audit). public_dest (str): the IP and port that limacharlie.io should use to connect to this object. name (str): name to use to register as an Output on limacharlie.io. ssl_cert (str): optional, path to file with (PEM) ssl cert to use to receive from the cloud, if not set generates self-signed certs. ssl_key (str): optional, path to the file with (PEM) ssl key to use to receive from the cloud, if not set generates self-signed certs. is_parse (bool): if set to True (default) the data will be parsed as JSON to native Python. max_buffer (int): the maximum number of messages to buffer in the queue. inv_id (str): only receive events marked with this investigation ID. tag (str): only receive Events from Sensors with this Tag. cat (str): only receive Detections of this Category. sid (str): only receive Detections and Events from this Sensor. ''' self._manager = manager self._listen_on = listen_on.split(':') if 1 < len(self._listen_on): self._listen_on_port = int(self._listen_on[1]) self._listen_on = self._listen_on[0] else: self._listen_on = self._listen_on[0] self._listen_on_port = 443 if '' == self._listen_on: self._listen_on = '0.0.0.0' self._data_type = data_type self._public_dest = public_dest if public_dest != '' else None self._name = name self._output_name = None self._is_parse = is_parse self._max_buffer = max_buffer self._dropped = 0 self._ssl_cert = ssl_cert self._ssl_key = ssl_key if self._ssl_cert is not None and not os.path.isfile(self._ssl_cert): raise LcApiException('No cert file at path: %s' % self._ssl_cert) if self._ssl_key is not None and not os.path.isfile(self._ssl_key): raise LcApiException('No key file at path: %s' % self._ssl_key) if self._data_type not in ('event', 'detect', 'audit'): raise LcApiException('Invalid data type: %s' % self._data_type) # Setup internal structures. self.queue = Queue(maxsize=self._max_buffer) if self._ssl_cert is None or self._ssl_key is None: # Generate certs. _, tmpKey = tempfile.mkstemp() _, tmpCert = tempfile.mkstemp() if 0 != os.system( 'openssl req -x509 -days 36500 -newkey rsa:4096 -keyout %s -out %s -nodes -sha256 -subj "/C=US/ST=CA/L=Mountain View/O=refractionPOINT/CN=limacharlie_firehose" > /dev/null 2>&1' % (tmpKey, tmpCert)): raise LcApiException( "Failed to generate self-signed certificate.") else: # Use the keys provided. tmpKey = self._ssl_key tmpCert = self._ssl_cert # Start the server. self._sslCtx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) self._sslCtx.load_cert_chain(certfile=tmpCert, keyfile=tmpKey) self._sslCtx.set_ciphers('ECDHE-RSA-AES128-GCM-SHA256') self._server = StreamServer((self._listen_on, self._listen_on_port), self._handleNewClient) self._server.start() self._manager._printDebug('Listening for connections.') # If the name is specified we assume the user wants us to register # the firehose directly using the API. # If the name is not present, we assume the user has registered it # manually somehow. if self._name is not None: self._manager._printDebug('Registration required.') self._output_name = 'py_lc_fh_%s' % self._name # Check if the output already exists. outputs = self._manager.outputs() if self._output_name not in outputs: # It's not there, register it. effectiveDest = self._public_dest if effectiveDest is None: effectiveDest = '%s:%s' % (self._getPublicIp(), self._listen_on_port) if (self._ssl_cert is not None) and (self._ssl_key is not None): isStrict = 'true' else: isStrict = 'false' kwOutputArgs = { 'dest_host': effectiveDest, 'is_tls': 'true', 'is_strict_tls': isStrict, 'is_no_header': 'true', } if inv_id is not None: kwOutputArgs['inv_id'] = inv_id if tag is not None: kwOutputArgs['tag'] = tag if cat is not None: kwOutputArgs['cat'] = cat if sid is not None: kwOutputArgs['sid'] = sid self._manager.add_output(self._output_name, 'syslog', self._data_type, **kwOutputArgs) self._manager._printDebug('Registration done.') else: self._manager._printDebug('Registration already done.') else: self._manager._printDebug('Registration not required.') def shutdown(self): '''Stop receiving data and potentially unregister the Output (if created here).''' if self._name is not None: self._manager._printDebug('Unregistering.') self._manager.del_output(self._output_name) self._server.close() self._manager._printDebug('Closed.') def getDropped(self): '''Get the number of messages dropped because queue was full.''' return self._dropped def resetDroppedCounter(self): '''Reset the counter of dropped messages.''' self._dropped = 0 def _getPublicIp(self): return json.load(urllib2.urlopen('http://jsonip.com'))['ip'] def _handleNewClient(self, sock, address): self._manager._printDebug('new firehose connection: %s' % (address, )) sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 5) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 10) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 2) try: sock = self._sslCtx.wrap_socket(sock, server_side=True, do_handshake_on_connect=True, suppress_ragged_eofs=True) except: print(traceback.format_exc()) self._manager._printDebug('firehose connection closed: %s' % (address, )) return curData = [] while True: try: data = sock.recv(8192) if not data: break if '\n' in data: chunks = data.split('\n') curData.append(chunks[0]) try: if self._is_parse: self.queue.put_nowait(json.loads(''.join(curData))) else: self.queue.put_nowait(''.join(curData)) except: self.dropped += 1 for c in chunks[1:-1]: try: if self._is_parse: self.queue.put_nowait(json.loads(c)) else: self.queue.put_nowait(c) except: self.dropped += 1 curData = [chunks[-1]] else: curData.append(data) except: self._manager._printDebug('error decoding data') self._manager._printDebug('firehose connection closed: %s' % (address, )) sock.close()
def run(self): servers = [] ssl_args = {} if self.cfg.is_ssl: ssl_args = dict(server_side=True, do_handshake_on_connect=False, **self.cfg.ssl_options) for s in self.sockets: s.setblocking(1) pool = Pool(self.worker_connections) if self.server_class is not None: server = self.server_class(s, application=self.wsgi, spawn=pool, log=self.log, handler_class=self.wsgi_handler, **ssl_args) else: hfun = partial(self.handle, s) server = StreamServer(s, handle=hfun, spawn=pool, **ssl_args) server.start() servers.append(server) try: while self.alive: self.notify() gevent.sleep(1.0) except KeyboardInterrupt: pass except: try: server.stop() except: pass raise try: # Stop accepting requests for server in servers: if hasattr(server, 'close'): # gevent 1.0 server.close() if hasattr(server, 'kill'): # gevent < 1.0 server.kill() # Handle current requests until graceful_timeout ts = time.time() while time.time() - ts <= self.cfg.graceful_timeout: accepting = 0 for server in servers: if server.pool.free_count() != server.pool.size: accepting += 1 # if no server is accepting a connection, we can exit if not accepting: return self.notify() gevent.sleep(1.0) # Force kill all active the handlers self.log.warning("Worker graceful timeout (pid:%s)" % self.pid) [server.stop(timeout=1) for server in servers] except: pass
class ReplayServer: def __init__(self, bind_address): self.server = StreamServer(bind_address, self.connect_handler) self.server.init_socket() self.server.socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) self.clients = set() self.replay_streams = WeakValueDictionary() self.replay_streamers = set() self.replay_peers = set() def run(self): log.info('Live Replay Server listening on %s:%s', self.server.server_host, self.server.server_port) self.server.serve_forever() def stop(self): log.info('Shutting down gracefully.') self.server.close() log.info('Listen socket closed.') self.server.stop() log.info('All streams finished.') def create_stream(self, game_id): "Gets or creates stream for posting." stream = self.replay_streams.get(game_id) if not stream: stream = ReplayStream(game_id) self.replay_streams[game_id] = stream file_peer = ReplayFilePeer(open(pjoin(config.REPLAY_FOLDER, '%d.scfareplay' % game_id), 'wb')) gevent.spawn(stream.stream_steps, file_peer) return stream def connect_handler(self, client_socket, client_address): self.clients.add(client_socket) log.info('Connection from %s. [%3d clients]', client_address, len(self.clients)) def readNulString(socket): s = b'' c = socket.recv(1) while c != b'\x00': s = s + c c = socket.recv(1) return s.decode() try: # gpgnet:// old-style connection gpg_head = readNulString(client_socket) replay_name = gpg_head[1:] game_id = int(replay_name.split("/")[1]) if gpg_head[0] == 'P': # 'P'osting log.info('%s POST %s', client_address, replay_name) if replay_name.endswith(".gwreplay"): galactic_war = True elif replay_name.endswith('.fafreplay'): galactic_war = False elif replay_name.endswith(".scfareplay"): log.exception("Can't handle .scfareplay: %s", replay_name) else: log.exception('Unknown replay extension: %s', replay_name) stream = self.create_stream(game_id) streamer = ReplayStreamer(stream, client_socket, replay_name, game_id) with keepref(streamer, self.replay_streamers): try: streamer.read_stream() except OSError: pass # Disconnected elif gpg_head[0] == 'G': # 'G'etting log.info('%s GET %s', client_address, replay_name) stream = self.replay_streams.get(game_id) if not stream: raise UnknownReplay('%s requested unknown replay: %s' % (client_address, replay_name)) log.info('Connecting %s to %s', client_address, stream) peer = ReplayPeer(client_socket) with keepref(peer, self.replay_peers): try: stream.stream_steps(peer) except BrokenPipeError: pass # Disconnected else: raise UnknownMethod('%s unknown method: %s' % (client_address, gpg_head)) finally: if not client_socket.closed: client_socket.shutdown(2) self.clients.remove(client_socket) log.info('%s disconnected. [%3d clients]', client_address, len(self.clients))
def close(self): if self.closed: sys.exit('Multiple exit signals received - aborting.') else: log('Closing listener socket') StreamServer.close(self)
class BaseServer(ABC): bytes_to_receive: int = 65535 buffer_size: int = io.DEFAULT_BUFFER_SIZE # noinspection PyTypeChecker def __init__(self, host: str, port: int): self._check_init_arguments(host, port) self._host = host self._port = port self._server: StreamServer = None self._ws = WSConnection(ConnectionType.SERVER) self._client: socket = None # client socket provided by the StreamServer self._running = True @staticmethod def _check_ws_headers(headers: Headers) -> None: if headers is None: return error_message = 'headers must of a list of tuples of the form [(bytes, bytes), ..]' if not isinstance(headers, list): raise TypeError(error_message) try: for key, value in headers: if not isinstance(key, bytes) or not isinstance(value, bytes): raise TypeError(error_message) except ValueError: # in case it is not a list of tuples raise TypeError(error_message) @abstractmethod def handle_request(self, request: Request) -> None: pass def accept_request(self, extra_headers: Headers = None, sub_protocol: str = None) -> None: self._check_ws_headers(extra_headers) if sub_protocol is not None and not isinstance(sub_protocol, str): raise TypeError('sub_protocol must be a string') extra_headers = extra_headers if extra_headers else [] self._client.sendall( self._ws.send( AcceptConnection(extra_headers=extra_headers, subprotocol=sub_protocol))) def reject_request(self, status_code: int = 400, reason: str = None) -> None: if not isinstance(status_code, int): raise TypeError('status_code must be an integer') if reason is not None and not isinstance(reason, str): raise TypeError('reason must be a string') if not reason: self._client.sendall( self._ws.send(RejectConnection(status_code=status_code))) else: data = bytearray( self._ws.send( RejectConnection(has_body=True, headers=[(b'Content-type', b'text/txt') ]))) data.extend(self._ws.send(RejectData(reason.encode()))) self._client.sendall(bytes(data)) def close_request(self, code: int = 1000, reason: str = None) -> None: if not isinstance(code, int): raise TypeError('code must be an integer') if not isinstance(reason, str): raise TypeError('reason must be a string') self._client.sendall(self._ws.send(CloseConnection(code, reason))) def _handle_close_event(self, event: CloseConnection) -> None: if self._ws.state is ConnectionState.REMOTE_CLOSING: self._client.sendall(self._ws.send(event.response())) def _handle_ping(self, event: Ping) -> None: self._client.sendall(self._ws.send(event.response())) @abstractmethod def receive_text(self, data: str) -> None: pass @abstractmethod def receive_json(self, data: Any) -> None: pass @abstractmethod def receive_bytes(self, data: bytes) -> None: pass @abstractmethod def handle_pong(self, data: bytes) -> None: pass def _send_data(self, data: AnyStr) -> None: if isinstance(data, str): io_object = io.StringIO(data) else: io_object = io.BytesIO(data) with io_object as f: chunk = f.read(self.buffer_size) while chunk: if len(chunk) < self.buffer_size: self._client.sendall( self._ws.send(Message(data, message_finished=True))) break else: self._client.sendall( self._ws.send(Message(data, message_finished=False))) chunk = f.read(self.buffer_size) def ping(self, data: bytes = b'hello') -> None: if not isinstance(data, bytes): raise TypeError('data must be bytes') self._client.sendall(self._ws.send(Ping(data))) def send(self, data: AnyStr) -> None: if not isinstance(data, (bytes, str)): raise TypeError('data must be either a string or binary data') self._send_data(data) def send_json(self, data: Any) -> None: self.send(json.dumps(data)) @staticmethod def _check_init_arguments(host: str, port: int) -> None: if not isinstance(host, str): raise TypeError('host must be a string') error_message = 'custom_port must a positive integer' if not isinstance(port, int): raise TypeError(error_message) if port < 0: raise TypeError(error_message) def _handler(self, client: socket, address: Tuple[str, int]) -> None: self._client = client text_message = [] binary_message = bytearray() while self._running: data = client.recv(self.bytes_to_receive) self._ws.receive_data(data) for event in self._ws.events(): if isinstance(event, Request): self.handle_request(event) elif isinstance(event, CloseConnection): self._handle_close_event(event) self._running = False elif isinstance(event, Ping): self._handle_ping(event) elif isinstance(event, Pong): self.handle_pong(event.payload) elif isinstance(event, TextMessage): text_message.append(event.data) if event.message_finished: str_data = ''.join(text_message) try: self.receive_json(json.loads(str_data)) except json.JSONDecodeError: self.receive_text(str_data) text_message.clear() elif isinstance(event, BytesMessage): binary_message.extend(event.data) if event.message_finished: self.receive_bytes(bytes(binary_message)) binary_message.clear() else: print('unknown event:', event) def run(self, backlog: int = 256, spawn: str = 'default', **kwargs) -> None: self._server = StreamServer((self._host, self._port), self._handler, backlog=backlog, spawn=spawn, **kwargs) self._server.serve_forever() def close(self) -> None: if self._server is not None: self._server.close()
class ThriftServer: def __init__(self, module, handler_class, addr, max_process=1, max_conn=1000): module.handler = handler_class() global service service = module self.proc = None self.workers = [] self.running = True pool = Pool(max_conn) self.server = StreamServer(addr, handle, spawn=pool) self.server.reuse_addr = 1 self.server.start() def signal_master_handler(signum, frame): log.warn("signal %d catched in master %d, wait for kill all worker", signum, os.getpid()) self.running = False for p in self.workers: p.terminate() def signal_worker_handler(signum, frame): log.warn("worker %d will exit after all request handled", os.getpid()) self.server.close() def server_start(): signal.signal(signal.SIGTERM, signal_worker_handler) log.warn('server started addr=%s:%d pid=%d', addr[0], addr[1], os.getpid()) if hasattr(service.handler, '_initial'): service.handler._initial() self.server.serve_forever() def _start_process(index): server_name = 'proc-%02d' % index p = multiprocessing.Process(target=server_start, name=server_name) p.start() return p def signal_child_handler(signum, frame): time.sleep(1) if self.running: log.warn("master recv worker exit, fork one") try: pinfo = os.waitpid(-1, 0) pid = pinfo[0] index = -1 for i in range(0, len(self.workers)): p = self.workers[i] if p.pid == pid: index = i break if index >= 0: self.workers[index] = _start_process(index) except OSError: log.info('waitpid error:') if max_process == 1: signal.signal(signal.SIGTERM, signal_worker_handler) gevent.spawn(self.forever) server_start() else: for i in range(0, max_process): self.workers.append(_start_process(i)) signal.signal(signal.SIGTERM, signal_master_handler) signal.signal(signal.SIGCHLD, signal_child_handler) def forever(self, report=None): try: while self.running: if len(self.workers) > 0: time.sleep(60) else: gevent.sleep(60) if report: report() log.warn('master exit') except Exception, e: log.warn(traceback.format_exc()) log.warn('master exception: %s', str(e)) finally:
def close(self): self.log.info("%-8s: %s@%s" % ('svrCLOSE', repr(self.listener), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) StreamServer.close(self)
def close(self): print 'Closing listener socket' StreamServer.close(self)
def close(self): if self.closed: logging.critical('Multiple exit signals received - aborting.') sys.exit('Multiple exit signals received - aborting.') else: StreamServer.close(self)
def close(self): if not self.closed: self.logger.info("Close Server") StreamServer.close(self)
class ThriftServer(ThriftBaseServer): def __init__(self, module, handler_class, addr, max_process=1, max_conn=1000): module.handler = handler_class() global service service = module self.addr = addr self.proc = None self.workers = [] self.running = True pool = Pool(max_conn) self.server = StreamServer(addr, handle, spawn=pool) self.server.reuse_addr = 1 self.server.start() def signal_master_handler(signum, frame): log.warn( "signal %d catched in master %d, wait for kill all worker", signum, os.getpid()) self.running = False for p in self.workers: p.terminate() def signal_worker_handler(signum, frame): log.warn("worker %d will exit after all request handled", os.getpid()) self.server.close() def signal_child_handler(signum, frame): time.sleep(1) log.warn("master recv worker exit") try: pinfo = os.waitpid(-1, 0) pid = pinfo[0] index = -1 for i in range(0, len(self.workers)): p = self.workers[i] if p.pid == pid: index = i break except OSError: log.info('waitpid error:', traceback.format_exc()) if self.running: log.warn("master fork one") if index >= 0: self.workers[index] = _start_process(index) else: log.warn("master del worker %d", pid) if index >= 0: self.workers.pop(index) def server_start(): signal.signal(signal.SIGTERM, signal_worker_handler) log.warn('server started addr=%s:%d pid=%d', self.addr[0], self.addr[1], os.getpid()) self.install() if hasattr(service.handler, '_initial'): service.handler._initial() self.server.serve_forever() def _start_process(index): server_name = 'proc-%02d' % index p = multiprocessing.Process(target=server_start, name=server_name) p.start() return p if max_process == 1: signal.signal(signal.SIGTERM, signal_worker_handler) #gevent.spawn(self.forever) server_start() else: for i in range(0, max_process): self.workers.append(_start_process(i)) signal.signal(signal.SIGTERM, signal_master_handler) signal.signal(signal.SIGCHLD, signal_child_handler)
class KyotoTycoonMockServer(object): def __init__(self): self.data = {} self.command_logs = deque([]) self.stream_server = None def __call__(self, sock, address): """StreamServer framework will call this instance as callable object""" self.sock = sock self.address = address self._run() def _run(self): """receive log data and put into each queue""" while True: # determine command type (command,) = struct.unpack('!B', self._read(1)) if command == MB_SET_BULK: # determine number of key-value pairs flag, num_data = struct.unpack('!II', self._read(4+4)) # receive pairs data_to_set = {} for i in range(num_data): # !HIIq => 2 + 4 + 4 + 8 kv_header = self._read(2+4+4+8) zero, key_len, val_len, lifetime = struct.unpack('!HIIq', kv_header) key = self.sock.recv(key_len) val = self.sock.recv(val_len) data_to_set[key] = val # update cache memory self.data.update(data_to_set) # log command self.command_logs.append(dict( command='set_bulk', num_data=num_data, values=data_to_set)) # reply if flag != FLAG_NOREPLY: reply = struct.pack('!BI', MB_SET_BULK, len(data_to_set)) self.sock.send(reply) elif command == MB_GET_BULK: # command: get bulk zero, num_keys = struct.unpack('!II', self._read(4+4)) keys = [] for i in range(num_keys): k_header = self._read(2+4) zero, key_len = struct.unpack('!HI', k_header) key = self.sock.recv(key_len) keys.append(key) self.command_logs.append(dict( command='get_bulk', num_keys=num_keys, keys=keys)) found_keys = [k for k in keys if k in self.data] # reply self.sock.send(struct.pack('!BI', MB_GET_BULK, len(found_keys))) for key in found_keys: value = self.data[key] kv_data = struct.pack('!HIIq', 0, len(key), len(value), 0) self.sock.send(kv_data) self.sock.send(key) self.sock.send(value) elif command == MB_REMOVE_BULK: # command: remove bulk flag, num_keys = struct.unpack('!II', self._read(4+4)) keys = [] for i in range(num_keys): k_header = self._read(2+4) zero, key_len = struct.unpack('!HI', k_header) key = self.sock.recv(key_len) keys.append(key) self.command_logs.append(dict( command='remove_bulk', num_keys=num_keys, keys=keys)) found_keys = [k for k in keys if k in self.data] for k in found_keys: del self.data[k] # reply if flag != FLAG_NOREPLY: self.sock.send(struct.pack('!BI', MB_REMOVE_BULK, len(found_keys))) else: pass # unsupported operation def _read(self, length): buf = '' readlen = 0 while len(buf) < length: chunk = self.sock.recv(length - readlen) if chunk == '': gevent.sleep() buf += chunk readlen += len(chunk) return buf def clear_data(self): self.data = {} self.command_logs = deque([]) def wait(self, n, timeout_msec=0): """wait until log data queue named with 'tag' is filled with 'n' items""" time_msec_start = int(time.time() * 1000) while len(self.data) < n: gevent.sleep() if 0 < timeout_msec and time_msec_start + timeout_msec < int(time.time() * 1000): raise KTMockTimeOutError('wait(n=%d) timed out' % n) def start(self, port=1978, bind_address='127.0.0.1'): if hasattr(self, 'stream_server') and self.stream_server is not None: raise StandardError('server is already started') self.stream_server = StreamServer((bind_address, port), self) self.stream_server.start() def stop(self): if self.stream_server is None: raise StandardError('server has not started yet') self.stream_server.stop() self.stream_server.close() self.stream_server = None
def run(self): servers = [] ssl_args = {} if self.cfg.is_ssl: ssl_args = dict(server_side=True, do_handshake_on_connect=False, **self.cfg.ssl_options) for s in self.sockets: s.setblocking(1) pool = Pool(self.worker_connections) if self.server_class is not None: server = self.server_class( s, application=self.wsgi, spawn=pool, log=self.log, handler_class=self.wsgi_handler, **ssl_args) else: hfun = partial(self.handle, s) server = StreamServer(s, handle=hfun, spawn=pool, **ssl_args) server.start() servers.append(server) try: while self.alive: self.notify() gevent.sleep(1.0) except KeyboardInterrupt: pass except: try: server.stop() except: pass raise try: # Stop accepting requests for server in servers: if hasattr(server, 'close'): # gevent 1.0 server.close() if hasattr(server, 'kill'): # gevent < 1.0 server.kill() # Handle current requests until graceful_timeout ts = time.time() while time.time() - ts <= self.cfg.graceful_timeout: accepting = 0 for server in servers: if server.pool.free_count() != server.pool.size: accepting += 1 # if no server is accepting a connection, we can exit if not accepting: return self.notify() gevent.sleep(1.0) # Force kill all active the handlers self.log.warning("Worker graceful timeout (pid:%s)" % self.pid) [server.stop(timeout=1) for server in servers] except: pass