def stop_session(fd): try: # unregister before we close s poller.unregister(fd) except BaseException as e: logger.error('unregister error:' + str(e)) session = self.fd_to_session.pop(fd) # this will close the socket session.stop()
def send_response(self, response): def default_decimal(obj): if isinstance(obj, Decimal): return float(obj) raise TypeError try: msg = json.dumps(response, default=default_decimal) + '\n' except BaseException as e: logger.error('send_response:' + str(e)) return self.response_queue.put(msg)
def run(self): if self.shared is None: raise TypeError("self.shared not set in Processor") while not self.shared.stopped(): session, request = self.pop_request() try: self.do_dispatch(session, request) except: logger.error('dispatch', exc_info=True) self.collect_garbage() self.stop()
def get_block(self, block_hash): block = self.lbrycrdd('getblock', (block_hash,)) while True: try: response = [self.lbrycrdd("getrawtransaction", (txid,)) for txid in block['tx']] except: logger.error("lbrycrdd error (getfullblock)") self.wait_on_lbrycrdd() continue block['tx'] = response return block
def get_block(self, block_hash): block = self.lbrycrdd('getblock', (block_hash, )) while True: try: response = [ self.lbrycrdd("getrawtransaction", (txid, )) for txid in block['tx'] ] except: logger.error("lbrycrdd error (getfullblock)") self.wait_on_lbrycrdd() continue block['tx'] = response return block
def add_request(self, session, request): # see if we can get if from cache. if not, add request to queue message_id = request.get('id') try: result = self.process(request, cache_only=False) except BaseException as e: print_log("Bad request from", session.address, str(type(e)), ":", str(e)) traceback.print_exc() self.push_response(session, {'id': message_id, 'error': str(e)}) return except: logger.error("process error", exc_info=True) print_log("error processing request from", session.address) print_log(str(request)) self.push_response(session, {'id': message_id, 'error': 'unknown error'}) if result == -1: self.queue.put((session, request)) else: self.push_response(session, {'id': message_id, 'result': result})
def add_request(self, session, request): # see if we can get if from cache. if not, add request to queue message_id = request.get('id') try: result = self.process(request, cache_only=False) except BaseException as e: print_log("Bad request from", session.address, str(type(e)), ":", str(e)) traceback.print_exc() self.push_response(session, {'id': message_id, 'error': str(e)}) return except: logger.error("process error", exc_info=True) print_log("error processing request from", session.address) print_log(str(request)) self.push_response(session, { 'id': message_id, 'error': 'unknown error' }) if result == -1: self.queue.put((session, request)) else: self.push_response(session, {'id': message_id, 'result': result})
def run(self): while not self.shared.stopped(): try: session, request = self.queue.get(True, timeout=1) msg_id = request.get('id') except: continue if session.stopped(): continue try: result = self.process(request) self.push_response(session, {'id': msg_id, 'result': result}) except BaseException, e: self.push_response(session, {'id': msg_id, 'error': str(e)}) except: logger.error("process error", exc_info=True) self.push_response(session, { 'id': msg_id, 'error': 'unknown error' }) self.close() class Dispatcher: def __init__(self, config): self.shared = Shared(config) self.request_dispatcher = RequestDispatcher(self.shared) self.request_dispatcher.start() self.response_dispatcher = \ ResponseDispatcher(self.shared, self.request_dispatcher)
def run(self): for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, cannonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.setblocking(0) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except socket.error: sock = None continue try: sock.bind(sa) sock.listen(5) except socket.error: sock.close() sock = None continue break host = sa[0] if af == socket.AF_INET6: host = "[%s]" % host if sock is None: print_log("could not open " + ("SSL" if self.use_ssl else "TCP") + " socket on %s:%d" % (host, self.port)) return print_log(("SSL" if self.use_ssl else "TCP") + " server started on %s:%d" % (host, self.port)) sock_fd = sock.fileno() poller = select.poll() poller.register(sock) def stop_session(fd): try: # unregister before we close s poller.unregister(fd) except BaseException as e: logger.error('unregister error:' + str(e)) session = self.fd_to_session.pop(fd) # this will close the socket session.stop() def check_do_handshake(session): if session.handshake: return try: session._connection.do_handshake() except ssl.SSLError as err: if err.args[0] == ssl.SSL_ERROR_WANT_READ: return elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: poller.modify(session.raw_connection, READ_WRITE) return else: raise BaseException(str(err)) poller.modify(session.raw_connection, READ_ONLY) session.handshake = True redo = [] while not self.shared.stopped(): if self.shared.paused(): sessions = self.fd_to_session.keys() if sessions: logger.info("closing %d sessions" % len(sessions)) for fd in sessions: stop_session(fd) time.sleep(1) continue if redo: events = redo redo = [] else: now = time.time() for fd, session in self.fd_to_session.items(): # Anti-DOS: wait 0.01 second between requests if now - session.time > 0.01 and session.message: cmd = session.parse_message() if not cmd: break if cmd == 'quit': data = False break session.time = now self.handle_command(cmd, session) # Anti-DOS: Stop reading if the session does not read responses if session.response_queue.empty(): mode = READ_ONLY elif session.response_queue.qsize() < 200: mode = READ_WRITE else: mode = WRITE_ONLY if mode != session.mode: poller.modify(session.raw_connection, mode) session.mode = mode # Collect garbage if now - session.time > session.timeout: stop_session(fd) events = poller.poll(TIMEOUT) for fd, flag in events: # open new session if fd == sock_fd: if flag & (select.POLLIN | select.POLLPRI): try: connection, address = sock.accept() session = TcpSession( self.dispatcher, connection, address, use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile) except BaseException as e: logger.error("cannot start TCP session" + str(e) + ' ' + repr(address)) connection.close() continue connection = session._connection connection.setblocking(False) self.fd_to_session[connection.fileno()] = session poller.register(connection, READ_ONLY) continue # existing session session = self.fd_to_session[fd] s = session._connection # non-blocking handshake try: check_do_handshake(session) except BaseException as e: # logger.error('handshake failure:' + str(e) + ' ' + repr(session.address)) stop_session(fd) continue # anti DOS now = time.time() if now - session.time < 0.01: continue # Read input messages. if flag & (select.POLLIN | select.POLLPRI): try: data = s.recv(self.buffer_size) except ssl.SSLError as x: if x.args[0] == ssl.SSL_ERROR_WANT_READ: pass elif x.args[0] == ssl.SSL_ERROR_SSL: pass else: logger.error('SSL recv error:' + repr(x)) continue except socket.error as x: if x.args[0] != 104: logger.error('recv error: ' + repr(x) + ' %d' % fd) stop_session(fd) continue except ValueError as e: logger.error('recv error: ' + str(e) + ' %d' % fd) stop_session(fd) continue if data: session.message += data if len(data) == self.buffer_size: redo.append((fd, flag)) if not data: stop_session(fd) continue elif flag & select.POLLHUP: print_log('client hung up', session.address) stop_session(fd) elif flag & select.POLLOUT: # Socket is ready to send data, if there is any to send. if session.retry_msg: next_msg = session.retry_msg else: try: next_msg = session.response_queue.get_nowait() except queue.Empty: continue try: sent = s.send(next_msg) except socket.error as x: logger.error("send error:" + str(x)) stop_session(fd) continue session.retry_msg = next_msg[sent:] elif flag & select.POLLERR: print_log('handling exceptional condition for', session.address) stop_session(fd) elif flag & select.POLLNVAL: print_log('invalid request', session.address) stop_session(fd) print_log('TCP thread terminating', self.shared.stopped())
def run(self): while not self.shared.stopped(): try: session, request = self.queue.get(True, timeout=1) msg_id = request.get('id') except: continue if session.stopped(): continue try: result = self.process(request) self.push_response(session, {'id': msg_id, 'result': result}) except BaseException, e: self.push_response(session, {'id': msg_id, 'error': str(e)}) except: logger.error("process error", exc_info=True) self.push_response(session, {'id': msg_id, 'error': 'unknown error'}) self.close() class Dispatcher: def __init__(self, config): self.shared = Shared(config) self.request_dispatcher = RequestDispatcher(self.shared) self.request_dispatcher.start() self.response_dispatcher = \ ResponseDispatcher(self.shared, self.request_dispatcher) self.response_dispatcher.start() def register(self, prefix, processor):
def run(self): for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, cannonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.setblocking(0) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except socket.error: sock = None continue try: sock.bind(sa) sock.listen(5) except socket.error: sock.close() sock = None continue break host = sa[0] if af == socket.AF_INET6: host = "[%s]" % host if sock is None: print_log("could not open " + ("SSL" if self.use_ssl else "TCP") + " socket on %s:%d" % (host, self.port)) return print_log(("SSL" if self.use_ssl else "TCP") + " server started on %s:%d" % (host, self.port)) if not self.started.called: self.started.callback(True) sock_fd = sock.fileno() poller = select.poll() poller.register(sock) def stop_session(fd): try: # unregister before we close s poller.unregister(fd) except BaseException as e: logger.error('unregister error:' + str(e)) session = self.fd_to_session.pop(fd) # this will close the socket session.stop() def check_do_handshake(session): if session.handshake: return try: session._connection.do_handshake() except ssl.SSLError as err: if err.args[0] == ssl.SSL_ERROR_WANT_READ: return elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: poller.modify(session.raw_connection, READ_WRITE) return else: raise BaseException(str(err)) poller.modify(session.raw_connection, READ_ONLY) session.handshake = True redo = [] while not self.shared.stopped(): if self.shared.paused(): sessions = self.fd_to_session.keys() if sessions: logger.info("closing %d sessions" % len(sessions)) for fd in sessions: stop_session(fd) time.sleep(1) continue if redo: events = redo redo = [] else: now = time.time() for fd, session in self.fd_to_session.items(): # Anti-DOS: wait 0.01 second between requests if now - session.time > 0.01 and session.message: cmd = session.parse_message() if not cmd: break if cmd == 'quit': data = False break session.time = now self.handle_command(cmd, session) # Anti-DOS: Stop reading if the session does not read responses if session.response_queue.empty(): mode = READ_ONLY elif session.response_queue.qsize() < 200: mode = READ_WRITE else: mode = WRITE_ONLY if mode != session.mode: poller.modify(session.raw_connection, mode) session.mode = mode # Collect garbage if now - session.time > session.timeout: stop_session(fd) try: events = poller.poll(TIMEOUT) except select.error as (code, msg): if code != errno.EINTR: raise events = [] for fd, flag in events: # open new session if fd == sock_fd: if flag & (select.POLLIN | select.POLLPRI): try: connection, address = sock.accept() session = TcpSession(self.dispatcher, connection, address, use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile) except BaseException as e: logger.error("cannot start TCP session" + str(e) + ' ' + repr(address)) connection.close() continue connection = session._connection connection.setblocking(False) self.fd_to_session[connection.fileno()] = session poller.register(connection, READ_ONLY) continue # existing session session = self.fd_to_session[fd] s = session._connection # non-blocking handshake try: check_do_handshake(session) except BaseException as e: # logger.error('handshake failure:' + str(e) + ' ' + repr(session.address)) stop_session(fd) continue # anti DOS now = time.time() if now - session.time < 0.01: continue # Read input messages. if flag & (select.POLLIN | select.POLLPRI): try: data = s.recv(self.buffer_size) except ssl.SSLError as x: if x.args[0] == ssl.SSL_ERROR_WANT_READ: pass elif x.args[0] == ssl.SSL_ERROR_SSL: pass else: logger.error('SSL recv error:' + repr(x)) continue except socket.error as x: if x.args[0] != 104: logger.error('recv error: ' + repr(x) + ' %d' % fd) stop_session(fd) continue except ValueError as e: logger.error('recv error: ' + str(e) + ' %d' % fd) stop_session(fd) continue if data: session.message += data if len(data) == self.buffer_size: redo.append((fd, flag)) if not data: stop_session(fd) continue elif flag & select.POLLHUP: print_log('client hung up', session.address) stop_session(fd) elif flag & select.POLLOUT: # Socket is ready to send data, if there is any to send. if session.retry_msg: next_msg = session.retry_msg else: try: next_msg = session.response_queue.get_nowait() except queue.Empty: continue try: sent = s.send(next_msg) except socket.error as x: logger.error("send error:" + str(x)) stop_session(fd) continue session.retry_msg = next_msg[sent:] elif flag & select.POLLERR: print_log('handling exceptional condition for', session.address) stop_session(fd) elif flag & select.POLLNVAL: print_log('invalid request', session.address) stop_session(fd)