def stop_session(fd):
try:
# unregister before we close s
poller.unregister(fd)
except BaseException as e:
logger.error('unregister error:' + str(e))
session = self.fd_to_session.pop(fd)
# this will close the socket
session.stop()
def stop_session(fd):
try:
# unregister before we close s
poller.unregister(fd)
except BaseException as e:
logger.error('unregister error:' + str(e))
session = self.fd_to_session.pop(fd)
# this will close the socket
session.stop()
def send_response(self, response):
def default_decimal(obj):
if isinstance(obj, Decimal):
return float(obj)
raise TypeError
try:
msg = json.dumps(response, default=default_decimal) + '\n'
except BaseException as e:
logger.error('send_response:' + str(e))
return
self.response_queue.put(msg)
def send_response(self, response):
def default_decimal(obj):
if isinstance(obj, Decimal):
return float(obj)
raise TypeError
try:
msg = json.dumps(response, default=default_decimal) + '\n'
except BaseException as e:
logger.error('send_response:' + str(e))
return
self.response_queue.put(msg)
def run(self):
if self.shared is None:
raise TypeError("self.shared not set in Processor")
while not self.shared.stopped():
session, request = self.pop_request()
try:
self.do_dispatch(session, request)
except:
logger.error('dispatch', exc_info=True)
self.collect_garbage()
self.stop()
def run(self):
if self.shared is None:
raise TypeError("self.shared not set in Processor")
while not self.shared.stopped():
session, request = self.pop_request()
try:
self.do_dispatch(session, request)
except:
logger.error('dispatch', exc_info=True)
self.collect_garbage()
self.stop()
def get_block(self, block_hash):
block = self.lbrycrdd('getblock', (block_hash,))
while True:
try:
response = [self.lbrycrdd("getrawtransaction", (txid,)) for txid in block['tx']]
except:
logger.error("lbrycrdd error (getfullblock)")
self.wait_on_lbrycrdd()
continue
block['tx'] = response
return block
def get_block(self, block_hash):
block = self.lbrycrdd('getblock', (block_hash, ))
while True:
try:
response = [
self.lbrycrdd("getrawtransaction", (txid, ))
for txid in block['tx']
]
except:
logger.error("lbrycrdd error (getfullblock)")
self.wait_on_lbrycrdd()
continue
block['tx'] = response
return block
def add_request(self, session, request):
# see if we can get if from cache. if not, add request to queue
message_id = request.get('id')
try:
result = self.process(request, cache_only=False)
except BaseException as e:
print_log("Bad request from", session.address, str(type(e)), ":", str(e))
traceback.print_exc()
self.push_response(session, {'id': message_id, 'error': str(e)})
return
except:
logger.error("process error", exc_info=True)
print_log("error processing request from", session.address)
print_log(str(request))
self.push_response(session, {'id': message_id, 'error': 'unknown error'})
if result == -1:
self.queue.put((session, request))
else:
self.push_response(session, {'id': message_id, 'result': result})
def add_request(self, session, request):
# see if we can get if from cache. if not, add request to queue
message_id = request.get('id')
try:
result = self.process(request, cache_only=False)
except BaseException as e:
print_log("Bad request from", session.address, str(type(e)), ":",
str(e))
traceback.print_exc()
self.push_response(session, {'id': message_id, 'error': str(e)})
return
except:
logger.error("process error", exc_info=True)
print_log("error processing request from", session.address)
print_log(str(request))
self.push_response(session, {
'id': message_id,
'error': 'unknown error'
})
if result == -1:
self.queue.put((session, request))
else:
self.push_response(session, {'id': message_id, 'result': result})
def run(self):
while not self.shared.stopped():
try:
session, request = self.queue.get(True, timeout=1)
msg_id = request.get('id')
except:
continue
if session.stopped():
continue
try:
result = self.process(request)
self.push_response(session, {'id': msg_id, 'result': result})
except BaseException, e:
self.push_response(session, {'id': msg_id, 'error': str(e)})
except:
logger.error("process error", exc_info=True)
self.push_response(session, {
'id': msg_id,
'error': 'unknown error'
})
self.close()
class Dispatcher:
def __init__(self, config):
self.shared = Shared(config)
self.request_dispatcher = RequestDispatcher(self.shared)
self.request_dispatcher.start()
self.response_dispatcher = \
ResponseDispatcher(self.shared, self.request_dispatcher)
def run(self):
for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC,
socket.SOCK_STREAM):
af, socktype, proto, cannonname, sa = res
try:
sock = socket.socket(af, socktype, proto)
sock.setblocking(0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
except socket.error:
sock = None
continue
try:
sock.bind(sa)
sock.listen(5)
except socket.error:
sock.close()
sock = None
continue
break
host = sa[0]
if af == socket.AF_INET6:
host = "[%s]" % host
if sock is None:
print_log("could not open " + ("SSL" if self.use_ssl else "TCP") +
" socket on %s:%d" % (host, self.port))
return
print_log(("SSL" if self.use_ssl else "TCP") +
" server started on %s:%d" % (host, self.port))
sock_fd = sock.fileno()
poller = select.poll()
poller.register(sock)
def stop_session(fd):
try:
# unregister before we close s
poller.unregister(fd)
except BaseException as e:
logger.error('unregister error:' + str(e))
session = self.fd_to_session.pop(fd)
# this will close the socket
session.stop()
def check_do_handshake(session):
if session.handshake:
return
try:
session._connection.do_handshake()
except ssl.SSLError as err:
if err.args[0] == ssl.SSL_ERROR_WANT_READ:
return
elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
poller.modify(session.raw_connection, READ_WRITE)
return
else:
raise BaseException(str(err))
poller.modify(session.raw_connection, READ_ONLY)
session.handshake = True
redo = []
while not self.shared.stopped():
if self.shared.paused():
sessions = self.fd_to_session.keys()
if sessions:
logger.info("closing %d sessions" % len(sessions))
for fd in sessions:
stop_session(fd)
time.sleep(1)
continue
if redo:
events = redo
redo = []
else:
now = time.time()
for fd, session in self.fd_to_session.items():
# Anti-DOS: wait 0.01 second between requests
if now - session.time > 0.01 and session.message:
cmd = session.parse_message()
if not cmd:
break
if cmd == 'quit':
data = False
break
session.time = now
self.handle_command(cmd, session)
# Anti-DOS: Stop reading if the session does not read responses
if session.response_queue.empty():
mode = READ_ONLY
elif session.response_queue.qsize() < 200:
mode = READ_WRITE
else:
mode = WRITE_ONLY
if mode != session.mode:
poller.modify(session.raw_connection, mode)
session.mode = mode
# Collect garbage
if now - session.time > session.timeout:
stop_session(fd)
events = poller.poll(TIMEOUT)
for fd, flag in events:
# open new session
if fd == sock_fd:
if flag & (select.POLLIN | select.POLLPRI):
try:
connection, address = sock.accept()
session = TcpSession(
self.dispatcher,
connection,
address,
use_ssl=self.use_ssl,
ssl_certfile=self.ssl_certfile,
ssl_keyfile=self.ssl_keyfile)
except BaseException as e:
logger.error("cannot start TCP session" + str(e) +
' ' + repr(address))
connection.close()
continue
connection = session._connection
connection.setblocking(False)
self.fd_to_session[connection.fileno()] = session
poller.register(connection, READ_ONLY)
continue
# existing session
session = self.fd_to_session[fd]
s = session._connection
# non-blocking handshake
try:
check_do_handshake(session)
except BaseException as e:
# logger.error('handshake failure:' + str(e) + ' ' + repr(session.address))
stop_session(fd)
continue
# anti DOS
now = time.time()
if now - session.time < 0.01:
continue
# Read input messages.
if flag & (select.POLLIN | select.POLLPRI):
try:
data = s.recv(self.buffer_size)
except ssl.SSLError as x:
if x.args[0] == ssl.SSL_ERROR_WANT_READ:
pass
elif x.args[0] == ssl.SSL_ERROR_SSL:
pass
else:
logger.error('SSL recv error:' + repr(x))
continue
except socket.error as x:
if x.args[0] != 104:
logger.error('recv error: ' + repr(x) + ' %d' % fd)
stop_session(fd)
continue
except ValueError as e:
logger.error('recv error: ' + str(e) + ' %d' % fd)
stop_session(fd)
continue
if data:
session.message += data
if len(data) == self.buffer_size:
redo.append((fd, flag))
if not data:
stop_session(fd)
continue
elif flag & select.POLLHUP:
print_log('client hung up', session.address)
stop_session(fd)
elif flag & select.POLLOUT:
# Socket is ready to send data, if there is any to send.
if session.retry_msg:
next_msg = session.retry_msg
else:
try:
next_msg = session.response_queue.get_nowait()
except queue.Empty:
continue
try:
sent = s.send(next_msg)
except socket.error as x:
logger.error("send error:" + str(x))
stop_session(fd)
continue
session.retry_msg = next_msg[sent:]
elif flag & select.POLLERR:
print_log('handling exceptional condition for',
session.address)
stop_session(fd)
elif flag & select.POLLNVAL:
print_log('invalid request', session.address)
stop_session(fd)
print_log('TCP thread terminating', self.shared.stopped())
def run(self):
while not self.shared.stopped():
try:
session, request = self.queue.get(True, timeout=1)
msg_id = request.get('id')
except:
continue
if session.stopped():
continue
try:
result = self.process(request)
self.push_response(session, {'id': msg_id, 'result': result})
except BaseException, e:
self.push_response(session, {'id': msg_id, 'error': str(e)})
except:
logger.error("process error", exc_info=True)
self.push_response(session, {'id': msg_id, 'error': 'unknown error'})
self.close()
class Dispatcher:
def __init__(self, config):
self.shared = Shared(config)
self.request_dispatcher = RequestDispatcher(self.shared)
self.request_dispatcher.start()
self.response_dispatcher = \
ResponseDispatcher(self.shared, self.request_dispatcher)
self.response_dispatcher.start()
def register(self, prefix, processor):
def run(self):
for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM):
af, socktype, proto, cannonname, sa = res
try:
sock = socket.socket(af, socktype, proto)
sock.setblocking(0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
except socket.error:
sock = None
continue
try:
sock.bind(sa)
sock.listen(5)
except socket.error:
sock.close()
sock = None
continue
break
host = sa[0]
if af == socket.AF_INET6:
host = "[%s]" % host
if sock is None:
print_log("could not open " + ("SSL" if self.use_ssl else "TCP") + " socket on %s:%d" % (host, self.port))
return
print_log(("SSL" if self.use_ssl else "TCP") + " server started on %s:%d" % (host, self.port))
if not self.started.called:
self.started.callback(True)
sock_fd = sock.fileno()
poller = select.poll()
poller.register(sock)
def stop_session(fd):
try:
# unregister before we close s
poller.unregister(fd)
except BaseException as e:
logger.error('unregister error:' + str(e))
session = self.fd_to_session.pop(fd)
# this will close the socket
session.stop()
def check_do_handshake(session):
if session.handshake:
return
try:
session._connection.do_handshake()
except ssl.SSLError as err:
if err.args[0] == ssl.SSL_ERROR_WANT_READ:
return
elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
poller.modify(session.raw_connection, READ_WRITE)
return
else:
raise BaseException(str(err))
poller.modify(session.raw_connection, READ_ONLY)
session.handshake = True
redo = []
while not self.shared.stopped():
if self.shared.paused():
sessions = self.fd_to_session.keys()
if sessions:
logger.info("closing %d sessions" % len(sessions))
for fd in sessions:
stop_session(fd)
time.sleep(1)
continue
if redo:
events = redo
redo = []
else:
now = time.time()
for fd, session in self.fd_to_session.items():
# Anti-DOS: wait 0.01 second between requests
if now - session.time > 0.01 and session.message:
cmd = session.parse_message()
if not cmd:
break
if cmd == 'quit':
data = False
break
session.time = now
self.handle_command(cmd, session)
# Anti-DOS: Stop reading if the session does not read responses
if session.response_queue.empty():
mode = READ_ONLY
elif session.response_queue.qsize() < 200:
mode = READ_WRITE
else:
mode = WRITE_ONLY
if mode != session.mode:
poller.modify(session.raw_connection, mode)
session.mode = mode
# Collect garbage
if now - session.time > session.timeout:
stop_session(fd)
try:
events = poller.poll(TIMEOUT)
except select.error as (code, msg):
if code != errno.EINTR:
raise
events = []
for fd, flag in events:
# open new session
if fd == sock_fd:
if flag & (select.POLLIN | select.POLLPRI):
try:
connection, address = sock.accept()
session = TcpSession(self.dispatcher, connection, address,
use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile,
ssl_keyfile=self.ssl_keyfile)
except BaseException as e:
logger.error("cannot start TCP session" + str(e) + ' ' + repr(address))
connection.close()
continue
connection = session._connection
connection.setblocking(False)
self.fd_to_session[connection.fileno()] = session
poller.register(connection, READ_ONLY)
continue
# existing session
session = self.fd_to_session[fd]
s = session._connection
# non-blocking handshake
try:
check_do_handshake(session)
except BaseException as e:
# logger.error('handshake failure:' + str(e) + ' ' + repr(session.address))
stop_session(fd)
continue
# anti DOS
now = time.time()
if now - session.time < 0.01:
continue
# Read input messages.
if flag & (select.POLLIN | select.POLLPRI):
try:
data = s.recv(self.buffer_size)
except ssl.SSLError as x:
if x.args[0] == ssl.SSL_ERROR_WANT_READ:
pass
elif x.args[0] == ssl.SSL_ERROR_SSL:
pass
else:
logger.error('SSL recv error:' + repr(x))
continue
except socket.error as x:
if x.args[0] != 104:
logger.error('recv error: ' + repr(x) + ' %d' % fd)
stop_session(fd)
continue
except ValueError as e:
logger.error('recv error: ' + str(e) + ' %d' % fd)
stop_session(fd)
continue
if data:
session.message += data
if len(data) == self.buffer_size:
redo.append((fd, flag))
if not data:
stop_session(fd)
continue
elif flag & select.POLLHUP:
print_log('client hung up', session.address)
stop_session(fd)
elif flag & select.POLLOUT:
# Socket is ready to send data, if there is any to send.
if session.retry_msg:
next_msg = session.retry_msg
else:
try:
next_msg = session.response_queue.get_nowait()
except queue.Empty:
continue
try:
sent = s.send(next_msg)
except socket.error as x:
logger.error("send error:" + str(x))
stop_session(fd)
continue
session.retry_msg = next_msg[sent:]
elif flag & select.POLLERR:
print_log('handling exceptional condition for', session.address)
stop_session(fd)
elif flag & select.POLLNVAL:
print_log('invalid request', session.address)
stop_session(fd)
